diff --git a/docs/src/data/stan-example.json b/docs/src/data/stan-example.json new file mode 100644 index 000000000..992b777f8 --- /dev/null +++ b/docs/src/data/stan-example.json @@ -0,0 +1,4 @@ +{ + "shape": ["triangle", "square", "circle", "square", "triangle", "square", "triangle", "circle", "circle", "square"], + "rate": [9.8870, 0.0130, 2.9010, 7.4670, 8.5910, 9.5310, 5.8240, 4.2370, 8.3350, 8.2430] +} diff --git a/docs/src/shapes-of-data.md b/docs/src/shapes-of-data.md index 491311211..9667bbd9c 100644 --- a/docs/src/shapes-of-data.md +++ b/docs/src/shapes-of-data.md @@ -474,3 +474,94 @@ output -- as any full transpose must. Thanks to @Fravadona on [issue 321](https://github.com/johnkerl/miller/issues/321) for the original version of this recipe. + +## Columns as JSON arrays + +Some downstream tools -- for example the [Stan](https://mc-stan.org/) modeling +language -- want their input as a JSON object whose values are arrays, one +array per column, rather than the more usual array-of-records shape: + +
+{
+ "shape": ["triangle", "square", "circle"],
+ "rate": [9.8870, 0.0130, 2.9010]
+}
+
+
+rather than
+
+
+[
+ {"shape": "triangle", "rate": 9.8870},
+ {"shape": "square", "rate": 0.0130},
+ {"shape": "circle", "rate": 2.9010}
+]
+
+
+There's no dedicated Miller file format for this -- it's just a particular
+shape of JSON, and the same [out-of-stream
+variable](reference-dsl-variables.md#out-of-stream-variables) technique from
+the transposing example above gets you there, keying by field name first and
+row number second (rather than the other way around). The
+[`arrayify`](reference-dsl-builtin-functions.md#arrayify) function turns the
+per-column maps (keyed `"1"`, `"2"`, ...) into real JSON arrays, and
+[`emit1`](reference-dsl-output-statements.md#emit1-and-emitemitpemitf) emits the
+whole thing as a single record rather than splitting it one-record-per-key
+the way plain `emit` would:
+
+
+mlr --icsv --ojson cut -f shape,rate then put -q '
+ for (k, v in $*) {
+ @output_record[k][NR] = v;
+ }
+ end {
+ emit1 arrayify(@output_record);
+ }
+' example.csv
+
+
+[
+{
+ "shape": ["triangle", "square", "circle", "square", "triangle", "square", "triangle", "circle", "circle", "square"],
+ "rate": [9.8870, 0.0130, 2.9010, 7.4670, 8.5910, 9.5310, 5.8240, 4.2370, 8.3350, 8.2430]
+}
+]
+
+
+To go the other way -- expanding column-arrays back into one record per row
+-- find the longest array in the record, then re-key by row index first and
+field name second:
+
+
+mlr --ijson --ocsv put -q '
+ n = 0;
+ for (k, v in $*) {
+ n = max(n, length(v));
+ }
+ keys = get_keys($*);
+ for (int i = 1; i <= n; i += 1) {
+ map row = {};
+ for (k in keys) {
+ row[k] = $[k][i];
+ }
+ emit row;
+ }
+' data/stan-example.json
+
++shape,rate +triangle,9.8870 +square,0.0130 +circle,2.9010 +square,7.4670 +triangle,8.5910 +square,9.5310 +triangle,5.8240 +circle,4.2370 +circle,8.3350 +square,8.2430 ++ +Save either of these as a `.mlr` file and pull it in with `put -q -f +mkstan.mlr` or `put -q -f unstan.mlr` to reuse them without retyping. See also +[issue 392](https://github.com/johnkerl/miller/issues/392). diff --git a/docs/src/shapes-of-data.md.in b/docs/src/shapes-of-data.md.in index 0b81d4c10..227ba32ac 100644 --- a/docs/src/shapes-of-data.md.in +++ b/docs/src/shapes-of-data.md.in @@ -260,3 +260,73 @@ output -- as any full transpose must. Thanks to @Fravadona on [issue 321](https://github.com/johnkerl/miller/issues/321) for the original version of this recipe. + +## Columns as JSON arrays + +Some downstream tools -- for example the [Stan](https://mc-stan.org/) modeling +language -- want their input as a JSON object whose values are arrays, one +array per column, rather than the more usual array-of-records shape: + +GENMD-CARDIFY +{ + "shape": ["triangle", "square", "circle"], + "rate": [9.8870, 0.0130, 2.9010] +} +GENMD-EOF + +rather than + +GENMD-CARDIFY +[ + {"shape": "triangle", "rate": 9.8870}, + {"shape": "square", "rate": 0.0130}, + {"shape": "circle", "rate": 2.9010} +] +GENMD-EOF + +There's no dedicated Miller file format for this -- it's just a particular +shape of JSON, and the same [out-of-stream +variable](reference-dsl-variables.md#out-of-stream-variables) technique from +the transposing example above gets you there, keying by field name first and +row number second (rather than the other way around). The +[`arrayify`](reference-dsl-builtin-functions.md#arrayify) function turns the +per-column maps (keyed `"1"`, `"2"`, ...) into real JSON arrays, and +[`emit1`](reference-dsl-output-statements.md#emit1-and-emitemitpemitf) emits the +whole thing as a single record rather than splitting it one-record-per-key +the way plain `emit` would: + +GENMD-RUN-COMMAND +mlr --icsv --ojson cut -f shape,rate then put -q ' + for (k, v in $*) { + @output_record[k][NR] = v; + } + end { + emit1 arrayify(@output_record); + } +' example.csv +GENMD-EOF + +To go the other way -- expanding column-arrays back into one record per row +-- find the longest array in the record, then re-key by row index first and +field name second: + +GENMD-RUN-COMMAND +mlr --ijson --ocsv put -q ' + n = 0; + for (k, v in $*) { + n = max(n, length(v)); + } + keys = get_keys($*); + for (int i = 1; i <= n; i += 1) { + map row = {}; + for (k in keys) { + row[k] = $[k][i]; + } + emit row; + } +' data/stan-example.json +GENMD-EOF + +Save either of these as a `.mlr` file and pull it in with `put -q -f +mkstan.mlr` or `put -q -f unstan.mlr` to reuse them without retyping. See also +[issue 392](https://github.com/johnkerl/miller/issues/392).