diff --git a/docs/src/data/stan-example.json b/docs/src/data/stan-example.json new file mode 100644 index 000000000..992b777f8 --- /dev/null +++ b/docs/src/data/stan-example.json @@ -0,0 +1,4 @@ +{ + "shape": ["triangle", "square", "circle", "square", "triangle", "square", "triangle", "circle", "circle", "square"], + "rate": [9.8870, 0.0130, 2.9010, 7.4670, 8.5910, 9.5310, 5.8240, 4.2370, 8.3350, 8.2430] +} diff --git a/docs/src/shapes-of-data.md b/docs/src/shapes-of-data.md index 491311211..9667bbd9c 100644 --- a/docs/src/shapes-of-data.md +++ b/docs/src/shapes-of-data.md @@ -474,3 +474,94 @@ output -- as any full transpose must. Thanks to @Fravadona on [issue 321](https://github.com/johnkerl/miller/issues/321) for the original version of this recipe. + +## Columns as JSON arrays + +Some downstream tools -- for example the [Stan](https://mc-stan.org/) modeling +language -- want their input as a JSON object whose values are arrays, one +array per column, rather than the more usual array-of-records shape: + +
+{
+  "shape": ["triangle", "square", "circle"],
+  "rate":  [9.8870, 0.0130, 2.9010]
+}
+
+ +rather than + +
+[
+  {"shape": "triangle", "rate": 9.8870},
+  {"shape": "square",   "rate": 0.0130},
+  {"shape": "circle",   "rate": 2.9010}
+]
+
+ +There's no dedicated Miller file format for this -- it's just a particular +shape of JSON, and the same [out-of-stream +variable](reference-dsl-variables.md#out-of-stream-variables) technique from +the transposing example above gets you there, keying by field name first and +row number second (rather than the other way around). The +[`arrayify`](reference-dsl-builtin-functions.md#arrayify) function turns the +per-column maps (keyed `"1"`, `"2"`, ...) into real JSON arrays, and +[`emit1`](reference-dsl-output-statements.md#emit1-and-emitemitpemitf) emits the +whole thing as a single record rather than splitting it one-record-per-key +the way plain `emit` would: + +
+mlr --icsv --ojson cut -f shape,rate then put -q '
+  for (k, v in $*) {
+    @output_record[k][NR] = v;
+  }
+  end {
+    emit1 arrayify(@output_record);
+  }
+' example.csv
+
+
+[
+{
+  "shape": ["triangle", "square", "circle", "square", "triangle", "square", "triangle", "circle", "circle", "square"],
+  "rate": [9.8870, 0.0130, 2.9010, 7.4670, 8.5910, 9.5310, 5.8240, 4.2370, 8.3350, 8.2430]
+}
+]
+
+ +To go the other way -- expanding column-arrays back into one record per row +-- find the longest array in the record, then re-key by row index first and +field name second: + +
+mlr --ijson --ocsv put -q '
+  n = 0;
+  for (k, v in $*) {
+    n = max(n, length(v));
+  }
+  keys = get_keys($*);
+  for (int i = 1; i <= n; i += 1) {
+    map row = {};
+    for (k in keys) {
+      row[k] = $[k][i];
+    }
+    emit row;
+  }
+' data/stan-example.json
+
+
+shape,rate
+triangle,9.8870
+square,0.0130
+circle,2.9010
+square,7.4670
+triangle,8.5910
+square,9.5310
+triangle,5.8240
+circle,4.2370
+circle,8.3350
+square,8.2430
+
+ +Save either of these as a `.mlr` file and pull it in with `put -q -f +mkstan.mlr` or `put -q -f unstan.mlr` to reuse them without retyping. See also +[issue 392](https://github.com/johnkerl/miller/issues/392). diff --git a/docs/src/shapes-of-data.md.in b/docs/src/shapes-of-data.md.in index 0b81d4c10..227ba32ac 100644 --- a/docs/src/shapes-of-data.md.in +++ b/docs/src/shapes-of-data.md.in @@ -260,3 +260,73 @@ output -- as any full transpose must. Thanks to @Fravadona on [issue 321](https://github.com/johnkerl/miller/issues/321) for the original version of this recipe. + +## Columns as JSON arrays + +Some downstream tools -- for example the [Stan](https://mc-stan.org/) modeling +language -- want their input as a JSON object whose values are arrays, one +array per column, rather than the more usual array-of-records shape: + +GENMD-CARDIFY +{ + "shape": ["triangle", "square", "circle"], + "rate": [9.8870, 0.0130, 2.9010] +} +GENMD-EOF + +rather than + +GENMD-CARDIFY +[ + {"shape": "triangle", "rate": 9.8870}, + {"shape": "square", "rate": 0.0130}, + {"shape": "circle", "rate": 2.9010} +] +GENMD-EOF + +There's no dedicated Miller file format for this -- it's just a particular +shape of JSON, and the same [out-of-stream +variable](reference-dsl-variables.md#out-of-stream-variables) technique from +the transposing example above gets you there, keying by field name first and +row number second (rather than the other way around). The +[`arrayify`](reference-dsl-builtin-functions.md#arrayify) function turns the +per-column maps (keyed `"1"`, `"2"`, ...) into real JSON arrays, and +[`emit1`](reference-dsl-output-statements.md#emit1-and-emitemitpemitf) emits the +whole thing as a single record rather than splitting it one-record-per-key +the way plain `emit` would: + +GENMD-RUN-COMMAND +mlr --icsv --ojson cut -f shape,rate then put -q ' + for (k, v in $*) { + @output_record[k][NR] = v; + } + end { + emit1 arrayify(@output_record); + } +' example.csv +GENMD-EOF + +To go the other way -- expanding column-arrays back into one record per row +-- find the longest array in the record, then re-key by row index first and +field name second: + +GENMD-RUN-COMMAND +mlr --ijson --ocsv put -q ' + n = 0; + for (k, v in $*) { + n = max(n, length(v)); + } + keys = get_keys($*); + for (int i = 1; i <= n; i += 1) { + map row = {}; + for (k in keys) { + row[k] = $[k][i]; + } + emit row; + } +' data/stan-example.json +GENMD-EOF + +Save either of these as a `.mlr` file and pull it in with `put -q -f +mkstan.mlr` or `put -q -f unstan.mlr` to reuse them without retyping. See also +[issue 392](https://github.com/johnkerl/miller/issues/392).