diff --git a/docs/src/10min.md b/docs/src/10min.md index 7a03b2ed5..5e7f566d0 100644 --- a/docs/src/10min.md +++ b/docs/src/10min.md @@ -88,6 +88,7 @@ purple,square,false,10,91,72.3735,8.2430 mlr --icsv --ojson tail -n 2 example.csv
+[
{
"color": "yellow",
"shape": "circle",
@@ -96,7 +97,7 @@ purple,square,false,10,91,72.3735,8.2430
"index": 87,
"quantity": 63.5058,
"rate": 8.3350
-}
+},
{
"color": "purple",
"shape": "square",
@@ -106,6 +107,7 @@ purple,square,false,10,91,72.3735,8.2430
"quantity": 72.3735,
"rate": 8.2430
}
+]
You can sort on a single field:
@@ -655,6 +657,7 @@ a matter of specifying input-format and output-format flags:
mlr --json cat example.json
+[
{
"color": "yellow",
"shape": "triangle",
@@ -663,7 +666,7 @@ a matter of specifying input-format and output-format flags:
"index": 11,
"quantity": 43.6498,
"rate": 9.8870
-}
+},
{
"color": "red",
"shape": "square",
@@ -673,6 +676,7 @@ a matter of specifying input-format and output-format flags:
"quantity": 79.2778,
"rate": 0.0130
}
+]
@@ -747,6 +751,7 @@ These transformations are reversible: mlr --ijson --oxtab cat data/server-log.json | mlr --ixtab --ojson cat
+[
{
"hostname": "localhost",
"pid": 12345,
@@ -768,6 +773,7 @@ These transformations are reversible:
}
}
}
+]
See the [flatten/unflatten page](flatten-unflatten.md) for more information.
diff --git a/docs/src/file-formats.md b/docs/src/file-formats.md
index 1899d72a8..d074f7a75 100644
--- a/docs/src/file-formats.md
+++ b/docs/src/file-formats.md
@@ -37,18 +37,18 @@ TSV: same but with tabs in places of commas
JSON (array of objects):
+---------------------+
| [ |
-| { |
-| "apple": 1, | Record 1: "apple":"1", "bat":"2", "cog":"3"
-| "bat": 2, |
-| "cog": 3 |
+| { |
+| "apple": 1, | Record 1: "apple":"1", "bat":"2", "cog":"3"
+| "bat": 2, |
+| "cog": 3 |
+| }, |
+| { |
+| "dish": { | Record 2: "dish.egg":"7",
+| "egg": 7, | "dish.flint":"8", "garlic":""
+| "flint": 8 |
| }, |
-| { |
-| "dish": { | Record 2: "dish.egg":"7",
-| "egg": 7, | "dish.flint":"8", "garlic":""
-| "flint": 8 |
-| }, |
-| "garlic": "" |
-| } |
+| "garlic": "" |
+| } |
| ] |
+---------------------+
@@ -159,30 +159,34 @@ An **array of single-level objects** is, quite simply, **a table**:
mlr --json head -n 2 then cut -f color,shape data/json-example-1.json
+[
{
"color": "yellow",
"shape": "triangle"
-}
+},
{
"color": "red",
"shape": "square"
}
+]
mlr --json head -n 2 then cut -f color,u,v data/json-example-1.json
+[
{
"color": "yellow",
"u": 0.632170,
"v": 0.988721
-}
+},
{
"color": "red",
"u": 0.219668,
"v": 0.001257
}
+]
Single-level JSON data goes back and forth between JSON and tabular formats
@@ -223,6 +227,7 @@ input as well as output in JSON format, JSON structure is preserved throughout t
mlr --json --jvstack head -n 2 data/json-example-2.json
+[
{
"flag": 1,
"i": 11,
@@ -236,7 +241,7 @@ input as well as output in JSON format, JSON structure is preserved throughout t
"w": 0.436498,
"x": 5.798188
}
-}
+},
{
"flag": 1,
"i": 15,
@@ -251,6 +256,7 @@ input as well as output in JSON format, JSON structure is preserved throughout t
"x": 2.944117
}
}
+]
But if the input format is JSON and the output format is not (or vice versa) then key-concatenation applies:
diff --git a/docs/src/flatten-unflatten.md b/docs/src/flatten-unflatten.md
index 4db94d896..7a3c138d2 100644
--- a/docs/src/flatten-unflatten.md
+++ b/docs/src/flatten-unflatten.md
@@ -181,13 +181,14 @@ a,b.x,b.y
mlr --ijson --ocsv cat data/map-values.json | mlr --icsv --ojson cat
+[
{
"a": 1,
"b": {
"x": 2,
"y": 3
}
-}
+},
{
"a": 4,
"b": {
@@ -195,6 +196,7 @@ a,b.x,b.y
"y": 6
}
}
+]
## Converting arrays between JSON and non-JSON
@@ -284,14 +286,16 @@ a,b.1,b.2
mlr --ijson --ocsv cat data/array-values.json | mlr --icsv --ojson cat
+[
{
"a": 1,
"b": [2, 3]
-}
+},
{
"a": 4,
"b": [5, 6]
}
+]
## Auto-inferencing of arrays on unflatten
@@ -314,9 +318,11 @@ a.1,a.2,a.3
mlr --c2j cat data/consecutive.csv
+[
{
"a": [4, 5, 6]
}
+]
@@ -331,6 +337,7 @@ a.1,a.3,a.5 mlr --c2j cat data/non-consecutive.csv
+[
{
"a": {
"1": 4,
@@ -338,6 +345,7 @@ a.1,a.3,a.5
"5": 6
}
}
+]
## Manual control
@@ -373,16 +381,18 @@ Using JSON output, we can see that `splita` has produced an array-valued field n
mlr --icsv --ojson --from data/hostnames.csv put '$components = splita($host, ".")'
+[
{
"host": "apoapsis.east.our.org",
"status": "up",
"components": ["apoapsis", "east", "our", "org"]
-}
+},
{
"host": "nadir.west.our.org",
"status": "down",
"components": ["nadir", "west", "our", "org"]
}
+]
Using CSV output, with default auto-flatten, we get `components.1` through `components.4`:
@@ -452,18 +462,20 @@ leave `b` JSON-stringified:
mlr --ixtab --ojson json-parse -f a data/hostnames.xtab
+[
{
"host": "apoapsis.east.our.org",
"status": "up",
"a": ["apoapsis", "east", "our", "org"],
"b": "[\"apoapsis\", \"east\", \"our\", \"org\"]"
-}
+},
{
"host": "nadir.west.our.org",
"status": "down",
"a": ["nadir", "west", "our", "org"],
"b": "[\"nadir\", \"west\", \"our\", \"org\"]"
}
+]
See also the
diff --git a/docs/src/keystroke-savers.md b/docs/src/keystroke-savers.md
index 268f92b0d..c6dc27f1a 100644
--- a/docs/src/keystroke-savers.md
+++ b/docs/src/keystroke-savers.md
@@ -33,6 +33,7 @@ red square true 2 15 79.2778 0.0130
mlr --c2j head -n 2 example.csv
+[
{
"color": "yellow",
"shape": "triangle",
@@ -41,7 +42,7 @@ red square true 2 15 79.2778 0.0130
"index": 11,
"quantity": 43.6498,
"rate": 9.8870
-}
+},
{
"color": "red",
"shape": "square",
@@ -51,6 +52,7 @@ red square true 2 15 79.2778 0.0130
"quantity": 79.2778,
"rate": 0.0130
}
+]
You can get the full list [here](file-formats.md#data-conversion-keystroke-savers).
diff --git a/docs/src/manpage.md b/docs/src/manpage.md
index b9549f948..8a9170dbd 100644
--- a/docs/src/manpage.md
+++ b/docs/src/manpage.md
@@ -73,18 +73,18 @@ FILE FORMATS
JSON (array of objects):
+---------------------+
| [ |
- | { |
- | "apple": 1, | Record 1: "apple":"1", "bat":"2", "cog":"3"
- | "bat": 2, |
- | "cog": 3 |
+ | { |
+ | "apple": 1, | Record 1: "apple":"1", "bat":"2", "cog":"3"
+ | "bat": 2, |
+ | "cog": 3 |
+ | }, |
+ | { |
+ | "dish": { | Record 2: "dish.egg":"7",
+ | "egg": 7, | "dish.flint":"8", "garlic":""
+ | "flint": 8 |
| }, |
- | { |
- | "dish": { | Record 2: "dish.egg":"7",
- | "egg": 7, | "dish.flint":"8", "garlic":""
- | "flint": 8 |
- | }, |
- | "garlic": "" |
- | } |
+ | "garlic": "" |
+ | } |
| ] |
+---------------------+
@@ -360,6 +360,7 @@ FILE-FORMAT FLAGS
--gen-step, and --gen-stop values. See also the
seqgen verb, which is more useful/intuitive.
--ijson Use JSON format for input data.
+ --ijsonl Use JSON Lines format for input data.
--inidx Use NIDX format for input data.
--io {format name} Use format name for input and output data. For
example: `--io csv` is the same as `--csv`.
@@ -369,12 +370,14 @@ FILE-FORMAT FLAGS
--iusv or --iusvlite Use USV format for input data.
--ixtab Use XTAB format for input data.
--json or -j Use JSON format for input and output data.
+ --jsonl Use JSON Lines format for input and output data.
--nidx Use NIDX format for input and output data.
--oasv or --oasvlite Use ASV format for output data.
--ocsv Use CSV format for output data.
--ocsvlite Use CSV-lite format for output data.
--odkvp Use DKVP format for output data.
--ojson Use JSON format for output data.
+ --ojsonl Use JSON Lines format for output data.
--omd Use markdown-tabular format for output data.
--onidx Use NIDX format for output data.
--opprint Use PPRINT format for output data.
diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt
index f380d8898..0c56b04eb 100644
--- a/docs/src/manpage.txt
+++ b/docs/src/manpage.txt
@@ -52,18 +52,18 @@ FILE FORMATS
JSON (array of objects):
+---------------------+
| [ |
- | { |
- | "apple": 1, | Record 1: "apple":"1", "bat":"2", "cog":"3"
- | "bat": 2, |
- | "cog": 3 |
+ | { |
+ | "apple": 1, | Record 1: "apple":"1", "bat":"2", "cog":"3"
+ | "bat": 2, |
+ | "cog": 3 |
+ | }, |
+ | { |
+ | "dish": { | Record 2: "dish.egg":"7",
+ | "egg": 7, | "dish.flint":"8", "garlic":""
+ | "flint": 8 |
| }, |
- | { |
- | "dish": { | Record 2: "dish.egg":"7",
- | "egg": 7, | "dish.flint":"8", "garlic":""
- | "flint": 8 |
- | }, |
- | "garlic": "" |
- | } |
+ | "garlic": "" |
+ | } |
| ] |
+---------------------+
@@ -339,6 +339,7 @@ FILE-FORMAT FLAGS
--gen-step, and --gen-stop values. See also the
seqgen verb, which is more useful/intuitive.
--ijson Use JSON format for input data.
+ --ijsonl Use JSON Lines format for input data.
--inidx Use NIDX format for input data.
--io {format name} Use format name for input and output data. For
example: `--io csv` is the same as `--csv`.
@@ -348,12 +349,14 @@ FILE-FORMAT FLAGS
--iusv or --iusvlite Use USV format for input data.
--ixtab Use XTAB format for input data.
--json or -j Use JSON format for input and output data.
+ --jsonl Use JSON Lines format for input and output data.
--nidx Use NIDX format for input and output data.
--oasv or --oasvlite Use ASV format for output data.
--ocsv Use CSV format for output data.
--ocsvlite Use CSV-lite format for output data.
--odkvp Use DKVP format for output data.
--ojson Use JSON format for output data.
+ --ojsonl Use JSON Lines format for output data.
--omd Use markdown-tabular format for output data.
--onidx Use NIDX format for output data.
--opprint Use PPRINT format for output data.
diff --git a/docs/src/miller-programming-language.md b/docs/src/miller-programming-language.md
index 8a3770d17..ed41a4f6c 100644
--- a/docs/src/miller-programming-language.md
+++ b/docs/src/miller-programming-language.md
@@ -162,9 +162,11 @@ sum
mlr --c2j --from example.csv put -q 'begin { @sum = 0 } @sum += $quantity; end {emit @sum}'
+[
{
"sum": 652.7185
}
+]
@@ -176,10 +178,12 @@ sum '
+[
{
"count": 10,
"sum": 652.7185
}
+]
We'll see in the documentation for [stats1](reference-verbs.md#stats1) that there's a lower-keystroking way to get counts and sums of things:
@@ -188,10 +192,12 @@ We'll see in the documentation for [stats1](reference-verbs.md#stats1) that ther
mlr --c2j --from example.csv stats1 -a sum,count -f quantity
+[
{
"quantity_sum": 652.7185,
"quantity_count": 10
}
+]
So, take this sum/count example as an indication of the kinds of things you can do using Miller's programming language.
@@ -397,35 +403,39 @@ For example, you can sum up all the `$a` values across records without having to
mlr --json cat absent-example.json
+[
{
"a": 1,
"b": 2
-}
+},
{
"c": 3
-}
+},
{
"a": 4,
"b": 5
}
+]
mlr --json put '@sum_of_a += $a; end {emit @sum_of_a}' absent-example.json
+[
{
"a": 1,
"b": 2
-}
+},
{
"c": 3
-}
+},
{
"a": 4,
"b": 5
-}
+},
{
"sum_of_a": 5
}
+]
diff --git a/docs/src/new-in-miller-6.md b/docs/src/new-in-miller-6.md
index 327c57e46..757c24670 100644
--- a/docs/src/new-in-miller-6.md
+++ b/docs/src/new-in-miller-6.md
@@ -210,19 +210,23 @@ For example (see [https://github.com/johnkerl/miller/issues/178](https://github.
echo '{ "a": "0123" }' | mlr --json cat
+[
{
"a": "0123"
}
+]
echo '{ "x": 1.230, "y": 1.230000000 }' | mlr --json cat
+[
{
"x": 1.230,
"y": 1.230000000
}
+]
### Regex support for IFS and IPS
diff --git a/docs/src/operating-on-all-records.md b/docs/src/operating-on-all-records.md
index 6da5b05a1..6663f1c18 100644
--- a/docs/src/operating-on-all-records.md
+++ b/docs/src/operating-on-all-records.md
@@ -68,22 +68,24 @@ after all the input is read.
'
+[
{
"word": "apple",
"value": 37
-}
+},
{
"word": "ball",
"value": 28
-}
+},
{
"word": "cat",
"value": 54
-}
+},
{
"count": 3,
"sum": 119
}
+]
And if all we want is the final output and not the input data, we can use `put
@@ -103,10 +105,12 @@ And if all we want is the final output and not the input data, we can use `put
'
+[
{
"count": 3,
"sum": 119
}
+]
As discussed a bit more on the page on [streaming processing and memory
@@ -163,10 +167,12 @@ cat,54
"value": 54
}
}
+[
{
"count": 3,
"sum": 119
}
+]
The downside to this, of course, is that this retains all records (plus data-structure overhead) in memory, so you're limited to processing files that fit in your computer's memory. The upside, though, is that you can do random access over the records using things like
@@ -220,10 +226,12 @@ The third option is to retain records in an [array](reference-main-arrays.md), t
"value": 54
}
]
+[
{
"count": 3,
"sum": 119
}
+]
Just as with the retain-as-map approach, the downside is the overhead of
@@ -266,6 +274,8 @@ array will have [null-gaps](reference-main-arrays.md) in it:
"value": 54
}
]
+[
+]
You can index `@records` by `@count` rather than `NR` to get a contiguous array:
@@ -303,10 +313,12 @@ You can index `@records` by `@count` rather than `NR` to get a contiguous array:
"value": 54
}
]
+[
{
"count": 2,
"sum": 91
}
+]
If you use a map to retain records, then this is a non-issue: maps can retain whatever values you like:
@@ -342,10 +354,12 @@ If you use a map to retain records, then this is a non-issue: maps can retain wh
"value": 54
}
}
+[
{
"count": 2,
"sum": 91
}
+]
Do note that Miller [maps](reference-main-maps.md) preserve insertion order, so
@@ -384,10 +398,12 @@ interested in:
"1": 37,
"3": 54
}
+[
{
"count": 2,
"sum": 91
}
+]
## Sorting
diff --git a/docs/src/record-heterogeneity.md b/docs/src/record-heterogeneity.md
index 2c796d1df..3aad90c38 100644
--- a/docs/src/record-heterogeneity.md
+++ b/docs/src/record-heterogeneity.md
@@ -49,9 +49,11 @@ It has three records (written here using JSON formatting):
mlr --icsv --ojson --no-jvstack cat data/het/hom.csv
-{"a": 1, "b": 2, "c": 3}
-{"a": 4, "b": 5, "c": 6}
+[
+{"a": 1, "b": 2, "c": 3},
+{"a": 4, "b": 5, "c": 6},
{"a": 7, "b": 8, "c": 9}
+]
Here every row has the same keys, in the same order: `a,b,c`.
@@ -86,9 +88,11 @@ a,b,c
mlr --icsv --ojson --no-jvstack cat data/het/fillable.csv
-{"a": 1, "b": 2, "c": 3}
-{"a": 4, "b": "", "c": 6}
+[
+{"a": 1, "b": 2, "c": 3},
+{"a": 4, "b": "", "c": 6},
{"a": "", "b": 8, "c": 9}
+]
This example is still homogeneous, though: every row has the same keys, in the same order: `a,b,c`.
@@ -144,22 +148,24 @@ with 1) for too-long rows:
mlr --icsv --ojson --allow-ragged-csv-input cat data/het/ragged.csv
+[
{
"a": 1,
"b": 2,
"c": 3
-}
+},
{
"a": 4,
"b": 5,
"c": ""
-}
+},
{
"a": 7,
"b": 8,
"c": 9,
"4": 10
}
+]
### Irregular data
@@ -194,9 +200,11 @@ the keys:
mlr --json --no-jvstack regularize data/het/irregular.json
-{"a": 1, "b": 2, "c": 3}
-{"a": 4, "b": 5, "c": 6}
+[
+{"a": 1, "b": 2, "c": 3},
+{"a": 4, "b": 5, "c": 6},
{"a": 7, "b": 8, "c": 9}
+]
The `regularize` verb tries to re-order subsequent rows to look like the first
@@ -213,21 +221,23 @@ data for items which are present, but won't log data for items which aren't.
mlr --json cat data/het/sparse.json
+[
{
"host": "xy01.east",
"status": "running",
"volume": "/dev/sda1"
-}
+},
{
"host": "xy92.west",
"status": "running"
-}
+},
{
"purpose": "failover",
"host": "xy55.east",
"volume": "/dev/sda1",
"reimaged": true
}
+]
This data is called **sparse** (from the [data-storage term](https://en.wikipedia.org/wiki/Sparse_matrix)).
@@ -239,20 +249,21 @@ every record has the same keys:
mlr --json unsparsify data/het/sparse.json
+[
{
"host": "xy01.east",
"status": "running",
"volume": "/dev/sda1",
"purpose": "",
"reimaged": ""
-}
+},
{
"host": "xy92.west",
"status": "running",
"volume": "",
"purpose": "",
"reimaged": ""
-}
+},
{
"host": "xy55.east",
"status": "",
@@ -260,6 +271,7 @@ every record has the same keys:
"purpose": "failover",
"reimaged": true
}
+]
Since this data is now homogeneous (rectangular), it pretty-prints nicely:
diff --git a/docs/src/reference-dsl-operators.md b/docs/src/reference-dsl-operators.md
index 9690b18eb..50628c56c 100644
--- a/docs/src/reference-dsl-operators.md
+++ b/docs/src/reference-dsl-operators.md
@@ -106,6 +106,8 @@ However, in Miller 6 it has optional use for map traversal. Example:
bar.baz bar.baz +[ +]This also works on the left-hand sides of assignment statements: @@ -116,6 +118,7 @@ This also works on the left-hand sides of assignment statements: '
+[
{
"hostname": "localhost",
"pid": 12345,
@@ -137,6 +140,7 @@ This also works on the left-hand sides of assignment statements:
}
}
}
+]
A few caveats:
@@ -150,6 +154,8 @@ A few caveats:
6989 +[ +]* However (awkwardly), if you want to use `.` for map-traversal as well as string-concatenation in the same statement, you'll need to insert parentheses, as the default associativity is left-to-right: @@ -161,6 +167,8 @@ A few caveats:
(error) +[ +]
@@ -170,4 +178,6 @@ A few caveats:
GET -- api/check +[ +]diff --git a/docs/src/reference-main-data-types.md b/docs/src/reference-main-data-types.md index a773a8c42..3c74f928b 100644 --- a/docs/src/reference-main-data-types.md +++ b/docs/src/reference-main-data-types.md @@ -155,27 +155,30 @@ id,blob mlr --icsv --ojson --from data/json-in-csv.csv cat
+[
{
"id": 100,
"blob": "{\"a\":1,\"b\":[2,3,4]}"
-}
+},
{
"id": 105,
"blob": "{\"a\":6,\"b\":[7,8,9]}"
}
+]
mlr --icsv --ojson --from data/json-in-csv.csv json-parse -f blob
+[
{
"id": 100,
"blob": {
"a": 1,
"b": [2, 3, 4]
}
-}
+},
{
"id": 105,
"blob": {
@@ -183,19 +186,21 @@ id,blob
"b": [7, 8, 9]
}
}
+]
mlr --icsv --ojson --from data/json-in-csv.csv put '$blob = json_parse($blob)'
+[
{
"id": 100,
"blob": {
"a": 1,
"b": [2, 3, 4]
}
-}
+},
{
"id": 105,
"blob": {
@@ -203,6 +208,7 @@ id,blob
"b": [7, 8, 9]
}
}
+]
These have their respective operations to convert back to string: the
diff --git a/docs/src/reference-main-flag-list.md b/docs/src/reference-main-flag-list.md
index 3b03fd50e..b3f06fdc2 100644
--- a/docs/src/reference-main-flag-list.md
+++ b/docs/src/reference-main-flag-list.md
@@ -22,6 +22,7 @@ Here are flags you can use when invoking Miller. For example, when you type
mlr --icsv --ojson head -n 1 example.csv
+[
{
"color": "yellow",
"shape": "triangle",
@@ -31,6 +32,7 @@ Here are flags you can use when invoking Miller. For example, when you type
"quantity": 43.6498,
"rate": 9.8870
}
+]
the `--icsv` and `--ojson` bits are _flags_. See the [Miller command
@@ -149,6 +151,7 @@ are overridden in all cases by setting output format to `format2`.
* `--idkvp`: Use DKVP format for input data.
* `--igen`: Ignore input files and instead generate sequential numeric input using --gen-field-name, --gen-start, --gen-step, and --gen-stop values. See also the seqgen verb, which is more useful/intuitive.
* `--ijson`: Use JSON format for input data.
+* `--ijsonl`: Use JSON Lines format for input data.
* `--inidx`: Use NIDX format for input data.
* `--io {format name}`: Use format name for input and output data. For example: `--io csv` is the same as `--csv`.
* `--ipprint`: Use PPRINT format for input data.
@@ -157,12 +160,14 @@ are overridden in all cases by setting output format to `format2`.
* `--iusv or --iusvlite`: Use USV format for input data.
* `--ixtab`: Use XTAB format for input data.
* `--json or -j`: Use JSON format for input and output data.
+* `--jsonl`: Use JSON Lines format for input and output data.
* `--nidx`: Use NIDX format for input and output data.
* `--oasv or --oasvlite`: Use ASV format for output data.
* `--ocsv`: Use CSV format for output data.
* `--ocsvlite`: Use CSV-lite format for output data.
* `--odkvp`: Use DKVP format for output data.
* `--ojson`: Use JSON format for output data.
+* `--ojsonl`: Use JSON Lines format for output data.
* `--omd`: Use markdown-tabular format for output data.
* `--onidx`: Use NIDX format for output data.
* `--opprint`: Use PPRINT format for output data.
diff --git a/docs/src/reference-verbs.md b/docs/src/reference-verbs.md
index eecdc27e8..5d4c6da8d 100644
--- a/docs/src/reference-verbs.md
+++ b/docs/src/reference-verbs.md
@@ -366,72 +366,80 @@ leave off -k as well as -v.
mlr --icsv --ojson cat data/clean-whitespace.csv
+[
{
" Name ": " Ann Simons",
" Preference ": " blue "
-}
+},
{
" Name ": "Bob Wang ",
" Preference ": " red "
-}
+},
{
" Name ": " Carol Vee",
" Preference ": " yellow"
}
+]
mlr --icsv --ojson clean-whitespace -k data/clean-whitespace.csv
+[
{
"Name": " Ann Simons",
"Preference": " blue "
-}
+},
{
"Name": "Bob Wang ",
"Preference": " red "
-}
+},
{
"Name": " Carol Vee",
"Preference": " yellow"
}
+]
mlr --icsv --ojson clean-whitespace -v data/clean-whitespace.csv
+[
{
" Name ": "Ann Simons",
" Preference ": "blue"
-}
+},
{
" Name ": "Bob Wang",
" Preference ": "red"
-}
+},
{
" Name ": "Carol Vee",
" Preference ": "yellow"
}
+]
mlr --icsv --ojson clean-whitespace data/clean-whitespace.csv
+[
{
"Name": "Ann Simons",
"Preference": "blue"
-}
+},
{
"Name": "Bob Wang",
"Preference": "red"
-}
+},
{
"Name": "Carol Vee",
"Preference": "yellow"
}
+]
Function links:
@@ -2941,21 +2949,23 @@ c b a
mlr --json sort-within-records data/sort-within-records.json
+[
{
"a": 1,
"b": 2,
"c": 3
-}
+},
{
"a": 5,
"b": 4,
"c": 6
-}
+},
{
"a": 9,
"b": 8,
"c": 7
}
+]
@@ -3874,6 +3884,7 @@ Examples: mlr --json unsparsify data/sparse.json
+[
{
"a": 1,
"b": 2,
@@ -3881,7 +3892,7 @@ Examples:
"u": "",
"x": "",
"w": ""
-}
+},
{
"a": "",
"b": 2,
@@ -3889,7 +3900,7 @@ Examples:
"u": 1,
"x": "",
"w": ""
-}
+},
{
"a": 1,
"b": "",
@@ -3897,7 +3908,7 @@ Examples:
"u": "",
"x": 3,
"w": ""
-}
+},
{
"a": "",
"b": "",
@@ -3906,6 +3917,7 @@ Examples:
"x": "",
"w": 2
}
+]
diff --git a/docs/src/repl.md b/docs/src/repl.md index db1864b66..8a23605d0 100644 --- a/docs/src/repl.md +++ b/docs/src/repl.md @@ -66,6 +66,7 @@ Using `put` and `filter`, you can do the following as we've seen above:
HELLO
+[
{
"color": "yellow",
"shape": "triangle",
@@ -75,7 +76,7 @@ HELLO
"quantity": 43.6498,
"rate": 9.8870,
"qr": 4.414868008496004
-}
+},
{
"color": "red",
"shape": "square",
@@ -85,8 +86,9 @@ HELLO
"quantity": 79.2778,
"rate": 0.0130,
"qr": 6098.292307692308
-}
-GOODBYE
+}GOODBYE
+
+]
## Using Miller with the REPL
diff --git a/docs/src/scripting.md b/docs/src/scripting.md
index 8757df9d5..aa67d8db0 100644
--- a/docs/src/scripting.md
+++ b/docs/src/scripting.md
@@ -88,37 +88,41 @@ circle 3 0.3
example-shell-script --ojson example.csv
+[
{
"shape": "triangle",
"count": 3,
"count_fraction": 0.3
-}
+},
{
"shape": "square",
"count": 4,
"count_fraction": 0.4
-}
+},
{
"shape": "circle",
"count": 3,
"count_fraction": 0.3
}
+]
example-shell-script --ojson then filter '$count == 3' example.csv
+[
{
"shape": "triangle",
"count": 3,
"count_fraction": 0.3
-}
+},
{
"shape": "circle",
"count": 3,
"count_fraction": 0.3
}
+]
etc.
@@ -172,37 +176,41 @@ circle 3 0.3
example-mlr-s-script --ojson example.csv
+[
{
"shape": "triangle",
"count": 3,
"count_fraction": 0.3
-}
+},
{
"shape": "square",
"count": 4,
"count_fraction": 0.4
-}
+},
{
"shape": "circle",
"count": 3,
"count_fraction": 0.3
}
+]
example-mlr-s-script --ojson then filter '$count == 3' example.csv
+[
{
"shape": "triangle",
"count": 3,
"count_fraction": 0.3
-}
+},
{
"shape": "circle",
"count": 3,
"count_fraction": 0.3
}
+]
## Miller scripts on Windows
@@ -245,37 +253,41 @@ circle 3 0.3
mlr -s example-mlr-s-script-no-shebang --ojson example.csv
+[
{
"shape": "triangle",
"count": 3,
"count_fraction": 0.3
-}
+},
{
"shape": "square",
"count": 4,
"count_fraction": 0.4
-}
+},
{
"shape": "circle",
"count": 3,
"count_fraction": 0.3
}
+]
mlr -s example-mlr-s-script-no-shebang --ojson then filter '$count == 3' example.csv
+[
{
"shape": "triangle",
"count": 3,
"count_fraction": 0.3
-}
+},
{
"shape": "circle",
"count": 3,
"count_fraction": 0.3
}
+]
and so on. See also [Miller on Windows](miller-on-windows.md).
diff --git a/docs/src/sorting.md b/docs/src/sorting.md
index 0675d18be..73af42cb3 100644
--- a/docs/src/sorting.md
+++ b/docs/src/sorting.md
@@ -407,6 +407,7 @@ Also note that, unlike the `sort-within-record` verb with its `-r` flag,
'
+[
{
"hostname": "localhost",
"pid": 12345,
@@ -428,6 +429,7 @@ Also note that, unlike the `sort-within-record` verb with its `-r` flag,
}
}
}
+]
## Simple sorting of maps across records
@@ -529,6 +531,7 @@ recaptiulate (for reference) what `sort` with default flags already does; the th
'
+[
{
"key": "alpha",
"values": "5;2;8;6;1;4;9;10;3;7",
@@ -536,6 +539,7 @@ recaptiulate (for reference) what `sort` with default flags already does; the th
"reverse": [10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
"even_then_odd": [2, 4, 6, 8, 10, 1, 3, 5, 7, 9]
}
+]
## Custom sorting of arrays across records
diff --git a/docs/src/special-symbols-and-formatting.md b/docs/src/special-symbols-and-formatting.md
index 02388f441..668b28930 100644
--- a/docs/src/special-symbols-and-formatting.md
+++ b/docs/src/special-symbols-and-formatting.md
@@ -35,14 +35,16 @@ Likewise [JSON](file-formats.md#json):
mlr --icsv --ojson cat commas.csv
+[
{
"Name": "Xiao, Lin",
"Role": "administrator"
-}
+},
{
"Name": "Khavari, Darius",
"Role": "tester"
}
+]
For Miller's [XTAB](file-formats.md#xtab-vertical-tabular) there is no escaping for carriage returns, but commas work fine:
diff --git a/docs/src/two-pass-algorithms.md b/docs/src/two-pass-algorithms.md
index bd8b9331a..146f3a81e 100644
--- a/docs/src/two-pass-algorithms.md
+++ b/docs/src/two-pass-algorithms.md
@@ -296,57 +296,59 @@ Then
mlr --json put -q -f data/feature-count.mlr data/features.json
+[
{
"record_count": 12
-}
+},
{
"key": "qoh",
"key_counts": 8
-}
+},
{
"key": "rate",
"key_counts": 8
-}
+},
{
"key": "latency",
"key_counts": 7
-}
+},
{
"key": "name",
"key_counts": 4
-}
+},
{
"key": "uid",
"key_counts": 3
-}
+},
{
"key": "uid2",
"key_counts": 1
-}
+},
{
"key": "qoh",
"key_fraction": 0.6666666666666666
-}
+},
{
"key": "rate",
"key_fraction": 0.6666666666666666
-}
+},
{
"key": "latency",
"key_fraction": 0.5833333333333334
-}
+},
{
"key": "name",
"key_fraction": 0.3333333333333333
-}
+},
{
"key": "uid",
"key_fraction": 0.25
-}
+},
{
"key": "uid2",
"key_fraction": 0.08333333333333333
}
+]
@@ -429,6 +431,7 @@ end {
mlr --json put -q -f data/unsparsify.mlr data/sparse.json
+[
{
"a": 1,
"b": 2,
@@ -436,7 +439,7 @@ end {
"u": "",
"x": "",
"w": ""
-}
+},
{
"a": "",
"b": 2,
@@ -444,7 +447,7 @@ end {
"u": 1,
"x": "",
"w": ""
-}
+},
{
"a": 1,
"b": "",
@@ -452,7 +455,7 @@ end {
"u": "",
"x": 3,
"w": ""
-}
+},
{
"a": "",
"b": "",
@@ -461,6 +464,7 @@ end {
"x": "",
"w": 2
}
+]
diff --git a/internal/pkg/auxents/help/entry.go b/internal/pkg/auxents/help/entry.go
index 5a93c0361..8b362f4c7 100644
--- a/internal/pkg/auxents/help/entry.go
+++ b/internal/pkg/auxents/help/entry.go
@@ -357,18 +357,18 @@ TSV: same but with tabs in places of commas
JSON (array of objects):
+---------------------+
| [ |
-| { |
-| "apple": 1, | Record 1: "apple":"1", "bat":"2", "cog":"3"
-| "bat": 2, |
-| "cog": 3 |
+| { |
+| "apple": 1, | Record 1: "apple":"1", "bat":"2", "cog":"3"
+| "bat": 2, |
+| "cog": 3 |
+| }, |
+| { |
+| "dish": { | Record 2: "dish.egg":"7",
+| "egg": 7, | "dish.flint":"8", "garlic":""
+| "flint": 8 |
| }, |
-| { |
-| "dish": { | Record 2: "dish.egg":"7",
-| "egg": 7, | "dish.flint":"8", "garlic":""
-| "flint": 8 |
-| }, |
-| "garlic": "" |
-| } |
+| "garlic": "" |
+| } |
| ] |
+---------------------+
diff --git a/internal/pkg/cli/option_parse.go b/internal/pkg/cli/option_parse.go
index 551a80666..de9fbcf9d 100644
--- a/internal/pkg/cli/option_parse.go
+++ b/internal/pkg/cli/option_parse.go
@@ -700,6 +700,15 @@ var FileFormatFlagSection = FlagSection{
},
},
+ {
+ name: "--ijsonl",
+ help: "Use JSON Lines format for input data.",
+ parser: func(args []string, argc int, pargi *int, options *TOptions) {
+ options.ReaderOptions.InputFileFormat = "json"
+ *pargi += 1
+ },
+ },
+
{
name: "--inidx",
help: "Use NIDX format for input data.",
@@ -892,6 +901,19 @@ var FileFormatFlagSection = FlagSection{
help: "Use JSON format for output data.",
parser: func(args []string, argc int, pargi *int, options *TOptions) {
options.WriterOptions.OutputFileFormat = "json"
+ options.WriterOptions.WrapJSONOutputInOuterList = true
+ options.WriterOptions.JSONOutputMultiline = true
+ *pargi += 1
+ },
+ },
+
+ {
+ name: "--ojsonl",
+ help: "Use JSON Lines format for output data.",
+ parser: func(args []string, argc int, pargi *int, options *TOptions) {
+ options.WriterOptions.OutputFileFormat = "json"
+ options.WriterOptions.WrapJSONOutputInOuterList = false
+ options.WriterOptions.JSONOutputMultiline = false
*pargi += 1
},
},
@@ -1046,9 +1068,22 @@ var FileFormatFlagSection = FlagSection{
help: "Use JSON format for input and output data.",
altNames: []string{"-j"},
parser: func(args []string, argc int, pargi *int, options *TOptions) {
-
options.ReaderOptions.InputFileFormat = "json"
options.WriterOptions.OutputFileFormat = "json"
+ options.WriterOptions.WrapJSONOutputInOuterList = true
+ options.WriterOptions.JSONOutputMultiline = true
+ *pargi += 1
+ },
+ },
+
+ {
+ name: "--jsonl",
+ help: "Use JSON Lines format for input and output data.",
+ parser: func(args []string, argc int, pargi *int, options *TOptions) {
+ options.ReaderOptions.InputFileFormat = "json"
+ options.WriterOptions.OutputFileFormat = "json"
+ options.WriterOptions.WrapJSONOutputInOuterList = false
+ options.WriterOptions.JSONOutputMultiline = false
*pargi += 1
},
},
@@ -1202,6 +1237,8 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
parser: func(args []string, argc int, pargi *int, options *TOptions) {
options.ReaderOptions.InputFileFormat = "csv"
options.WriterOptions.OutputFileFormat = "json"
+ options.WriterOptions.WrapJSONOutputInOuterList = true
+ options.WriterOptions.JSONOutputMultiline = true
options.ReaderOptions.irsWasSpecified = true
*pargi += 1
},
@@ -1318,6 +1355,8 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
options.ReaderOptions.InputFileFormat = "csv"
options.ReaderOptions.IFS = "\t"
options.WriterOptions.OutputFileFormat = "json"
+ options.WriterOptions.WrapJSONOutputInOuterList = true
+ options.WriterOptions.JSONOutputMultiline = true
options.ReaderOptions.ifsWasSpecified = true
options.ReaderOptions.irsWasSpecified = true
*pargi += 1
@@ -1435,6 +1474,8 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
parser: func(args []string, argc int, pargi *int, options *TOptions) {
options.ReaderOptions.InputFileFormat = "dkvp"
options.WriterOptions.OutputFileFormat = "json"
+ options.WriterOptions.WrapJSONOutputInOuterList = true
+ options.WriterOptions.JSONOutputMultiline = true
*pargi += 1
},
},
@@ -1537,6 +1578,8 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
parser: func(args []string, argc int, pargi *int, options *TOptions) {
options.ReaderOptions.InputFileFormat = "nidx"
options.WriterOptions.OutputFileFormat = "json"
+ options.WriterOptions.WrapJSONOutputInOuterList = true
+ options.WriterOptions.JSONOutputMultiline = true
*pargi += 1
},
},
@@ -1762,6 +1805,8 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
options.ReaderOptions.InputFileFormat = "pprint"
options.ReaderOptions.IFS = " "
options.WriterOptions.OutputFileFormat = "json"
+ options.WriterOptions.WrapJSONOutputInOuterList = true
+ options.WriterOptions.JSONOutputMultiline = true
options.ReaderOptions.ifsWasSpecified = true
*pargi += 1
},
@@ -1856,6 +1901,8 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
parser: func(args []string, argc int, pargi *int, options *TOptions) {
options.ReaderOptions.InputFileFormat = "xtab"
options.WriterOptions.OutputFileFormat = "json"
+ options.WriterOptions.WrapJSONOutputInOuterList = true
+ options.WriterOptions.JSONOutputMultiline = true
*pargi += 1
},
},
diff --git a/man/manpage.txt b/man/manpage.txt
index f380d8898..0c56b04eb 100644
--- a/man/manpage.txt
+++ b/man/manpage.txt
@@ -52,18 +52,18 @@ FILE FORMATS
JSON (array of objects):
+---------------------+
| [ |
- | { |
- | "apple": 1, | Record 1: "apple":"1", "bat":"2", "cog":"3"
- | "bat": 2, |
- | "cog": 3 |
+ | { |
+ | "apple": 1, | Record 1: "apple":"1", "bat":"2", "cog":"3"
+ | "bat": 2, |
+ | "cog": 3 |
+ | }, |
+ | { |
+ | "dish": { | Record 2: "dish.egg":"7",
+ | "egg": 7, | "dish.flint":"8", "garlic":""
+ | "flint": 8 |
| }, |
- | { |
- | "dish": { | Record 2: "dish.egg":"7",
- | "egg": 7, | "dish.flint":"8", "garlic":""
- | "flint": 8 |
- | }, |
- | "garlic": "" |
- | } |
+ | "garlic": "" |
+ | } |
| ] |
+---------------------+
@@ -339,6 +339,7 @@ FILE-FORMAT FLAGS
--gen-step, and --gen-stop values. See also the
seqgen verb, which is more useful/intuitive.
--ijson Use JSON format for input data.
+ --ijsonl Use JSON Lines format for input data.
--inidx Use NIDX format for input data.
--io {format name} Use format name for input and output data. For
example: `--io csv` is the same as `--csv`.
@@ -348,12 +349,14 @@ FILE-FORMAT FLAGS
--iusv or --iusvlite Use USV format for input data.
--ixtab Use XTAB format for input data.
--json or -j Use JSON format for input and output data.
+ --jsonl Use JSON Lines format for input and output data.
--nidx Use NIDX format for input and output data.
--oasv or --oasvlite Use ASV format for output data.
--ocsv Use CSV format for output data.
--ocsvlite Use CSV-lite format for output data.
--odkvp Use DKVP format for output data.
--ojson Use JSON format for output data.
+ --ojsonl Use JSON Lines format for output data.
--omd Use markdown-tabular format for output data.
--onidx Use NIDX format for output data.
--opprint Use PPRINT format for output data.
diff --git a/man/mlr.1 b/man/mlr.1
index a73efa972..4ad489038 100644
--- a/man/mlr.1
+++ b/man/mlr.1
@@ -81,18 +81,18 @@ TSV: same but with tabs in places of commas
JSON (array of objects):
+---------------------+
| [ |
-| { |
-| "apple": 1, | Record 1: "apple":"1", "bat":"2", "cog":"3"
-| "bat": 2, |
-| "cog": 3 |
+| { |
+| "apple": 1, | Record 1: "apple":"1", "bat":"2", "cog":"3"
+| "bat": 2, |
+| "cog": 3 |
+| }, |
+| { |
+| "dish": { | Record 2: "dish.egg":"7",
+| "egg": 7, | "dish.flint":"8", "garlic":""
+| "flint": 8 |
| }, |
-| { |
-| "dish": { | Record 2: "dish.egg":"7",
-| "egg": 7, | "dish.flint":"8", "garlic":""
-| "flint": 8 |
-| }, |
-| "garlic": "" |
-| } |
+| "garlic": "" |
+| } |
| ] |
+---------------------+
@@ -418,6 +418,7 @@ are overridden in all cases by setting output format to `format2`.
--gen-step, and --gen-stop values. See also the
seqgen verb, which is more useful/intuitive.
--ijson Use JSON format for input data.
+--ijsonl Use JSON Lines format for input data.
--inidx Use NIDX format for input data.
--io {format name} Use format name for input and output data. For
example: `--io csv` is the same as `--csv`.
@@ -427,12 +428,14 @@ are overridden in all cases by setting output format to `format2`.
--iusv or --iusvlite Use USV format for input data.
--ixtab Use XTAB format for input data.
--json or -j Use JSON format for input and output data.
+--jsonl Use JSON Lines format for input and output data.
--nidx Use NIDX format for input and output data.
--oasv or --oasvlite Use ASV format for output data.
--ocsv Use CSV format for output data.
--ocsvlite Use CSV-lite format for output data.
--odkvp Use DKVP format for output data.
--ojson Use JSON format for output data.
+--ojsonl Use JSON Lines format for output data.
--omd Use markdown-tabular format for output data.
--onidx Use NIDX format for output data.
--opprint Use PPRINT format for output data.
diff --git a/todo.txt b/todo.txt
index 3f2a14963..a8aeb3da6 100644
--- a/todo.txt
+++ b/todo.txt
@@ -11,6 +11,7 @@ JSON LINES
https://jsonlines.org/
o glossary
* olh
+* RT cases
* contact re https://jsonlines.org/on_the_web/
----------------------------------------------------------------