mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-17 16:38:54 +00:00
John Kerl
GitHub
parent
98ac3f99c1
commit
1b9526e585
35 changed files with 85 additions and 92 deletions
|
|
@ -0,0 +1,8 @@
|
|||
denom
|
||||
inTerm
|
||||
inout
|
||||
iput
|
||||
nd
|
||||
nin
|
||||
numer
|
||||
Wit
|
||||
1
.github/workflows/codespell.yml
vendored
1
.github/workflows/codespell.yml
vendored
|
|
@ -32,4 +32,5 @@ jobs:
|
|||
check_filenames: true
|
||||
ignore_words_file: .codespellignore
|
||||
exclude_file: docs/src/data/english-words.txt
|
||||
exclude_file: docs/src/data/colours.csv
|
||||
|
||||
|
|
|
|||
|
|
@ -244,7 +244,7 @@ code {
|
|||
}
|
||||
|
||||
th, dl.field-list > dt {
|
||||
background-color: #ede;
|
||||
background-color: #e0d0e0;
|
||||
}
|
||||
|
||||
.warning code {
|
||||
|
|
|
|||
|
|
@ -206,7 +206,7 @@ purple square 0 91 0.259926 0.824322 0.723735 6.854221
|
|||
|
||||
### Nested JSON objects
|
||||
|
||||
Additionally, Miller can **tabularize nested objects by concatentating keys**. If your processing has
|
||||
Additionally, Miller can **tabularize nested objects by concatenating keys**. If your processing has
|
||||
input as well as output in JSON format, JSON structure is preserved throughout the processing:
|
||||
|
||||
<pre class="pre-highlight-in-pair">
|
||||
|
|
@ -466,7 +466,7 @@ resource=/some/other/path,loadsec=0.97,ok=false
|
|||
etc. and I just log them as needed. Then later, I can use `grep`, `mlr --opprint group-like`, etc.
|
||||
to analyze my logs.
|
||||
|
||||
See the [seprators page](reference-main-separators.md) regarding how to specify separators other than the default equals-sign and comma.
|
||||
See the [separators page](reference-main-separators.md) regarding how to specify separators other than the default equals-sign and comma.
|
||||
|
||||
## NIDX: Index-numbered (toolkit style)
|
||||
|
||||
|
|
@ -550,24 +550,6 @@ No help found for "format-conversion". Please try 'mlr help find format-conversi
|
|||
See also 'mlr help topics'.
|
||||
</pre>
|
||||
|
||||
<!---
|
||||
TODO: probably entirely unsupport this feature in Miller6.
|
||||
|
||||
## Autodetect of line endings
|
||||
|
||||
Default line endings (`--irs` and `--ors`) are `'auto'` which means **autodetect from the input file format**, as long as the input file(s) have lines ending in either LF (also known as linefeed, `'\n'`, `0x0a`, Unix-style) or CRLF (also known as carriage-return/linefeed pairs, `'\r\n'`, `0x0d 0x0a`, Windows style).
|
||||
|
||||
**If both IRS and ORS are auto (which is the default) then LF input will lead to LF output and CRLF input will lead to CRLF output, regardless of the platform you're running on.**
|
||||
|
||||
The line-ending autodetector triggers on the first line ending detected in the input stream. E.g. if you specify a CRLF-terminated file on the command line followed by an LF-terminated file then autodetected line endings will be CRLF.
|
||||
|
||||
If you use `--ors {something else}` with (default or explicitly specified) `--irs auto` then line endings are autodetected on input and set to what you specify on output.
|
||||
|
||||
If you use `--irs {something else}` with (default or explicitly specified) `--ors auto` then the output line endings used are LF on Unix/Linux/BSD/MacOS X, and CRLF on Windows.
|
||||
|
||||
See also the [separators page](reference-main-separators.md) for more information about record/field/pair separators.
|
||||
--->
|
||||
|
||||
## Comments in data
|
||||
|
||||
You can include comments within your data files, and either have them ignored, or passed directly through to the standard output as soon as they are encountered:
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ GENMD-EOF
|
|||
|
||||
### Nested JSON objects
|
||||
|
||||
Additionally, Miller can **tabularize nested objects by concatentating keys**. If your processing has
|
||||
Additionally, Miller can **tabularize nested objects by concatenating keys**. If your processing has
|
||||
input as well as output in JSON format, JSON structure is preserved throughout the processing:
|
||||
|
||||
GENMD-RUN-COMMAND
|
||||
|
|
@ -262,7 +262,7 @@ GENMD-EOF
|
|||
etc. and I just log them as needed. Then later, I can use `grep`, `mlr --opprint group-like`, etc.
|
||||
to analyze my logs.
|
||||
|
||||
See the [seprators page](reference-main-separators.md) regarding how to specify separators other than the default equals-sign and comma.
|
||||
See the [separators page](reference-main-separators.md) regarding how to specify separators other than the default equals-sign and comma.
|
||||
|
||||
## NIDX: Index-numbered (toolkit style)
|
||||
|
||||
|
|
@ -308,24 +308,6 @@ GENMD-RUN-COMMAND
|
|||
mlr help format-conversion
|
||||
GENMD-EOF
|
||||
|
||||
<!---
|
||||
TODO: probably entirely unsupport this feature in Miller6.
|
||||
|
||||
## Autodetect of line endings
|
||||
|
||||
Default line endings (`--irs` and `--ors`) are `'auto'` which means **autodetect from the input file format**, as long as the input file(s) have lines ending in either LF (also known as linefeed, `'\n'`, `0x0a`, Unix-style) or CRLF (also known as carriage-return/linefeed pairs, `'\r\n'`, `0x0d 0x0a`, Windows style).
|
||||
|
||||
**If both IRS and ORS are auto (which is the default) then LF input will lead to LF output and CRLF input will lead to CRLF output, regardless of the platform you're running on.**
|
||||
|
||||
The line-ending autodetector triggers on the first line ending detected in the input stream. E.g. if you specify a CRLF-terminated file on the command line followed by an LF-terminated file then autodetected line endings will be CRLF.
|
||||
|
||||
If you use `--ors {something else}` with (default or explicitly specified) `--irs auto` then line endings are autodetected on input and set to what you specify on output.
|
||||
|
||||
If you use `--irs {something else}` with (default or explicitly specified) `--ors auto` then the output line endings used are LF on Unix/Linux/BSD/MacOS X, and CRLF on Windows.
|
||||
|
||||
See also the [separators page](reference-main-separators.md) for more information about record/field/pair separators.
|
||||
--->
|
||||
|
||||
## Comments in data
|
||||
|
||||
You can include comments within your data files, and either have them ignored, or passed directly through to the standard output as soon as they are encountered:
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ end
|
|||
# Our job is to read foo.md.in and write foo.md. Then Mkdocs will create HTML
|
||||
# in the build directory. However, Mkdocs writes not foo.html but
|
||||
# foo/index.html. This is true for all pages except the base index.html. Also,
|
||||
# in this funciton we're writing raw HTML not Markdown so we can't leverage
|
||||
# in this function we're writing raw HTML not Markdown so we can't leverage
|
||||
# Mkdocs foo.md -> foo/index.html renaming. This also means we need to be
|
||||
# mindful of directory structure. Given input
|
||||
# index.md.in
|
||||
|
|
|
|||
|
|
@ -325,7 +325,7 @@ A [keyword](#keyword) used for defining a
|
|||
|
||||
A bit of callable code in the
|
||||
[Miller programming language](miller-programming-language.md)
|
||||
which takes zero or more arguments, and optionall returns a value.
|
||||
which takes zero or more arguments, and optionally returns a value.
|
||||
|
||||
See the [page on built-in functions](reference-dsl-builtin-functions.md) to see
|
||||
functions which are present in Miller.
|
||||
|
|
@ -456,7 +456,7 @@ entire [record stream](#streaming). See the
|
|||
## manpage / manual page
|
||||
|
||||
A form of on-line help which is [common in Unix-like operating
|
||||
systems](https://en.wikipedia.org/wiki/Man_page), includig MacOS and BSD
|
||||
systems](https://en.wikipedia.org/wiki/Man_page), including MacOS and BSD
|
||||
variants.
|
||||
|
||||
If you've [installed Miller](installing-miller.md) using your system's package-install tools
|
||||
|
|
|
|||
|
|
@ -309,7 +309,7 @@ A [keyword](#keyword) used for defining a
|
|||
|
||||
A bit of callable code in the
|
||||
[Miller programming language](miller-programming-language.md)
|
||||
which takes zero or more arguments, and optionall returns a value.
|
||||
which takes zero or more arguments, and optionally returns a value.
|
||||
|
||||
See the [page on built-in functions](reference-dsl-builtin-functions.md) to see
|
||||
functions which are present in Miller.
|
||||
|
|
@ -440,7 +440,7 @@ entire [record stream](#streaming). See the
|
|||
## manpage / manual page
|
||||
|
||||
A form of on-line help which is [common in Unix-like operating
|
||||
systems](https://en.wikipedia.org/wiki/Man_page), includig MacOS and BSD
|
||||
systems](https://en.wikipedia.org/wiki/Man_page), including MacOS and BSD
|
||||
variants.
|
||||
|
||||
If you've [installed Miller](installing-miller.md) using your system's package-install tools
|
||||
|
|
|
|||
|
|
@ -383,7 +383,7 @@ FILE-FORMAT FLAGS
|
|||
csv` is the same as `--ocsv`.
|
||||
|
||||
FLATTEN-UNFLATTEN FLAGS
|
||||
These flags control how Miller converts record values which are maps or arrays, when input is JSON and ouput is non-JSON (flattening) or input is non-JSON and output is JSON (unflattening).
|
||||
These flags control how Miller converts record values which are maps or arrays, when input is JSON and output is non-JSON (flattening) or input is non-JSON and output is JSON (unflattening).
|
||||
|
||||
See the Flatten/unflatten doc page for more information.
|
||||
|
||||
|
|
@ -530,7 +530,7 @@ MISCELLANEOUS FLAGS
|
|||
--nr-progress-mod {m} With m a positive integer: print filename and record
|
||||
count to os.Stderr every m input records.
|
||||
--ofmt {format} E.g. `%.18f`, `%.0f`, `%9.6e`. Please use
|
||||
sprintf-style codes for floating-point nummbers. If
|
||||
sprintf-style codes for floating-point numbers. If
|
||||
not specified, default formatting is used. See also
|
||||
the `fmtnum` function and the `format-values` verb.
|
||||
--records-per-batch {n} This is an internal parameter for maximum number of
|
||||
|
|
@ -3040,5 +3040,5 @@ SEE ALSO
|
|||
|
||||
|
||||
|
||||
2021-12-30 MILLER(1)
|
||||
2022-01-04 MILLER(1)
|
||||
</pre>
|
||||
|
|
|
|||
|
|
@ -362,7 +362,7 @@ FILE-FORMAT FLAGS
|
|||
csv` is the same as `--ocsv`.
|
||||
|
||||
FLATTEN-UNFLATTEN FLAGS
|
||||
These flags control how Miller converts record values which are maps or arrays, when input is JSON and ouput is non-JSON (flattening) or input is non-JSON and output is JSON (unflattening).
|
||||
These flags control how Miller converts record values which are maps or arrays, when input is JSON and output is non-JSON (flattening) or input is non-JSON and output is JSON (unflattening).
|
||||
|
||||
See the Flatten/unflatten doc page for more information.
|
||||
|
||||
|
|
@ -509,7 +509,7 @@ MISCELLANEOUS FLAGS
|
|||
--nr-progress-mod {m} With m a positive integer: print filename and record
|
||||
count to os.Stderr every m input records.
|
||||
--ofmt {format} E.g. `%.18f`, `%.0f`, `%9.6e`. Please use
|
||||
sprintf-style codes for floating-point nummbers. If
|
||||
sprintf-style codes for floating-point numbers. If
|
||||
not specified, default formatting is used. See also
|
||||
the `fmtnum` function and the `format-values` verb.
|
||||
--records-per-batch {n} This is an internal parameter for maximum number of
|
||||
|
|
@ -3019,4 +3019,4 @@ SEE ALSO
|
|||
|
||||
|
||||
|
||||
2021-12-30 MILLER(1)
|
||||
2022-01-04 MILLER(1)
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ Miller can do many kinds of processing on key-value-pair data using elapsed time
|
|||
## Some examples
|
||||
|
||||
This is some data from [https://community.opencellid.org](https://community.opencellid.org): approximately 40
|
||||
million records, 0.9GB compressed, 3.2GB uncommpressed.
|
||||
million records, 0.9GB compressed, 3.2GB uncompressed.
|
||||
|
||||
First we see that decompression is much cheaper than compression: 10 seconds vs. 2.5 minutes:
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ Miller can do many kinds of processing on key-value-pair data using elapsed time
|
|||
## Some examples
|
||||
|
||||
This is some data from [https://community.opencellid.org](https://community.opencellid.org): approximately 40
|
||||
million records, 0.9GB compressed, 3.2GB uncommpressed.
|
||||
million records, 0.9GB compressed, 3.2GB uncompressed.
|
||||
|
||||
First we see that decompression is much cheaper than compression: 10 seconds vs. 2.5 minutes:
|
||||
|
||||
|
|
|
|||
|
|
@ -546,7 +546,7 @@ a=wye,b=pan,i=5,x=0.573288,y=0.863624
|
|||
x_sum=2.26476
|
||||
</pre>
|
||||
|
||||
Since uninitialized out-of-stream variables default to 0 for addition/substraction and 1 for multiplication when they appear on expression right-hand sides (not quite as in `awk`, where they'd default to 0 either way), the above can be written more succinctly as
|
||||
Since uninitialized out-of-stream variables default to 0 for addition/subtraction and 1 for multiplication when they appear on expression right-hand sides (not quite as in `awk`, where they'd default to 0 either way), the above can be written more succinctly as
|
||||
|
||||
<pre class="pre-highlight-in-pair">
|
||||
<b>mlr put '</b>
|
||||
|
|
|
|||
|
|
@ -355,7 +355,7 @@ mlr put '
|
|||
' ./data/small
|
||||
GENMD-EOF
|
||||
|
||||
Since uninitialized out-of-stream variables default to 0 for addition/substraction and 1 for multiplication when they appear on expression right-hand sides (not quite as in `awk`, where they'd default to 0 either way), the above can be written more succinctly as
|
||||
Since uninitialized out-of-stream variables default to 0 for addition/subtraction and 1 for multiplication when they appear on expression right-hand sides (not quite as in `awk`, where they'd default to 0 either way), the above can be written more succinctly as
|
||||
|
||||
GENMD-RUN-COMMAND
|
||||
mlr put '
|
||||
|
|
|
|||
|
|
@ -360,7 +360,7 @@ parser not being able to efficiently disambiguate all the punctuational
|
|||
opportunities.
|
||||
|
||||
So, `emit1` can handle syntactic richness in the one thing being emitted;
|
||||
`emitf`, `emit`, and `emitp` can handle syntatic richness in the side-by-side
|
||||
`emitf`, `emit`, and `emitp` can handle syntactic richness in the side-by-side
|
||||
placement, indexing, and redirection.
|
||||
|
||||
(Mnemonic: If all you want is to insert a new record into the record stream, `emit1` is probably the one you want.)
|
||||
|
|
|
|||
|
|
@ -171,7 +171,7 @@ parser not being able to efficiently disambiguate all the punctuational
|
|||
opportunities.
|
||||
|
||||
So, `emit1` can handle syntactic richness in the one thing being emitted;
|
||||
`emitf`, `emit`, and `emitp` can handle syntatic richness in the side-by-side
|
||||
`emitf`, `emit`, and `emitp` can handle syntactic richness in the side-by-side
|
||||
placement, indexing, and redirection.
|
||||
|
||||
(Mnemonic: If all you want is to insert a new record into the record stream, `emit1` is probably the one you want.)
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ Three flags control input-scanning for numbers: `mlr -O`, `mlr -A`, and `mlr -S`
|
|||
Prefix `0x` means hexadecimal, e.g. `0xcafe`; prefix `0b` means binary, e.g.
|
||||
`0b1011`; prefix `0o` means octal, e.g. `0o377`. Numbers in data files with
|
||||
leading zeroes, e.g. `0377` or `06789`, are treated as strings in Miller,
|
||||
unless you specifiy `mlr -O`: then `0377` will scan as an octal integer (with
|
||||
unless you specify `mlr -O`: then `0377` will scan as an octal integer (with
|
||||
value 255), and `06789` will scan as a decimal integer (with value 6789).
|
||||
|
||||
If you want all numbers from data files to be treated as floats, then you may
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ Three flags control input-scanning for numbers: `mlr -O`, `mlr -A`, and `mlr -S`
|
|||
Prefix `0x` means hexadecimal, e.g. `0xcafe`; prefix `0b` means binary, e.g.
|
||||
`0b1011`; prefix `0o` means octal, e.g. `0o377`. Numbers in data files with
|
||||
leading zeroes, e.g. `0377` or `06789`, are treated as strings in Miller,
|
||||
unless you specifiy `mlr -O`: then `0377` will scan as an octal integer (with
|
||||
unless you specify `mlr -O`: then `0377` will scan as an octal integer (with
|
||||
value 255), and `06789` will scan as a decimal integer (with value 6789).
|
||||
|
||||
If you want all numbers from data files to be treated as floats, then you may
|
||||
|
|
|
|||
|
|
@ -180,7 +180,7 @@ are overridden in all cases by setting output format to `format2`.
|
|||
|
||||
## Flatten-unflatten flags
|
||||
|
||||
These flags control how Miller converts record values which are maps or arrays, when input is JSON and ouput is non-JSON (flattening) or input is non-JSON and output is JSON (unflattening).
|
||||
These flags control how Miller converts record values which are maps or arrays, when input is JSON and output is non-JSON (flattening) or input is non-JSON and output is JSON (unflattening).
|
||||
|
||||
See the Flatten/unflatten doc page for more information.
|
||||
|
||||
|
|
@ -268,7 +268,7 @@ These are flags which don't fit into any other category.
|
|||
* `--no-fflush`: Let buffered output not be written after every output record. The default is flush output after every record if the output is to the terminal, or less often if the output is to a file or a pipe. The default is a significant performance optimization for large files. Use this flag to allow less-frequent updates when output is to the terminal. This is unlikely to be a noticeable performance improvement, since direct-to-screen output for large files has its own overhead.
|
||||
* `--no-hash-records`: See --hash-records.
|
||||
* `--nr-progress-mod {m}`: With m a positive integer: print filename and record count to os.Stderr every m input records.
|
||||
* `--ofmt {format}`: E.g. `%.18f`, `%.0f`, `%9.6e`. Please use sprintf-style codes for floating-point nummbers. If not specified, default formatting is used. See also the `fmtnum` function and the `format-values` verb.
|
||||
* `--ofmt {format}`: E.g. `%.18f`, `%.0f`, `%9.6e`. Please use sprintf-style codes for floating-point numbers. If not specified, default formatting is used. See also the `fmtnum` function and the `format-values` verb.
|
||||
* `--records-per-batch {n}`: This is an internal parameter for maximum number of records in a batch size. Normally this does not need to be modified.
|
||||
* `--seed {n}`: with `n` of the form `12345678` or `0xcafefeed`. For `put`/`filter` `urand`, `urandint`, and `urand32`.
|
||||
* `--tz {timezone}`: Specify timezone, overriding `$TZ` environment variable (if any).
|
||||
|
|
|
|||
|
|
@ -26,9 +26,9 @@ Miller has three kinds of null data:
|
|||
|
||||
* **Absent (key not present)**: a field name is not present, e.g. input record is `x=1,y=2` and a `put` or `filter` expression refers to `$z`. Or, reading an out-of-stream variable which hasn't been assigned a value yet, e.g. `mlr put -q '@sum += $x; end{emit @sum}'` or `mlr put -q '@sum[$a][$b] += $x; end{emit @sum, "a", "b"}'`.
|
||||
|
||||
* **JSON null**: The main purpose of this is to support reading the `null` type in JSON files. The [Miller programming language](miller-programming-language.md) has a `null` keyword as well, so you can also write the null type using `$x = null`. Addtionally, though, when you write past the end of an array, leaving gaps -- e.g. writing `a[12]` when the array `a` has length 10 -- JSON-null is used to fill the gaps. See also the [arrays page](reference-main-arrays.md#auto-extend-and-null-gaps).
|
||||
* **JSON null**: The main purpose of this is to support reading the `null` type in JSON files. The [Miller programming language](miller-programming-language.md) has a `null` keyword as well, so you can also write the null type using `$x = null`. Additionally, though, when you write past the end of an array, leaving gaps -- e.g. writing `a[12]` when the array `a` has length 10 -- JSON-null is used to fill the gaps. See also the [arrays page](reference-main-arrays.md#auto-extend-and-null-gaps).
|
||||
|
||||
You can test these programatically using the functions `is_empty`/`is_not_empty`, `is_absent`/`is_present`, and `is_null`/`is_not_null`. For the last pair, note that null means either empty or absent. Here is a full list of such functions:
|
||||
You can test these programmatically using the functions `is_empty`/`is_not_empty`, `is_absent`/`is_present`, and `is_null`/`is_not_null`. For the last pair, note that null means either empty or absent. Here is a full list of such functions:
|
||||
|
||||
<pre class="pre-highlight-in-pair">
|
||||
<b>mlr -f | grep is_</b>
|
||||
|
|
@ -146,7 +146,7 @@ The reasoning is as follows:
|
|||
|
||||
* Empty values are explicit in the data so they should explicitly affect accumulations: `mlr put '@sum += $x'` should accumulate numeric `x` values into the sum but an empty `x`, when encountered in the input data stream, should make the sum non-numeric. To work around this you can use the `is_not_null` function as follows: `mlr put 'is_not_null($x) { @sum += $x }'`
|
||||
|
||||
* Absent stream-record values should not break accumulations, since Miller by design handles heterogenous data: the running `@sum` in `mlr put '@sum += $x'` should not be invalidated for records which have no `x`.
|
||||
* Absent stream-record values should not break accumulations, since Miller by design handles heterogeneous data: the running `@sum` in `mlr put '@sum += $x'` should not be invalidated for records which have no `x`.
|
||||
|
||||
* Absent out-of-stream-variable values are precisely what allow you to write `mlr put '@sum += $x'`. Otherwise you would have to write `mlr put 'begin{@sum = 0}; @sum += $x'` -- which is tolerable -- but for `mlr put 'begin{...}; @sum[$a][$b] += $x'` you'd have to pre-initialize `@sum` for all values of `$a` and `$b` in your input data stream, which is intolerable.
|
||||
|
||||
|
|
@ -154,7 +154,7 @@ The reasoning is as follows:
|
|||
|
||||
## Absent-test functions
|
||||
|
||||
Since absent plus absent is absent (and likewise for other operators), accumulations such as `@sum += $x` work correctly on heterogenous data, as do within-record formulas if both operands are absent. If one operand is present, you may get behavior you don't desire. To work around this -- namely, to set an output field only for records which have all the inputs present -- you can use a pattern-action block with `is_present`:
|
||||
Since absent plus absent is absent (and likewise for other operators), accumulations such as `@sum += $x` work correctly on heterogeneous data, as do within-record formulas if both operands are absent. If one operand is present, you may get behavior you don't desire. To work around this -- namely, to set an output field only for records which have all the inputs present -- you can use a pattern-action block with `is_present`:
|
||||
|
||||
<pre class="pre-highlight-in-pair">
|
||||
<b>mlr cat data/het.dkvp</b>
|
||||
|
|
|
|||
|
|
@ -10,9 +10,9 @@ Miller has three kinds of null data:
|
|||
|
||||
* **Absent (key not present)**: a field name is not present, e.g. input record is `x=1,y=2` and a `put` or `filter` expression refers to `$z`. Or, reading an out-of-stream variable which hasn't been assigned a value yet, e.g. `mlr put -q '@sum += $x; end{emit @sum}'` or `mlr put -q '@sum[$a][$b] += $x; end{emit @sum, "a", "b"}'`.
|
||||
|
||||
* **JSON null**: The main purpose of this is to support reading the `null` type in JSON files. The [Miller programming language](miller-programming-language.md) has a `null` keyword as well, so you can also write the null type using `$x = null`. Addtionally, though, when you write past the end of an array, leaving gaps -- e.g. writing `a[12]` when the array `a` has length 10 -- JSON-null is used to fill the gaps. See also the [arrays page](reference-main-arrays.md#auto-extend-and-null-gaps).
|
||||
* **JSON null**: The main purpose of this is to support reading the `null` type in JSON files. The [Miller programming language](miller-programming-language.md) has a `null` keyword as well, so you can also write the null type using `$x = null`. Additionally, though, when you write past the end of an array, leaving gaps -- e.g. writing `a[12]` when the array `a` has length 10 -- JSON-null is used to fill the gaps. See also the [arrays page](reference-main-arrays.md#auto-extend-and-null-gaps).
|
||||
|
||||
You can test these programatically using the functions `is_empty`/`is_not_empty`, `is_absent`/`is_present`, and `is_null`/`is_not_null`. For the last pair, note that null means either empty or absent. Here is a full list of such functions:
|
||||
You can test these programmatically using the functions `is_empty`/`is_not_empty`, `is_absent`/`is_present`, and `is_null`/`is_not_null`. For the last pair, note that null means either empty or absent. Here is a full list of such functions:
|
||||
|
||||
GENMD-RUN-COMMAND
|
||||
mlr -f | grep is_
|
||||
|
|
@ -70,7 +70,7 @@ The reasoning is as follows:
|
|||
|
||||
* Empty values are explicit in the data so they should explicitly affect accumulations: `mlr put '@sum += $x'` should accumulate numeric `x` values into the sum but an empty `x`, when encountered in the input data stream, should make the sum non-numeric. To work around this you can use the `is_not_null` function as follows: `mlr put 'is_not_null($x) { @sum += $x }'`
|
||||
|
||||
* Absent stream-record values should not break accumulations, since Miller by design handles heterogenous data: the running `@sum` in `mlr put '@sum += $x'` should not be invalidated for records which have no `x`.
|
||||
* Absent stream-record values should not break accumulations, since Miller by design handles heterogeneous data: the running `@sum` in `mlr put '@sum += $x'` should not be invalidated for records which have no `x`.
|
||||
|
||||
* Absent out-of-stream-variable values are precisely what allow you to write `mlr put '@sum += $x'`. Otherwise you would have to write `mlr put 'begin{@sum = 0}; @sum += $x'` -- which is tolerable -- but for `mlr put 'begin{...}; @sum[$a][$b] += $x'` you'd have to pre-initialize `@sum` for all values of `$a` and `$b` in your input data stream, which is intolerable.
|
||||
|
||||
|
|
@ -78,7 +78,7 @@ The reasoning is as follows:
|
|||
|
||||
## Absent-test functions
|
||||
|
||||
Since absent plus absent is absent (and likewise for other operators), accumulations such as `@sum += $x` work correctly on heterogenous data, as do within-record formulas if both operands are absent. If one operand is present, you may get behavior you don't desire. To work around this -- namely, to set an output field only for records which have all the inputs present -- you can use a pattern-action block with `is_present`:
|
||||
Since absent plus absent is absent (and likewise for other operators), accumulations such as `@sum += $x` work correctly on heterogeneous data, as do within-record formulas if both operands are absent. If one operand is present, you may get behavior you don't desire. To work around this -- namely, to set an output field only for records which have all the inputs present -- you can use a pattern-action block with `is_present`:
|
||||
|
||||
GENMD-RUN-COMMAND
|
||||
mlr cat data/het.dkvp
|
||||
|
|
|
|||
|
|
@ -662,7 +662,7 @@ an array. Here the map keys will be `NR` values `"1"`, `"2"`, etc.
|
|||
Why would we do this? When we're operating across all records and keeping all
|
||||
of them -- densely -- accumulating them in an array is fine. If we're only
|
||||
taking a subset -- sparsely -- and we want to retain the original `NR` as keys,
|
||||
using a map is handy, since we don't need continguous keys.
|
||||
using a map is handy, since we don't need contiguous keys.
|
||||
|
||||
<pre class="pre-highlight-in-pair">
|
||||
<b>mlr --icsv --opprint --from example.csv put -q '</b>
|
||||
|
|
|
|||
|
|
@ -393,7 +393,7 @@ an array. Here the map keys will be `NR` values `"1"`, `"2"`, etc.
|
|||
Why would we do this? When we're operating across all records and keeping all
|
||||
of them -- densely -- accumulating them in an array is fine. If we're only
|
||||
taking a subset -- sparsely -- and we want to retain the original `NR` as keys,
|
||||
using a map is handy, since we don't need continguous keys.
|
||||
using a map is handy, since we don't need contiguous keys.
|
||||
|
||||
GENMD-RUN-COMMAND
|
||||
mlr --icsv --opprint --from example.csv put -q '
|
||||
|
|
|
|||
|
|
@ -166,7 +166,7 @@ input data -- and they wait for end of input to produce their output.
|
|||
Any `end` blocks you provide will not be executed until end of stream; otherwise these
|
||||
don't want for end of stream. Similarly, if you write logic to retain all records
|
||||
(see also the page on [operating on all records](operating-on-all-records.md.in))
|
||||
these will be memory-unfriendly; otherwhise they are memory-friendly.
|
||||
these will be memory-unfriendly; otherwise they are memory-friendly.
|
||||
|
||||
Most simple operations such as `mlr put '$z = $x + $y'` are fully streaming.
|
||||
|
||||
|
|
|
|||
|
|
@ -150,7 +150,7 @@ input data -- and they wait for end of input to produce their output.
|
|||
Any `end` blocks you provide will not be executed until end of stream; otherwise these
|
||||
don't want for end of stream. Similarly, if you write logic to retain all records
|
||||
(see also the page on [operating on all records](operating-on-all-records.md.in))
|
||||
these will be memory-unfriendly; otherwhise they are memory-friendly.
|
||||
these will be memory-unfriendly; otherwise they are memory-friendly.
|
||||
|
||||
Most simple operations such as `mlr put '$z = $x + $y'` are fully streaming.
|
||||
|
||||
|
|
|
|||
|
|
@ -2358,7 +2358,7 @@ var OutputColorizationFlagSection = FlagSection{
|
|||
// FLATTEN/UNFLATTEN FLAGS
|
||||
|
||||
func FlattenUnflattenPrintInfo() {
|
||||
fmt.Println("These flags control how Miller converts record values which are maps or arrays, when input is JSON and ouput is non-JSON (flattening) or input is non-JSON and output is JSON (unflattening).")
|
||||
fmt.Println("These flags control how Miller converts record values which are maps or arrays, when input is JSON and output is non-JSON (flattening) or input is non-JSON and output is JSON (unflattening).")
|
||||
fmt.Println()
|
||||
fmt.Println("See the Flatten/unflatten doc page for more information.")
|
||||
}
|
||||
|
|
@ -2519,7 +2519,7 @@ var MiscFlagSection = FlagSection{
|
|||
{
|
||||
name: "--ofmt",
|
||||
arg: "{format}",
|
||||
help: "E.g. `%.18f`, `%.0f`, `%9.6e`. Please use sprintf-style codes for floating-point nummbers. If not specified, default formatting is used. See also the `fmtnum` function and the `format-values` verb.",
|
||||
help: "E.g. `%.18f`, `%.0f`, `%9.6e`. Please use sprintf-style codes for floating-point numbers. If not specified, default formatting is used. See also the `fmtnum` function and the `format-values` verb.",
|
||||
parser: func(args []string, argc int, pargi *int, options *TOptions) {
|
||||
CheckArgCount(args, *pargi, argc, 2)
|
||||
options.WriterOptions.FPOFMT = args[*pargi+1]
|
||||
|
|
|
|||
|
|
@ -328,7 +328,7 @@ func (node *EmitXStatementNode) Execute(state *runtime.State) (*BlockExitPayload
|
|||
// Example:
|
||||
// DSL expression: @sum[$a][$b] += $n; end { dump; emit @sum }
|
||||
// Name: "sum"
|
||||
// Values: single array contaning the map
|
||||
// Values: single array containing the map
|
||||
// {
|
||||
// "sum": {
|
||||
// "vee": {
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@
|
|||
// where the '=~' sets the captures and the "\2:\1" uses them. (Note that
|
||||
// https://github.com/johnkerl/miller/issues/388 has a better suggestion
|
||||
// which would make the captures explicit as variables, rather than implicit
|
||||
// within CST state -- regardless, the current syntax will still be supprted
|
||||
// for backward compatability and so is here to stay.) Here we make use of Go
|
||||
// within CST state -- regardless, the current syntax will still be supported
|
||||
// for backward compatibility and so is here to stay.) Here we make use of Go
|
||||
// regexp-library functions to write to, and then later interpolate from, a
|
||||
// captures array which is stored within CST state. (See the `runtime.State`
|
||||
// object.)
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ THE SOFTWARE.
|
|||
// supported specifier (i.e. there must be intervening text to match first)
|
||||
|
||||
// Local mods (johnkerl 2021-10-17): ParseTZ and strptime_tz supporting
|
||||
// Miller's idiosyncracies.
|
||||
// Miller's idiosyncrasies.
|
||||
|
||||
package strptime
|
||||
|
||||
|
|
@ -214,7 +214,7 @@ var (
|
|||
// ErrFormatMismatch means that intervening text in the strptime format string did not
|
||||
// match within the parsed string.
|
||||
ErrFormatMismatch = errors.New("date format mismatch")
|
||||
// ErrFormatUnsupported means that the format string includes unsupport percent-escapes.
|
||||
// ErrFormatUnsupported means that the format string includes unsupported percent-escapes.
|
||||
ErrFormatUnsupported = errors.New("date format contains unsupported percent-encodings")
|
||||
|
||||
formatMap = map[int]string{
|
||||
|
|
|
|||
|
|
@ -6,13 +6,13 @@ type ScanType int
|
|||
|
||||
const (
|
||||
scanTypeString ScanType = 0
|
||||
scanTypeDecimalInt = 1
|
||||
scanTypeLeadingZeroDecimalInt = 2
|
||||
scanTypeOctalInt = 3
|
||||
scanTypeLeadingZeroOctalInt = 4
|
||||
scanTypeHexInt = 5
|
||||
scanTypeBinaryInt = 6
|
||||
scanTypeMaybeFloat = 7
|
||||
scanTypeDecimalInt ScanType = 1
|
||||
scanTypeLeadingZeroDecimalInt ScanType = 2
|
||||
scanTypeOctalInt ScanType = 3
|
||||
scanTypeLeadingZeroOctalInt ScanType = 4
|
||||
scanTypeHexInt ScanType = 5
|
||||
scanTypeBinaryInt ScanType = 6
|
||||
scanTypeMaybeFloat ScanType = 7
|
||||
)
|
||||
|
||||
const typeNameString = "string"
|
||||
|
|
|
|||
|
|
@ -71,6 +71,16 @@ func transformerHeadParseCLI(
|
|||
} else if opt == "-n" {
|
||||
headCount = cli.VerbGetIntArgOrDie(verb, opt, args, &argi, argc)
|
||||
|
||||
// This is a bit of a hack. In our Getoptify routine we preprocess
|
||||
// the command line sending '-xyz' to '-x -y -z', but leaving
|
||||
// '--xyz' as-is. Also, Unix-like tools often support 'head -n4'
|
||||
// and 'tail -n4' in addition to 'head -n 4' and 'tail -n 4'. Our
|
||||
// getoptify paradigm, combined with syntax familiar to users,
|
||||
// means we get '-n -4' here. So, take the absolute value to handle this.
|
||||
if headCount < 0 {
|
||||
headCount = -headCount
|
||||
}
|
||||
|
||||
} else if opt == "-g" {
|
||||
groupByFieldNames = cli.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc)
|
||||
|
||||
|
|
|
|||
|
|
@ -71,6 +71,16 @@ func transformerTailParseCLI(
|
|||
} else if opt == "-n" {
|
||||
tailCount = cli.VerbGetIntArgOrDie(verb, opt, args, &argi, argc)
|
||||
|
||||
// This is a bit of a hack. In our Getoptify routine we preprocess
|
||||
// the command line sending '-xyz' to '-x -y -z', but leaving
|
||||
// '--xyz' as-is. Also, Unix-like tools often support 'head -n4'
|
||||
// and 'tail -n4' in addition to 'head -n 4' and 'tail -n 4'. Our
|
||||
// getoptify paradigm, combined with syntax familiar to users,
|
||||
// means we get '-n -4' here. So, take the absolute value to handle this.
|
||||
if tailCount < 0 {
|
||||
tailCount = -tailCount
|
||||
}
|
||||
|
||||
} else if opt == "-g" {
|
||||
groupByFieldNames = cli.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
// ================================================================
|
||||
// Support for things like 'num x = $a + $b' in the DSL, wherein we check types
|
||||
// at assigment time.
|
||||
// at assignment time.
|
||||
// ================================================================
|
||||
|
||||
package types
|
||||
|
|
|
|||
|
|
@ -3019,4 +3019,4 @@ SEE ALSO
|
|||
|
||||
|
||||
|
||||
2021-12-30 MILLER(1)
|
||||
2022-01-04 MILLER(1)
|
||||
|
|
|
|||
|
|
@ -2,12 +2,12 @@
|
|||
.\" Title: mlr
|
||||
.\" Author: [see the "AUTHOR" section]
|
||||
.\" Generator: ./mkman.rb
|
||||
.\" Date: 2021-12-30
|
||||
.\" Date: 2022-01-04
|
||||
.\" Manual: \ \&
|
||||
.\" Source: \ \&
|
||||
.\" Language: English
|
||||
.\"
|
||||
.TH "MILLER" "1" "2021-12-30" "\ \&" "\ \&"
|
||||
.TH "MILLER" "1" "2022-01-04" "\ \&" "\ \&"
|
||||
.\" -----------------------------------------------------------------
|
||||
.\" * Portability definitions
|
||||
.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue