diff --git a/docs/src/manpage.md b/docs/src/manpage.md index b12f1506b..1dd08f9ea 100644 --- a/docs/src/manpage.md +++ b/docs/src/manpage.md @@ -584,6 +584,12 @@ This is simply a copy of what you should see on running `man mlr` at a command p 1mMISCELLANEOUS FLAGS0m These are flags which don't fit into any other category. + --errors-json Emit parse errors as a JSON object to stderr instead + of a plain text message. Intended for AI agents and + scripts that branch on error kind rather than + regex-matching prose. Equivalent to setting the + `MLR_ERRORS_JSON` environment variable to a truthy + value. --fflush Force buffered output to be written after every output record. The default is flush output after every record if the output is to the terminal, or @@ -998,57 +1004,58 @@ This is simply a copy of what you should see on running `man mlr` at a command p Replaces a numeric field with a number of asterisks, allowing for cheesy bar plots. These align best with --opprint or --oxtab output format. Options: - -f {a,b,c} Field names to convert to bars. - --lo {lo} Lower-limit value for min-width bar: default '0.000000'. - --hi {hi} Upper-limit value for max-width bar: default '100.000000'. - -w {n} Bar-field width: default '40'. - --auto Automatically computes limits, ignoring --lo and --hi. - Holds all records in memory before producing any output. - -c {character} Fill character: default '*'. - -x {character} Out-of-bounds character: default '#'. - -b {character} Blank character: default '.'. + -f {a,b,c} Field names to convert to bars. + --lo {lo} Lower-limit value for min-width bar: default '0.000000'. + --hi {hi} Upper-limit value for max-width bar: default '100.000000'. + -w {n} Bar-field width: default '40'. + --auto Automatically computes limits, ignoring --lo and --hi. Holds all + records in memory before producing any output. + -c {character} Fill character: default '*'. + -x {character} Out-of-bounds character: default '#'. + -b {character} Blank character: default '.'. + -h|--help Show this message. Nominally the fill, out-of-bounds, and blank characters will be strings of length 1. However you can make them all longer if you so desire. - -h|--help Show this message. 1mbootstrap0m Usage: mlr bootstrap [options] Emits an n-sample, with replacement, of the input records. See also mlr sample and mlr shuffle. Options: - -n Number of samples to output. Defaults to number of input records. - Must be non-negative. + -n {n} Number of samples to output. Defaults to number of input records. Must + be non-negative. -h|--help Show this message. 1mcase0m Usage: mlr case [options] Uppercases strings in record keys and/or values. Options: - -k Case only keys, not keys and values. - -v Case only values, not keys and values. - -f {a,b,c} Specify which field names to case (default: all) - -u Convert to uppercase - -l Convert to lowercase - -s Convert to sentence case (capitalize first letter) - -t Convert to title case (capitalize words) - -h|--help Show this message. + -k Case only keys, not keys and values. + -v Case only values, not keys and values. + -f {a,b,c} Specify which field names to case (default: all). + -u Convert to uppercase. + -l Convert to lowercase. + -s Convert to sentence case (capitalize first letter). + -t Convert to title case (capitalize words). + -h|--help Show this message. 1mcat0m Usage: mlr cat [options] Passes input records directly to output. Most useful for format conversion. Options: -n Prepend field "n" to each record with record-counter starting at 1. - -N {name} Prepend field {name} to each record with record-counter starting at 1. - -g {a,b,c} Optional group-by-field names for counters, e.g. a,b,c + -N {name} Prepend field {name} to each record with record-counter starting at + 1. + -g {a,b,c} Optional group-by-field names for counters, e.g. a,b,c. --filename Prepend current filename to each record. --filenum Prepend current filenum (1-up) to each record. - -h|--help Show this message. + -h|--help Show this message. 1mcheck0m Usage: mlr check [options] Consumes records without printing any output, - Useful for doing a well-formatted check on input data. with the exception that warnings are printed to stderr. + Useful for doing a well-formatted check on input data. Current checks are: * Data are parseable * If any key is the empty string @@ -1064,11 +1071,11 @@ This is simply a copy of what you should see on running `man mlr` at a command p and clean_whitespace. Options: - -k|--keys-only Do not touch values. - -v|--values-only Do not touch keys. + -k|--keys-only Do not touch values. + -v|--values-only Do not touch keys. + -h|--help Show this message. It is an error to specify -k as well as -v -- to clean keys and values, leave off -k as well as -v. - -h|--help Show this message. 1mcount-distinct0m Usage: mlr count-distinct [options] @@ -1076,25 +1083,26 @@ This is simply a copy of what you should see on running `man mlr` at a command p Same as uniq -c. Options: - -f {a,b,c} Field names for distinct count. - -x {a,b,c} Field names to exclude for distinct count: use each record's others instead. - -n Show only the number of distinct values. Not compatible with -u. - -o {name} Field name for output count. Default "count". - Ignored with -u. - -u Do unlashed counts for multiple field names. With -f a,b and - without -u, computes counts for distinct combinations of a - and b field values. With -f a,b and with -u, computes counts - for distinct a field values and counts for distinct b field - values separately. + -f {a,b,c} Field names for distinct count (synonym for -g). + -g {a,b,c} Field names for distinct count. + -x {a,b,c} Field names to exclude for distinct count; use each record's other + fields instead. + -n Show only the number of distinct values. Not compatible with -u. + -o {name} Field name for output count. Default "count". Ignored with -u. + -u Do unlashed counts for multiple field names. With -f a,b and without + -u, computes counts for distinct combinations of a and b field + values. With -f a,b and with -u, computes counts for distinct a field + values and counts for distinct b field values separately. + -h|--help Show this message. 1mcount0m Usage: mlr count [options] Prints number of records, optionally grouped by distinct values for specified field names. Options: -g {a,b,c} Optional group-by-field names for counts, e.g. a,b,c - -n {n} Show only the number of distinct values. Not interesting without -g. - -o {name} Field name for output-count. Default "count". - -h|--help Show this message. + -n Show only the number of distinct values. Not interesting without -g. + -o {name} Field name for output-count. Default "count". + -h|--help Show this message. 1mcount-similar0m Usage: mlr count-similar [options] @@ -1102,22 +1110,22 @@ This is simply a copy of what you should see on running `man mlr` at a command p the number of other records having the same group-by field values. Options: -g {a,b,c} Group-by-field names for counts, e.g. a,b,c - -o {name} Field name for output-counts. Defaults to "count". - -h|--help Show this message. + -o {name} Field name for output-counts. Defaults to "count". + -h|--help Show this message. 1mcut0m Usage: mlr cut [options] Passes through input records with specified fields included/excluded. Options: - -f {a,b,c} Comma-separated field names for cut, e.g. a,b,c. - -o Retain fields in the order specified here in the argument list. - Default is to retain them in the order found in the input data. - -x|--complement Exclude, rather than include, field names specified by -f. - -r Treat field names as regular expressions. "ab", "a.*b" will - match any field name containing the substring "ab" or matching - "a.*b", respectively; anchors of the form "^ab$", "^a.*b$" may - be used. - -h|--help Show this message. + -f {a,b,c} Comma-separated field names to include or exclude, e.g. a,b,c. + -o Retain fields in the order specified by -f rather than in + input-record order. + -x|--complement Exclude, rather than include, the field names specified by -f. + -r Treat field names as regular expressions. "ab", "a.*b" will + match any field name containing the substring "ab" or matching + "a.*b", respectively; anchors of the form "^ab$", "^a.*b$" may + be used. + -h|--help Show this message. Examples: mlr cut -f hostname,status mlr cut -x -f hostname,status @@ -1129,11 +1137,11 @@ This is simply a copy of what you should see on running `man mlr` at a command p Usage: mlr decimate [options] Passes through one of every n records, optionally by category. Options: - -b Decimate by printing first of every n. - -e Decimate by printing last of every n (default). - -g {a,b,c} Optional group-by-field names for decimate counts, e.g. a,b,c. - -n {n} Decimation factor (default 10). - -h|--help Show this message. + -b Decimate by printing first of every n. + -e Decimate by printing last of every n (default). + -g {a,b,c} Optional group-by-field names for decimate counts, e.g. a,b,c. + -n {n} Decimation factor (default 10). + -h|--help Show this message. 1mfill-down0m Usage: mlr fill-down [options] @@ -1143,13 +1151,14 @@ This is simply a copy of what you should see on running `man mlr` at a command p With -a, a field is 'missing' only if it is absent. Options: - --all Operate on all fields in the input. - -a|--only-if-absent If a given record has a missing value for a given field, - fill that from the corresponding value from a previous record, if any. - By default, a 'missing' field either is absent, or has the empty-string value. - With -a, a field is 'missing' only if it is absent. - -f Field names for fill-down. - -h|--help Show this message. + --all Operate on all fields in the input. + -a|--only-if-absent If a given record has a missing value for a given field, + fill that from the corresponding value from a previous + record, if any. By default, a 'missing' field either is + absent, or has the empty-string value. With -a, a field is + 'missing' only if it is absent. + -f {a,b,c} Field names for fill-down. + -h|--help Show this message. 1mfill-empty0m Usage: mlr fill-empty [options] @@ -1157,6 +1166,7 @@ This is simply a copy of what you should see on running `man mlr` at a command p Options: -v {string} Fill-value: defaults to "N/A" -S Don't infer type -- so '-v 0' would fill string 0 not int 0. + -h|--help Show this message. 1mfilter0m Usage: mlr filter [options] {DSL expression} @@ -1165,56 +1175,53 @@ This is simply a copy of what you should see on running `man mlr` at a command p See also: https://miller.readthedocs.io/en/latest/reference-verbs Options: - -f {file name} File containing a DSL expression (see examples below). If the filename - is a directory, all *.mlr files in that directory are loaded. + -f {file name} File containing a DSL expression (see examples below). If the + filename is a directory, all *.mlr files in that directory are + loaded. + -e {expression} DSL expression to evaluate. You can use this after -f to add an + expression. Example use case: define functions/subroutines in a + file you specify with -f, then call them with an expression you + specify with -e. + -s {name=value} Predefines out-of-stream variable @name to have the given value. + Thus mlr put -s foo=97 '$column += @foo' is like mlr put 'begin + {@foo = 97} $column += @foo'. The value part is subject to + type-inferencing. May be specified more than once, e.g. -s + name1=value1 -s name2=value2. Note: the value may be an + environment variable, e.g. -s sequence=$SEQUENCE. + -x Prints records for which {expression} evaluates to false, not + true, i.e. invert the sense of the filter expression. Default + false. + -q Does not include the modified record in the output stream. + Useful for when all desired output is in begin and/or end + blocks. + -S No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. + -F No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. + -w Print warnings about things like uninitialized variables. + -W Same as -w, but exit the process if there are any warnings. + -p Prints the expression's AST (abstract syntax tree), which gives + full transparency on the precedence and associativity rules of + Miller's grammar, to stdout. + -d Like -p but uses a parenthesized-expression format for the AST. + -D Like -d but with output all on one line. + -E Echo DSL expression before printing parse-tree. + -v Same as -E -p. + -X Exit after parsing but before stream-processing. Useful with + -v/-d/-D, if you only want to look at parser information. + --explain Parse and type-check the DSL expression, report whether it is + valid, and exit without reading the input stream. Exit status is + 0 if the expression is valid and non-zero otherwise; combine + with --errors-json for a machine-readable error. + -h|--help Show this message. - -e {expression} You can use this after -f to add an expression. Example use - case: define functions/subroutines in a file you specify with -f, then call - them with an expression you specify with -e. - - (If you mix -e and -f then the expressions are evaluated in the order encountered. + If you mix -e and -f then the expressions are evaluated in the order encountered. Since the expression pieces are simply concatenated, please be sure to use intervening - semicolons to separate expressions.) + semicolons to separate expressions. - -s name=value: Predefines out-of-stream variable @name to have - Thus mlr put -s foo=97 '$column += @foo' is like - mlr put 'begin {@foo = 97} $column += @foo'. - The value part is subject to type-inferencing. - May be specified more than once, e.g. -s name1=value1 -s name2=value2. - Note: the value may be an environment variable, e.g. -s sequence=$SEQUENCE - - -x (default false) Prints records for which {expression} evaluates to false, not true, - i.e. invert the sense of the filter expression. - - -q Does not include the modified record in the output stream. - Useful for when all desired output is in begin and/or end blocks. - - -S and -F: There are no-ops in Miller 6 and above, since now type-inferencing is done - by the record-readers before filter/put is executed. Supported as no-op pass-through - flags for backward compatibility. - - -h|--help Show this message. - - Parser-info options: - - -w Print warnings about things like uninitialized variables. - - -W Same as -w, but exit the process if there are any warnings. - - -p Prints the expressions's AST (abstract syntax tree), which gives full - transparency on the precedence and associativity rules of Miller's grammar, - to stdout. - - -d Like -p but uses a parenthesized-expression format for the AST. - - -D Like -d but with output all on one line. - - -E Echo DSL expression before printing parse-tree - - -v Same as -E -p. - - -X Exit after parsing but before stream-processing. Useful with -v/-d/-D, if you - only want to look at parser information. + Parser-info options are -w, -W, -p, -d, -D, -E, -v, and -X. Records will pass the filter depending on the last bare-boolean statement in the DSL expression. That can be the result of <, ==, >, etc., the return value of a function call @@ -1252,9 +1259,9 @@ This is simply a copy of what you should see on running `man mlr` at a command p Flattens multi-level maps to single-level ones. Example: field with name 'a' and value '{"b": { "c": 4 }}' becomes name 'a.b.c' and value 4. Options: - -f Comma-separated list of field names to flatten (default all). - -s Separator, defaulting to mlr --flatsep value. - -h|--help Show this message. + -f Comma-separated list of field names to flatten (default all). + -s {string} Separator, defaulting to mlr --flatsep value. + -h|--help Show this message. 1mformat-values0m Usage: mlr format-values [options] @@ -1270,22 +1277,21 @@ This is simply a copy of what you should see on running `man mlr` at a command p undefined behavior and/or program crashes. See your system's "man printf". Options: - -i {integer format} Defaults to "%d". - Examples: "%06lld", "%08llx". - Note that Miller integers are long long so you must use - formats which apply to long long, e.g. with ll in them. - Undefined behavior results otherwise. - -f {float format} Defaults to "%f". - Examples: "%8.3lf", "%.6le". - Note that Miller floats are double-precision so you must - use formats which apply to double, e.g. with l[efg] in them. - Undefined behavior results otherwise. - -s {string format} Defaults to "%s". - Examples: "_%s", "%08s". - Note that you must use formats which apply to string, e.g. - with s in them. Undefined behavior results otherwise. + -i {integer format} Integer format string; defaults to "%d". Examples: "%06lld", + "%08llx". Note that Miller integers are long long so you + must use formats which apply to long long, e.g. with ll in + them. Undefined behavior results otherwise. + -f {float format} Float format string; defaults to "%f". Examples: "%8.3lf", + "%.6le". Note that Miller floats are double-precision so you + must use formats which apply to double, e.g. with l[efg] in + them. Undefined behavior results otherwise. + -s {string format} String format string; defaults to "%s". Examples: "_%s", + "%08s". Note that you must use formats which apply to + string, e.g. with s in them. Undefined behavior results + otherwise. -n Coerce field values autodetected as int to float, and then apply the float format. + -h|--help Show this message. 1mfraction0m Usage: mlr fraction [options] @@ -1299,34 +1305,33 @@ This is simply a copy of what you should see on running `man mlr` at a command p and emits output records. This means it produces no output until all input is read. Options: - -f {a,b,c} Field name(s) for fraction calculation - -g {d,e,f} Optional group-by-field name(s) for fraction counts - -p Produce percents [0..100], not fractions [0..1]. Output field names - end with "_percent" rather than "_fraction" - -c Produce cumulative distributions, i.e. running sums: each output - value folds in the sum of the previous for the specified group - E.g. with input records x=1 x=2 x=3 and x=4, emits output records - x=1,x_cumulative_fraction=0.1 x=2,x_cumulative_fraction=0.3 - x=3,x_cumulative_fraction=0.6 and x=4,x_cumulative_fraction=1.0 + -f {a,b,c} Field name(s) for fraction calculation + -g {d,e,f} Optional group-by-field name(s) for fraction counts + -p Produce percents [0..100], not fractions [0..1]. Output field names + end with "_percent" rather than "_fraction" + -c Produce cumulative distributions, i.e. running sums: each output + value folds in the sum of the previous for the specified group. E.g. + with input records x=1 x=2 x=3 and x=4, emits output records + x=1,x_cumulative_fraction=0.1 x=2,x_cumulative_fraction=0.3 + x=3,x_cumulative_fraction=0.6 and x=4,x_cumulative_fraction=1.0 + -h|--help Show this message. 1mgap0m Usage: mlr gap [options] Emits an empty record every n records, or when certain values change. + One of -n or -g is required. Options: - Emits an empty record every n records, or when certain values change. -g {a,b,c} Print a gap whenever values of these fields (e.g. a,b,c) changes. - -n {n} Print a gap every n records. - One of -f or -g is required. - -n is ignored if -g is present. - -h|--help Show this message. + -n {n} Print a gap every n records. Ignored if -g is present. + -h|--help Show this message. 1mgrep0m Usage: mlr grep [options] {regular expression} Passes through records which match the regular expression. Options: - -i Use case-insensitive search. - -v Invert: pass through records which do not match the regex. - -a Only grep for values, not keys and values. + -i Use case-insensitive search. + -v Invert: pass through records which do not match the regex. + -a Only grep for values, not keys and values. -h|--help Show this message. Note that "mlr filter" is more powerful, but requires you to know field names. By contrast, "mlr grep" allows you to regex-match the entire record. It does this @@ -1342,7 +1347,8 @@ This is simply a copy of what you should see on running `man mlr` at a command p 1mgroup-by0m Usage: mlr group-by [options] {comma-separated field names} - Outputs records in batches having identical values at specified field names.Options: + Outputs records in batches having identical values at specified field names. + Options: -h|--help Show this message. 1mgroup-like0m @@ -1359,21 +1365,28 @@ This is simply a copy of what you should see on running `man mlr` at a command p and \x1f. Write \\ to get a literal backslash. See also the `sub` and `ssub` verbs. Options: - -f {a,b,c} Field names to convert. - -r {regex} Regular expression for field names to convert. - -a Convert all fields. - -h|--help Show this message. + -f {a,b,c} Field names to apply substitution to. + -r {regex} Regular expression for field names to apply substitution to. + -a Apply substitution to all fields. + -h|--help Show this message. 1mhaving-fields0m Usage: mlr having-fields [options] Conditionally passes through records depending on each record's field names. Options: - --at-least {comma-separated names} - --which-are {comma-separated names} - --at-most {comma-separated names} - --all-matching {regular expression} - --any-matching {regular expression} - --none-matching {regular expression} + --at-least {comma-separated names} Pass records that have at least these field + names. + --which-are {comma-separated names} Pass records whose field names are exactly + these. + --at-most {comma-separated names} Pass records that have at most these field + names. + --all-matching {regular expression} Pass records where all field names match + the regex. + --any-matching {regular expression} Pass records where any field name matches + the regex. + --none-matching {regular expression} Pass records where no field name matches + the regex. + -h|--help Show this message. Examples: mlr having-fields --which-are amount,status,owner mlr having-fields --any-matching 'sda[0-9]' @@ -1386,40 +1399,41 @@ This is simply a copy of what you should see on running `man mlr` at a command p Without -g, ceases consuming more input (i.e. is fast) when n records have been read. Options: -g {a,b,c} Optional group-by-field names for head counts, e.g. a,b,c. - -n {n} Head-count to print. Default 10. - A negative count, e.g. -n -2, passes through all but the last n records, - optionally by category. - -h|--help Show this message. + -n {n} Head-count to print. Default 10. A negative count, e.g. -n -2, passes + through all but the last n records, optionally by category. + -h|--help Show this message. 1mhistogram0m Just a histogram. Input values < lo or > hi are not counted. Usage: mlr histogram [options] - -f {a,b,c} Value-field names for histogram counts - --lo {lo} Histogram low value - --hi {hi} Histogram high value - --nbins {n} Number of histogram bins. Defaults to 20. - --auto Automatically computes limits, ignoring --lo and --hi. - Holds all values in memory before producing any output. - -o {prefix} Prefix for output field name. Default: no prefix. - -h|--help Show this message. + Options: + -f {a,b,c} Value-field names for histogram counts. + --lo {lo} Histogram low value. + --hi {hi} Histogram high value. + --nbins {n} Number of histogram bins. Defaults to 20. + --auto Automatically computes limits, ignoring --lo and --hi. Holds all + values in memory before producing any output. + -o {prefix} Prefix for output field name. Default: no prefix. + -h|--help Show this message. 1mjson-parse0m Usage: mlr json-parse [options] Tries to convert string field values to parsed JSON, e.g. "[1,2,3]" -> [1,2,3]. Options: - -f {...} Comma-separated list of field names to json-parse (default all). - -k If supplied, then on parse fail for any cell, keep the (unparsable) - input value for the cell. - -h|--help Show this message. + -f {a,b,c} Comma-separated list of field names to json-parse (default all). + -k If supplied, then on parse fail for any cell, keep the (unparsable) + input value for the cell. + -h|--help Show this message. 1mjson-stringify0m Usage: mlr json-stringify [options] Produces string field values from field-value data, e.g. [1,2,3] -> "[1,2,3]". Options: - -f {...} Comma-separated list of field names to json-parse (default all). - --jvstack Produce multi-line JSON output. + -f {a,b,c} Comma-separated list of field names to json-stringify (default + all). + --jvstack Produce multi-line JSON output. --no-jvstack Produce single-line JSON output per record (default). - -h|--help Show this message. + -h|--help Show this message. 1mjoin0m Usage: mlr join [options] @@ -1428,34 +1442,53 @@ This is simply a copy of what you should see on running `man mlr` at a command p Functionality is essentially the same as the system "join" command, but for record streams. Options: - -f {left file name} - -j {a,b,c} Comma-separated join-field names for output - -l {a,b,c} Comma-separated join-field names for left input file; - defaults to -j values if omitted. - -r {a,b,c} Comma-separated join-field names for right input file(s); - defaults to -j values if omitted. - --lk|--left-keep-field-names {a,b,c} If supplied, this means keep only the specified field - names from the left file. Automatically includes the join-field name(s). Helpful - for when you only want a limited subset of information from the left file. - Tip: you can use --lk "": this means the left file becomes solely a row-selector - for the input files. - --lp {text} Additional prefix for non-join output field names from - the left file. Applies to paired and unpaired output records. - --rp {text} Additional prefix for non-join output field names from - the right file(s). Applies to paired and unpaired output records. - --np Do not emit paired records - --ul Emit unpaired records from the left file - --ur Emit unpaired records from the right file(s) - -s|--sorted-input Require sorted input: records must be sorted - lexically by their join-field names, else not all records will - be paired. The only likely use case for this is with a left - file which is too big to fit into system memory otherwise. - -u Enable unsorted input. (This is the default even without -u.) - In this case, the entire left file will be loaded into memory. - --prepipe {command} As in main input options; see mlr --help for details. - If you wish to use a prepipe command for the main input as well - as here, it must be specified there as well as here. - --prepipex {command} Likewise. + -f {left file name} Left file name for join. + -j {a,b,c} Comma-separated join-field names for + output. + -l {a,b,c} Comma-separated join-field names for left + input file; defaults to -j values if + omitted. + -r {a,b,c} Comma-separated join-field names for right + input file(s); defaults to -j values if + omitted. + --lk|--left-keep-field-names {a,b,c} If supplied, this means keep only the + specified field names from the left file. + Automatically includes the join-field + name(s). Helpful for when you only want a + limited subset of information from the left + file. Tip: you can use --lk "": this means + the left file becomes solely a row-selector + for the input files. + --lp {text} Additional prefix for non-join output field + names from the left file. Applies to paired + and unpaired output records. + --rp {text} Additional prefix for non-join output field + names from the right file(s). Applies to + paired and unpaired output records. + --np Do not emit paired records. + --ul Emit unpaired records from the left file. + --ur Emit unpaired records from the right + file(s). + -s|--sorted-input Require sorted input: records must be + sorted lexically by their join-field names, + else not all records will be paired. The + only likely use case for this is with a + left file which is too big to fit into + system memory otherwise. + -u Enable unsorted input. (This is the default + even without -u.) In this case, the entire + left file will be loaded into memory. + --prepipe {command} Shell command to prepipe the left-file + input through. As in main input options; + see mlr --help for details. If you wish to + use a prepipe command for the main input as + well as here, it must be specified there as + well as here. + --prepipex {command} Shell command to prepipe the left-file + input through (no shell quoting). As in + main input options; see mlr --help for + details. + -h|--help Show this message. File-format options default to those for the right file names on the Miller argument list, but may be overridden for the left file as follows. Please see the main "mlr --help" for more information on syntax for these arguments: @@ -1498,10 +1531,11 @@ This is simply a copy of what you should see on running `man mlr` at a command p Shows the least frequently occurring distinct values for specified field names. The first entry is the statistical anti-mode; the remaining are runners-up. Options: - -f {one or more comma-separated field names}. Required flag. - -n {count}. Optional flag defaulting to 10. - -b Suppress counts; show only field values. - -o {name} Field name for output count. Default "count". + -f {a,b,c} One or more comma-separated field names to group by. Required flag. + -n {count} Maximum number of results to output. Optional flag defaulting to 10. + -b Suppress counts; show only field values. + -o {name} Field name for output count. Default "count". + -h|--help Show this message. See also "mlr most-frequent". 1mmerge-fields0m @@ -1509,7 +1543,25 @@ This is simply a copy of what you should see on running `man mlr` at a command p Computes univariate statistics for each input record, accumulated across specified fields. Options: - -a {sum,count,...} Names of accumulators. One or more of: + -a {sum,count,...} Names of accumulators: one or more of the accumulators listed + below. + -f {a,b,c} Value-field names on which to compute statistics. Requires + -o. + -r {a,b,c} Regular expressions for value-field names on which to compute + statistics. Requires -o. + -c {a,b,c} Substrings for collapse mode: all fields which have the same + names after removing substrings will be accumulated together. + Please see examples below. + -i Use interpolated percentiles, like R's type=7; default like + type=1. Not sensical for string-valued fields. + -o {name} Output field basename for -f/-r. + -k Keep the input fields which contributed to the output + statistics; the default is to omit them. + -S No-op flag for backward compatibility with Miller 5. + -F No-op flag for backward compatibility with Miller 5. + -h|--help Show this message. + + Accumulators for -a: count Count instances of fields null_count Count number of empty-string/JSON-null instances per field distinct_count Count number of distinct values per field @@ -1527,17 +1579,6 @@ This is simply a copy of what you should see on running `man mlr` at a command p max Compute maximum values of specified fields minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields - -f {a,b,c} Value-field names on which to compute statistics. Requires -o. - -r {a,b,c} Regular expressions for value-field names on which to compute - statistics. Requires -o. - -c {a,b,c} Substrings for collapse mode. All fields which have the same names - after removing substrings will be accumulated together. Please see - examples below. - -i Use interpolated percentiles, like R's type=7; default like type=1. - Not sensical for string-valued fields. - -o {name} Output field basename for -f/-r. - -k Keep the input fields which contributed to the output statistics; - the default is to omit them. String-valued data make sense unless arithmetic on them is required, e.g. for sum, mean, interpolated percentiles, etc. In case of mixed data, @@ -1559,26 +1600,40 @@ This is simply a copy of what you should see on running `man mlr` at a command p Shows the most frequently occurring distinct values for specified field names. The first entry is the statistical mode; the remaining are runners-up. Options: - -f {one or more comma-separated field names}. Required flag. - -n {count}. Optional flag defaulting to 10. - -b Suppress counts; show only field values. - -o {name} Field name for output count. Default "count". + -f {a,b,c} One or more comma-separated field names to group by. Required flag. + -n {count} Maximum number of results to output. Optional flag defaulting to 10. + -b Suppress counts; show only field values. + -o {name} Field name for output count. Default "count". + -h|--help Show this message. See also "mlr least-frequent". 1mnest0m Usage: mlr nest [options] Explodes specified field values into separate fields/records, or reverses this. Options: - --explode,--implode One is required. - --values,--pairs One is required. - --across-records,--across-fields One is required. - -f {field name} Required. - -r {field names} Like -f but treat arguments as a regular expression. Match all - field names and operate on each in record order. Example: `-r '^[xy]$`'. - --nested-fs {string} Defaults to ";". Field separator for nested values. - --nested-ps {string} Defaults to ":". Pair separator for nested key-value pairs. - --evar {string} Shorthand for --explode --values --across-records --nested-fs {string} - --ivar {string} Shorthand for --implode --values --across-records --nested-fs {string} + --explode Explode field values into separate fields/records. One of + --explode or --implode is required. + --implode Reverse of --explode. One of --explode or --implode is + required. + --values Operate on field values. One of --values or --pairs is + required. + --pairs Operate on field key-value pairs. One of --values or + --pairs is required. + --across-records Explode/implode across records. One of --across-records or + --across-fields is required. + --across-fields Explode/implode across fields. One of --across-records or + --across-fields is required. + -f {field name} Required field name to operate on. + -r {field names} Like -f but treat arguments as a regular expression. Match + all field names and operate on each in record order. + Example: -r '^[xy]$'. + --nested-fs {string} Field separator for nested values. Defaults to ";". + --nested-ps {string} Pair separator for nested key-value pairs. Defaults to ":". + --evar {string} Shorthand for --explode --values --across-records + --nested-fs {string}. + --ivar {string} Shorthand for --implode --values --across-records + --nested-fs {string}. + -h|--help Show this message. Please use "mlr --usage-separator-options" for information on specifying separators. Examples: @@ -1630,56 +1685,53 @@ This is simply a copy of what you should see on running `man mlr` at a command p See also: https://miller.readthedocs.io/en/latest/reference-verbs Options: - -f {file name} File containing a DSL expression (see examples below). If the filename - is a directory, all *.mlr files in that directory are loaded. + -f {file name} File containing a DSL expression (see examples below). If the + filename is a directory, all *.mlr files in that directory are + loaded. + -e {expression} DSL expression to evaluate. You can use this after -f to add an + expression. Example use case: define functions/subroutines in a + file you specify with -f, then call them with an expression you + specify with -e. + -s {name=value} Predefines out-of-stream variable @name to have the given value. + Thus mlr put -s foo=97 '$column += @foo' is like mlr put 'begin + {@foo = 97} $column += @foo'. The value part is subject to + type-inferencing. May be specified more than once, e.g. -s + name1=value1 -s name2=value2. Note: the value may be an + environment variable, e.g. -s sequence=$SEQUENCE. + -x Prints records for which {expression} evaluates to false, not + true, i.e. invert the sense of the filter expression. Default + false. + -q Does not include the modified record in the output stream. + Useful for when all desired output is in begin and/or end + blocks. + -S No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. + -F No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. + -w Print warnings about things like uninitialized variables. + -W Same as -w, but exit the process if there are any warnings. + -p Prints the expression's AST (abstract syntax tree), which gives + full transparency on the precedence and associativity rules of + Miller's grammar, to stdout. + -d Like -p but uses a parenthesized-expression format for the AST. + -D Like -d but with output all on one line. + -E Echo DSL expression before printing parse-tree. + -v Same as -E -p. + -X Exit after parsing but before stream-processing. Useful with + -v/-d/-D, if you only want to look at parser information. + --explain Parse and type-check the DSL expression, report whether it is + valid, and exit without reading the input stream. Exit status is + 0 if the expression is valid and non-zero otherwise; combine + with --errors-json for a machine-readable error. + -h|--help Show this message. - -e {expression} You can use this after -f to add an expression. Example use - case: define functions/subroutines in a file you specify with -f, then call - them with an expression you specify with -e. - - (If you mix -e and -f then the expressions are evaluated in the order encountered. + If you mix -e and -f then the expressions are evaluated in the order encountered. Since the expression pieces are simply concatenated, please be sure to use intervening - semicolons to separate expressions.) + semicolons to separate expressions. - -s name=value: Predefines out-of-stream variable @name to have - Thus mlr put -s foo=97 '$column += @foo' is like - mlr put 'begin {@foo = 97} $column += @foo'. - The value part is subject to type-inferencing. - May be specified more than once, e.g. -s name1=value1 -s name2=value2. - Note: the value may be an environment variable, e.g. -s sequence=$SEQUENCE - - -x (default false) Prints records for which {expression} evaluates to false, not true, - i.e. invert the sense of the filter expression. - - -q Does not include the modified record in the output stream. - Useful for when all desired output is in begin and/or end blocks. - - -S and -F: There are no-ops in Miller 6 and above, since now type-inferencing is done - by the record-readers before filter/put is executed. Supported as no-op pass-through - flags for backward compatibility. - - -h|--help Show this message. - - Parser-info options: - - -w Print warnings about things like uninitialized variables. - - -W Same as -w, but exit the process if there are any warnings. - - -p Prints the expressions's AST (abstract syntax tree), which gives full - transparency on the precedence and associativity rules of Miller's grammar, - to stdout. - - -d Like -p but uses a parenthesized-expression format for the AST. - - -D Like -d but with output all on one line. - - -E Echo DSL expression before printing parse-tree - - -v Same as -E -p. - - -X Exit after parsing but before stream-processing. Useful with -v/-d/-D, if you - only want to look at parser information. + Parser-info options are -w, -W, -p, -d, -D, -E, -v, and -X. Examples: mlr --from example.csv put '$qr = $quantity * $rate' @@ -1723,16 +1775,15 @@ This is simply a copy of what you should see on running `man mlr` at a command p Usage: mlr rename [options] {old1,new1,old2,new2,...} Renames specified fields. Options: - -r Treat old field names as regular expressions. "ab", "a.*b" - will match any field name containing the substring "ab" or - matching "a.*b", respectively; anchors of the form "^ab$", - "^a.*b$" may be used. New field names may be plain strings, - or may contain capture groups of the form "\1" through - "\9". Wrapping the regex in double quotes is optional, but - is required if you wish to follow it with 'i' to indicate - case-insensitivity. - -g Do global replacement within each field name rather than - first-match replacement. + -r Treat old field names as regular expressions. "ab", "a.*b" will match + any field name containing the substring "ab" or matching "a.*b", + respectively; anchors of the form "^ab$", "^a.*b$" may be used. New + field names may be plain strings, or may contain capture groups of the + form "\1" through "\9". Wrapping the regex in double quotes is + optional, but is required if you wish to follow it with 'i' to + indicate case-insensitivity. + -g Do global replacement within each field name rather than first-match + replacement. -h|--help Show this message. Examples: mlr rename old_name,new_name @@ -1746,18 +1797,19 @@ This is simply a copy of what you should see on running `man mlr` at a command p Usage: mlr reorder [options] Moves specified names to start of record, or end of record. Options: - -e Put specified field names at record end: default is to put them at record start. + -e Put specified field names at record end: default is to put them at + record start. -f {a,b,c} Field names to reorder. - -r Treat field names as regular expressions. Matched fields are moved - to start or end in record order. Example: -r '^YYY,^XXX' puts all - YYY- and XXX-prefixed fields first (in record order), then the rest. + -r {a,b,c} Treat field names as regular expressions. Matched fields are moved to + start or end in record order. Example: -r '^YYY,^XXX' puts all YYY- + and XXX-prefixed fields first (in record order), then the rest. -b {x} Put field names specified with -f before field name specified by {x}, if any. If {x} isn't present in a given record, the specified fields will not be moved. -a {x} Put field names specified with -f after field name specified by {x}, if any. If {x} isn't present in a given record, the specified fields will not be moved. - -h|--help Show this message. + -h|--help Show this message. Examples: mlr reorder -f a,b sends input record "d=4,b=2,a=1,c=3" to "a=1,b=2,d=4,c=3". @@ -1767,11 +1819,12 @@ This is simply a copy of what you should see on running `man mlr` at a command p 1mrepeat0m Usage: mlr repeat [options] Copies input records to output records multiple times. - Options must be exactly one of the following: - -n {repeat count} Repeat each input record this many times. - -f {field name} Same, but take the repeat count from the specified - field name of each input record. - -h|--help Show this message. + Options must be exactly one of -n or -f. + Options: + -n {repeat count} Repeat each input record this many times. + -f {field name} Same as -n, but take the repeat count from the specified field + name of each input record. + -h|--help Show this message. Example: echo x=0 | mlr repeat -n 4 then put '$x=urand()' produces: @@ -1808,6 +1861,19 @@ This is simply a copy of what you should see on running `man mlr` at a command p These pivot/reshape the input data to undo the wide-to-long operation. Note: this does not work with tail -f; it produces output records only after all input records have been read. + Options: + -i {input field names} Input field names for wide-to-long reshape. + Use with -o. + -r {input field regex} Input field regex for wide-to-long reshape. + May be repeated. Use with -o. If you have + multiple regexes, please specify them using + multiple -r, since regexes can contain + commas within them. + -o {key-field name,value-field name} Output key-field and value-field names for + wide-to-long reshape. Requires -i or -r. + -s {key-field name,value-field name} Key-field and value-field names for + long-to-wide reshape. + -h|--help Show this message. Examples: @@ -1857,17 +1923,20 @@ This is simply a copy of what you should see on running `man mlr` at a command p See also mlr bootstrap and mlr shuffle. Options: -g {a,b,c} Optional: group-by-field names for samples, e.g. a,b,c. - -k {k} Required: number of records to output in total, or by group if using -g. - -h|--help Show this message. + -k {k} Required: number of records to output in total, or by group if using + -g. + -h|--help Show this message. 1msec2gmtdate0m - Usage: ../c/mlr sec2gmtdate {comma-separated list of field names} + Usage: mlr sec2gmtdate {comma-separated list of field names} Replaces a numeric field representing seconds since the epoch with the corresponding GMT year-month-day timestamp; leaves non-numbers as-is. This is nothing more than a keystroke-saver for the sec2gmtdate function: - ../c/mlr sec2gmtdate time1,time2 + mlr sec2gmtdate time1,time2 is the same as - ../c/mlr put '$time1=sec2gmtdate($time1);$time2=sec2gmtdate($time2)' + mlr put '$time1=sec2gmtdate($time1);$time2=sec2gmtdate($time2)' + Options: + -h|--help Show this message. 1msec2gmt0m Usage: mlr sec2gmt [options] {comma-separated list of field names} @@ -1878,24 +1947,32 @@ This is simply a copy of what you should see on running `man mlr` at a command p is the same as mlr put '$time1 = sec2gmt($time1); $time2 = sec2gmt($time2)' Options: - -1 through -9: format the seconds using 1..9 decimal places, respectively. - --millis Input numbers are treated as milliseconds since the epoch. - --micros Input numbers are treated as microseconds since the epoch. - --nanos Input numbers are treated as nanoseconds since the epoch. + -1 Format seconds with 1 decimal place. + -2 Format seconds with 2 decimal places. + -3 Format seconds with 3 decimal places. + -4 Format seconds with 4 decimal places. + -5 Format seconds with 5 decimal places. + -6 Format seconds with 6 decimal places. + -7 Format seconds with 7 decimal places. + -8 Format seconds with 8 decimal places. + -9 Format seconds with 9 decimal places. + --millis Input numbers are treated as milliseconds since the epoch. + --micros Input numbers are treated as microseconds since the epoch. + --nanos Input numbers are treated as nanoseconds since the epoch. -h|--help Show this message. 1mseqgen0m Usage: mlr seqgen [options] - Passes input records directly to output. Most useful for format conversion. Produces a sequence of counters. Discards the input record stream. Produces - output as specified by the options + output as specified by the options. Options: - -f {name} (default "i") Field name for counters. - --start {value} (default 1) Inclusive start value. - --step {value} (default 1) Step value. - --stop {value} (default 100) Inclusive stop value. - -h|--help Show this message. + -f {name} Field name for counters. Default "i". + --start {value} Inclusive start value. Default 1. + --step {value} Step value. Default 1. May be negative but not zero (unless + start == stop). + --stop {value} Inclusive stop value. Default 100. + -h|--help Show this message. Start, stop, and/or step may be floating-point. Output is integer if start, stop, and step are all integers. Step may be negative. It may not be zero unless start == stop. @@ -1923,17 +2000,20 @@ This is simply a copy of what you should see on running `man mlr` at a command p in the order they were encountered in the input record stream. Options: - -f {comma-separated field names} Lexical ascending - -r {comma-separated field names} Lexical descending - -c {comma-separated field names} Case-folded lexical ascending - -cr {comma-separated field names} Case-folded lexical descending - -n {comma-separated field names} Numerical ascending; nulls sort last - -nf {comma-separated field names} Same as -n - -nr {comma-separated field names} Numerical descending; nulls sort first - -t {comma-separated field names} Natural ascending - -b Move sort fields to start of record, as in reorder -b - -tr|-rt {comma-separated field names} Natural descending - -h|--help Show this message. + -f {a,b,c} Lexical ascending sort on the specified field names. + -r {a,b,c} Lexical descending sort on the specified field names. + -c {a,b,c} Case-folded lexical ascending sort on the specified field names. + -cr {a,b,c} Case-folded lexical descending sort on the specified field + names. + -n {a,b,c} Numerical ascending sort on the specified field names; nulls + sort last. + -nf {a,b,c} Same as -n. + -nr {a,b,c} Numerical descending sort on the specified field names; nulls + sort first. + -t {a,b,c} Natural ascending sort on the specified field names. + -b Move sort fields to start of record, as in reorder -b. + -tr|-rt {a,b,c} Natural descending sort on the specified field names. + -h|--help Show this message. Example: mlr sort -f a,b -nr x,y,z @@ -1944,13 +2024,13 @@ This is simply a copy of what you should see on running `man mlr` at a command p Usage: mlr sort-within-records [options] Outputs records sorted lexically ascending by keys. Options: - -f {names} Sort only these keys; others preserve record order. - -r {regex} Sort only keys matching this regex; others preserve record order. - Example: -r '^[xy]' sorts keys starting with x or y. - With no regex argument, -r recursively sorts subobjects/submaps - (e.g. for JSON input), or combines with -f to treat names as regex. - -n Sort field names naturally (e.g. 2 before 12). Combines with -f/-r. - -h|--help Show this message. + -f {names} Sort only these keys; others preserve record order. + -r {regex} Sort only keys matching this regex; others preserve record order. + Example: -r '^[xy]' sorts keys starting with x or y. With no regex + argument, -r recursively sorts subobjects/submaps (e.g. for JSON + input), or combines with -f to treat names as regexes. + -n Sort field names naturally (e.g. 2 before 12). Combines with -f/-r. + -h|--help Show this message. 1msparsify0m Usage: mlr sparsify [options] @@ -1958,26 +2038,28 @@ This is simply a copy of what you should see on running `man mlr` at a command p specified value). Only makes sense with output format not being CSV or TSV. Options: -s {filler string} What values to remove. Defaults to the empty string. - -f {a,b,c} Specify field names to be operated on; any other fields won't be - modified. The default is to modify all fields. - -h|--help Show this message. + -f {a,b,c} Specify field names to be operated on; any other fields won't + be modified. The default is to modify all fields. + -h|--help Show this message. Example: if input is a=1,b=,c=3 then output is a=1,c=3. 1msplit0m Usage: mlr split [options] {filename} Options: - -n {n}: Cap file sizes at N records. - -m {m}: Produce M files, round-robining records among them. - -g {a,b,c}: Write separate files with records having distinct values for fields named a,b,c. - Exactly one of -m, -n, or -g must be supplied. - --prefix {p} Specify filename prefix; default "split". - --suffix {s} Specify filename suffix; default is from mlr output format, e.g. "csv". - --folder {f} Specify output directory; default is current directory. - -a Append to existing file(s), if any, rather than overwriting. - -v Send records along to downstream verbs as well as splitting to files. + -n {n} Cap output file sizes at N records. + -m {m} Produce M files, round-robining records among them. + -g {a,b,c} Write separate files with records having distinct values for the + specified field names. + --prefix {p} Output filename prefix. Default "split". + --suffix {s} Output filename suffix. Default is from the output format, e.g. + "csv". + --folder {f} Output directory. Default is current directory. + -a Append to existing files rather than overwriting. + -v Send records downstream as well as splitting to files. -e Do NOT URL-escape names of output files. - -j {J} Use string J to join filename parts; default "_". + -j {J} String used to join filename parts. Default "_". -h|--help Show this message. + Exactly one of -m, -n, or -g must be supplied. Any of the output-format command-line flags (see mlr -h). For example, using mlr --icsv --from myfile.csv split --ojson -n 1000 the input is CSV, but the output files are JSON. @@ -2016,17 +2098,42 @@ This is simply a copy of what you should see on running `man mlr` at a command p as \n, \t, and \x1f. Write \\ to get a literal backslash. See also the `gsub` and `sub` verbs. Options: - -f {a,b,c} Field names to convert. - -r {regex} Regular expression for field names to convert. - -a Convert all fields. - -h|--help Show this message. + -f {a,b,c} Field names to apply substitution to. + -r {regex} Regular expression for field names to apply substitution to. + -a Apply substitution to all fields. + -h|--help Show this message. 1mstats10m Usage: mlr stats1 [options] Computes univariate statistics for one or more given fields, accumulated across the input record stream. Options: - -a {sum,count,...} Names of accumulators: one or more of: + -a {sum,count,...} Names of accumulators: one or more of the listed values. Also + accepts median (same as p50) and percentiles p{n} for n in + 0..100, e.g. p10 p25.2 p50 p98 p100. + -f {a,b,c} Value-field names on which to compute statistics. + --fr {regex} Regex for value-field names on which to compute statistics + (compute statistics on values in all field names matching the + regex). + --fx {regex} Inverted regex for value-field names on which to compute + statistics (compute statistics on values in all field names + not matching the regex). + -g {d,e,f} Optional group-by-field names. + --gr {regex} Regex for optional group-by-field names (group by values in + field names matching the regex). + --gx {regex} Inverted regex for optional group-by-field names (group by + values in field names not matching the regex). + --grfx {regex} Shorthand for --gr {regex} --fx {that same regex}. + -i Use interpolated percentiles, like R's type=7; default like + type=1. Not sensical for string-valued fields. + -s Print iterative stats. Useful in tail -f contexts, in which + case please avoid pprint-format output since end of input + stream will never be seen. Likewise, if input is coming from + `tail -f` be sure to use `--records-per-batch 1`. + -S No-op flag for backward compatibility with Miller 5. + -F No-op flag for backward compatibility with Miller 5. + -h|--help Show this message. + Names of accumulators for -a, one or more of: median This is the same as p50 p10 p25.2 p50 p98 p100 etc. count Count instances of fields @@ -2046,28 +2153,6 @@ This is simply a copy of what you should see on running `man mlr` at a command p max Compute maximum values of specified fields minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields - - -f {a,b,c} Value-field names on which to compute statistics - --fr {regex} Regex for value-field names on which to compute statistics - (compute statistics on values in all field names matching regex - --fx {regex} Inverted regex for value-field names on which to compute statistics - (compute statistics on values in all field names not matching regex) - - -g {d,e,f} Optional group-by-field names - --gr {regex} Regex for optional group-by-field names - (group by values in field names matching regex) - --gx {regex} Inverted regex for optional group-by-field names - (group by values in field names not matching regex) - - --grfx {regex} Shorthand for --gr {regex} --fx {that same regex} - - -i Use interpolated percentiles, like R's type=7; default like type=1. - Not sensical for string-valued fields.\n"); - -s Print iterative stats. Useful in tail -f contexts, in which - case please avoid pprint-format output since end of input - stream will never be seen. Likewise, if input is coming from `tail -f` - be sure to use `--records-per-batch 1`. - -h|--help Show this message. Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape Example: mlr stats1 -a count,mode -f size Example: mlr stats1 -a count,mode -f size -g shape @@ -2090,7 +2175,27 @@ This is simply a copy of what you should see on running `man mlr` at a command p Usage: mlr stats2 [options] Computes bivariate statistics for one or more given field-name pairs, accumulated across the input record stream. - -a {linreg-ols,corr,...} Names of accumulators: one or more of: + Options: + -a {linreg-ols,corr,...} Names of accumulators: one or more of the listed + values. + -f {a,b,c,d} Value-field name-pairs on which to compute statistics. + There must be an even number of names. + -g {e,f,g} Optional group-by-field names. + -v Print additional output for linreg-pca. + -s Print iterative stats. Useful in tail -f contexts, in + which case please avoid pprint-format output since end + of input stream will never be seen. Likewise, if input + is coming from `tail -f`, be sure to use + `--records-per-batch 1`. + --fit Rather than printing regression parameters, applies + them to the input data to compute new fit fields. All + input records are held in memory until end of input + stream. Has effect only for linreg-ols, linreg-pca, and + logireg. + -S No-op flag for backward compatibility with Miller 5. + -F No-op flag for backward compatibility with Miller 5. + -h|--help Show this message. + Names of accumulators for -a, one or more of: linreg-ols Linear regression using ordinary least squares linreg-pca Linear regression using principal component analysis r2 Quality metric for linreg-ols (linreg-pca emits its own) @@ -2098,18 +2203,6 @@ This is simply a copy of what you should see on running `man mlr` at a command p corr Sample correlation cov Sample covariance covx Sample-covariance matrix - -f {a,b,c,d} Value-field name-pairs on which to compute statistics. - There must be an even number of names. - -g {e,f,g} Optional group-by-field names. - -v Print additional output for linreg-pca. - -s Print iterative stats. Useful in tail -f contexts, in which - case please avoid pprint-format output since end of input - stream will never be seen. Likewise, if input is coming from - `tail -f`, be sure to use `--records-per-batch 1`. - --fit Rather than printing regression parameters, applies them to - the input data to compute new fit fields. All input records are - held in memory until end of input stream. Has effect only for - linreg-ols, linreg-pca, and logireg. Only one of -s or --fit may be used. Example: mlr stats2 -a linreg-pca -f x,y Example: mlr stats2 -a linreg-ols,r2 -f x,y -g size,shape @@ -2119,7 +2212,25 @@ This is simply a copy of what you should see on running `man mlr` at a command p Usage: mlr step [options] Computes values dependent on earlier/later records, optionally grouped by category. Options: - -a {delta,rsum,...} Names of steppers: comma-separated, one or more of: + -a {delta,rsum,...} Names of steppers: comma-separated, one or more of the + listed values. + -f {a,b,c} Value-field names on which to compute statistics. + -g {d,e,f} Optional group-by-field names. + -F Computes integerable things (e.g. counter) in floating + point. As of Miller 6 this happens automatically, but the + flag is accepted as a no-op for backward compatibility with + Miller 5 and below. + -d {x,y,z} Weights for EWMA. 1 means current sample gets all weight (no + smoothing), near under 1 is light smoothing, near over 0 is + heavy smoothing. Multiple weights may be specified, e.g. + "mlr step -a ewma -f sys_load -d 0.01,0.1,0.9". Default if + omitted is "-d 0.5". + -o {a,b,c} Custom suffixes for EWMA output fields. If omitted, these + default to the -d values. If supplied, the number of -o + values must be the same as the number of -d values. + -h|--help Show this message. + + Names of steppers for -a, comma-separated, one or more of: counter Count instances of field(s) between successive records delta Compute differences in field(s) between successive records ewma Exponentially weighted moving average over successive records @@ -2132,21 +2243,6 @@ This is simply a copy of what you should see on running `man mlr` at a command p shift_lead Include value(s) in field(s) from the next record, if any slwin Sliding-window averages over m records back and n forward. E.g. slwin_7_2 for 7 back and 2 forward. - -f {a,b,c} Value-field names on which to compute statistics - -g {d,e,f} Optional group-by-field names - -F Computes integerable things (e.g. counter) in floating point. - As of Miller 6 this happens automatically, but the flag is accepted - as a no-op for backward compatibility with Miller 5 and below. - -d {x,y,z} Weights for EWMA. 1 means current sample gets all weight (no - smoothing), near under 1 is light smoothing, near over 0 is - heavy smoothing. Multiple weights may be specified, e.g. - "mlr step -a ewma -f sys_load -d 0.01,0.1,0.9". Default if omitted - is "-d 0.5". - -o {a,b,c} Custom suffixes for EWMA output fields. If omitted, these default to - the -d values. If supplied, the number of -o values must be the same - as the number of -d values. - -h|--help Show this message. - Examples: mlr step -a rsum -f request_size mlr step -a delta -f request_size -g hostname @@ -2167,10 +2263,10 @@ This is simply a copy of what you should see on running `man mlr` at a command p and \x1f. Write \\ to get a literal backslash. See also the `gsub` and `ssub` verbs. Options: - -f {a,b,c} Field names to convert. - -r {regex} Regular expression for field names to convert. - -a Convert all fields. - -h|--help Show this message. + -f {a,b,c} Field names to apply substitution to. + -r {regex} Regular expression for field names to apply substitution to. + -a Apply substitution to all fields. + -h|--help Show this message. 1msummary0m Usage: mlr summary [options] @@ -2210,19 +2306,19 @@ This is simply a copy of what you should see on running `man mlr` at a command p Options: -a {mean,sum,etc.} Use only the specified summarizers. - -x {mean,sum,etc.} Use all summarizers, except the specified ones. + -x {mean,sum,etc.} Use all summarizers except the specified ones. --all Use all available summarizers. - --transpose Show output with field names as column names.. - -h|--help Show this message. + --transpose Show output with field names as column names. + -h|--help Show this message. 1msurv0m Usage: mlr surv -d {duration-field} -s {status-field} Estimate Kaplan-Meier survival curve (right-censored). Options: - -d {field} Name of duration field (time-to-event or censoring). - -s {field} Name of status field (0=censored, 1=event). - -h, --help Show this message. + -d {field} Name of duration field (time-to-event or censoring). + -s {field} Name of status field (0=censored, 1=event). + -h|--help Show this message. 1mtac0m Usage: mlr tac [options] @@ -2235,34 +2331,35 @@ This is simply a copy of what you should see on running `man mlr` at a command p Passes through the last n records, optionally by category. Options: -g {a,b,c} Optional group-by-field names for tail counts, e.g. a,b,c. - -n {n} Tail-count to print. Default 10. - A leading '+' means start at the nth record rather than print - the last n: e.g. -n +3 passes through all but the first 2 - records, optionally by category. - -h|--help Show this message. + -n {n} Tail-count to print. Default 10. A leading '+' means start at the nth + record rather than print the last n: e.g. -n +3 passes through all + but the first 2 records, optionally by category. + -h|--help Show this message. 1mtee0m Usage: mlr tee [options] {filename} Options: - -a Append to existing file, if any, rather than overwriting. - -p Treat filename as a pipe-to command. + -a Append to existing file, if any, rather than overwriting. + -p Treat filename as a pipe-to command. + -h|--help Show this message. Any of the output-format command-line flags (see mlr -h). Example: using mlr --icsv --opprint put '...' then tee --ojson ./mytap.dat then stats1 ... the input is CSV, the output is pretty-print tabular, but the tee-file output is written in JSON format. - -h|--help Show this message. - 1mtemplate0m Usage: mlr template [options] Places input-record fields in the order specified by list of column names. If the input record is missing a specified field, it will be filled with the fill-with. If the input record possesses an unspecified field, it will be discarded. Options: - -f {a,b,c} Comma-separated field names for template, e.g. a,b,c. - -t {filename} CSV file whose header line will be used for template. - --fill-with {filler string} What to fill absent fields with. Defaults to the empty string. - -h|--help Show this message. + -f {a,b,c} Comma-separated field names for template, e.g. + a,b,c. + -t {filename} CSV file whose header line will be used for + template. + --fill-with {filler string} What to fill absent fields with. Defaults to the + empty string. + -h|--help Show this message. Example: * Specified fields are a,b,c. * Input record is c=3,a=1,f=6. @@ -2270,16 +2367,19 @@ This is simply a copy of what you should see on running `man mlr` at a command p 1mtop0m Usage: mlr top [options] - -f {a,b,c} Value-field names for top counts. - -g {d,e,f} Optional group-by-field names for top counts. - -n {count} How many records to print per category; default 1. - -a Print all fields for top-value records; default is - to print only value and group-by fields. Requires a single - value-field name only. - --min Print top smallest values; default is top largest values. - -F Keep top values as floats even if they look like integers. - -o {name} Field name for output indices. Default "top_idx". - This is ignored if -a is used. + Options: + -f {a,b,c} Value-field names for top counts. + -g {d,e,f} Optional group-by-field names for top counts. + -n {count} How many records to print per category; default 1. + -a Print all fields for top-value records; default is to print only + value and group-by fields. Requires a single value-field name only. + --max Print top largest values. This is the default. + --min Print top smallest values; default is top largest values. + -F Keep top values as floats even if they look like integers (ignored in + Miller 6, kept for backward compatibility). + -o {name} Field name for output indices. Default "top_idx". Ignored if -a is + used. + -h|--help Show this message. Prints the n records with smallest/largest values at specified fields, optionally by category. If -a is given, then the top records are emitted with the same fields as they appeared in the input. Without -a, only fields @@ -2288,7 +2388,7 @@ This is simply a copy of what you should see on running `man mlr` at a command p 1mutf8-to-latin10m Usage: mlr utf8-to-latin1, with no options. - Recursively converts record strings from Latin-1 to UTF-8. + Recursively converts record strings from UTF-8 to Latin-1. For field-level control, please see the utf8_to_latin1 DSL function. Options: -h|--help Show this message. @@ -2298,9 +2398,9 @@ This is simply a copy of what you should see on running `man mlr` at a command p Reverses flatten. Example: field with name 'a.b.c' and value 4 becomes name 'a' and value '{"b": { "c": 4 }}'. Options: - -f {a,b,c} Comma-separated list of field names to unflatten (default all). + -f {a,b,c} Comma-separated list of field names to unflatten (default all). -s {string} Separator, defaulting to mlr --flatsep value. - -h|--help Show this message. + -h|--help Show this message. 1muniq0m Usage: mlr uniq [options] @@ -2308,21 +2408,24 @@ This is simply a copy of what you should see on running `man mlr` at a command p count-distinct. For uniq, -f is a synonym for -g. Options: - -g {d,e,f} Group-by-field names for uniq counts. - -x {a,b,c} Field names to exclude for uniq: use each record's others instead. - -c Show repeat counts in addition to unique values. - -n Show only the number of distinct values. - -o {name} Field name for output count. Default "count". - -a Output each unique record only once. Incompatible with -g. - With -c, produces unique records, with repeat counts for each. - With -n, produces only one record which is the unique-record count. - With neither -c nor -n, produces unique records. + -g {d,e,f} Group-by field names for uniq counts. + -f {d,e,f} Synonym for -g. + -x {a,b,c} Field names to exclude for uniq; use each record's other fields + instead. + -c Show repeat counts in addition to unique values. + -n Show only the number of distinct values. + -o {name} Field name for output count. Default "count". + -a Output each unique record only once. Incompatible with -g. With -c, + produces unique records, with repeat counts for each. With -n, + produces only one record which is the unique-record count. With + neither -c nor -n, produces unique records. + -h|--help Show this message. 1munspace0m Usage: mlr unspace [options] Replaces spaces in record keys and/or values with _. This is helpful for PPRINT output. Options: - -f {x} Replace spaces with specified filler character. + -f {x} Replace spaces with specified filler character. Default "_". -k Unspace only keys, not keys and values. -v Unspace only values, not keys and values. -h|--help Show this message. @@ -2333,11 +2436,11 @@ This is simply a copy of what you should see on running `man mlr` at a command p For field names absent in a given record but present in others, fills in a value. This verb retains all input before producing any output. Options: - --fill-with {filler string} What to fill absent fields with. Defaults to - the empty string. - -f {a,b,c} Specify field names to be operated on. Any other fields won't be - modified, and operation will be streaming. - -h|--help Show this message. + --fill-with {filler string} What to fill absent fields with. Defaults to the + empty string. + -f {a,b,c} Specify field names to be operated on; others are + not modified and operation is streaming. + -h|--help Show this message. Example: if the input is two records, one being 'a=1,b=2' and the other being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and 'a=,b=3,c=4'. @@ -3899,5 +4002,5 @@ This is simply a copy of what you should see on running `man mlr` at a command p MIME Type for Comma-Separated Values (CSV) Files, the Miller docsite https://miller.readthedocs.io - 2026-07-01 4mMILLER24m(1) + 2026-07-03 4mMILLER24m(1) diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt index 30058ef3b..7dd245f9a 100644 --- a/docs/src/manpage.txt +++ b/docs/src/manpage.txt @@ -563,6 +563,12 @@ 1mMISCELLANEOUS FLAGS0m These are flags which don't fit into any other category. + --errors-json Emit parse errors as a JSON object to stderr instead + of a plain text message. Intended for AI agents and + scripts that branch on error kind rather than + regex-matching prose. Equivalent to setting the + `MLR_ERRORS_JSON` environment variable to a truthy + value. --fflush Force buffered output to be written after every output record. The default is flush output after every record if the output is to the terminal, or @@ -977,57 +983,58 @@ Replaces a numeric field with a number of asterisks, allowing for cheesy bar plots. These align best with --opprint or --oxtab output format. Options: - -f {a,b,c} Field names to convert to bars. - --lo {lo} Lower-limit value for min-width bar: default '0.000000'. - --hi {hi} Upper-limit value for max-width bar: default '100.000000'. - -w {n} Bar-field width: default '40'. - --auto Automatically computes limits, ignoring --lo and --hi. - Holds all records in memory before producing any output. - -c {character} Fill character: default '*'. - -x {character} Out-of-bounds character: default '#'. - -b {character} Blank character: default '.'. + -f {a,b,c} Field names to convert to bars. + --lo {lo} Lower-limit value for min-width bar: default '0.000000'. + --hi {hi} Upper-limit value for max-width bar: default '100.000000'. + -w {n} Bar-field width: default '40'. + --auto Automatically computes limits, ignoring --lo and --hi. Holds all + records in memory before producing any output. + -c {character} Fill character: default '*'. + -x {character} Out-of-bounds character: default '#'. + -b {character} Blank character: default '.'. + -h|--help Show this message. Nominally the fill, out-of-bounds, and blank characters will be strings of length 1. However you can make them all longer if you so desire. - -h|--help Show this message. 1mbootstrap0m Usage: mlr bootstrap [options] Emits an n-sample, with replacement, of the input records. See also mlr sample and mlr shuffle. Options: - -n Number of samples to output. Defaults to number of input records. - Must be non-negative. + -n {n} Number of samples to output. Defaults to number of input records. Must + be non-negative. -h|--help Show this message. 1mcase0m Usage: mlr case [options] Uppercases strings in record keys and/or values. Options: - -k Case only keys, not keys and values. - -v Case only values, not keys and values. - -f {a,b,c} Specify which field names to case (default: all) - -u Convert to uppercase - -l Convert to lowercase - -s Convert to sentence case (capitalize first letter) - -t Convert to title case (capitalize words) - -h|--help Show this message. + -k Case only keys, not keys and values. + -v Case only values, not keys and values. + -f {a,b,c} Specify which field names to case (default: all). + -u Convert to uppercase. + -l Convert to lowercase. + -s Convert to sentence case (capitalize first letter). + -t Convert to title case (capitalize words). + -h|--help Show this message. 1mcat0m Usage: mlr cat [options] Passes input records directly to output. Most useful for format conversion. Options: -n Prepend field "n" to each record with record-counter starting at 1. - -N {name} Prepend field {name} to each record with record-counter starting at 1. - -g {a,b,c} Optional group-by-field names for counters, e.g. a,b,c + -N {name} Prepend field {name} to each record with record-counter starting at + 1. + -g {a,b,c} Optional group-by-field names for counters, e.g. a,b,c. --filename Prepend current filename to each record. --filenum Prepend current filenum (1-up) to each record. - -h|--help Show this message. + -h|--help Show this message. 1mcheck0m Usage: mlr check [options] Consumes records without printing any output, - Useful for doing a well-formatted check on input data. with the exception that warnings are printed to stderr. + Useful for doing a well-formatted check on input data. Current checks are: * Data are parseable * If any key is the empty string @@ -1043,11 +1050,11 @@ and clean_whitespace. Options: - -k|--keys-only Do not touch values. - -v|--values-only Do not touch keys. + -k|--keys-only Do not touch values. + -v|--values-only Do not touch keys. + -h|--help Show this message. It is an error to specify -k as well as -v -- to clean keys and values, leave off -k as well as -v. - -h|--help Show this message. 1mcount-distinct0m Usage: mlr count-distinct [options] @@ -1055,25 +1062,26 @@ Same as uniq -c. Options: - -f {a,b,c} Field names for distinct count. - -x {a,b,c} Field names to exclude for distinct count: use each record's others instead. - -n Show only the number of distinct values. Not compatible with -u. - -o {name} Field name for output count. Default "count". - Ignored with -u. - -u Do unlashed counts for multiple field names. With -f a,b and - without -u, computes counts for distinct combinations of a - and b field values. With -f a,b and with -u, computes counts - for distinct a field values and counts for distinct b field - values separately. + -f {a,b,c} Field names for distinct count (synonym for -g). + -g {a,b,c} Field names for distinct count. + -x {a,b,c} Field names to exclude for distinct count; use each record's other + fields instead. + -n Show only the number of distinct values. Not compatible with -u. + -o {name} Field name for output count. Default "count". Ignored with -u. + -u Do unlashed counts for multiple field names. With -f a,b and without + -u, computes counts for distinct combinations of a and b field + values. With -f a,b and with -u, computes counts for distinct a field + values and counts for distinct b field values separately. + -h|--help Show this message. 1mcount0m Usage: mlr count [options] Prints number of records, optionally grouped by distinct values for specified field names. Options: -g {a,b,c} Optional group-by-field names for counts, e.g. a,b,c - -n {n} Show only the number of distinct values. Not interesting without -g. - -o {name} Field name for output-count. Default "count". - -h|--help Show this message. + -n Show only the number of distinct values. Not interesting without -g. + -o {name} Field name for output-count. Default "count". + -h|--help Show this message. 1mcount-similar0m Usage: mlr count-similar [options] @@ -1081,22 +1089,22 @@ the number of other records having the same group-by field values. Options: -g {a,b,c} Group-by-field names for counts, e.g. a,b,c - -o {name} Field name for output-counts. Defaults to "count". - -h|--help Show this message. + -o {name} Field name for output-counts. Defaults to "count". + -h|--help Show this message. 1mcut0m Usage: mlr cut [options] Passes through input records with specified fields included/excluded. Options: - -f {a,b,c} Comma-separated field names for cut, e.g. a,b,c. - -o Retain fields in the order specified here in the argument list. - Default is to retain them in the order found in the input data. - -x|--complement Exclude, rather than include, field names specified by -f. - -r Treat field names as regular expressions. "ab", "a.*b" will - match any field name containing the substring "ab" or matching - "a.*b", respectively; anchors of the form "^ab$", "^a.*b$" may - be used. - -h|--help Show this message. + -f {a,b,c} Comma-separated field names to include or exclude, e.g. a,b,c. + -o Retain fields in the order specified by -f rather than in + input-record order. + -x|--complement Exclude, rather than include, the field names specified by -f. + -r Treat field names as regular expressions. "ab", "a.*b" will + match any field name containing the substring "ab" or matching + "a.*b", respectively; anchors of the form "^ab$", "^a.*b$" may + be used. + -h|--help Show this message. Examples: mlr cut -f hostname,status mlr cut -x -f hostname,status @@ -1108,11 +1116,11 @@ Usage: mlr decimate [options] Passes through one of every n records, optionally by category. Options: - -b Decimate by printing first of every n. - -e Decimate by printing last of every n (default). - -g {a,b,c} Optional group-by-field names for decimate counts, e.g. a,b,c. - -n {n} Decimation factor (default 10). - -h|--help Show this message. + -b Decimate by printing first of every n. + -e Decimate by printing last of every n (default). + -g {a,b,c} Optional group-by-field names for decimate counts, e.g. a,b,c. + -n {n} Decimation factor (default 10). + -h|--help Show this message. 1mfill-down0m Usage: mlr fill-down [options] @@ -1122,13 +1130,14 @@ With -a, a field is 'missing' only if it is absent. Options: - --all Operate on all fields in the input. - -a|--only-if-absent If a given record has a missing value for a given field, - fill that from the corresponding value from a previous record, if any. - By default, a 'missing' field either is absent, or has the empty-string value. - With -a, a field is 'missing' only if it is absent. - -f Field names for fill-down. - -h|--help Show this message. + --all Operate on all fields in the input. + -a|--only-if-absent If a given record has a missing value for a given field, + fill that from the corresponding value from a previous + record, if any. By default, a 'missing' field either is + absent, or has the empty-string value. With -a, a field is + 'missing' only if it is absent. + -f {a,b,c} Field names for fill-down. + -h|--help Show this message. 1mfill-empty0m Usage: mlr fill-empty [options] @@ -1136,6 +1145,7 @@ Options: -v {string} Fill-value: defaults to "N/A" -S Don't infer type -- so '-v 0' would fill string 0 not int 0. + -h|--help Show this message. 1mfilter0m Usage: mlr filter [options] {DSL expression} @@ -1144,56 +1154,53 @@ See also: https://miller.readthedocs.io/en/latest/reference-verbs Options: - -f {file name} File containing a DSL expression (see examples below). If the filename - is a directory, all *.mlr files in that directory are loaded. + -f {file name} File containing a DSL expression (see examples below). If the + filename is a directory, all *.mlr files in that directory are + loaded. + -e {expression} DSL expression to evaluate. You can use this after -f to add an + expression. Example use case: define functions/subroutines in a + file you specify with -f, then call them with an expression you + specify with -e. + -s {name=value} Predefines out-of-stream variable @name to have the given value. + Thus mlr put -s foo=97 '$column += @foo' is like mlr put 'begin + {@foo = 97} $column += @foo'. The value part is subject to + type-inferencing. May be specified more than once, e.g. -s + name1=value1 -s name2=value2. Note: the value may be an + environment variable, e.g. -s sequence=$SEQUENCE. + -x Prints records for which {expression} evaluates to false, not + true, i.e. invert the sense of the filter expression. Default + false. + -q Does not include the modified record in the output stream. + Useful for when all desired output is in begin and/or end + blocks. + -S No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. + -F No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. + -w Print warnings about things like uninitialized variables. + -W Same as -w, but exit the process if there are any warnings. + -p Prints the expression's AST (abstract syntax tree), which gives + full transparency on the precedence and associativity rules of + Miller's grammar, to stdout. + -d Like -p but uses a parenthesized-expression format for the AST. + -D Like -d but with output all on one line. + -E Echo DSL expression before printing parse-tree. + -v Same as -E -p. + -X Exit after parsing but before stream-processing. Useful with + -v/-d/-D, if you only want to look at parser information. + --explain Parse and type-check the DSL expression, report whether it is + valid, and exit without reading the input stream. Exit status is + 0 if the expression is valid and non-zero otherwise; combine + with --errors-json for a machine-readable error. + -h|--help Show this message. - -e {expression} You can use this after -f to add an expression. Example use - case: define functions/subroutines in a file you specify with -f, then call - them with an expression you specify with -e. - - (If you mix -e and -f then the expressions are evaluated in the order encountered. + If you mix -e and -f then the expressions are evaluated in the order encountered. Since the expression pieces are simply concatenated, please be sure to use intervening - semicolons to separate expressions.) + semicolons to separate expressions. - -s name=value: Predefines out-of-stream variable @name to have - Thus mlr put -s foo=97 '$column += @foo' is like - mlr put 'begin {@foo = 97} $column += @foo'. - The value part is subject to type-inferencing. - May be specified more than once, e.g. -s name1=value1 -s name2=value2. - Note: the value may be an environment variable, e.g. -s sequence=$SEQUENCE - - -x (default false) Prints records for which {expression} evaluates to false, not true, - i.e. invert the sense of the filter expression. - - -q Does not include the modified record in the output stream. - Useful for when all desired output is in begin and/or end blocks. - - -S and -F: There are no-ops in Miller 6 and above, since now type-inferencing is done - by the record-readers before filter/put is executed. Supported as no-op pass-through - flags for backward compatibility. - - -h|--help Show this message. - - Parser-info options: - - -w Print warnings about things like uninitialized variables. - - -W Same as -w, but exit the process if there are any warnings. - - -p Prints the expressions's AST (abstract syntax tree), which gives full - transparency on the precedence and associativity rules of Miller's grammar, - to stdout. - - -d Like -p but uses a parenthesized-expression format for the AST. - - -D Like -d but with output all on one line. - - -E Echo DSL expression before printing parse-tree - - -v Same as -E -p. - - -X Exit after parsing but before stream-processing. Useful with -v/-d/-D, if you - only want to look at parser information. + Parser-info options are -w, -W, -p, -d, -D, -E, -v, and -X. Records will pass the filter depending on the last bare-boolean statement in the DSL expression. That can be the result of <, ==, >, etc., the return value of a function call @@ -1231,9 +1238,9 @@ Flattens multi-level maps to single-level ones. Example: field with name 'a' and value '{"b": { "c": 4 }}' becomes name 'a.b.c' and value 4. Options: - -f Comma-separated list of field names to flatten (default all). - -s Separator, defaulting to mlr --flatsep value. - -h|--help Show this message. + -f Comma-separated list of field names to flatten (default all). + -s {string} Separator, defaulting to mlr --flatsep value. + -h|--help Show this message. 1mformat-values0m Usage: mlr format-values [options] @@ -1249,22 +1256,21 @@ undefined behavior and/or program crashes. See your system's "man printf". Options: - -i {integer format} Defaults to "%d". - Examples: "%06lld", "%08llx". - Note that Miller integers are long long so you must use - formats which apply to long long, e.g. with ll in them. - Undefined behavior results otherwise. - -f {float format} Defaults to "%f". - Examples: "%8.3lf", "%.6le". - Note that Miller floats are double-precision so you must - use formats which apply to double, e.g. with l[efg] in them. - Undefined behavior results otherwise. - -s {string format} Defaults to "%s". - Examples: "_%s", "%08s". - Note that you must use formats which apply to string, e.g. - with s in them. Undefined behavior results otherwise. + -i {integer format} Integer format string; defaults to "%d". Examples: "%06lld", + "%08llx". Note that Miller integers are long long so you + must use formats which apply to long long, e.g. with ll in + them. Undefined behavior results otherwise. + -f {float format} Float format string; defaults to "%f". Examples: "%8.3lf", + "%.6le". Note that Miller floats are double-precision so you + must use formats which apply to double, e.g. with l[efg] in + them. Undefined behavior results otherwise. + -s {string format} String format string; defaults to "%s". Examples: "_%s", + "%08s". Note that you must use formats which apply to + string, e.g. with s in them. Undefined behavior results + otherwise. -n Coerce field values autodetected as int to float, and then apply the float format. + -h|--help Show this message. 1mfraction0m Usage: mlr fraction [options] @@ -1278,34 +1284,33 @@ and emits output records. This means it produces no output until all input is read. Options: - -f {a,b,c} Field name(s) for fraction calculation - -g {d,e,f} Optional group-by-field name(s) for fraction counts - -p Produce percents [0..100], not fractions [0..1]. Output field names - end with "_percent" rather than "_fraction" - -c Produce cumulative distributions, i.e. running sums: each output - value folds in the sum of the previous for the specified group - E.g. with input records x=1 x=2 x=3 and x=4, emits output records - x=1,x_cumulative_fraction=0.1 x=2,x_cumulative_fraction=0.3 - x=3,x_cumulative_fraction=0.6 and x=4,x_cumulative_fraction=1.0 + -f {a,b,c} Field name(s) for fraction calculation + -g {d,e,f} Optional group-by-field name(s) for fraction counts + -p Produce percents [0..100], not fractions [0..1]. Output field names + end with "_percent" rather than "_fraction" + -c Produce cumulative distributions, i.e. running sums: each output + value folds in the sum of the previous for the specified group. E.g. + with input records x=1 x=2 x=3 and x=4, emits output records + x=1,x_cumulative_fraction=0.1 x=2,x_cumulative_fraction=0.3 + x=3,x_cumulative_fraction=0.6 and x=4,x_cumulative_fraction=1.0 + -h|--help Show this message. 1mgap0m Usage: mlr gap [options] Emits an empty record every n records, or when certain values change. + One of -n or -g is required. Options: - Emits an empty record every n records, or when certain values change. -g {a,b,c} Print a gap whenever values of these fields (e.g. a,b,c) changes. - -n {n} Print a gap every n records. - One of -f or -g is required. - -n is ignored if -g is present. - -h|--help Show this message. + -n {n} Print a gap every n records. Ignored if -g is present. + -h|--help Show this message. 1mgrep0m Usage: mlr grep [options] {regular expression} Passes through records which match the regular expression. Options: - -i Use case-insensitive search. - -v Invert: pass through records which do not match the regex. - -a Only grep for values, not keys and values. + -i Use case-insensitive search. + -v Invert: pass through records which do not match the regex. + -a Only grep for values, not keys and values. -h|--help Show this message. Note that "mlr filter" is more powerful, but requires you to know field names. By contrast, "mlr grep" allows you to regex-match the entire record. It does this @@ -1321,7 +1326,8 @@ 1mgroup-by0m Usage: mlr group-by [options] {comma-separated field names} - Outputs records in batches having identical values at specified field names.Options: + Outputs records in batches having identical values at specified field names. + Options: -h|--help Show this message. 1mgroup-like0m @@ -1338,21 +1344,28 @@ and \x1f. Write \\ to get a literal backslash. See also the `sub` and `ssub` verbs. Options: - -f {a,b,c} Field names to convert. - -r {regex} Regular expression for field names to convert. - -a Convert all fields. - -h|--help Show this message. + -f {a,b,c} Field names to apply substitution to. + -r {regex} Regular expression for field names to apply substitution to. + -a Apply substitution to all fields. + -h|--help Show this message. 1mhaving-fields0m Usage: mlr having-fields [options] Conditionally passes through records depending on each record's field names. Options: - --at-least {comma-separated names} - --which-are {comma-separated names} - --at-most {comma-separated names} - --all-matching {regular expression} - --any-matching {regular expression} - --none-matching {regular expression} + --at-least {comma-separated names} Pass records that have at least these field + names. + --which-are {comma-separated names} Pass records whose field names are exactly + these. + --at-most {comma-separated names} Pass records that have at most these field + names. + --all-matching {regular expression} Pass records where all field names match + the regex. + --any-matching {regular expression} Pass records where any field name matches + the regex. + --none-matching {regular expression} Pass records where no field name matches + the regex. + -h|--help Show this message. Examples: mlr having-fields --which-are amount,status,owner mlr having-fields --any-matching 'sda[0-9]' @@ -1365,40 +1378,41 @@ Without -g, ceases consuming more input (i.e. is fast) when n records have been read. Options: -g {a,b,c} Optional group-by-field names for head counts, e.g. a,b,c. - -n {n} Head-count to print. Default 10. - A negative count, e.g. -n -2, passes through all but the last n records, - optionally by category. - -h|--help Show this message. + -n {n} Head-count to print. Default 10. A negative count, e.g. -n -2, passes + through all but the last n records, optionally by category. + -h|--help Show this message. 1mhistogram0m Just a histogram. Input values < lo or > hi are not counted. Usage: mlr histogram [options] - -f {a,b,c} Value-field names for histogram counts - --lo {lo} Histogram low value - --hi {hi} Histogram high value - --nbins {n} Number of histogram bins. Defaults to 20. - --auto Automatically computes limits, ignoring --lo and --hi. - Holds all values in memory before producing any output. - -o {prefix} Prefix for output field name. Default: no prefix. - -h|--help Show this message. + Options: + -f {a,b,c} Value-field names for histogram counts. + --lo {lo} Histogram low value. + --hi {hi} Histogram high value. + --nbins {n} Number of histogram bins. Defaults to 20. + --auto Automatically computes limits, ignoring --lo and --hi. Holds all + values in memory before producing any output. + -o {prefix} Prefix for output field name. Default: no prefix. + -h|--help Show this message. 1mjson-parse0m Usage: mlr json-parse [options] Tries to convert string field values to parsed JSON, e.g. "[1,2,3]" -> [1,2,3]. Options: - -f {...} Comma-separated list of field names to json-parse (default all). - -k If supplied, then on parse fail for any cell, keep the (unparsable) - input value for the cell. - -h|--help Show this message. + -f {a,b,c} Comma-separated list of field names to json-parse (default all). + -k If supplied, then on parse fail for any cell, keep the (unparsable) + input value for the cell. + -h|--help Show this message. 1mjson-stringify0m Usage: mlr json-stringify [options] Produces string field values from field-value data, e.g. [1,2,3] -> "[1,2,3]". Options: - -f {...} Comma-separated list of field names to json-parse (default all). - --jvstack Produce multi-line JSON output. + -f {a,b,c} Comma-separated list of field names to json-stringify (default + all). + --jvstack Produce multi-line JSON output. --no-jvstack Produce single-line JSON output per record (default). - -h|--help Show this message. + -h|--help Show this message. 1mjoin0m Usage: mlr join [options] @@ -1407,34 +1421,53 @@ Functionality is essentially the same as the system "join" command, but for record streams. Options: - -f {left file name} - -j {a,b,c} Comma-separated join-field names for output - -l {a,b,c} Comma-separated join-field names for left input file; - defaults to -j values if omitted. - -r {a,b,c} Comma-separated join-field names for right input file(s); - defaults to -j values if omitted. - --lk|--left-keep-field-names {a,b,c} If supplied, this means keep only the specified field - names from the left file. Automatically includes the join-field name(s). Helpful - for when you only want a limited subset of information from the left file. - Tip: you can use --lk "": this means the left file becomes solely a row-selector - for the input files. - --lp {text} Additional prefix for non-join output field names from - the left file. Applies to paired and unpaired output records. - --rp {text} Additional prefix for non-join output field names from - the right file(s). Applies to paired and unpaired output records. - --np Do not emit paired records - --ul Emit unpaired records from the left file - --ur Emit unpaired records from the right file(s) - -s|--sorted-input Require sorted input: records must be sorted - lexically by their join-field names, else not all records will - be paired. The only likely use case for this is with a left - file which is too big to fit into system memory otherwise. - -u Enable unsorted input. (This is the default even without -u.) - In this case, the entire left file will be loaded into memory. - --prepipe {command} As in main input options; see mlr --help for details. - If you wish to use a prepipe command for the main input as well - as here, it must be specified there as well as here. - --prepipex {command} Likewise. + -f {left file name} Left file name for join. + -j {a,b,c} Comma-separated join-field names for + output. + -l {a,b,c} Comma-separated join-field names for left + input file; defaults to -j values if + omitted. + -r {a,b,c} Comma-separated join-field names for right + input file(s); defaults to -j values if + omitted. + --lk|--left-keep-field-names {a,b,c} If supplied, this means keep only the + specified field names from the left file. + Automatically includes the join-field + name(s). Helpful for when you only want a + limited subset of information from the left + file. Tip: you can use --lk "": this means + the left file becomes solely a row-selector + for the input files. + --lp {text} Additional prefix for non-join output field + names from the left file. Applies to paired + and unpaired output records. + --rp {text} Additional prefix for non-join output field + names from the right file(s). Applies to + paired and unpaired output records. + --np Do not emit paired records. + --ul Emit unpaired records from the left file. + --ur Emit unpaired records from the right + file(s). + -s|--sorted-input Require sorted input: records must be + sorted lexically by their join-field names, + else not all records will be paired. The + only likely use case for this is with a + left file which is too big to fit into + system memory otherwise. + -u Enable unsorted input. (This is the default + even without -u.) In this case, the entire + left file will be loaded into memory. + --prepipe {command} Shell command to prepipe the left-file + input through. As in main input options; + see mlr --help for details. If you wish to + use a prepipe command for the main input as + well as here, it must be specified there as + well as here. + --prepipex {command} Shell command to prepipe the left-file + input through (no shell quoting). As in + main input options; see mlr --help for + details. + -h|--help Show this message. File-format options default to those for the right file names on the Miller argument list, but may be overridden for the left file as follows. Please see the main "mlr --help" for more information on syntax for these arguments: @@ -1477,10 +1510,11 @@ Shows the least frequently occurring distinct values for specified field names. The first entry is the statistical anti-mode; the remaining are runners-up. Options: - -f {one or more comma-separated field names}. Required flag. - -n {count}. Optional flag defaulting to 10. - -b Suppress counts; show only field values. - -o {name} Field name for output count. Default "count". + -f {a,b,c} One or more comma-separated field names to group by. Required flag. + -n {count} Maximum number of results to output. Optional flag defaulting to 10. + -b Suppress counts; show only field values. + -o {name} Field name for output count. Default "count". + -h|--help Show this message. See also "mlr most-frequent". 1mmerge-fields0m @@ -1488,7 +1522,25 @@ Computes univariate statistics for each input record, accumulated across specified fields. Options: - -a {sum,count,...} Names of accumulators. One or more of: + -a {sum,count,...} Names of accumulators: one or more of the accumulators listed + below. + -f {a,b,c} Value-field names on which to compute statistics. Requires + -o. + -r {a,b,c} Regular expressions for value-field names on which to compute + statistics. Requires -o. + -c {a,b,c} Substrings for collapse mode: all fields which have the same + names after removing substrings will be accumulated together. + Please see examples below. + -i Use interpolated percentiles, like R's type=7; default like + type=1. Not sensical for string-valued fields. + -o {name} Output field basename for -f/-r. + -k Keep the input fields which contributed to the output + statistics; the default is to omit them. + -S No-op flag for backward compatibility with Miller 5. + -F No-op flag for backward compatibility with Miller 5. + -h|--help Show this message. + + Accumulators for -a: count Count instances of fields null_count Count number of empty-string/JSON-null instances per field distinct_count Count number of distinct values per field @@ -1506,17 +1558,6 @@ max Compute maximum values of specified fields minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields - -f {a,b,c} Value-field names on which to compute statistics. Requires -o. - -r {a,b,c} Regular expressions for value-field names on which to compute - statistics. Requires -o. - -c {a,b,c} Substrings for collapse mode. All fields which have the same names - after removing substrings will be accumulated together. Please see - examples below. - -i Use interpolated percentiles, like R's type=7; default like type=1. - Not sensical for string-valued fields. - -o {name} Output field basename for -f/-r. - -k Keep the input fields which contributed to the output statistics; - the default is to omit them. String-valued data make sense unless arithmetic on them is required, e.g. for sum, mean, interpolated percentiles, etc. In case of mixed data, @@ -1538,26 +1579,40 @@ Shows the most frequently occurring distinct values for specified field names. The first entry is the statistical mode; the remaining are runners-up. Options: - -f {one or more comma-separated field names}. Required flag. - -n {count}. Optional flag defaulting to 10. - -b Suppress counts; show only field values. - -o {name} Field name for output count. Default "count". + -f {a,b,c} One or more comma-separated field names to group by. Required flag. + -n {count} Maximum number of results to output. Optional flag defaulting to 10. + -b Suppress counts; show only field values. + -o {name} Field name for output count. Default "count". + -h|--help Show this message. See also "mlr least-frequent". 1mnest0m Usage: mlr nest [options] Explodes specified field values into separate fields/records, or reverses this. Options: - --explode,--implode One is required. - --values,--pairs One is required. - --across-records,--across-fields One is required. - -f {field name} Required. - -r {field names} Like -f but treat arguments as a regular expression. Match all - field names and operate on each in record order. Example: `-r '^[xy]$`'. - --nested-fs {string} Defaults to ";". Field separator for nested values. - --nested-ps {string} Defaults to ":". Pair separator for nested key-value pairs. - --evar {string} Shorthand for --explode --values --across-records --nested-fs {string} - --ivar {string} Shorthand for --implode --values --across-records --nested-fs {string} + --explode Explode field values into separate fields/records. One of + --explode or --implode is required. + --implode Reverse of --explode. One of --explode or --implode is + required. + --values Operate on field values. One of --values or --pairs is + required. + --pairs Operate on field key-value pairs. One of --values or + --pairs is required. + --across-records Explode/implode across records. One of --across-records or + --across-fields is required. + --across-fields Explode/implode across fields. One of --across-records or + --across-fields is required. + -f {field name} Required field name to operate on. + -r {field names} Like -f but treat arguments as a regular expression. Match + all field names and operate on each in record order. + Example: -r '^[xy]$'. + --nested-fs {string} Field separator for nested values. Defaults to ";". + --nested-ps {string} Pair separator for nested key-value pairs. Defaults to ":". + --evar {string} Shorthand for --explode --values --across-records + --nested-fs {string}. + --ivar {string} Shorthand for --implode --values --across-records + --nested-fs {string}. + -h|--help Show this message. Please use "mlr --usage-separator-options" for information on specifying separators. Examples: @@ -1609,56 +1664,53 @@ See also: https://miller.readthedocs.io/en/latest/reference-verbs Options: - -f {file name} File containing a DSL expression (see examples below). If the filename - is a directory, all *.mlr files in that directory are loaded. + -f {file name} File containing a DSL expression (see examples below). If the + filename is a directory, all *.mlr files in that directory are + loaded. + -e {expression} DSL expression to evaluate. You can use this after -f to add an + expression. Example use case: define functions/subroutines in a + file you specify with -f, then call them with an expression you + specify with -e. + -s {name=value} Predefines out-of-stream variable @name to have the given value. + Thus mlr put -s foo=97 '$column += @foo' is like mlr put 'begin + {@foo = 97} $column += @foo'. The value part is subject to + type-inferencing. May be specified more than once, e.g. -s + name1=value1 -s name2=value2. Note: the value may be an + environment variable, e.g. -s sequence=$SEQUENCE. + -x Prints records for which {expression} evaluates to false, not + true, i.e. invert the sense of the filter expression. Default + false. + -q Does not include the modified record in the output stream. + Useful for when all desired output is in begin and/or end + blocks. + -S No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. + -F No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. + -w Print warnings about things like uninitialized variables. + -W Same as -w, but exit the process if there are any warnings. + -p Prints the expression's AST (abstract syntax tree), which gives + full transparency on the precedence and associativity rules of + Miller's grammar, to stdout. + -d Like -p but uses a parenthesized-expression format for the AST. + -D Like -d but with output all on one line. + -E Echo DSL expression before printing parse-tree. + -v Same as -E -p. + -X Exit after parsing but before stream-processing. Useful with + -v/-d/-D, if you only want to look at parser information. + --explain Parse and type-check the DSL expression, report whether it is + valid, and exit without reading the input stream. Exit status is + 0 if the expression is valid and non-zero otherwise; combine + with --errors-json for a machine-readable error. + -h|--help Show this message. - -e {expression} You can use this after -f to add an expression. Example use - case: define functions/subroutines in a file you specify with -f, then call - them with an expression you specify with -e. - - (If you mix -e and -f then the expressions are evaluated in the order encountered. + If you mix -e and -f then the expressions are evaluated in the order encountered. Since the expression pieces are simply concatenated, please be sure to use intervening - semicolons to separate expressions.) + semicolons to separate expressions. - -s name=value: Predefines out-of-stream variable @name to have - Thus mlr put -s foo=97 '$column += @foo' is like - mlr put 'begin {@foo = 97} $column += @foo'. - The value part is subject to type-inferencing. - May be specified more than once, e.g. -s name1=value1 -s name2=value2. - Note: the value may be an environment variable, e.g. -s sequence=$SEQUENCE - - -x (default false) Prints records for which {expression} evaluates to false, not true, - i.e. invert the sense of the filter expression. - - -q Does not include the modified record in the output stream. - Useful for when all desired output is in begin and/or end blocks. - - -S and -F: There are no-ops in Miller 6 and above, since now type-inferencing is done - by the record-readers before filter/put is executed. Supported as no-op pass-through - flags for backward compatibility. - - -h|--help Show this message. - - Parser-info options: - - -w Print warnings about things like uninitialized variables. - - -W Same as -w, but exit the process if there are any warnings. - - -p Prints the expressions's AST (abstract syntax tree), which gives full - transparency on the precedence and associativity rules of Miller's grammar, - to stdout. - - -d Like -p but uses a parenthesized-expression format for the AST. - - -D Like -d but with output all on one line. - - -E Echo DSL expression before printing parse-tree - - -v Same as -E -p. - - -X Exit after parsing but before stream-processing. Useful with -v/-d/-D, if you - only want to look at parser information. + Parser-info options are -w, -W, -p, -d, -D, -E, -v, and -X. Examples: mlr --from example.csv put '$qr = $quantity * $rate' @@ -1702,16 +1754,15 @@ Usage: mlr rename [options] {old1,new1,old2,new2,...} Renames specified fields. Options: - -r Treat old field names as regular expressions. "ab", "a.*b" - will match any field name containing the substring "ab" or - matching "a.*b", respectively; anchors of the form "^ab$", - "^a.*b$" may be used. New field names may be plain strings, - or may contain capture groups of the form "\1" through - "\9". Wrapping the regex in double quotes is optional, but - is required if you wish to follow it with 'i' to indicate - case-insensitivity. - -g Do global replacement within each field name rather than - first-match replacement. + -r Treat old field names as regular expressions. "ab", "a.*b" will match + any field name containing the substring "ab" or matching "a.*b", + respectively; anchors of the form "^ab$", "^a.*b$" may be used. New + field names may be plain strings, or may contain capture groups of the + form "\1" through "\9". Wrapping the regex in double quotes is + optional, but is required if you wish to follow it with 'i' to + indicate case-insensitivity. + -g Do global replacement within each field name rather than first-match + replacement. -h|--help Show this message. Examples: mlr rename old_name,new_name @@ -1725,18 +1776,19 @@ Usage: mlr reorder [options] Moves specified names to start of record, or end of record. Options: - -e Put specified field names at record end: default is to put them at record start. + -e Put specified field names at record end: default is to put them at + record start. -f {a,b,c} Field names to reorder. - -r Treat field names as regular expressions. Matched fields are moved - to start or end in record order. Example: -r '^YYY,^XXX' puts all - YYY- and XXX-prefixed fields first (in record order), then the rest. + -r {a,b,c} Treat field names as regular expressions. Matched fields are moved to + start or end in record order. Example: -r '^YYY,^XXX' puts all YYY- + and XXX-prefixed fields first (in record order), then the rest. -b {x} Put field names specified with -f before field name specified by {x}, if any. If {x} isn't present in a given record, the specified fields will not be moved. -a {x} Put field names specified with -f after field name specified by {x}, if any. If {x} isn't present in a given record, the specified fields will not be moved. - -h|--help Show this message. + -h|--help Show this message. Examples: mlr reorder -f a,b sends input record "d=4,b=2,a=1,c=3" to "a=1,b=2,d=4,c=3". @@ -1746,11 +1798,12 @@ 1mrepeat0m Usage: mlr repeat [options] Copies input records to output records multiple times. - Options must be exactly one of the following: - -n {repeat count} Repeat each input record this many times. - -f {field name} Same, but take the repeat count from the specified - field name of each input record. - -h|--help Show this message. + Options must be exactly one of -n or -f. + Options: + -n {repeat count} Repeat each input record this many times. + -f {field name} Same as -n, but take the repeat count from the specified field + name of each input record. + -h|--help Show this message. Example: echo x=0 | mlr repeat -n 4 then put '$x=urand()' produces: @@ -1787,6 +1840,19 @@ These pivot/reshape the input data to undo the wide-to-long operation. Note: this does not work with tail -f; it produces output records only after all input records have been read. + Options: + -i {input field names} Input field names for wide-to-long reshape. + Use with -o. + -r {input field regex} Input field regex for wide-to-long reshape. + May be repeated. Use with -o. If you have + multiple regexes, please specify them using + multiple -r, since regexes can contain + commas within them. + -o {key-field name,value-field name} Output key-field and value-field names for + wide-to-long reshape. Requires -i or -r. + -s {key-field name,value-field name} Key-field and value-field names for + long-to-wide reshape. + -h|--help Show this message. Examples: @@ -1836,17 +1902,20 @@ See also mlr bootstrap and mlr shuffle. Options: -g {a,b,c} Optional: group-by-field names for samples, e.g. a,b,c. - -k {k} Required: number of records to output in total, or by group if using -g. - -h|--help Show this message. + -k {k} Required: number of records to output in total, or by group if using + -g. + -h|--help Show this message. 1msec2gmtdate0m - Usage: ../c/mlr sec2gmtdate {comma-separated list of field names} + Usage: mlr sec2gmtdate {comma-separated list of field names} Replaces a numeric field representing seconds since the epoch with the corresponding GMT year-month-day timestamp; leaves non-numbers as-is. This is nothing more than a keystroke-saver for the sec2gmtdate function: - ../c/mlr sec2gmtdate time1,time2 + mlr sec2gmtdate time1,time2 is the same as - ../c/mlr put '$time1=sec2gmtdate($time1);$time2=sec2gmtdate($time2)' + mlr put '$time1=sec2gmtdate($time1);$time2=sec2gmtdate($time2)' + Options: + -h|--help Show this message. 1msec2gmt0m Usage: mlr sec2gmt [options] {comma-separated list of field names} @@ -1857,24 +1926,32 @@ is the same as mlr put '$time1 = sec2gmt($time1); $time2 = sec2gmt($time2)' Options: - -1 through -9: format the seconds using 1..9 decimal places, respectively. - --millis Input numbers are treated as milliseconds since the epoch. - --micros Input numbers are treated as microseconds since the epoch. - --nanos Input numbers are treated as nanoseconds since the epoch. + -1 Format seconds with 1 decimal place. + -2 Format seconds with 2 decimal places. + -3 Format seconds with 3 decimal places. + -4 Format seconds with 4 decimal places. + -5 Format seconds with 5 decimal places. + -6 Format seconds with 6 decimal places. + -7 Format seconds with 7 decimal places. + -8 Format seconds with 8 decimal places. + -9 Format seconds with 9 decimal places. + --millis Input numbers are treated as milliseconds since the epoch. + --micros Input numbers are treated as microseconds since the epoch. + --nanos Input numbers are treated as nanoseconds since the epoch. -h|--help Show this message. 1mseqgen0m Usage: mlr seqgen [options] - Passes input records directly to output. Most useful for format conversion. Produces a sequence of counters. Discards the input record stream. Produces - output as specified by the options + output as specified by the options. Options: - -f {name} (default "i") Field name for counters. - --start {value} (default 1) Inclusive start value. - --step {value} (default 1) Step value. - --stop {value} (default 100) Inclusive stop value. - -h|--help Show this message. + -f {name} Field name for counters. Default "i". + --start {value} Inclusive start value. Default 1. + --step {value} Step value. Default 1. May be negative but not zero (unless + start == stop). + --stop {value} Inclusive stop value. Default 100. + -h|--help Show this message. Start, stop, and/or step may be floating-point. Output is integer if start, stop, and step are all integers. Step may be negative. It may not be zero unless start == stop. @@ -1902,17 +1979,20 @@ in the order they were encountered in the input record stream. Options: - -f {comma-separated field names} Lexical ascending - -r {comma-separated field names} Lexical descending - -c {comma-separated field names} Case-folded lexical ascending - -cr {comma-separated field names} Case-folded lexical descending - -n {comma-separated field names} Numerical ascending; nulls sort last - -nf {comma-separated field names} Same as -n - -nr {comma-separated field names} Numerical descending; nulls sort first - -t {comma-separated field names} Natural ascending - -b Move sort fields to start of record, as in reorder -b - -tr|-rt {comma-separated field names} Natural descending - -h|--help Show this message. + -f {a,b,c} Lexical ascending sort on the specified field names. + -r {a,b,c} Lexical descending sort on the specified field names. + -c {a,b,c} Case-folded lexical ascending sort on the specified field names. + -cr {a,b,c} Case-folded lexical descending sort on the specified field + names. + -n {a,b,c} Numerical ascending sort on the specified field names; nulls + sort last. + -nf {a,b,c} Same as -n. + -nr {a,b,c} Numerical descending sort on the specified field names; nulls + sort first. + -t {a,b,c} Natural ascending sort on the specified field names. + -b Move sort fields to start of record, as in reorder -b. + -tr|-rt {a,b,c} Natural descending sort on the specified field names. + -h|--help Show this message. Example: mlr sort -f a,b -nr x,y,z @@ -1923,13 +2003,13 @@ Usage: mlr sort-within-records [options] Outputs records sorted lexically ascending by keys. Options: - -f {names} Sort only these keys; others preserve record order. - -r {regex} Sort only keys matching this regex; others preserve record order. - Example: -r '^[xy]' sorts keys starting with x or y. - With no regex argument, -r recursively sorts subobjects/submaps - (e.g. for JSON input), or combines with -f to treat names as regex. - -n Sort field names naturally (e.g. 2 before 12). Combines with -f/-r. - -h|--help Show this message. + -f {names} Sort only these keys; others preserve record order. + -r {regex} Sort only keys matching this regex; others preserve record order. + Example: -r '^[xy]' sorts keys starting with x or y. With no regex + argument, -r recursively sorts subobjects/submaps (e.g. for JSON + input), or combines with -f to treat names as regexes. + -n Sort field names naturally (e.g. 2 before 12). Combines with -f/-r. + -h|--help Show this message. 1msparsify0m Usage: mlr sparsify [options] @@ -1937,26 +2017,28 @@ specified value). Only makes sense with output format not being CSV or TSV. Options: -s {filler string} What values to remove. Defaults to the empty string. - -f {a,b,c} Specify field names to be operated on; any other fields won't be - modified. The default is to modify all fields. - -h|--help Show this message. + -f {a,b,c} Specify field names to be operated on; any other fields won't + be modified. The default is to modify all fields. + -h|--help Show this message. Example: if input is a=1,b=,c=3 then output is a=1,c=3. 1msplit0m Usage: mlr split [options] {filename} Options: - -n {n}: Cap file sizes at N records. - -m {m}: Produce M files, round-robining records among them. - -g {a,b,c}: Write separate files with records having distinct values for fields named a,b,c. - Exactly one of -m, -n, or -g must be supplied. - --prefix {p} Specify filename prefix; default "split". - --suffix {s} Specify filename suffix; default is from mlr output format, e.g. "csv". - --folder {f} Specify output directory; default is current directory. - -a Append to existing file(s), if any, rather than overwriting. - -v Send records along to downstream verbs as well as splitting to files. + -n {n} Cap output file sizes at N records. + -m {m} Produce M files, round-robining records among them. + -g {a,b,c} Write separate files with records having distinct values for the + specified field names. + --prefix {p} Output filename prefix. Default "split". + --suffix {s} Output filename suffix. Default is from the output format, e.g. + "csv". + --folder {f} Output directory. Default is current directory. + -a Append to existing files rather than overwriting. + -v Send records downstream as well as splitting to files. -e Do NOT URL-escape names of output files. - -j {J} Use string J to join filename parts; default "_". + -j {J} String used to join filename parts. Default "_". -h|--help Show this message. + Exactly one of -m, -n, or -g must be supplied. Any of the output-format command-line flags (see mlr -h). For example, using mlr --icsv --from myfile.csv split --ojson -n 1000 the input is CSV, but the output files are JSON. @@ -1995,17 +2077,42 @@ as \n, \t, and \x1f. Write \\ to get a literal backslash. See also the `gsub` and `sub` verbs. Options: - -f {a,b,c} Field names to convert. - -r {regex} Regular expression for field names to convert. - -a Convert all fields. - -h|--help Show this message. + -f {a,b,c} Field names to apply substitution to. + -r {regex} Regular expression for field names to apply substitution to. + -a Apply substitution to all fields. + -h|--help Show this message. 1mstats10m Usage: mlr stats1 [options] Computes univariate statistics for one or more given fields, accumulated across the input record stream. Options: - -a {sum,count,...} Names of accumulators: one or more of: + -a {sum,count,...} Names of accumulators: one or more of the listed values. Also + accepts median (same as p50) and percentiles p{n} for n in + 0..100, e.g. p10 p25.2 p50 p98 p100. + -f {a,b,c} Value-field names on which to compute statistics. + --fr {regex} Regex for value-field names on which to compute statistics + (compute statistics on values in all field names matching the + regex). + --fx {regex} Inverted regex for value-field names on which to compute + statistics (compute statistics on values in all field names + not matching the regex). + -g {d,e,f} Optional group-by-field names. + --gr {regex} Regex for optional group-by-field names (group by values in + field names matching the regex). + --gx {regex} Inverted regex for optional group-by-field names (group by + values in field names not matching the regex). + --grfx {regex} Shorthand for --gr {regex} --fx {that same regex}. + -i Use interpolated percentiles, like R's type=7; default like + type=1. Not sensical for string-valued fields. + -s Print iterative stats. Useful in tail -f contexts, in which + case please avoid pprint-format output since end of input + stream will never be seen. Likewise, if input is coming from + `tail -f` be sure to use `--records-per-batch 1`. + -S No-op flag for backward compatibility with Miller 5. + -F No-op flag for backward compatibility with Miller 5. + -h|--help Show this message. + Names of accumulators for -a, one or more of: median This is the same as p50 p10 p25.2 p50 p98 p100 etc. count Count instances of fields @@ -2025,28 +2132,6 @@ max Compute maximum values of specified fields minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields - - -f {a,b,c} Value-field names on which to compute statistics - --fr {regex} Regex for value-field names on which to compute statistics - (compute statistics on values in all field names matching regex - --fx {regex} Inverted regex for value-field names on which to compute statistics - (compute statistics on values in all field names not matching regex) - - -g {d,e,f} Optional group-by-field names - --gr {regex} Regex for optional group-by-field names - (group by values in field names matching regex) - --gx {regex} Inverted regex for optional group-by-field names - (group by values in field names not matching regex) - - --grfx {regex} Shorthand for --gr {regex} --fx {that same regex} - - -i Use interpolated percentiles, like R's type=7; default like type=1. - Not sensical for string-valued fields.\n"); - -s Print iterative stats. Useful in tail -f contexts, in which - case please avoid pprint-format output since end of input - stream will never be seen. Likewise, if input is coming from `tail -f` - be sure to use `--records-per-batch 1`. - -h|--help Show this message. Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape Example: mlr stats1 -a count,mode -f size Example: mlr stats1 -a count,mode -f size -g shape @@ -2069,7 +2154,27 @@ Usage: mlr stats2 [options] Computes bivariate statistics for one or more given field-name pairs, accumulated across the input record stream. - -a {linreg-ols,corr,...} Names of accumulators: one or more of: + Options: + -a {linreg-ols,corr,...} Names of accumulators: one or more of the listed + values. + -f {a,b,c,d} Value-field name-pairs on which to compute statistics. + There must be an even number of names. + -g {e,f,g} Optional group-by-field names. + -v Print additional output for linreg-pca. + -s Print iterative stats. Useful in tail -f contexts, in + which case please avoid pprint-format output since end + of input stream will never be seen. Likewise, if input + is coming from `tail -f`, be sure to use + `--records-per-batch 1`. + --fit Rather than printing regression parameters, applies + them to the input data to compute new fit fields. All + input records are held in memory until end of input + stream. Has effect only for linreg-ols, linreg-pca, and + logireg. + -S No-op flag for backward compatibility with Miller 5. + -F No-op flag for backward compatibility with Miller 5. + -h|--help Show this message. + Names of accumulators for -a, one or more of: linreg-ols Linear regression using ordinary least squares linreg-pca Linear regression using principal component analysis r2 Quality metric for linreg-ols (linreg-pca emits its own) @@ -2077,18 +2182,6 @@ corr Sample correlation cov Sample covariance covx Sample-covariance matrix - -f {a,b,c,d} Value-field name-pairs on which to compute statistics. - There must be an even number of names. - -g {e,f,g} Optional group-by-field names. - -v Print additional output for linreg-pca. - -s Print iterative stats. Useful in tail -f contexts, in which - case please avoid pprint-format output since end of input - stream will never be seen. Likewise, if input is coming from - `tail -f`, be sure to use `--records-per-batch 1`. - --fit Rather than printing regression parameters, applies them to - the input data to compute new fit fields. All input records are - held in memory until end of input stream. Has effect only for - linreg-ols, linreg-pca, and logireg. Only one of -s or --fit may be used. Example: mlr stats2 -a linreg-pca -f x,y Example: mlr stats2 -a linreg-ols,r2 -f x,y -g size,shape @@ -2098,7 +2191,25 @@ Usage: mlr step [options] Computes values dependent on earlier/later records, optionally grouped by category. Options: - -a {delta,rsum,...} Names of steppers: comma-separated, one or more of: + -a {delta,rsum,...} Names of steppers: comma-separated, one or more of the + listed values. + -f {a,b,c} Value-field names on which to compute statistics. + -g {d,e,f} Optional group-by-field names. + -F Computes integerable things (e.g. counter) in floating + point. As of Miller 6 this happens automatically, but the + flag is accepted as a no-op for backward compatibility with + Miller 5 and below. + -d {x,y,z} Weights for EWMA. 1 means current sample gets all weight (no + smoothing), near under 1 is light smoothing, near over 0 is + heavy smoothing. Multiple weights may be specified, e.g. + "mlr step -a ewma -f sys_load -d 0.01,0.1,0.9". Default if + omitted is "-d 0.5". + -o {a,b,c} Custom suffixes for EWMA output fields. If omitted, these + default to the -d values. If supplied, the number of -o + values must be the same as the number of -d values. + -h|--help Show this message. + + Names of steppers for -a, comma-separated, one or more of: counter Count instances of field(s) between successive records delta Compute differences in field(s) between successive records ewma Exponentially weighted moving average over successive records @@ -2111,21 +2222,6 @@ shift_lead Include value(s) in field(s) from the next record, if any slwin Sliding-window averages over m records back and n forward. E.g. slwin_7_2 for 7 back and 2 forward. - -f {a,b,c} Value-field names on which to compute statistics - -g {d,e,f} Optional group-by-field names - -F Computes integerable things (e.g. counter) in floating point. - As of Miller 6 this happens automatically, but the flag is accepted - as a no-op for backward compatibility with Miller 5 and below. - -d {x,y,z} Weights for EWMA. 1 means current sample gets all weight (no - smoothing), near under 1 is light smoothing, near over 0 is - heavy smoothing. Multiple weights may be specified, e.g. - "mlr step -a ewma -f sys_load -d 0.01,0.1,0.9". Default if omitted - is "-d 0.5". - -o {a,b,c} Custom suffixes for EWMA output fields. If omitted, these default to - the -d values. If supplied, the number of -o values must be the same - as the number of -d values. - -h|--help Show this message. - Examples: mlr step -a rsum -f request_size mlr step -a delta -f request_size -g hostname @@ -2146,10 +2242,10 @@ and \x1f. Write \\ to get a literal backslash. See also the `gsub` and `ssub` verbs. Options: - -f {a,b,c} Field names to convert. - -r {regex} Regular expression for field names to convert. - -a Convert all fields. - -h|--help Show this message. + -f {a,b,c} Field names to apply substitution to. + -r {regex} Regular expression for field names to apply substitution to. + -a Apply substitution to all fields. + -h|--help Show this message. 1msummary0m Usage: mlr summary [options] @@ -2189,19 +2285,19 @@ Options: -a {mean,sum,etc.} Use only the specified summarizers. - -x {mean,sum,etc.} Use all summarizers, except the specified ones. + -x {mean,sum,etc.} Use all summarizers except the specified ones. --all Use all available summarizers. - --transpose Show output with field names as column names.. - -h|--help Show this message. + --transpose Show output with field names as column names. + -h|--help Show this message. 1msurv0m Usage: mlr surv -d {duration-field} -s {status-field} Estimate Kaplan-Meier survival curve (right-censored). Options: - -d {field} Name of duration field (time-to-event or censoring). - -s {field} Name of status field (0=censored, 1=event). - -h, --help Show this message. + -d {field} Name of duration field (time-to-event or censoring). + -s {field} Name of status field (0=censored, 1=event). + -h|--help Show this message. 1mtac0m Usage: mlr tac [options] @@ -2214,34 +2310,35 @@ Passes through the last n records, optionally by category. Options: -g {a,b,c} Optional group-by-field names for tail counts, e.g. a,b,c. - -n {n} Tail-count to print. Default 10. - A leading '+' means start at the nth record rather than print - the last n: e.g. -n +3 passes through all but the first 2 - records, optionally by category. - -h|--help Show this message. + -n {n} Tail-count to print. Default 10. A leading '+' means start at the nth + record rather than print the last n: e.g. -n +3 passes through all + but the first 2 records, optionally by category. + -h|--help Show this message. 1mtee0m Usage: mlr tee [options] {filename} Options: - -a Append to existing file, if any, rather than overwriting. - -p Treat filename as a pipe-to command. + -a Append to existing file, if any, rather than overwriting. + -p Treat filename as a pipe-to command. + -h|--help Show this message. Any of the output-format command-line flags (see mlr -h). Example: using mlr --icsv --opprint put '...' then tee --ojson ./mytap.dat then stats1 ... the input is CSV, the output is pretty-print tabular, but the tee-file output is written in JSON format. - -h|--help Show this message. - 1mtemplate0m Usage: mlr template [options] Places input-record fields in the order specified by list of column names. If the input record is missing a specified field, it will be filled with the fill-with. If the input record possesses an unspecified field, it will be discarded. Options: - -f {a,b,c} Comma-separated field names for template, e.g. a,b,c. - -t {filename} CSV file whose header line will be used for template. - --fill-with {filler string} What to fill absent fields with. Defaults to the empty string. - -h|--help Show this message. + -f {a,b,c} Comma-separated field names for template, e.g. + a,b,c. + -t {filename} CSV file whose header line will be used for + template. + --fill-with {filler string} What to fill absent fields with. Defaults to the + empty string. + -h|--help Show this message. Example: * Specified fields are a,b,c. * Input record is c=3,a=1,f=6. @@ -2249,16 +2346,19 @@ 1mtop0m Usage: mlr top [options] - -f {a,b,c} Value-field names for top counts. - -g {d,e,f} Optional group-by-field names for top counts. - -n {count} How many records to print per category; default 1. - -a Print all fields for top-value records; default is - to print only value and group-by fields. Requires a single - value-field name only. - --min Print top smallest values; default is top largest values. - -F Keep top values as floats even if they look like integers. - -o {name} Field name for output indices. Default "top_idx". - This is ignored if -a is used. + Options: + -f {a,b,c} Value-field names for top counts. + -g {d,e,f} Optional group-by-field names for top counts. + -n {count} How many records to print per category; default 1. + -a Print all fields for top-value records; default is to print only + value and group-by fields. Requires a single value-field name only. + --max Print top largest values. This is the default. + --min Print top smallest values; default is top largest values. + -F Keep top values as floats even if they look like integers (ignored in + Miller 6, kept for backward compatibility). + -o {name} Field name for output indices. Default "top_idx". Ignored if -a is + used. + -h|--help Show this message. Prints the n records with smallest/largest values at specified fields, optionally by category. If -a is given, then the top records are emitted with the same fields as they appeared in the input. Without -a, only fields @@ -2267,7 +2367,7 @@ 1mutf8-to-latin10m Usage: mlr utf8-to-latin1, with no options. - Recursively converts record strings from Latin-1 to UTF-8. + Recursively converts record strings from UTF-8 to Latin-1. For field-level control, please see the utf8_to_latin1 DSL function. Options: -h|--help Show this message. @@ -2277,9 +2377,9 @@ Reverses flatten. Example: field with name 'a.b.c' and value 4 becomes name 'a' and value '{"b": { "c": 4 }}'. Options: - -f {a,b,c} Comma-separated list of field names to unflatten (default all). + -f {a,b,c} Comma-separated list of field names to unflatten (default all). -s {string} Separator, defaulting to mlr --flatsep value. - -h|--help Show this message. + -h|--help Show this message. 1muniq0m Usage: mlr uniq [options] @@ -2287,21 +2387,24 @@ count-distinct. For uniq, -f is a synonym for -g. Options: - -g {d,e,f} Group-by-field names for uniq counts. - -x {a,b,c} Field names to exclude for uniq: use each record's others instead. - -c Show repeat counts in addition to unique values. - -n Show only the number of distinct values. - -o {name} Field name for output count. Default "count". - -a Output each unique record only once. Incompatible with -g. - With -c, produces unique records, with repeat counts for each. - With -n, produces only one record which is the unique-record count. - With neither -c nor -n, produces unique records. + -g {d,e,f} Group-by field names for uniq counts. + -f {d,e,f} Synonym for -g. + -x {a,b,c} Field names to exclude for uniq; use each record's other fields + instead. + -c Show repeat counts in addition to unique values. + -n Show only the number of distinct values. + -o {name} Field name for output count. Default "count". + -a Output each unique record only once. Incompatible with -g. With -c, + produces unique records, with repeat counts for each. With -n, + produces only one record which is the unique-record count. With + neither -c nor -n, produces unique records. + -h|--help Show this message. 1munspace0m Usage: mlr unspace [options] Replaces spaces in record keys and/or values with _. This is helpful for PPRINT output. Options: - -f {x} Replace spaces with specified filler character. + -f {x} Replace spaces with specified filler character. Default "_". -k Unspace only keys, not keys and values. -v Unspace only values, not keys and values. -h|--help Show this message. @@ -2312,11 +2415,11 @@ For field names absent in a given record but present in others, fills in a value. This verb retains all input before producing any output. Options: - --fill-with {filler string} What to fill absent fields with. Defaults to - the empty string. - -f {a,b,c} Specify field names to be operated on. Any other fields won't be - modified, and operation will be streaming. - -h|--help Show this message. + --fill-with {filler string} What to fill absent fields with. Defaults to the + empty string. + -f {a,b,c} Specify field names to be operated on; others are + not modified and operation is streaming. + -h|--help Show this message. Example: if the input is two records, one being 'a=1,b=2' and the other being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and 'a=,b=3,c=4'. @@ -3878,4 +3981,4 @@ MIME Type for Comma-Separated Values (CSV) Files, the Miller docsite https://miller.readthedocs.io - 2026-07-01 4mMILLER24m(1) + 2026-07-03 4mMILLER24m(1) diff --git a/docs/src/reference-verbs.md b/docs/src/reference-verbs.md index 6e51545b8..a4e7f21f4 100644 --- a/docs/src/reference-verbs.md +++ b/docs/src/reference-verbs.md @@ -1035,6 +1035,10 @@ Options: -v Same as -E -p. -X Exit after parsing but before stream-processing. Useful with -v/-d/-D, if you only want to look at parser information. +--explain Parse and type-check the DSL expression, report whether it is + valid, and exit without reading the input stream. Exit status is + 0 if the expression is valid and non-zero otherwise; combine + with --errors-json for a machine-readable error. -h|--help Show this message. If you mix -e and -f then the expressions are evaluated in the order encountered. @@ -2436,6 +2440,10 @@ Options: -v Same as -E -p. -X Exit after parsing but before stream-processing. Useful with -v/-d/-D, if you only want to look at parser information. +--explain Parse and type-check the DSL expression, report whether it is + valid, and exit without reading the input stream. Exit status is + 0 if the expression is valid and non-zero otherwise; combine + with --errors-json for a machine-readable error. -h|--help Show this message. If you mix -e and -f then the expressions are evaluated in the order encountered. diff --git a/man/manpage.txt b/man/manpage.txt index 30058ef3b..7dd245f9a 100644 --- a/man/manpage.txt +++ b/man/manpage.txt @@ -563,6 +563,12 @@ 1mMISCELLANEOUS FLAGS0m These are flags which don't fit into any other category. + --errors-json Emit parse errors as a JSON object to stderr instead + of a plain text message. Intended for AI agents and + scripts that branch on error kind rather than + regex-matching prose. Equivalent to setting the + `MLR_ERRORS_JSON` environment variable to a truthy + value. --fflush Force buffered output to be written after every output record. The default is flush output after every record if the output is to the terminal, or @@ -977,57 +983,58 @@ Replaces a numeric field with a number of asterisks, allowing for cheesy bar plots. These align best with --opprint or --oxtab output format. Options: - -f {a,b,c} Field names to convert to bars. - --lo {lo} Lower-limit value for min-width bar: default '0.000000'. - --hi {hi} Upper-limit value for max-width bar: default '100.000000'. - -w {n} Bar-field width: default '40'. - --auto Automatically computes limits, ignoring --lo and --hi. - Holds all records in memory before producing any output. - -c {character} Fill character: default '*'. - -x {character} Out-of-bounds character: default '#'. - -b {character} Blank character: default '.'. + -f {a,b,c} Field names to convert to bars. + --lo {lo} Lower-limit value for min-width bar: default '0.000000'. + --hi {hi} Upper-limit value for max-width bar: default '100.000000'. + -w {n} Bar-field width: default '40'. + --auto Automatically computes limits, ignoring --lo and --hi. Holds all + records in memory before producing any output. + -c {character} Fill character: default '*'. + -x {character} Out-of-bounds character: default '#'. + -b {character} Blank character: default '.'. + -h|--help Show this message. Nominally the fill, out-of-bounds, and blank characters will be strings of length 1. However you can make them all longer if you so desire. - -h|--help Show this message. 1mbootstrap0m Usage: mlr bootstrap [options] Emits an n-sample, with replacement, of the input records. See also mlr sample and mlr shuffle. Options: - -n Number of samples to output. Defaults to number of input records. - Must be non-negative. + -n {n} Number of samples to output. Defaults to number of input records. Must + be non-negative. -h|--help Show this message. 1mcase0m Usage: mlr case [options] Uppercases strings in record keys and/or values. Options: - -k Case only keys, not keys and values. - -v Case only values, not keys and values. - -f {a,b,c} Specify which field names to case (default: all) - -u Convert to uppercase - -l Convert to lowercase - -s Convert to sentence case (capitalize first letter) - -t Convert to title case (capitalize words) - -h|--help Show this message. + -k Case only keys, not keys and values. + -v Case only values, not keys and values. + -f {a,b,c} Specify which field names to case (default: all). + -u Convert to uppercase. + -l Convert to lowercase. + -s Convert to sentence case (capitalize first letter). + -t Convert to title case (capitalize words). + -h|--help Show this message. 1mcat0m Usage: mlr cat [options] Passes input records directly to output. Most useful for format conversion. Options: -n Prepend field "n" to each record with record-counter starting at 1. - -N {name} Prepend field {name} to each record with record-counter starting at 1. - -g {a,b,c} Optional group-by-field names for counters, e.g. a,b,c + -N {name} Prepend field {name} to each record with record-counter starting at + 1. + -g {a,b,c} Optional group-by-field names for counters, e.g. a,b,c. --filename Prepend current filename to each record. --filenum Prepend current filenum (1-up) to each record. - -h|--help Show this message. + -h|--help Show this message. 1mcheck0m Usage: mlr check [options] Consumes records without printing any output, - Useful for doing a well-formatted check on input data. with the exception that warnings are printed to stderr. + Useful for doing a well-formatted check on input data. Current checks are: * Data are parseable * If any key is the empty string @@ -1043,11 +1050,11 @@ and clean_whitespace. Options: - -k|--keys-only Do not touch values. - -v|--values-only Do not touch keys. + -k|--keys-only Do not touch values. + -v|--values-only Do not touch keys. + -h|--help Show this message. It is an error to specify -k as well as -v -- to clean keys and values, leave off -k as well as -v. - -h|--help Show this message. 1mcount-distinct0m Usage: mlr count-distinct [options] @@ -1055,25 +1062,26 @@ Same as uniq -c. Options: - -f {a,b,c} Field names for distinct count. - -x {a,b,c} Field names to exclude for distinct count: use each record's others instead. - -n Show only the number of distinct values. Not compatible with -u. - -o {name} Field name for output count. Default "count". - Ignored with -u. - -u Do unlashed counts for multiple field names. With -f a,b and - without -u, computes counts for distinct combinations of a - and b field values. With -f a,b and with -u, computes counts - for distinct a field values and counts for distinct b field - values separately. + -f {a,b,c} Field names for distinct count (synonym for -g). + -g {a,b,c} Field names for distinct count. + -x {a,b,c} Field names to exclude for distinct count; use each record's other + fields instead. + -n Show only the number of distinct values. Not compatible with -u. + -o {name} Field name for output count. Default "count". Ignored with -u. + -u Do unlashed counts for multiple field names. With -f a,b and without + -u, computes counts for distinct combinations of a and b field + values. With -f a,b and with -u, computes counts for distinct a field + values and counts for distinct b field values separately. + -h|--help Show this message. 1mcount0m Usage: mlr count [options] Prints number of records, optionally grouped by distinct values for specified field names. Options: -g {a,b,c} Optional group-by-field names for counts, e.g. a,b,c - -n {n} Show only the number of distinct values. Not interesting without -g. - -o {name} Field name for output-count. Default "count". - -h|--help Show this message. + -n Show only the number of distinct values. Not interesting without -g. + -o {name} Field name for output-count. Default "count". + -h|--help Show this message. 1mcount-similar0m Usage: mlr count-similar [options] @@ -1081,22 +1089,22 @@ the number of other records having the same group-by field values. Options: -g {a,b,c} Group-by-field names for counts, e.g. a,b,c - -o {name} Field name for output-counts. Defaults to "count". - -h|--help Show this message. + -o {name} Field name for output-counts. Defaults to "count". + -h|--help Show this message. 1mcut0m Usage: mlr cut [options] Passes through input records with specified fields included/excluded. Options: - -f {a,b,c} Comma-separated field names for cut, e.g. a,b,c. - -o Retain fields in the order specified here in the argument list. - Default is to retain them in the order found in the input data. - -x|--complement Exclude, rather than include, field names specified by -f. - -r Treat field names as regular expressions. "ab", "a.*b" will - match any field name containing the substring "ab" or matching - "a.*b", respectively; anchors of the form "^ab$", "^a.*b$" may - be used. - -h|--help Show this message. + -f {a,b,c} Comma-separated field names to include or exclude, e.g. a,b,c. + -o Retain fields in the order specified by -f rather than in + input-record order. + -x|--complement Exclude, rather than include, the field names specified by -f. + -r Treat field names as regular expressions. "ab", "a.*b" will + match any field name containing the substring "ab" or matching + "a.*b", respectively; anchors of the form "^ab$", "^a.*b$" may + be used. + -h|--help Show this message. Examples: mlr cut -f hostname,status mlr cut -x -f hostname,status @@ -1108,11 +1116,11 @@ Usage: mlr decimate [options] Passes through one of every n records, optionally by category. Options: - -b Decimate by printing first of every n. - -e Decimate by printing last of every n (default). - -g {a,b,c} Optional group-by-field names for decimate counts, e.g. a,b,c. - -n {n} Decimation factor (default 10). - -h|--help Show this message. + -b Decimate by printing first of every n. + -e Decimate by printing last of every n (default). + -g {a,b,c} Optional group-by-field names for decimate counts, e.g. a,b,c. + -n {n} Decimation factor (default 10). + -h|--help Show this message. 1mfill-down0m Usage: mlr fill-down [options] @@ -1122,13 +1130,14 @@ With -a, a field is 'missing' only if it is absent. Options: - --all Operate on all fields in the input. - -a|--only-if-absent If a given record has a missing value for a given field, - fill that from the corresponding value from a previous record, if any. - By default, a 'missing' field either is absent, or has the empty-string value. - With -a, a field is 'missing' only if it is absent. - -f Field names for fill-down. - -h|--help Show this message. + --all Operate on all fields in the input. + -a|--only-if-absent If a given record has a missing value for a given field, + fill that from the corresponding value from a previous + record, if any. By default, a 'missing' field either is + absent, or has the empty-string value. With -a, a field is + 'missing' only if it is absent. + -f {a,b,c} Field names for fill-down. + -h|--help Show this message. 1mfill-empty0m Usage: mlr fill-empty [options] @@ -1136,6 +1145,7 @@ Options: -v {string} Fill-value: defaults to "N/A" -S Don't infer type -- so '-v 0' would fill string 0 not int 0. + -h|--help Show this message. 1mfilter0m Usage: mlr filter [options] {DSL expression} @@ -1144,56 +1154,53 @@ See also: https://miller.readthedocs.io/en/latest/reference-verbs Options: - -f {file name} File containing a DSL expression (see examples below). If the filename - is a directory, all *.mlr files in that directory are loaded. + -f {file name} File containing a DSL expression (see examples below). If the + filename is a directory, all *.mlr files in that directory are + loaded. + -e {expression} DSL expression to evaluate. You can use this after -f to add an + expression. Example use case: define functions/subroutines in a + file you specify with -f, then call them with an expression you + specify with -e. + -s {name=value} Predefines out-of-stream variable @name to have the given value. + Thus mlr put -s foo=97 '$column += @foo' is like mlr put 'begin + {@foo = 97} $column += @foo'. The value part is subject to + type-inferencing. May be specified more than once, e.g. -s + name1=value1 -s name2=value2. Note: the value may be an + environment variable, e.g. -s sequence=$SEQUENCE. + -x Prints records for which {expression} evaluates to false, not + true, i.e. invert the sense of the filter expression. Default + false. + -q Does not include the modified record in the output stream. + Useful for when all desired output is in begin and/or end + blocks. + -S No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. + -F No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. + -w Print warnings about things like uninitialized variables. + -W Same as -w, but exit the process if there are any warnings. + -p Prints the expression's AST (abstract syntax tree), which gives + full transparency on the precedence and associativity rules of + Miller's grammar, to stdout. + -d Like -p but uses a parenthesized-expression format for the AST. + -D Like -d but with output all on one line. + -E Echo DSL expression before printing parse-tree. + -v Same as -E -p. + -X Exit after parsing but before stream-processing. Useful with + -v/-d/-D, if you only want to look at parser information. + --explain Parse and type-check the DSL expression, report whether it is + valid, and exit without reading the input stream. Exit status is + 0 if the expression is valid and non-zero otherwise; combine + with --errors-json for a machine-readable error. + -h|--help Show this message. - -e {expression} You can use this after -f to add an expression. Example use - case: define functions/subroutines in a file you specify with -f, then call - them with an expression you specify with -e. - - (If you mix -e and -f then the expressions are evaluated in the order encountered. + If you mix -e and -f then the expressions are evaluated in the order encountered. Since the expression pieces are simply concatenated, please be sure to use intervening - semicolons to separate expressions.) + semicolons to separate expressions. - -s name=value: Predefines out-of-stream variable @name to have - Thus mlr put -s foo=97 '$column += @foo' is like - mlr put 'begin {@foo = 97} $column += @foo'. - The value part is subject to type-inferencing. - May be specified more than once, e.g. -s name1=value1 -s name2=value2. - Note: the value may be an environment variable, e.g. -s sequence=$SEQUENCE - - -x (default false) Prints records for which {expression} evaluates to false, not true, - i.e. invert the sense of the filter expression. - - -q Does not include the modified record in the output stream. - Useful for when all desired output is in begin and/or end blocks. - - -S and -F: There are no-ops in Miller 6 and above, since now type-inferencing is done - by the record-readers before filter/put is executed. Supported as no-op pass-through - flags for backward compatibility. - - -h|--help Show this message. - - Parser-info options: - - -w Print warnings about things like uninitialized variables. - - -W Same as -w, but exit the process if there are any warnings. - - -p Prints the expressions's AST (abstract syntax tree), which gives full - transparency on the precedence and associativity rules of Miller's grammar, - to stdout. - - -d Like -p but uses a parenthesized-expression format for the AST. - - -D Like -d but with output all on one line. - - -E Echo DSL expression before printing parse-tree - - -v Same as -E -p. - - -X Exit after parsing but before stream-processing. Useful with -v/-d/-D, if you - only want to look at parser information. + Parser-info options are -w, -W, -p, -d, -D, -E, -v, and -X. Records will pass the filter depending on the last bare-boolean statement in the DSL expression. That can be the result of <, ==, >, etc., the return value of a function call @@ -1231,9 +1238,9 @@ Flattens multi-level maps to single-level ones. Example: field with name 'a' and value '{"b": { "c": 4 }}' becomes name 'a.b.c' and value 4. Options: - -f Comma-separated list of field names to flatten (default all). - -s Separator, defaulting to mlr --flatsep value. - -h|--help Show this message. + -f Comma-separated list of field names to flatten (default all). + -s {string} Separator, defaulting to mlr --flatsep value. + -h|--help Show this message. 1mformat-values0m Usage: mlr format-values [options] @@ -1249,22 +1256,21 @@ undefined behavior and/or program crashes. See your system's "man printf". Options: - -i {integer format} Defaults to "%d". - Examples: "%06lld", "%08llx". - Note that Miller integers are long long so you must use - formats which apply to long long, e.g. with ll in them. - Undefined behavior results otherwise. - -f {float format} Defaults to "%f". - Examples: "%8.3lf", "%.6le". - Note that Miller floats are double-precision so you must - use formats which apply to double, e.g. with l[efg] in them. - Undefined behavior results otherwise. - -s {string format} Defaults to "%s". - Examples: "_%s", "%08s". - Note that you must use formats which apply to string, e.g. - with s in them. Undefined behavior results otherwise. + -i {integer format} Integer format string; defaults to "%d". Examples: "%06lld", + "%08llx". Note that Miller integers are long long so you + must use formats which apply to long long, e.g. with ll in + them. Undefined behavior results otherwise. + -f {float format} Float format string; defaults to "%f". Examples: "%8.3lf", + "%.6le". Note that Miller floats are double-precision so you + must use formats which apply to double, e.g. with l[efg] in + them. Undefined behavior results otherwise. + -s {string format} String format string; defaults to "%s". Examples: "_%s", + "%08s". Note that you must use formats which apply to + string, e.g. with s in them. Undefined behavior results + otherwise. -n Coerce field values autodetected as int to float, and then apply the float format. + -h|--help Show this message. 1mfraction0m Usage: mlr fraction [options] @@ -1278,34 +1284,33 @@ and emits output records. This means it produces no output until all input is read. Options: - -f {a,b,c} Field name(s) for fraction calculation - -g {d,e,f} Optional group-by-field name(s) for fraction counts - -p Produce percents [0..100], not fractions [0..1]. Output field names - end with "_percent" rather than "_fraction" - -c Produce cumulative distributions, i.e. running sums: each output - value folds in the sum of the previous for the specified group - E.g. with input records x=1 x=2 x=3 and x=4, emits output records - x=1,x_cumulative_fraction=0.1 x=2,x_cumulative_fraction=0.3 - x=3,x_cumulative_fraction=0.6 and x=4,x_cumulative_fraction=1.0 + -f {a,b,c} Field name(s) for fraction calculation + -g {d,e,f} Optional group-by-field name(s) for fraction counts + -p Produce percents [0..100], not fractions [0..1]. Output field names + end with "_percent" rather than "_fraction" + -c Produce cumulative distributions, i.e. running sums: each output + value folds in the sum of the previous for the specified group. E.g. + with input records x=1 x=2 x=3 and x=4, emits output records + x=1,x_cumulative_fraction=0.1 x=2,x_cumulative_fraction=0.3 + x=3,x_cumulative_fraction=0.6 and x=4,x_cumulative_fraction=1.0 + -h|--help Show this message. 1mgap0m Usage: mlr gap [options] Emits an empty record every n records, or when certain values change. + One of -n or -g is required. Options: - Emits an empty record every n records, or when certain values change. -g {a,b,c} Print a gap whenever values of these fields (e.g. a,b,c) changes. - -n {n} Print a gap every n records. - One of -f or -g is required. - -n is ignored if -g is present. - -h|--help Show this message. + -n {n} Print a gap every n records. Ignored if -g is present. + -h|--help Show this message. 1mgrep0m Usage: mlr grep [options] {regular expression} Passes through records which match the regular expression. Options: - -i Use case-insensitive search. - -v Invert: pass through records which do not match the regex. - -a Only grep for values, not keys and values. + -i Use case-insensitive search. + -v Invert: pass through records which do not match the regex. + -a Only grep for values, not keys and values. -h|--help Show this message. Note that "mlr filter" is more powerful, but requires you to know field names. By contrast, "mlr grep" allows you to regex-match the entire record. It does this @@ -1321,7 +1326,8 @@ 1mgroup-by0m Usage: mlr group-by [options] {comma-separated field names} - Outputs records in batches having identical values at specified field names.Options: + Outputs records in batches having identical values at specified field names. + Options: -h|--help Show this message. 1mgroup-like0m @@ -1338,21 +1344,28 @@ and \x1f. Write \\ to get a literal backslash. See also the `sub` and `ssub` verbs. Options: - -f {a,b,c} Field names to convert. - -r {regex} Regular expression for field names to convert. - -a Convert all fields. - -h|--help Show this message. + -f {a,b,c} Field names to apply substitution to. + -r {regex} Regular expression for field names to apply substitution to. + -a Apply substitution to all fields. + -h|--help Show this message. 1mhaving-fields0m Usage: mlr having-fields [options] Conditionally passes through records depending on each record's field names. Options: - --at-least {comma-separated names} - --which-are {comma-separated names} - --at-most {comma-separated names} - --all-matching {regular expression} - --any-matching {regular expression} - --none-matching {regular expression} + --at-least {comma-separated names} Pass records that have at least these field + names. + --which-are {comma-separated names} Pass records whose field names are exactly + these. + --at-most {comma-separated names} Pass records that have at most these field + names. + --all-matching {regular expression} Pass records where all field names match + the regex. + --any-matching {regular expression} Pass records where any field name matches + the regex. + --none-matching {regular expression} Pass records where no field name matches + the regex. + -h|--help Show this message. Examples: mlr having-fields --which-are amount,status,owner mlr having-fields --any-matching 'sda[0-9]' @@ -1365,40 +1378,41 @@ Without -g, ceases consuming more input (i.e. is fast) when n records have been read. Options: -g {a,b,c} Optional group-by-field names for head counts, e.g. a,b,c. - -n {n} Head-count to print. Default 10. - A negative count, e.g. -n -2, passes through all but the last n records, - optionally by category. - -h|--help Show this message. + -n {n} Head-count to print. Default 10. A negative count, e.g. -n -2, passes + through all but the last n records, optionally by category. + -h|--help Show this message. 1mhistogram0m Just a histogram. Input values < lo or > hi are not counted. Usage: mlr histogram [options] - -f {a,b,c} Value-field names for histogram counts - --lo {lo} Histogram low value - --hi {hi} Histogram high value - --nbins {n} Number of histogram bins. Defaults to 20. - --auto Automatically computes limits, ignoring --lo and --hi. - Holds all values in memory before producing any output. - -o {prefix} Prefix for output field name. Default: no prefix. - -h|--help Show this message. + Options: + -f {a,b,c} Value-field names for histogram counts. + --lo {lo} Histogram low value. + --hi {hi} Histogram high value. + --nbins {n} Number of histogram bins. Defaults to 20. + --auto Automatically computes limits, ignoring --lo and --hi. Holds all + values in memory before producing any output. + -o {prefix} Prefix for output field name. Default: no prefix. + -h|--help Show this message. 1mjson-parse0m Usage: mlr json-parse [options] Tries to convert string field values to parsed JSON, e.g. "[1,2,3]" -> [1,2,3]. Options: - -f {...} Comma-separated list of field names to json-parse (default all). - -k If supplied, then on parse fail for any cell, keep the (unparsable) - input value for the cell. - -h|--help Show this message. + -f {a,b,c} Comma-separated list of field names to json-parse (default all). + -k If supplied, then on parse fail for any cell, keep the (unparsable) + input value for the cell. + -h|--help Show this message. 1mjson-stringify0m Usage: mlr json-stringify [options] Produces string field values from field-value data, e.g. [1,2,3] -> "[1,2,3]". Options: - -f {...} Comma-separated list of field names to json-parse (default all). - --jvstack Produce multi-line JSON output. + -f {a,b,c} Comma-separated list of field names to json-stringify (default + all). + --jvstack Produce multi-line JSON output. --no-jvstack Produce single-line JSON output per record (default). - -h|--help Show this message. + -h|--help Show this message. 1mjoin0m Usage: mlr join [options] @@ -1407,34 +1421,53 @@ Functionality is essentially the same as the system "join" command, but for record streams. Options: - -f {left file name} - -j {a,b,c} Comma-separated join-field names for output - -l {a,b,c} Comma-separated join-field names for left input file; - defaults to -j values if omitted. - -r {a,b,c} Comma-separated join-field names for right input file(s); - defaults to -j values if omitted. - --lk|--left-keep-field-names {a,b,c} If supplied, this means keep only the specified field - names from the left file. Automatically includes the join-field name(s). Helpful - for when you only want a limited subset of information from the left file. - Tip: you can use --lk "": this means the left file becomes solely a row-selector - for the input files. - --lp {text} Additional prefix for non-join output field names from - the left file. Applies to paired and unpaired output records. - --rp {text} Additional prefix for non-join output field names from - the right file(s). Applies to paired and unpaired output records. - --np Do not emit paired records - --ul Emit unpaired records from the left file - --ur Emit unpaired records from the right file(s) - -s|--sorted-input Require sorted input: records must be sorted - lexically by their join-field names, else not all records will - be paired. The only likely use case for this is with a left - file which is too big to fit into system memory otherwise. - -u Enable unsorted input. (This is the default even without -u.) - In this case, the entire left file will be loaded into memory. - --prepipe {command} As in main input options; see mlr --help for details. - If you wish to use a prepipe command for the main input as well - as here, it must be specified there as well as here. - --prepipex {command} Likewise. + -f {left file name} Left file name for join. + -j {a,b,c} Comma-separated join-field names for + output. + -l {a,b,c} Comma-separated join-field names for left + input file; defaults to -j values if + omitted. + -r {a,b,c} Comma-separated join-field names for right + input file(s); defaults to -j values if + omitted. + --lk|--left-keep-field-names {a,b,c} If supplied, this means keep only the + specified field names from the left file. + Automatically includes the join-field + name(s). Helpful for when you only want a + limited subset of information from the left + file. Tip: you can use --lk "": this means + the left file becomes solely a row-selector + for the input files. + --lp {text} Additional prefix for non-join output field + names from the left file. Applies to paired + and unpaired output records. + --rp {text} Additional prefix for non-join output field + names from the right file(s). Applies to + paired and unpaired output records. + --np Do not emit paired records. + --ul Emit unpaired records from the left file. + --ur Emit unpaired records from the right + file(s). + -s|--sorted-input Require sorted input: records must be + sorted lexically by their join-field names, + else not all records will be paired. The + only likely use case for this is with a + left file which is too big to fit into + system memory otherwise. + -u Enable unsorted input. (This is the default + even without -u.) In this case, the entire + left file will be loaded into memory. + --prepipe {command} Shell command to prepipe the left-file + input through. As in main input options; + see mlr --help for details. If you wish to + use a prepipe command for the main input as + well as here, it must be specified there as + well as here. + --prepipex {command} Shell command to prepipe the left-file + input through (no shell quoting). As in + main input options; see mlr --help for + details. + -h|--help Show this message. File-format options default to those for the right file names on the Miller argument list, but may be overridden for the left file as follows. Please see the main "mlr --help" for more information on syntax for these arguments: @@ -1477,10 +1510,11 @@ Shows the least frequently occurring distinct values for specified field names. The first entry is the statistical anti-mode; the remaining are runners-up. Options: - -f {one or more comma-separated field names}. Required flag. - -n {count}. Optional flag defaulting to 10. - -b Suppress counts; show only field values. - -o {name} Field name for output count. Default "count". + -f {a,b,c} One or more comma-separated field names to group by. Required flag. + -n {count} Maximum number of results to output. Optional flag defaulting to 10. + -b Suppress counts; show only field values. + -o {name} Field name for output count. Default "count". + -h|--help Show this message. See also "mlr most-frequent". 1mmerge-fields0m @@ -1488,7 +1522,25 @@ Computes univariate statistics for each input record, accumulated across specified fields. Options: - -a {sum,count,...} Names of accumulators. One or more of: + -a {sum,count,...} Names of accumulators: one or more of the accumulators listed + below. + -f {a,b,c} Value-field names on which to compute statistics. Requires + -o. + -r {a,b,c} Regular expressions for value-field names on which to compute + statistics. Requires -o. + -c {a,b,c} Substrings for collapse mode: all fields which have the same + names after removing substrings will be accumulated together. + Please see examples below. + -i Use interpolated percentiles, like R's type=7; default like + type=1. Not sensical for string-valued fields. + -o {name} Output field basename for -f/-r. + -k Keep the input fields which contributed to the output + statistics; the default is to omit them. + -S No-op flag for backward compatibility with Miller 5. + -F No-op flag for backward compatibility with Miller 5. + -h|--help Show this message. + + Accumulators for -a: count Count instances of fields null_count Count number of empty-string/JSON-null instances per field distinct_count Count number of distinct values per field @@ -1506,17 +1558,6 @@ max Compute maximum values of specified fields minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields - -f {a,b,c} Value-field names on which to compute statistics. Requires -o. - -r {a,b,c} Regular expressions for value-field names on which to compute - statistics. Requires -o. - -c {a,b,c} Substrings for collapse mode. All fields which have the same names - after removing substrings will be accumulated together. Please see - examples below. - -i Use interpolated percentiles, like R's type=7; default like type=1. - Not sensical for string-valued fields. - -o {name} Output field basename for -f/-r. - -k Keep the input fields which contributed to the output statistics; - the default is to omit them. String-valued data make sense unless arithmetic on them is required, e.g. for sum, mean, interpolated percentiles, etc. In case of mixed data, @@ -1538,26 +1579,40 @@ Shows the most frequently occurring distinct values for specified field names. The first entry is the statistical mode; the remaining are runners-up. Options: - -f {one or more comma-separated field names}. Required flag. - -n {count}. Optional flag defaulting to 10. - -b Suppress counts; show only field values. - -o {name} Field name for output count. Default "count". + -f {a,b,c} One or more comma-separated field names to group by. Required flag. + -n {count} Maximum number of results to output. Optional flag defaulting to 10. + -b Suppress counts; show only field values. + -o {name} Field name for output count. Default "count". + -h|--help Show this message. See also "mlr least-frequent". 1mnest0m Usage: mlr nest [options] Explodes specified field values into separate fields/records, or reverses this. Options: - --explode,--implode One is required. - --values,--pairs One is required. - --across-records,--across-fields One is required. - -f {field name} Required. - -r {field names} Like -f but treat arguments as a regular expression. Match all - field names and operate on each in record order. Example: `-r '^[xy]$`'. - --nested-fs {string} Defaults to ";". Field separator for nested values. - --nested-ps {string} Defaults to ":". Pair separator for nested key-value pairs. - --evar {string} Shorthand for --explode --values --across-records --nested-fs {string} - --ivar {string} Shorthand for --implode --values --across-records --nested-fs {string} + --explode Explode field values into separate fields/records. One of + --explode or --implode is required. + --implode Reverse of --explode. One of --explode or --implode is + required. + --values Operate on field values. One of --values or --pairs is + required. + --pairs Operate on field key-value pairs. One of --values or + --pairs is required. + --across-records Explode/implode across records. One of --across-records or + --across-fields is required. + --across-fields Explode/implode across fields. One of --across-records or + --across-fields is required. + -f {field name} Required field name to operate on. + -r {field names} Like -f but treat arguments as a regular expression. Match + all field names and operate on each in record order. + Example: -r '^[xy]$'. + --nested-fs {string} Field separator for nested values. Defaults to ";". + --nested-ps {string} Pair separator for nested key-value pairs. Defaults to ":". + --evar {string} Shorthand for --explode --values --across-records + --nested-fs {string}. + --ivar {string} Shorthand for --implode --values --across-records + --nested-fs {string}. + -h|--help Show this message. Please use "mlr --usage-separator-options" for information on specifying separators. Examples: @@ -1609,56 +1664,53 @@ See also: https://miller.readthedocs.io/en/latest/reference-verbs Options: - -f {file name} File containing a DSL expression (see examples below). If the filename - is a directory, all *.mlr files in that directory are loaded. + -f {file name} File containing a DSL expression (see examples below). If the + filename is a directory, all *.mlr files in that directory are + loaded. + -e {expression} DSL expression to evaluate. You can use this after -f to add an + expression. Example use case: define functions/subroutines in a + file you specify with -f, then call them with an expression you + specify with -e. + -s {name=value} Predefines out-of-stream variable @name to have the given value. + Thus mlr put -s foo=97 '$column += @foo' is like mlr put 'begin + {@foo = 97} $column += @foo'. The value part is subject to + type-inferencing. May be specified more than once, e.g. -s + name1=value1 -s name2=value2. Note: the value may be an + environment variable, e.g. -s sequence=$SEQUENCE. + -x Prints records for which {expression} evaluates to false, not + true, i.e. invert the sense of the filter expression. Default + false. + -q Does not include the modified record in the output stream. + Useful for when all desired output is in begin and/or end + blocks. + -S No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. + -F No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. + -w Print warnings about things like uninitialized variables. + -W Same as -w, but exit the process if there are any warnings. + -p Prints the expression's AST (abstract syntax tree), which gives + full transparency on the precedence and associativity rules of + Miller's grammar, to stdout. + -d Like -p but uses a parenthesized-expression format for the AST. + -D Like -d but with output all on one line. + -E Echo DSL expression before printing parse-tree. + -v Same as -E -p. + -X Exit after parsing but before stream-processing. Useful with + -v/-d/-D, if you only want to look at parser information. + --explain Parse and type-check the DSL expression, report whether it is + valid, and exit without reading the input stream. Exit status is + 0 if the expression is valid and non-zero otherwise; combine + with --errors-json for a machine-readable error. + -h|--help Show this message. - -e {expression} You can use this after -f to add an expression. Example use - case: define functions/subroutines in a file you specify with -f, then call - them with an expression you specify with -e. - - (If you mix -e and -f then the expressions are evaluated in the order encountered. + If you mix -e and -f then the expressions are evaluated in the order encountered. Since the expression pieces are simply concatenated, please be sure to use intervening - semicolons to separate expressions.) + semicolons to separate expressions. - -s name=value: Predefines out-of-stream variable @name to have - Thus mlr put -s foo=97 '$column += @foo' is like - mlr put 'begin {@foo = 97} $column += @foo'. - The value part is subject to type-inferencing. - May be specified more than once, e.g. -s name1=value1 -s name2=value2. - Note: the value may be an environment variable, e.g. -s sequence=$SEQUENCE - - -x (default false) Prints records for which {expression} evaluates to false, not true, - i.e. invert the sense of the filter expression. - - -q Does not include the modified record in the output stream. - Useful for when all desired output is in begin and/or end blocks. - - -S and -F: There are no-ops in Miller 6 and above, since now type-inferencing is done - by the record-readers before filter/put is executed. Supported as no-op pass-through - flags for backward compatibility. - - -h|--help Show this message. - - Parser-info options: - - -w Print warnings about things like uninitialized variables. - - -W Same as -w, but exit the process if there are any warnings. - - -p Prints the expressions's AST (abstract syntax tree), which gives full - transparency on the precedence and associativity rules of Miller's grammar, - to stdout. - - -d Like -p but uses a parenthesized-expression format for the AST. - - -D Like -d but with output all on one line. - - -E Echo DSL expression before printing parse-tree - - -v Same as -E -p. - - -X Exit after parsing but before stream-processing. Useful with -v/-d/-D, if you - only want to look at parser information. + Parser-info options are -w, -W, -p, -d, -D, -E, -v, and -X. Examples: mlr --from example.csv put '$qr = $quantity * $rate' @@ -1702,16 +1754,15 @@ Usage: mlr rename [options] {old1,new1,old2,new2,...} Renames specified fields. Options: - -r Treat old field names as regular expressions. "ab", "a.*b" - will match any field name containing the substring "ab" or - matching "a.*b", respectively; anchors of the form "^ab$", - "^a.*b$" may be used. New field names may be plain strings, - or may contain capture groups of the form "\1" through - "\9". Wrapping the regex in double quotes is optional, but - is required if you wish to follow it with 'i' to indicate - case-insensitivity. - -g Do global replacement within each field name rather than - first-match replacement. + -r Treat old field names as regular expressions. "ab", "a.*b" will match + any field name containing the substring "ab" or matching "a.*b", + respectively; anchors of the form "^ab$", "^a.*b$" may be used. New + field names may be plain strings, or may contain capture groups of the + form "\1" through "\9". Wrapping the regex in double quotes is + optional, but is required if you wish to follow it with 'i' to + indicate case-insensitivity. + -g Do global replacement within each field name rather than first-match + replacement. -h|--help Show this message. Examples: mlr rename old_name,new_name @@ -1725,18 +1776,19 @@ Usage: mlr reorder [options] Moves specified names to start of record, or end of record. Options: - -e Put specified field names at record end: default is to put them at record start. + -e Put specified field names at record end: default is to put them at + record start. -f {a,b,c} Field names to reorder. - -r Treat field names as regular expressions. Matched fields are moved - to start or end in record order. Example: -r '^YYY,^XXX' puts all - YYY- and XXX-prefixed fields first (in record order), then the rest. + -r {a,b,c} Treat field names as regular expressions. Matched fields are moved to + start or end in record order. Example: -r '^YYY,^XXX' puts all YYY- + and XXX-prefixed fields first (in record order), then the rest. -b {x} Put field names specified with -f before field name specified by {x}, if any. If {x} isn't present in a given record, the specified fields will not be moved. -a {x} Put field names specified with -f after field name specified by {x}, if any. If {x} isn't present in a given record, the specified fields will not be moved. - -h|--help Show this message. + -h|--help Show this message. Examples: mlr reorder -f a,b sends input record "d=4,b=2,a=1,c=3" to "a=1,b=2,d=4,c=3". @@ -1746,11 +1798,12 @@ 1mrepeat0m Usage: mlr repeat [options] Copies input records to output records multiple times. - Options must be exactly one of the following: - -n {repeat count} Repeat each input record this many times. - -f {field name} Same, but take the repeat count from the specified - field name of each input record. - -h|--help Show this message. + Options must be exactly one of -n or -f. + Options: + -n {repeat count} Repeat each input record this many times. + -f {field name} Same as -n, but take the repeat count from the specified field + name of each input record. + -h|--help Show this message. Example: echo x=0 | mlr repeat -n 4 then put '$x=urand()' produces: @@ -1787,6 +1840,19 @@ These pivot/reshape the input data to undo the wide-to-long operation. Note: this does not work with tail -f; it produces output records only after all input records have been read. + Options: + -i {input field names} Input field names for wide-to-long reshape. + Use with -o. + -r {input field regex} Input field regex for wide-to-long reshape. + May be repeated. Use with -o. If you have + multiple regexes, please specify them using + multiple -r, since regexes can contain + commas within them. + -o {key-field name,value-field name} Output key-field and value-field names for + wide-to-long reshape. Requires -i or -r. + -s {key-field name,value-field name} Key-field and value-field names for + long-to-wide reshape. + -h|--help Show this message. Examples: @@ -1836,17 +1902,20 @@ See also mlr bootstrap and mlr shuffle. Options: -g {a,b,c} Optional: group-by-field names for samples, e.g. a,b,c. - -k {k} Required: number of records to output in total, or by group if using -g. - -h|--help Show this message. + -k {k} Required: number of records to output in total, or by group if using + -g. + -h|--help Show this message. 1msec2gmtdate0m - Usage: ../c/mlr sec2gmtdate {comma-separated list of field names} + Usage: mlr sec2gmtdate {comma-separated list of field names} Replaces a numeric field representing seconds since the epoch with the corresponding GMT year-month-day timestamp; leaves non-numbers as-is. This is nothing more than a keystroke-saver for the sec2gmtdate function: - ../c/mlr sec2gmtdate time1,time2 + mlr sec2gmtdate time1,time2 is the same as - ../c/mlr put '$time1=sec2gmtdate($time1);$time2=sec2gmtdate($time2)' + mlr put '$time1=sec2gmtdate($time1);$time2=sec2gmtdate($time2)' + Options: + -h|--help Show this message. 1msec2gmt0m Usage: mlr sec2gmt [options] {comma-separated list of field names} @@ -1857,24 +1926,32 @@ is the same as mlr put '$time1 = sec2gmt($time1); $time2 = sec2gmt($time2)' Options: - -1 through -9: format the seconds using 1..9 decimal places, respectively. - --millis Input numbers are treated as milliseconds since the epoch. - --micros Input numbers are treated as microseconds since the epoch. - --nanos Input numbers are treated as nanoseconds since the epoch. + -1 Format seconds with 1 decimal place. + -2 Format seconds with 2 decimal places. + -3 Format seconds with 3 decimal places. + -4 Format seconds with 4 decimal places. + -5 Format seconds with 5 decimal places. + -6 Format seconds with 6 decimal places. + -7 Format seconds with 7 decimal places. + -8 Format seconds with 8 decimal places. + -9 Format seconds with 9 decimal places. + --millis Input numbers are treated as milliseconds since the epoch. + --micros Input numbers are treated as microseconds since the epoch. + --nanos Input numbers are treated as nanoseconds since the epoch. -h|--help Show this message. 1mseqgen0m Usage: mlr seqgen [options] - Passes input records directly to output. Most useful for format conversion. Produces a sequence of counters. Discards the input record stream. Produces - output as specified by the options + output as specified by the options. Options: - -f {name} (default "i") Field name for counters. - --start {value} (default 1) Inclusive start value. - --step {value} (default 1) Step value. - --stop {value} (default 100) Inclusive stop value. - -h|--help Show this message. + -f {name} Field name for counters. Default "i". + --start {value} Inclusive start value. Default 1. + --step {value} Step value. Default 1. May be negative but not zero (unless + start == stop). + --stop {value} Inclusive stop value. Default 100. + -h|--help Show this message. Start, stop, and/or step may be floating-point. Output is integer if start, stop, and step are all integers. Step may be negative. It may not be zero unless start == stop. @@ -1902,17 +1979,20 @@ in the order they were encountered in the input record stream. Options: - -f {comma-separated field names} Lexical ascending - -r {comma-separated field names} Lexical descending - -c {comma-separated field names} Case-folded lexical ascending - -cr {comma-separated field names} Case-folded lexical descending - -n {comma-separated field names} Numerical ascending; nulls sort last - -nf {comma-separated field names} Same as -n - -nr {comma-separated field names} Numerical descending; nulls sort first - -t {comma-separated field names} Natural ascending - -b Move sort fields to start of record, as in reorder -b - -tr|-rt {comma-separated field names} Natural descending - -h|--help Show this message. + -f {a,b,c} Lexical ascending sort on the specified field names. + -r {a,b,c} Lexical descending sort on the specified field names. + -c {a,b,c} Case-folded lexical ascending sort on the specified field names. + -cr {a,b,c} Case-folded lexical descending sort on the specified field + names. + -n {a,b,c} Numerical ascending sort on the specified field names; nulls + sort last. + -nf {a,b,c} Same as -n. + -nr {a,b,c} Numerical descending sort on the specified field names; nulls + sort first. + -t {a,b,c} Natural ascending sort on the specified field names. + -b Move sort fields to start of record, as in reorder -b. + -tr|-rt {a,b,c} Natural descending sort on the specified field names. + -h|--help Show this message. Example: mlr sort -f a,b -nr x,y,z @@ -1923,13 +2003,13 @@ Usage: mlr sort-within-records [options] Outputs records sorted lexically ascending by keys. Options: - -f {names} Sort only these keys; others preserve record order. - -r {regex} Sort only keys matching this regex; others preserve record order. - Example: -r '^[xy]' sorts keys starting with x or y. - With no regex argument, -r recursively sorts subobjects/submaps - (e.g. for JSON input), or combines with -f to treat names as regex. - -n Sort field names naturally (e.g. 2 before 12). Combines with -f/-r. - -h|--help Show this message. + -f {names} Sort only these keys; others preserve record order. + -r {regex} Sort only keys matching this regex; others preserve record order. + Example: -r '^[xy]' sorts keys starting with x or y. With no regex + argument, -r recursively sorts subobjects/submaps (e.g. for JSON + input), or combines with -f to treat names as regexes. + -n Sort field names naturally (e.g. 2 before 12). Combines with -f/-r. + -h|--help Show this message. 1msparsify0m Usage: mlr sparsify [options] @@ -1937,26 +2017,28 @@ specified value). Only makes sense with output format not being CSV or TSV. Options: -s {filler string} What values to remove. Defaults to the empty string. - -f {a,b,c} Specify field names to be operated on; any other fields won't be - modified. The default is to modify all fields. - -h|--help Show this message. + -f {a,b,c} Specify field names to be operated on; any other fields won't + be modified. The default is to modify all fields. + -h|--help Show this message. Example: if input is a=1,b=,c=3 then output is a=1,c=3. 1msplit0m Usage: mlr split [options] {filename} Options: - -n {n}: Cap file sizes at N records. - -m {m}: Produce M files, round-robining records among them. - -g {a,b,c}: Write separate files with records having distinct values for fields named a,b,c. - Exactly one of -m, -n, or -g must be supplied. - --prefix {p} Specify filename prefix; default "split". - --suffix {s} Specify filename suffix; default is from mlr output format, e.g. "csv". - --folder {f} Specify output directory; default is current directory. - -a Append to existing file(s), if any, rather than overwriting. - -v Send records along to downstream verbs as well as splitting to files. + -n {n} Cap output file sizes at N records. + -m {m} Produce M files, round-robining records among them. + -g {a,b,c} Write separate files with records having distinct values for the + specified field names. + --prefix {p} Output filename prefix. Default "split". + --suffix {s} Output filename suffix. Default is from the output format, e.g. + "csv". + --folder {f} Output directory. Default is current directory. + -a Append to existing files rather than overwriting. + -v Send records downstream as well as splitting to files. -e Do NOT URL-escape names of output files. - -j {J} Use string J to join filename parts; default "_". + -j {J} String used to join filename parts. Default "_". -h|--help Show this message. + Exactly one of -m, -n, or -g must be supplied. Any of the output-format command-line flags (see mlr -h). For example, using mlr --icsv --from myfile.csv split --ojson -n 1000 the input is CSV, but the output files are JSON. @@ -1995,17 +2077,42 @@ as \n, \t, and \x1f. Write \\ to get a literal backslash. See also the `gsub` and `sub` verbs. Options: - -f {a,b,c} Field names to convert. - -r {regex} Regular expression for field names to convert. - -a Convert all fields. - -h|--help Show this message. + -f {a,b,c} Field names to apply substitution to. + -r {regex} Regular expression for field names to apply substitution to. + -a Apply substitution to all fields. + -h|--help Show this message. 1mstats10m Usage: mlr stats1 [options] Computes univariate statistics for one or more given fields, accumulated across the input record stream. Options: - -a {sum,count,...} Names of accumulators: one or more of: + -a {sum,count,...} Names of accumulators: one or more of the listed values. Also + accepts median (same as p50) and percentiles p{n} for n in + 0..100, e.g. p10 p25.2 p50 p98 p100. + -f {a,b,c} Value-field names on which to compute statistics. + --fr {regex} Regex for value-field names on which to compute statistics + (compute statistics on values in all field names matching the + regex). + --fx {regex} Inverted regex for value-field names on which to compute + statistics (compute statistics on values in all field names + not matching the regex). + -g {d,e,f} Optional group-by-field names. + --gr {regex} Regex for optional group-by-field names (group by values in + field names matching the regex). + --gx {regex} Inverted regex for optional group-by-field names (group by + values in field names not matching the regex). + --grfx {regex} Shorthand for --gr {regex} --fx {that same regex}. + -i Use interpolated percentiles, like R's type=7; default like + type=1. Not sensical for string-valued fields. + -s Print iterative stats. Useful in tail -f contexts, in which + case please avoid pprint-format output since end of input + stream will never be seen. Likewise, if input is coming from + `tail -f` be sure to use `--records-per-batch 1`. + -S No-op flag for backward compatibility with Miller 5. + -F No-op flag for backward compatibility with Miller 5. + -h|--help Show this message. + Names of accumulators for -a, one or more of: median This is the same as p50 p10 p25.2 p50 p98 p100 etc. count Count instances of fields @@ -2025,28 +2132,6 @@ max Compute maximum values of specified fields minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields - - -f {a,b,c} Value-field names on which to compute statistics - --fr {regex} Regex for value-field names on which to compute statistics - (compute statistics on values in all field names matching regex - --fx {regex} Inverted regex for value-field names on which to compute statistics - (compute statistics on values in all field names not matching regex) - - -g {d,e,f} Optional group-by-field names - --gr {regex} Regex for optional group-by-field names - (group by values in field names matching regex) - --gx {regex} Inverted regex for optional group-by-field names - (group by values in field names not matching regex) - - --grfx {regex} Shorthand for --gr {regex} --fx {that same regex} - - -i Use interpolated percentiles, like R's type=7; default like type=1. - Not sensical for string-valued fields.\n"); - -s Print iterative stats. Useful in tail -f contexts, in which - case please avoid pprint-format output since end of input - stream will never be seen. Likewise, if input is coming from `tail -f` - be sure to use `--records-per-batch 1`. - -h|--help Show this message. Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape Example: mlr stats1 -a count,mode -f size Example: mlr stats1 -a count,mode -f size -g shape @@ -2069,7 +2154,27 @@ Usage: mlr stats2 [options] Computes bivariate statistics for one or more given field-name pairs, accumulated across the input record stream. - -a {linreg-ols,corr,...} Names of accumulators: one or more of: + Options: + -a {linreg-ols,corr,...} Names of accumulators: one or more of the listed + values. + -f {a,b,c,d} Value-field name-pairs on which to compute statistics. + There must be an even number of names. + -g {e,f,g} Optional group-by-field names. + -v Print additional output for linreg-pca. + -s Print iterative stats. Useful in tail -f contexts, in + which case please avoid pprint-format output since end + of input stream will never be seen. Likewise, if input + is coming from `tail -f`, be sure to use + `--records-per-batch 1`. + --fit Rather than printing regression parameters, applies + them to the input data to compute new fit fields. All + input records are held in memory until end of input + stream. Has effect only for linreg-ols, linreg-pca, and + logireg. + -S No-op flag for backward compatibility with Miller 5. + -F No-op flag for backward compatibility with Miller 5. + -h|--help Show this message. + Names of accumulators for -a, one or more of: linreg-ols Linear regression using ordinary least squares linreg-pca Linear regression using principal component analysis r2 Quality metric for linreg-ols (linreg-pca emits its own) @@ -2077,18 +2182,6 @@ corr Sample correlation cov Sample covariance covx Sample-covariance matrix - -f {a,b,c,d} Value-field name-pairs on which to compute statistics. - There must be an even number of names. - -g {e,f,g} Optional group-by-field names. - -v Print additional output for linreg-pca. - -s Print iterative stats. Useful in tail -f contexts, in which - case please avoid pprint-format output since end of input - stream will never be seen. Likewise, if input is coming from - `tail -f`, be sure to use `--records-per-batch 1`. - --fit Rather than printing regression parameters, applies them to - the input data to compute new fit fields. All input records are - held in memory until end of input stream. Has effect only for - linreg-ols, linreg-pca, and logireg. Only one of -s or --fit may be used. Example: mlr stats2 -a linreg-pca -f x,y Example: mlr stats2 -a linreg-ols,r2 -f x,y -g size,shape @@ -2098,7 +2191,25 @@ Usage: mlr step [options] Computes values dependent on earlier/later records, optionally grouped by category. Options: - -a {delta,rsum,...} Names of steppers: comma-separated, one or more of: + -a {delta,rsum,...} Names of steppers: comma-separated, one or more of the + listed values. + -f {a,b,c} Value-field names on which to compute statistics. + -g {d,e,f} Optional group-by-field names. + -F Computes integerable things (e.g. counter) in floating + point. As of Miller 6 this happens automatically, but the + flag is accepted as a no-op for backward compatibility with + Miller 5 and below. + -d {x,y,z} Weights for EWMA. 1 means current sample gets all weight (no + smoothing), near under 1 is light smoothing, near over 0 is + heavy smoothing. Multiple weights may be specified, e.g. + "mlr step -a ewma -f sys_load -d 0.01,0.1,0.9". Default if + omitted is "-d 0.5". + -o {a,b,c} Custom suffixes for EWMA output fields. If omitted, these + default to the -d values. If supplied, the number of -o + values must be the same as the number of -d values. + -h|--help Show this message. + + Names of steppers for -a, comma-separated, one or more of: counter Count instances of field(s) between successive records delta Compute differences in field(s) between successive records ewma Exponentially weighted moving average over successive records @@ -2111,21 +2222,6 @@ shift_lead Include value(s) in field(s) from the next record, if any slwin Sliding-window averages over m records back and n forward. E.g. slwin_7_2 for 7 back and 2 forward. - -f {a,b,c} Value-field names on which to compute statistics - -g {d,e,f} Optional group-by-field names - -F Computes integerable things (e.g. counter) in floating point. - As of Miller 6 this happens automatically, but the flag is accepted - as a no-op for backward compatibility with Miller 5 and below. - -d {x,y,z} Weights for EWMA. 1 means current sample gets all weight (no - smoothing), near under 1 is light smoothing, near over 0 is - heavy smoothing. Multiple weights may be specified, e.g. - "mlr step -a ewma -f sys_load -d 0.01,0.1,0.9". Default if omitted - is "-d 0.5". - -o {a,b,c} Custom suffixes for EWMA output fields. If omitted, these default to - the -d values. If supplied, the number of -o values must be the same - as the number of -d values. - -h|--help Show this message. - Examples: mlr step -a rsum -f request_size mlr step -a delta -f request_size -g hostname @@ -2146,10 +2242,10 @@ and \x1f. Write \\ to get a literal backslash. See also the `gsub` and `ssub` verbs. Options: - -f {a,b,c} Field names to convert. - -r {regex} Regular expression for field names to convert. - -a Convert all fields. - -h|--help Show this message. + -f {a,b,c} Field names to apply substitution to. + -r {regex} Regular expression for field names to apply substitution to. + -a Apply substitution to all fields. + -h|--help Show this message. 1msummary0m Usage: mlr summary [options] @@ -2189,19 +2285,19 @@ Options: -a {mean,sum,etc.} Use only the specified summarizers. - -x {mean,sum,etc.} Use all summarizers, except the specified ones. + -x {mean,sum,etc.} Use all summarizers except the specified ones. --all Use all available summarizers. - --transpose Show output with field names as column names.. - -h|--help Show this message. + --transpose Show output with field names as column names. + -h|--help Show this message. 1msurv0m Usage: mlr surv -d {duration-field} -s {status-field} Estimate Kaplan-Meier survival curve (right-censored). Options: - -d {field} Name of duration field (time-to-event or censoring). - -s {field} Name of status field (0=censored, 1=event). - -h, --help Show this message. + -d {field} Name of duration field (time-to-event or censoring). + -s {field} Name of status field (0=censored, 1=event). + -h|--help Show this message. 1mtac0m Usage: mlr tac [options] @@ -2214,34 +2310,35 @@ Passes through the last n records, optionally by category. Options: -g {a,b,c} Optional group-by-field names for tail counts, e.g. a,b,c. - -n {n} Tail-count to print. Default 10. - A leading '+' means start at the nth record rather than print - the last n: e.g. -n +3 passes through all but the first 2 - records, optionally by category. - -h|--help Show this message. + -n {n} Tail-count to print. Default 10. A leading '+' means start at the nth + record rather than print the last n: e.g. -n +3 passes through all + but the first 2 records, optionally by category. + -h|--help Show this message. 1mtee0m Usage: mlr tee [options] {filename} Options: - -a Append to existing file, if any, rather than overwriting. - -p Treat filename as a pipe-to command. + -a Append to existing file, if any, rather than overwriting. + -p Treat filename as a pipe-to command. + -h|--help Show this message. Any of the output-format command-line flags (see mlr -h). Example: using mlr --icsv --opprint put '...' then tee --ojson ./mytap.dat then stats1 ... the input is CSV, the output is pretty-print tabular, but the tee-file output is written in JSON format. - -h|--help Show this message. - 1mtemplate0m Usage: mlr template [options] Places input-record fields in the order specified by list of column names. If the input record is missing a specified field, it will be filled with the fill-with. If the input record possesses an unspecified field, it will be discarded. Options: - -f {a,b,c} Comma-separated field names for template, e.g. a,b,c. - -t {filename} CSV file whose header line will be used for template. - --fill-with {filler string} What to fill absent fields with. Defaults to the empty string. - -h|--help Show this message. + -f {a,b,c} Comma-separated field names for template, e.g. + a,b,c. + -t {filename} CSV file whose header line will be used for + template. + --fill-with {filler string} What to fill absent fields with. Defaults to the + empty string. + -h|--help Show this message. Example: * Specified fields are a,b,c. * Input record is c=3,a=1,f=6. @@ -2249,16 +2346,19 @@ 1mtop0m Usage: mlr top [options] - -f {a,b,c} Value-field names for top counts. - -g {d,e,f} Optional group-by-field names for top counts. - -n {count} How many records to print per category; default 1. - -a Print all fields for top-value records; default is - to print only value and group-by fields. Requires a single - value-field name only. - --min Print top smallest values; default is top largest values. - -F Keep top values as floats even if they look like integers. - -o {name} Field name for output indices. Default "top_idx". - This is ignored if -a is used. + Options: + -f {a,b,c} Value-field names for top counts. + -g {d,e,f} Optional group-by-field names for top counts. + -n {count} How many records to print per category; default 1. + -a Print all fields for top-value records; default is to print only + value and group-by fields. Requires a single value-field name only. + --max Print top largest values. This is the default. + --min Print top smallest values; default is top largest values. + -F Keep top values as floats even if they look like integers (ignored in + Miller 6, kept for backward compatibility). + -o {name} Field name for output indices. Default "top_idx". Ignored if -a is + used. + -h|--help Show this message. Prints the n records with smallest/largest values at specified fields, optionally by category. If -a is given, then the top records are emitted with the same fields as they appeared in the input. Without -a, only fields @@ -2267,7 +2367,7 @@ 1mutf8-to-latin10m Usage: mlr utf8-to-latin1, with no options. - Recursively converts record strings from Latin-1 to UTF-8. + Recursively converts record strings from UTF-8 to Latin-1. For field-level control, please see the utf8_to_latin1 DSL function. Options: -h|--help Show this message. @@ -2277,9 +2377,9 @@ Reverses flatten. Example: field with name 'a.b.c' and value 4 becomes name 'a' and value '{"b": { "c": 4 }}'. Options: - -f {a,b,c} Comma-separated list of field names to unflatten (default all). + -f {a,b,c} Comma-separated list of field names to unflatten (default all). -s {string} Separator, defaulting to mlr --flatsep value. - -h|--help Show this message. + -h|--help Show this message. 1muniq0m Usage: mlr uniq [options] @@ -2287,21 +2387,24 @@ count-distinct. For uniq, -f is a synonym for -g. Options: - -g {d,e,f} Group-by-field names for uniq counts. - -x {a,b,c} Field names to exclude for uniq: use each record's others instead. - -c Show repeat counts in addition to unique values. - -n Show only the number of distinct values. - -o {name} Field name for output count. Default "count". - -a Output each unique record only once. Incompatible with -g. - With -c, produces unique records, with repeat counts for each. - With -n, produces only one record which is the unique-record count. - With neither -c nor -n, produces unique records. + -g {d,e,f} Group-by field names for uniq counts. + -f {d,e,f} Synonym for -g. + -x {a,b,c} Field names to exclude for uniq; use each record's other fields + instead. + -c Show repeat counts in addition to unique values. + -n Show only the number of distinct values. + -o {name} Field name for output count. Default "count". + -a Output each unique record only once. Incompatible with -g. With -c, + produces unique records, with repeat counts for each. With -n, + produces only one record which is the unique-record count. With + neither -c nor -n, produces unique records. + -h|--help Show this message. 1munspace0m Usage: mlr unspace [options] Replaces spaces in record keys and/or values with _. This is helpful for PPRINT output. Options: - -f {x} Replace spaces with specified filler character. + -f {x} Replace spaces with specified filler character. Default "_". -k Unspace only keys, not keys and values. -v Unspace only values, not keys and values. -h|--help Show this message. @@ -2312,11 +2415,11 @@ For field names absent in a given record but present in others, fills in a value. This verb retains all input before producing any output. Options: - --fill-with {filler string} What to fill absent fields with. Defaults to - the empty string. - -f {a,b,c} Specify field names to be operated on. Any other fields won't be - modified, and operation will be streaming. - -h|--help Show this message. + --fill-with {filler string} What to fill absent fields with. Defaults to the + empty string. + -f {a,b,c} Specify field names to be operated on; others are + not modified and operation is streaming. + -h|--help Show this message. Example: if the input is two records, one being 'a=1,b=2' and the other being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and 'a=,b=3,c=4'. @@ -3878,4 +3981,4 @@ MIME Type for Comma-Separated Values (CSV) Files, the Miller docsite https://miller.readthedocs.io - 2026-07-01 4mMILLER24m(1) + 2026-07-03 4mMILLER24m(1) diff --git a/man/mlr.1 b/man/mlr.1 index d06d45163..e3644fc34 100644 --- a/man/mlr.1 +++ b/man/mlr.1 @@ -2,12 +2,12 @@ .\" Title: mlr .\" Author: [see the "AUTHOR" section] .\" Generator: ./mkman.rb -.\" Date: 2026-07-01 +.\" Date: 2026-07-03 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "MILLER" "1" "2026-07-01" "\ \&" "\ \&" +.TH "MILLER" "1" "2026-07-03" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Portability definitions .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -699,6 +699,12 @@ These are flags which are applicable to markdown-tabular format. .\} .nf These are flags which don't fit into any other category. +--errors-json Emit parse errors as a JSON object to stderr instead + of a plain text message. Intended for AI agents and + scripts that branch on error kind rather than + regex-matching prose. Equivalent to setting the + `MLR_ERRORS_JSON` environment variable to a truthy + value. --fflush Force buffered output to be written after every output record. The default is flush output after every record if the output is to the terminal, or @@ -1177,18 +1183,18 @@ Usage: mlr bar [options] Replaces a numeric field with a number of asterisks, allowing for cheesy bar plots. These align best with --opprint or --oxtab output format. Options: --f {a,b,c} Field names to convert to bars. ---lo {lo} Lower-limit value for min-width bar: default '0.000000'. ---hi {hi} Upper-limit value for max-width bar: default '100.000000'. --w {n} Bar-field width: default '40'. ---auto Automatically computes limits, ignoring --lo and --hi. - Holds all records in memory before producing any output. --c {character} Fill character: default '*'. --x {character} Out-of-bounds character: default '#'. --b {character} Blank character: default '.'. +-f {a,b,c} Field names to convert to bars. +--lo {lo} Lower-limit value for min-width bar: default '0.000000'. +--hi {hi} Upper-limit value for max-width bar: default '100.000000'. +-w {n} Bar-field width: default '40'. +--auto Automatically computes limits, ignoring --lo and --hi. Holds all + records in memory before producing any output. +-c {character} Fill character: default '*'. +-x {character} Out-of-bounds character: default '#'. +-b {character} Blank character: default '.'. +-h|--help Show this message. Nominally the fill, out-of-bounds, and blank characters will be strings of length 1. However you can make them all longer if you so desire. --h|--help Show this message. .fi .if n \{\ .RE @@ -1201,8 +1207,8 @@ Usage: mlr bootstrap [options] Emits an n-sample, with replacement, of the input records. See also mlr sample and mlr shuffle. Options: - -n Number of samples to output. Defaults to number of input records. - Must be non-negative. +-n {n} Number of samples to output. Defaults to number of input records. Must + be non-negative. -h|--help Show this message. .fi .if n \{\ @@ -1215,14 +1221,14 @@ Options: Usage: mlr case [options] Uppercases strings in record keys and/or values. Options: --k Case only keys, not keys and values. --v Case only values, not keys and values. --f {a,b,c} Specify which field names to case (default: all) --u Convert to uppercase --l Convert to lowercase --s Convert to sentence case (capitalize first letter) --t Convert to title case (capitalize words) --h|--help Show this message. +-k Case only keys, not keys and values. +-v Case only values, not keys and values. +-f {a,b,c} Specify which field names to case (default: all). +-u Convert to uppercase. +-l Convert to lowercase. +-s Convert to sentence case (capitalize first letter). +-t Convert to title case (capitalize words). +-h|--help Show this message. .fi .if n \{\ .RE @@ -1235,11 +1241,12 @@ Usage: mlr cat [options] Passes input records directly to output. Most useful for format conversion. Options: -n Prepend field "n" to each record with record-counter starting at 1. --N {name} Prepend field {name} to each record with record-counter starting at 1. --g {a,b,c} Optional group-by-field names for counters, e.g. a,b,c +-N {name} Prepend field {name} to each record with record-counter starting at + 1. +-g {a,b,c} Optional group-by-field names for counters, e.g. a,b,c. --filename Prepend current filename to each record. --filenum Prepend current filenum (1-up) to each record. --h|--help Show this message. +-h|--help Show this message. .fi .if n \{\ .RE @@ -1250,8 +1257,8 @@ Options: .nf Usage: mlr check [options] Consumes records without printing any output, -Useful for doing a well-formatted check on input data. with the exception that warnings are printed to stderr. +Useful for doing a well-formatted check on input data. Current checks are: * Data are parseable * If any key is the empty string @@ -1273,11 +1280,11 @@ please see the DSL functions lstrip, rstrip, strip, collapse_whitespace, and clean_whitespace. Options: --k|--keys-only Do not touch values. --v|--values-only Do not touch keys. +-k|--keys-only Do not touch values. +-v|--values-only Do not touch keys. +-h|--help Show this message. It is an error to specify -k as well as -v -- to clean keys and values, leave off -k as well as -v. --h|--help Show this message. .fi .if n \{\ .RE @@ -1291,16 +1298,17 @@ Prints number of records having distinct values for specified field names. Same as uniq -c. Options: --f {a,b,c} Field names for distinct count. --x {a,b,c} Field names to exclude for distinct count: use each record's others instead. --n Show only the number of distinct values. Not compatible with -u. --o {name} Field name for output count. Default "count". - Ignored with -u. --u Do unlashed counts for multiple field names. With -f a,b and - without -u, computes counts for distinct combinations of a - and b field values. With -f a,b and with -u, computes counts - for distinct a field values and counts for distinct b field - values separately. +-f {a,b,c} Field names for distinct count (synonym for -g). +-g {a,b,c} Field names for distinct count. +-x {a,b,c} Field names to exclude for distinct count; use each record's other + fields instead. +-n Show only the number of distinct values. Not compatible with -u. +-o {name} Field name for output count. Default "count". Ignored with -u. +-u Do unlashed counts for multiple field names. With -f a,b and without + -u, computes counts for distinct combinations of a and b field + values. With -f a,b and with -u, computes counts for distinct a field + values and counts for distinct b field values separately. +-h|--help Show this message. .fi .if n \{\ .RE @@ -1313,9 +1321,9 @@ Usage: mlr count [options] Prints number of records, optionally grouped by distinct values for specified field names. Options: -g {a,b,c} Optional group-by-field names for counts, e.g. a,b,c --n {n} Show only the number of distinct values. Not interesting without -g. --o {name} Field name for output-count. Default "count". --h|--help Show this message. +-n Show only the number of distinct values. Not interesting without -g. +-o {name} Field name for output-count. Default "count". +-h|--help Show this message. .fi .if n \{\ .RE @@ -1329,8 +1337,8 @@ Ingests all records, then emits each record augmented by a count of the number of other records having the same group-by field values. Options: -g {a,b,c} Group-by-field names for counts, e.g. a,b,c --o {name} Field name for output-counts. Defaults to "count". --h|--help Show this message. +-o {name} Field name for output-counts. Defaults to "count". +-h|--help Show this message. .fi .if n \{\ .RE @@ -1342,15 +1350,15 @@ Options: Usage: mlr cut [options] Passes through input records with specified fields included/excluded. Options: - -f {a,b,c} Comma-separated field names for cut, e.g. a,b,c. - -o Retain fields in the order specified here in the argument list. - Default is to retain them in the order found in the input data. - -x|--complement Exclude, rather than include, field names specified by -f. - -r Treat field names as regular expressions. "ab", "a.*b" will - match any field name containing the substring "ab" or matching - "a.*b", respectively; anchors of the form "^ab$", "^a.*b$" may - be used. --h|--help Show this message. +-f {a,b,c} Comma-separated field names to include or exclude, e.g. a,b,c. +-o Retain fields in the order specified by -f rather than in + input-record order. +-x|--complement Exclude, rather than include, the field names specified by -f. +-r Treat field names as regular expressions. "ab", "a.*b" will + match any field name containing the substring "ab" or matching + "a.*b", respectively; anchors of the form "^ab$", "^a.*b$" may + be used. +-h|--help Show this message. Examples: mlr cut -f hostname,status mlr cut -x -f hostname,status @@ -1368,11 +1376,11 @@ Examples: Usage: mlr decimate [options] Passes through one of every n records, optionally by category. Options: - -b Decimate by printing first of every n. - -e Decimate by printing last of every n (default). - -g {a,b,c} Optional group-by-field names for decimate counts, e.g. a,b,c. - -n {n} Decimation factor (default 10). --h|--help Show this message. +-b Decimate by printing first of every n. +-e Decimate by printing last of every n (default). +-g {a,b,c} Optional group-by-field names for decimate counts, e.g. a,b,c. +-n {n} Decimation factor (default 10). +-h|--help Show this message. .fi .if n \{\ .RE @@ -1388,13 +1396,14 @@ By default, a 'missing' field either is absent, or has the empty-string value. With -a, a field is 'missing' only if it is absent. Options: - --all Operate on all fields in the input. - -a|--only-if-absent If a given record has a missing value for a given field, - fill that from the corresponding value from a previous record, if any. - By default, a 'missing' field either is absent, or has the empty-string value. - With -a, a field is 'missing' only if it is absent. - -f Field names for fill-down. - -h|--help Show this message. +--all Operate on all fields in the input. +-a|--only-if-absent If a given record has a missing value for a given field, + fill that from the corresponding value from a previous + record, if any. By default, a 'missing' field either is + absent, or has the empty-string value. With -a, a field is + 'missing' only if it is absent. +-f {a,b,c} Field names for fill-down. +-h|--help Show this message. .fi .if n \{\ .RE @@ -1408,6 +1417,7 @@ Fills empty-string fields with specified fill-value. Options: -v {string} Fill-value: defaults to "N/A" -S Don't infer type -- so '-v 0' would fill string 0 not int 0. +-h|--help Show this message. .fi .if n \{\ .RE @@ -1422,56 +1432,53 @@ stream records will be output. See also: https://miller.readthedocs.io/en/latest/reference-verbs Options: --f {file name} File containing a DSL expression (see examples below). If the filename - is a directory, all *.mlr files in that directory are loaded. +-f {file name} File containing a DSL expression (see examples below). If the + filename is a directory, all *.mlr files in that directory are + loaded. +-e {expression} DSL expression to evaluate. You can use this after -f to add an + expression. Example use case: define functions/subroutines in a + file you specify with -f, then call them with an expression you + specify with -e. +-s {name=value} Predefines out-of-stream variable @name to have the given value. + Thus mlr put -s foo=97 '$column += @foo' is like mlr put 'begin + {@foo = 97} $column += @foo'. The value part is subject to + type-inferencing. May be specified more than once, e.g. -s + name1=value1 -s name2=value2. Note: the value may be an + environment variable, e.g. -s sequence=$SEQUENCE. +-x Prints records for which {expression} evaluates to false, not + true, i.e. invert the sense of the filter expression. Default + false. +-q Does not include the modified record in the output stream. + Useful for when all desired output is in begin and/or end + blocks. +-S No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. +-F No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. +-w Print warnings about things like uninitialized variables. +-W Same as -w, but exit the process if there are any warnings. +-p Prints the expression's AST (abstract syntax tree), which gives + full transparency on the precedence and associativity rules of + Miller's grammar, to stdout. +-d Like -p but uses a parenthesized-expression format for the AST. +-D Like -d but with output all on one line. +-E Echo DSL expression before printing parse-tree. +-v Same as -E -p. +-X Exit after parsing but before stream-processing. Useful with + -v/-d/-D, if you only want to look at parser information. +--explain Parse and type-check the DSL expression, report whether it is + valid, and exit without reading the input stream. Exit status is + 0 if the expression is valid and non-zero otherwise; combine + with --errors-json for a machine-readable error. +-h|--help Show this message. --e {expression} You can use this after -f to add an expression. Example use - case: define functions/subroutines in a file you specify with -f, then call - them with an expression you specify with -e. - -(If you mix -e and -f then the expressions are evaluated in the order encountered. +If you mix -e and -f then the expressions are evaluated in the order encountered. Since the expression pieces are simply concatenated, please be sure to use intervening -semicolons to separate expressions.) +semicolons to separate expressions. --s name=value: Predefines out-of-stream variable @name to have - Thus mlr put -s foo=97 '$column += @foo' is like - mlr put 'begin {@foo = 97} $column += @foo'. - The value part is subject to type-inferencing. - May be specified more than once, e.g. -s name1=value1 -s name2=value2. - Note: the value may be an environment variable, e.g. -s sequence=$SEQUENCE - --x (default false) Prints records for which {expression} evaluates to false, not true, - i.e. invert the sense of the filter expression. - --q Does not include the modified record in the output stream. - Useful for when all desired output is in begin and/or end blocks. - --S and -F: There are no-ops in Miller 6 and above, since now type-inferencing is done - by the record-readers before filter/put is executed. Supported as no-op pass-through - flags for backward compatibility. - --h|--help Show this message. - -Parser-info options: - --w Print warnings about things like uninitialized variables. - --W Same as -w, but exit the process if there are any warnings. - --p Prints the expressions's AST (abstract syntax tree), which gives full - transparency on the precedence and associativity rules of Miller's grammar, - to stdout. - --d Like -p but uses a parenthesized-expression format for the AST. - --D Like -d but with output all on one line. - --E Echo DSL expression before printing parse-tree - --v Same as -E -p. - --X Exit after parsing but before stream-processing. Useful with -v/-d/-D, if you - only want to look at parser information. +Parser-info options are -w, -W, -p, -d, -D, -E, -v, and -X. Records will pass the filter depending on the last bare-boolean statement in the DSL expression. That can be the result of <, ==, >, etc., the return value of a function call @@ -1515,9 +1522,9 @@ Usage: mlr flatten [options] Flattens multi-level maps to single-level ones. Example: field with name 'a' and value '{"b": { "c": 4 }}' becomes name 'a.b.c' and value 4. Options: --f Comma-separated list of field names to flatten (default all). --s Separator, defaulting to mlr --flatsep value. --h|--help Show this message. +-f Comma-separated list of field names to flatten (default all). +-s {string} Separator, defaulting to mlr --flatsep value. +-h|--help Show this message. .fi .if n \{\ .RE @@ -1539,22 +1546,21 @@ Note: this verb lets you apply arbitrary format strings, which can produce undefined behavior and/or program crashes. See your system's "man printf". Options: --i {integer format} Defaults to "%d". - Examples: "%06lld", "%08llx". - Note that Miller integers are long long so you must use - formats which apply to long long, e.g. with ll in them. - Undefined behavior results otherwise. --f {float format} Defaults to "%f". - Examples: "%8.3lf", "%.6le". - Note that Miller floats are double-precision so you must - use formats which apply to double, e.g. with l[efg] in them. - Undefined behavior results otherwise. --s {string format} Defaults to "%s". - Examples: "_%s", "%08s". - Note that you must use formats which apply to string, e.g. - with s in them. Undefined behavior results otherwise. +-i {integer format} Integer format string; defaults to "%d". Examples: "%06lld", + "%08llx". Note that Miller integers are long long so you + must use formats which apply to long long, e.g. with ll in + them. Undefined behavior results otherwise. +-f {float format} Float format string; defaults to "%f". Examples: "%8.3lf", + "%.6le". Note that Miller floats are double-precision so you + must use formats which apply to double, e.g. with l[efg] in + them. Undefined behavior results otherwise. +-s {string format} String format string; defaults to "%s". Examples: "_%s", + "%08s". Note that you must use formats which apply to + string, e.g. with s in them. Undefined behavior results + otherwise. -n Coerce field values autodetected as int to float, and then apply the float format. +-h|--help Show this message. .fi .if n \{\ .RE @@ -1574,15 +1580,16 @@ input records and accumulates sums; on the second pass it computes quotients and emits output records. This means it produces no output until all input is read. Options: --f {a,b,c} Field name(s) for fraction calculation --g {d,e,f} Optional group-by-field name(s) for fraction counts --p Produce percents [0..100], not fractions [0..1]. Output field names - end with "_percent" rather than "_fraction" --c Produce cumulative distributions, i.e. running sums: each output - value folds in the sum of the previous for the specified group - E.g. with input records x=1 x=2 x=3 and x=4, emits output records - x=1,x_cumulative_fraction=0.1 x=2,x_cumulative_fraction=0.3 - x=3,x_cumulative_fraction=0.6 and x=4,x_cumulative_fraction=1.0 +-f {a,b,c} Field name(s) for fraction calculation +-g {d,e,f} Optional group-by-field name(s) for fraction counts +-p Produce percents [0..100], not fractions [0..1]. Output field names + end with "_percent" rather than "_fraction" +-c Produce cumulative distributions, i.e. running sums: each output + value folds in the sum of the previous for the specified group. E.g. + with input records x=1 x=2 x=3 and x=4, emits output records + x=1,x_cumulative_fraction=0.1 x=2,x_cumulative_fraction=0.3 + x=3,x_cumulative_fraction=0.6 and x=4,x_cumulative_fraction=1.0 +-h|--help Show this message. .fi .if n \{\ .RE @@ -1593,13 +1600,11 @@ Options: .nf Usage: mlr gap [options] Emits an empty record every n records, or when certain values change. +One of -n or -g is required. Options: -Emits an empty record every n records, or when certain values change. -g {a,b,c} Print a gap whenever values of these fields (e.g. a,b,c) changes. --n {n} Print a gap every n records. -One of -f or -g is required. --n is ignored if -g is present. --h|--help Show this message. +-n {n} Print a gap every n records. Ignored if -g is present. +-h|--help Show this message. .fi .if n \{\ .RE @@ -1611,9 +1616,9 @@ One of -f or -g is required. Usage: mlr grep [options] {regular expression} Passes through records which match the regular expression. Options: --i Use case-insensitive search. --v Invert: pass through records which do not match the regex. --a Only grep for values, not keys and values. +-i Use case-insensitive search. +-v Invert: pass through records which do not match the regex. +-a Only grep for values, not keys and values. -h|--help Show this message. Note that "mlr filter" is more powerful, but requires you to know field names. By contrast, "mlr grep" allows you to regex-match the entire record. It does this @@ -1635,7 +1640,8 @@ features of system grep, you can do .\} .nf Usage: mlr group-by [options] {comma-separated field names} -Outputs records in batches having identical values at specified field names.Options: +Outputs records in batches having identical values at specified field names. +Options: -h|--help Show this message. .fi .if n \{\ @@ -1664,10 +1670,10 @@ The replacement string supports C-style backslash escapes such as \en, \et, and \ex1f. Write \e\e to get a literal backslash. See also the `sub` and `ssub` verbs. Options: --f {a,b,c} Field names to convert. --r {regex} Regular expression for field names to convert. --a Convert all fields. --h|--help Show this message. +-f {a,b,c} Field names to apply substitution to. +-r {regex} Regular expression for field names to apply substitution to. +-a Apply substitution to all fields. +-h|--help Show this message. .fi .if n \{\ .RE @@ -1679,12 +1685,19 @@ Options: Usage: mlr having-fields [options] Conditionally passes through records depending on each record's field names. Options: - --at-least {comma-separated names} - --which-are {comma-separated names} - --at-most {comma-separated names} - --all-matching {regular expression} - --any-matching {regular expression} - --none-matching {regular expression} +--at-least {comma-separated names} Pass records that have at least these field + names. +--which-are {comma-separated names} Pass records whose field names are exactly + these. +--at-most {comma-separated names} Pass records that have at most these field + names. +--all-matching {regular expression} Pass records where all field names match + the regex. +--any-matching {regular expression} Pass records where any field name matches + the regex. +--none-matching {regular expression} Pass records where no field name matches + the regex. +-h|--help Show this message. Examples: mlr having-fields --which-are amount,status,owner mlr having-fields --any-matching 'sda[0-9]' @@ -1703,10 +1716,9 @@ Passes through the first n records, optionally by category. Without -g, ceases consuming more input (i.e. is fast) when n records have been read. Options: -g {a,b,c} Optional group-by-field names for head counts, e.g. a,b,c. --n {n} Head-count to print. Default 10. - A negative count, e.g. -n -2, passes through all but the last n records, - optionally by category. --h|--help Show this message. +-n {n} Head-count to print. Default 10. A negative count, e.g. -n -2, passes + through all but the last n records, optionally by category. +-h|--help Show this message. .fi .if n \{\ .RE @@ -1717,14 +1729,15 @@ Options: .nf Just a histogram. Input values < lo or > hi are not counted. Usage: mlr histogram [options] --f {a,b,c} Value-field names for histogram counts ---lo {lo} Histogram low value ---hi {hi} Histogram high value ---nbins {n} Number of histogram bins. Defaults to 20. ---auto Automatically computes limits, ignoring --lo and --hi. - Holds all values in memory before producing any output. --o {prefix} Prefix for output field name. Default: no prefix. --h|--help Show this message. +Options: +-f {a,b,c} Value-field names for histogram counts. +--lo {lo} Histogram low value. +--hi {hi} Histogram high value. +--nbins {n} Number of histogram bins. Defaults to 20. +--auto Automatically computes limits, ignoring --lo and --hi. Holds all + values in memory before producing any output. +-o {prefix} Prefix for output field name. Default: no prefix. +-h|--help Show this message. .fi .if n \{\ .RE @@ -1736,10 +1749,10 @@ Usage: mlr histogram [options] Usage: mlr json-parse [options] Tries to convert string field values to parsed JSON, e.g. "[1,2,3]" -> [1,2,3]. Options: --f {...} Comma-separated list of field names to json-parse (default all). --k If supplied, then on parse fail for any cell, keep the (unparsable) - input value for the cell. --h|--help Show this message. +-f {a,b,c} Comma-separated list of field names to json-parse (default all). +-k If supplied, then on parse fail for any cell, keep the (unparsable) + input value for the cell. +-h|--help Show this message. .fi .if n \{\ .RE @@ -1751,10 +1764,11 @@ Options: Usage: mlr json-stringify [options] Produces string field values from field-value data, e.g. [1,2,3] -> "[1,2,3]". Options: --f {...} Comma-separated list of field names to json-parse (default all). ---jvstack Produce multi-line JSON output. +-f {a,b,c} Comma-separated list of field names to json-stringify (default + all). +--jvstack Produce multi-line JSON output. --no-jvstack Produce single-line JSON output per record (default). --h|--help Show this message. +-h|--help Show this message. .fi .if n \{\ .RE @@ -1769,34 +1783,53 @@ at the end of the Miller argument list. Functionality is essentially the same as the system "join" command, but for record streams. Options: - -f {left file name} - -j {a,b,c} Comma-separated join-field names for output - -l {a,b,c} Comma-separated join-field names for left input file; - defaults to -j values if omitted. - -r {a,b,c} Comma-separated join-field names for right input file(s); - defaults to -j values if omitted. - --lk|--left-keep-field-names {a,b,c} If supplied, this means keep only the specified field - names from the left file. Automatically includes the join-field name(s). Helpful - for when you only want a limited subset of information from the left file. - Tip: you can use --lk "": this means the left file becomes solely a row-selector - for the input files. - --lp {text} Additional prefix for non-join output field names from - the left file. Applies to paired and unpaired output records. - --rp {text} Additional prefix for non-join output field names from - the right file(s). Applies to paired and unpaired output records. - --np Do not emit paired records - --ul Emit unpaired records from the left file - --ur Emit unpaired records from the right file(s) - -s|--sorted-input Require sorted input: records must be sorted - lexically by their join-field names, else not all records will - be paired. The only likely use case for this is with a left - file which is too big to fit into system memory otherwise. - -u Enable unsorted input. (This is the default even without -u.) - In this case, the entire left file will be loaded into memory. - --prepipe {command} As in main input options; see mlr --help for details. - If you wish to use a prepipe command for the main input as well - as here, it must be specified there as well as here. - --prepipex {command} Likewise. +-f {left file name} Left file name for join. +-j {a,b,c} Comma-separated join-field names for + output. +-l {a,b,c} Comma-separated join-field names for left + input file; defaults to -j values if + omitted. +-r {a,b,c} Comma-separated join-field names for right + input file(s); defaults to -j values if + omitted. +--lk|--left-keep-field-names {a,b,c} If supplied, this means keep only the + specified field names from the left file. + Automatically includes the join-field + name(s). Helpful for when you only want a + limited subset of information from the left + file. Tip: you can use --lk "": this means + the left file becomes solely a row-selector + for the input files. +--lp {text} Additional prefix for non-join output field + names from the left file. Applies to paired + and unpaired output records. +--rp {text} Additional prefix for non-join output field + names from the right file(s). Applies to + paired and unpaired output records. +--np Do not emit paired records. +--ul Emit unpaired records from the left file. +--ur Emit unpaired records from the right + file(s). +-s|--sorted-input Require sorted input: records must be + sorted lexically by their join-field names, + else not all records will be paired. The + only likely use case for this is with a + left file which is too big to fit into + system memory otherwise. +-u Enable unsorted input. (This is the default + even without -u.) In this case, the entire + left file will be loaded into memory. +--prepipe {command} Shell command to prepipe the left-file + input through. As in main input options; + see mlr --help for details. If you wish to + use a prepipe command for the main input as + well as here, it must be specified there as + well as here. +--prepipex {command} Shell command to prepipe the left-file + input through (no shell quoting). As in + main input options; see mlr --help for + details. +-h|--help Show this message. File-format options default to those for the right file names on the Miller argument list, but may be overridden for the left file as follows. Please see the main "mlr --help" for more information on syntax for these arguments: @@ -1857,10 +1890,11 @@ Usage: mlr least-frequent [options] Shows the least frequently occurring distinct values for specified field names. The first entry is the statistical anti-mode; the remaining are runners-up. Options: --f {one or more comma-separated field names}. Required flag. --n {count}. Optional flag defaulting to 10. --b Suppress counts; show only field values. --o {name} Field name for output count. Default "count". +-f {a,b,c} One or more comma-separated field names to group by. Required flag. +-n {count} Maximum number of results to output. Optional flag defaulting to 10. +-b Suppress counts; show only field values. +-o {name} Field name for output count. Default "count". +-h|--help Show this message. See also "mlr most-frequent". .fi .if n \{\ @@ -1874,7 +1908,25 @@ Usage: mlr merge-fields [options] Computes univariate statistics for each input record, accumulated across specified fields. Options: --a {sum,count,...} Names of accumulators. One or more of: +-a {sum,count,...} Names of accumulators: one or more of the accumulators listed + below. +-f {a,b,c} Value-field names on which to compute statistics. Requires + -o. +-r {a,b,c} Regular expressions for value-field names on which to compute + statistics. Requires -o. +-c {a,b,c} Substrings for collapse mode: all fields which have the same + names after removing substrings will be accumulated together. + Please see examples below. +-i Use interpolated percentiles, like R's type=7; default like + type=1. Not sensical for string-valued fields. +-o {name} Output field basename for -f/-r. +-k Keep the input fields which contributed to the output + statistics; the default is to omit them. +-S No-op flag for backward compatibility with Miller 5. +-F No-op flag for backward compatibility with Miller 5. +-h|--help Show this message. + +Accumulators for -a: count Count instances of fields null_count Count number of empty-string/JSON-null instances per field distinct_count Count number of distinct values per field @@ -1892,17 +1944,6 @@ Options: max Compute maximum values of specified fields minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields --f {a,b,c} Value-field names on which to compute statistics. Requires -o. --r {a,b,c} Regular expressions for value-field names on which to compute - statistics. Requires -o. --c {a,b,c} Substrings for collapse mode. All fields which have the same names - after removing substrings will be accumulated together. Please see - examples below. --i Use interpolated percentiles, like R's type=7; default like type=1. - Not sensical for string-valued fields. --o {name} Output field basename for -f/-r. --k Keep the input fields which contributed to the output statistics; - the default is to omit them. String-valued data make sense unless arithmetic on them is required, e.g. for sum, mean, interpolated percentiles, etc. In case of mixed data, @@ -1930,10 +1971,11 @@ Usage: mlr most-frequent [options] Shows the most frequently occurring distinct values for specified field names. The first entry is the statistical mode; the remaining are runners-up. Options: --f {one or more comma-separated field names}. Required flag. --n {count}. Optional flag defaulting to 10. --b Suppress counts; show only field values. --o {name} Field name for output count. Default "count". +-f {a,b,c} One or more comma-separated field names to group by. Required flag. +-n {count} Maximum number of results to output. Optional flag defaulting to 10. +-b Suppress counts; show only field values. +-o {name} Field name for output count. Default "count". +-h|--help Show this message. See also "mlr least-frequent". .fi .if n \{\ @@ -1946,16 +1988,29 @@ See also "mlr least-frequent". Usage: mlr nest [options] Explodes specified field values into separate fields/records, or reverses this. Options: - --explode,--implode One is required. - --values,--pairs One is required. - --across-records,--across-fields One is required. - -f {field name} Required. - -r {field names} Like -f but treat arguments as a regular expression. Match all - field names and operate on each in record order. Example: `-r '^[xy]$`'. - --nested-fs {string} Defaults to ";". Field separator for nested values. - --nested-ps {string} Defaults to ":". Pair separator for nested key-value pairs. - --evar {string} Shorthand for --explode --values --across-records --nested-fs {string} - --ivar {string} Shorthand for --implode --values --across-records --nested-fs {string} +--explode Explode field values into separate fields/records. One of + --explode or --implode is required. +--implode Reverse of --explode. One of --explode or --implode is + required. +--values Operate on field values. One of --values or --pairs is + required. +--pairs Operate on field key-value pairs. One of --values or + --pairs is required. +--across-records Explode/implode across records. One of --across-records or + --across-fields is required. +--across-fields Explode/implode across fields. One of --across-records or + --across-fields is required. +-f {field name} Required field name to operate on. +-r {field names} Like -f but treat arguments as a regular expression. Match + all field names and operate on each in record order. + Example: -r '^[xy]$'. +--nested-fs {string} Field separator for nested values. Defaults to ";". +--nested-ps {string} Pair separator for nested key-value pairs. Defaults to ":". +--evar {string} Shorthand for --explode --values --across-records + --nested-fs {string}. +--ivar {string} Shorthand for --implode --values --across-records + --nested-fs {string}. +-h|--help Show this message. Please use "mlr --usage-separator-options" for information on specifying separators. Examples: @@ -2019,56 +2074,53 @@ Lets you use a domain-specific language to programmatically alter stream records See also: https://miller.readthedocs.io/en/latest/reference-verbs Options: --f {file name} File containing a DSL expression (see examples below). If the filename - is a directory, all *.mlr files in that directory are loaded. +-f {file name} File containing a DSL expression (see examples below). If the + filename is a directory, all *.mlr files in that directory are + loaded. +-e {expression} DSL expression to evaluate. You can use this after -f to add an + expression. Example use case: define functions/subroutines in a + file you specify with -f, then call them with an expression you + specify with -e. +-s {name=value} Predefines out-of-stream variable @name to have the given value. + Thus mlr put -s foo=97 '$column += @foo' is like mlr put 'begin + {@foo = 97} $column += @foo'. The value part is subject to + type-inferencing. May be specified more than once, e.g. -s + name1=value1 -s name2=value2. Note: the value may be an + environment variable, e.g. -s sequence=$SEQUENCE. +-x Prints records for which {expression} evaluates to false, not + true, i.e. invert the sense of the filter expression. Default + false. +-q Does not include the modified record in the output stream. + Useful for when all desired output is in begin and/or end + blocks. +-S No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. +-F No-op in Miller 6 and above, since type-inferencing is now done + by the record-readers before filter/put is executed. Supported + as a no-op pass-through flag for backward compatibility. +-w Print warnings about things like uninitialized variables. +-W Same as -w, but exit the process if there are any warnings. +-p Prints the expression's AST (abstract syntax tree), which gives + full transparency on the precedence and associativity rules of + Miller's grammar, to stdout. +-d Like -p but uses a parenthesized-expression format for the AST. +-D Like -d but with output all on one line. +-E Echo DSL expression before printing parse-tree. +-v Same as -E -p. +-X Exit after parsing but before stream-processing. Useful with + -v/-d/-D, if you only want to look at parser information. +--explain Parse and type-check the DSL expression, report whether it is + valid, and exit without reading the input stream. Exit status is + 0 if the expression is valid and non-zero otherwise; combine + with --errors-json for a machine-readable error. +-h|--help Show this message. --e {expression} You can use this after -f to add an expression. Example use - case: define functions/subroutines in a file you specify with -f, then call - them with an expression you specify with -e. - -(If you mix -e and -f then the expressions are evaluated in the order encountered. +If you mix -e and -f then the expressions are evaluated in the order encountered. Since the expression pieces are simply concatenated, please be sure to use intervening -semicolons to separate expressions.) +semicolons to separate expressions. --s name=value: Predefines out-of-stream variable @name to have - Thus mlr put -s foo=97 '$column += @foo' is like - mlr put 'begin {@foo = 97} $column += @foo'. - The value part is subject to type-inferencing. - May be specified more than once, e.g. -s name1=value1 -s name2=value2. - Note: the value may be an environment variable, e.g. -s sequence=$SEQUENCE - --x (default false) Prints records for which {expression} evaluates to false, not true, - i.e. invert the sense of the filter expression. - --q Does not include the modified record in the output stream. - Useful for when all desired output is in begin and/or end blocks. - --S and -F: There are no-ops in Miller 6 and above, since now type-inferencing is done - by the record-readers before filter/put is executed. Supported as no-op pass-through - flags for backward compatibility. - --h|--help Show this message. - -Parser-info options: - --w Print warnings about things like uninitialized variables. - --W Same as -w, but exit the process if there are any warnings. - --p Prints the expressions's AST (abstract syntax tree), which gives full - transparency on the precedence and associativity rules of Miller's grammar, - to stdout. - --d Like -p but uses a parenthesized-expression format for the AST. - --D Like -d but with output all on one line. - --E Echo DSL expression before printing parse-tree - --v Same as -E -p. - --X Exit after parsing but before stream-processing. Useful with -v/-d/-D, if you - only want to look at parser information. +Parser-info options are -w, -W, -p, -d, -D, -E, -v, and -X. Examples: mlr --from example.csv put '$qr = $quantity * $rate' @@ -2130,16 +2182,15 @@ Options: Usage: mlr rename [options] {old1,new1,old2,new2,...} Renames specified fields. Options: --r Treat old field names as regular expressions. "ab", "a.*b" - will match any field name containing the substring "ab" or - matching "a.*b", respectively; anchors of the form "^ab$", - "^a.*b$" may be used. New field names may be plain strings, - or may contain capture groups of the form "\e1" through - "\e9". Wrapping the regex in double quotes is optional, but - is required if you wish to follow it with 'i' to indicate - case-insensitivity. --g Do global replacement within each field name rather than - first-match replacement. +-r Treat old field names as regular expressions. "ab", "a.*b" will match + any field name containing the substring "ab" or matching "a.*b", + respectively; anchors of the form "^ab$", "^a.*b$" may be used. New + field names may be plain strings, or may contain capture groups of the + form "\e1" through "\e9". Wrapping the regex in double quotes is + optional, but is required if you wish to follow it with 'i' to + indicate case-insensitivity. +-g Do global replacement within each field name rather than first-match + replacement. -h|--help Show this message. Examples: mlr rename old_name,new_name @@ -2159,18 +2210,19 @@ mlr rename -r '"name"i,Name' Rename "name", "Name", "NAME", etc. to "Name" Usage: mlr reorder [options] Moves specified names to start of record, or end of record. Options: --e Put specified field names at record end: default is to put them at record start. +-e Put specified field names at record end: default is to put them at + record start. -f {a,b,c} Field names to reorder. --r Treat field names as regular expressions. Matched fields are moved - to start or end in record order. Example: -r '^YYY,^XXX' puts all - YYY- and XXX-prefixed fields first (in record order), then the rest. +-r {a,b,c} Treat field names as regular expressions. Matched fields are moved to + start or end in record order. Example: -r '^YYY,^XXX' puts all YYY- + and XXX-prefixed fields first (in record order), then the rest. -b {x} Put field names specified with -f before field name specified by {x}, if any. If {x} isn't present in a given record, the specified fields will not be moved. -a {x} Put field names specified with -f after field name specified by {x}, if any. If {x} isn't present in a given record, the specified fields will not be moved. --h|--help Show this message. +-h|--help Show this message. Examples: mlr reorder -f a,b sends input record "d=4,b=2,a=1,c=3" to "a=1,b=2,d=4,c=3". @@ -2186,11 +2238,12 @@ mlr reorder -r '^YYY,^XXX' puts YYY- and XXX-prefixed fields first (record order .nf Usage: mlr repeat [options] Copies input records to output records multiple times. -Options must be exactly one of the following: --n {repeat count} Repeat each input record this many times. --f {field name} Same, but take the repeat count from the specified - field name of each input record. --h|--help Show this message. +Options must be exactly one of -n or -f. +Options: +-n {repeat count} Repeat each input record this many times. +-f {field name} Same as -n, but take the repeat count from the specified field + name of each input record. +-h|--help Show this message. Example: echo x=0 | mlr repeat -n 4 then put '$x=urand()' produces: @@ -2233,6 +2286,19 @@ Long-to-wide options: These pivot/reshape the input data to undo the wide-to-long operation. Note: this does not work with tail -f; it produces output records only after all input records have been read. +Options: +-i {input field names} Input field names for wide-to-long reshape. + Use with -o. +-r {input field regex} Input field regex for wide-to-long reshape. + May be repeated. Use with -o. If you have + multiple regexes, please specify them using + multiple -r, since regexes can contain + commas within them. +-o {key-field name,value-field name} Output key-field and value-field names for + wide-to-long reshape. Requires -i or -r. +-s {key-field name,value-field name} Key-field and value-field names for + long-to-wide reshape. +-h|--help Show this message. Examples: @@ -2288,8 +2354,9 @@ Reservoir sampling (subsampling without replacement), optionally by category. See also mlr bootstrap and mlr shuffle. Options: -g {a,b,c} Optional: group-by-field names for samples, e.g. a,b,c. --k {k} Required: number of records to output in total, or by group if using -g. --h|--help Show this message. +-k {k} Required: number of records to output in total, or by group if using + -g. +-h|--help Show this message. .fi .if n \{\ .RE @@ -2298,13 +2365,15 @@ Options: .RS 0 .\} .nf -Usage: ../c/mlr sec2gmtdate {comma-separated list of field names} +Usage: mlr sec2gmtdate {comma-separated list of field names} Replaces a numeric field representing seconds since the epoch with the corresponding GMT year-month-day timestamp; leaves non-numbers as-is. This is nothing more than a keystroke-saver for the sec2gmtdate function: - ../c/mlr sec2gmtdate time1,time2 + mlr sec2gmtdate time1,time2 is the same as - ../c/mlr put '$time1=sec2gmtdate($time1);$time2=sec2gmtdate($time2)' + mlr put '$time1=sec2gmtdate($time1);$time2=sec2gmtdate($time2)' +Options: +-h|--help Show this message. .fi .if n \{\ .RE @@ -2321,10 +2390,18 @@ more than a keystroke-saver for the sec2gmt function: is the same as mlr put '$time1 = sec2gmt($time1); $time2 = sec2gmt($time2)' Options: --1 through -9: format the seconds using 1..9 decimal places, respectively. ---millis Input numbers are treated as milliseconds since the epoch. ---micros Input numbers are treated as microseconds since the epoch. ---nanos Input numbers are treated as nanoseconds since the epoch. +-1 Format seconds with 1 decimal place. +-2 Format seconds with 2 decimal places. +-3 Format seconds with 3 decimal places. +-4 Format seconds with 4 decimal places. +-5 Format seconds with 5 decimal places. +-6 Format seconds with 6 decimal places. +-7 Format seconds with 7 decimal places. +-8 Format seconds with 8 decimal places. +-9 Format seconds with 9 decimal places. +--millis Input numbers are treated as milliseconds since the epoch. +--micros Input numbers are treated as microseconds since the epoch. +--nanos Input numbers are treated as nanoseconds since the epoch. -h|--help Show this message. .fi .if n \{\ @@ -2335,16 +2412,16 @@ Options: .\} .nf Usage: mlr seqgen [options] -Passes input records directly to output. Most useful for format conversion. Produces a sequence of counters. Discards the input record stream. Produces -output as specified by the options +output as specified by the options. Options: --f {name} (default "i") Field name for counters. ---start {value} (default 1) Inclusive start value. ---step {value} (default 1) Step value. ---stop {value} (default 100) Inclusive stop value. --h|--help Show this message. +-f {name} Field name for counters. Default "i". +--start {value} Inclusive start value. Default 1. +--step {value} Step value. Default 1. May be negative but not zero (unless + start == stop). +--stop {value} Inclusive stop value. Default 100. +-h|--help Show this message. Start, stop, and/or step may be floating-point. Output is integer if start, stop, and step are all integers. Step may be negative. It may not be zero unless start == stop. @@ -2390,17 +2467,20 @@ specified sort order.) The sort is stable: records that compare equal will sort in the order they were encountered in the input record stream. Options: --f {comma-separated field names} Lexical ascending --r {comma-separated field names} Lexical descending --c {comma-separated field names} Case-folded lexical ascending --cr {comma-separated field names} Case-folded lexical descending --n {comma-separated field names} Numerical ascending; nulls sort last --nf {comma-separated field names} Same as -n --nr {comma-separated field names} Numerical descending; nulls sort first --t {comma-separated field names} Natural ascending --b Move sort fields to start of record, as in reorder -b --tr|-rt {comma-separated field names} Natural descending --h|--help Show this message. +-f {a,b,c} Lexical ascending sort on the specified field names. +-r {a,b,c} Lexical descending sort on the specified field names. +-c {a,b,c} Case-folded lexical ascending sort on the specified field names. +-cr {a,b,c} Case-folded lexical descending sort on the specified field + names. +-n {a,b,c} Numerical ascending sort on the specified field names; nulls + sort last. +-nf {a,b,c} Same as -n. +-nr {a,b,c} Numerical descending sort on the specified field names; nulls + sort first. +-t {a,b,c} Natural ascending sort on the specified field names. +-b Move sort fields to start of record, as in reorder -b. +-tr|-rt {a,b,c} Natural descending sort on the specified field names. +-h|--help Show this message. Example: mlr sort -f a,b -nr x,y,z @@ -2417,13 +2497,13 @@ which is the same as: Usage: mlr sort-within-records [options] Outputs records sorted lexically ascending by keys. Options: --f {names} Sort only these keys; others preserve record order. --r {regex} Sort only keys matching this regex; others preserve record order. - Example: -r '^[xy]' sorts keys starting with x or y. - With no regex argument, -r recursively sorts subobjects/submaps - (e.g. for JSON input), or combines with -f to treat names as regex. --n Sort field names naturally (e.g. 2 before 12). Combines with -f/-r. --h|--help Show this message. +-f {names} Sort only these keys; others preserve record order. +-r {regex} Sort only keys matching this regex; others preserve record order. + Example: -r '^[xy]' sorts keys starting with x or y. With no regex + argument, -r recursively sorts subobjects/submaps (e.g. for JSON + input), or combines with -f to treat names as regexes. +-n Sort field names naturally (e.g. 2 before 12). Combines with -f/-r. +-h|--help Show this message. .fi .if n \{\ .RE @@ -2437,9 +2517,9 @@ Unsets fields for which the key is the empty string (or, optionally, another specified value). Only makes sense with output format not being CSV or TSV. Options: -s {filler string} What values to remove. Defaults to the empty string. --f {a,b,c} Specify field names to be operated on; any other fields won't be - modified. The default is to modify all fields. --h|--help Show this message. +-f {a,b,c} Specify field names to be operated on; any other fields won't + be modified. The default is to modify all fields. +-h|--help Show this message. Example: if input is a=1,b=,c=3 then output is a=1,c=3. .fi .if n \{\ @@ -2451,18 +2531,20 @@ Example: if input is a=1,b=,c=3 then output is a=1,c=3. .nf Usage: mlr split [options] {filename} Options: --n {n}: Cap file sizes at N records. --m {m}: Produce M files, round-robining records among them. --g {a,b,c}: Write separate files with records having distinct values for fields named a,b,c. -Exactly one of -m, -n, or -g must be supplied. ---prefix {p} Specify filename prefix; default "split". ---suffix {s} Specify filename suffix; default is from mlr output format, e.g. "csv". ---folder {f} Specify output directory; default is current directory. --a Append to existing file(s), if any, rather than overwriting. --v Send records along to downstream verbs as well as splitting to files. +-n {n} Cap output file sizes at N records. +-m {m} Produce M files, round-robining records among them. +-g {a,b,c} Write separate files with records having distinct values for the + specified field names. +--prefix {p} Output filename prefix. Default "split". +--suffix {s} Output filename suffix. Default is from the output format, e.g. + "csv". +--folder {f} Output directory. Default is current directory. +-a Append to existing files rather than overwriting. +-v Send records downstream as well as splitting to files. -e Do NOT URL-escape names of output files. --j {J} Use string J to join filename parts; default "_". +-j {J} String used to join filename parts. Default "_". -h|--help Show this message. +Exactly one of -m, -n, or -g must be supplied. Any of the output-format command-line flags (see mlr -h). For example, using mlr --icsv --from myfile.csv split --ojson -n 1000 the input is CSV, but the output files are JSON. @@ -2507,10 +2589,10 @@ Both the search and replacement strings support C-style backslash escapes such as \en, \et, and \ex1f. Write \e\e to get a literal backslash. See also the `gsub` and `sub` verbs. Options: --f {a,b,c} Field names to convert. --r {regex} Regular expression for field names to convert. --a Convert all fields. --h|--help Show this message. +-f {a,b,c} Field names to apply substitution to. +-r {regex} Regular expression for field names to apply substitution to. +-a Apply substitution to all fields. +-h|--help Show this message. .fi .if n \{\ .RE @@ -2523,7 +2605,32 @@ Usage: mlr stats1 [options] Computes univariate statistics for one or more given fields, accumulated across the input record stream. Options: --a {sum,count,...} Names of accumulators: one or more of: +-a {sum,count,...} Names of accumulators: one or more of the listed values. Also + accepts median (same as p50) and percentiles p{n} for n in + 0..100, e.g. p10 p25.2 p50 p98 p100. +-f {a,b,c} Value-field names on which to compute statistics. +--fr {regex} Regex for value-field names on which to compute statistics + (compute statistics on values in all field names matching the + regex). +--fx {regex} Inverted regex for value-field names on which to compute + statistics (compute statistics on values in all field names + not matching the regex). +-g {d,e,f} Optional group-by-field names. +--gr {regex} Regex for optional group-by-field names (group by values in + field names matching the regex). +--gx {regex} Inverted regex for optional group-by-field names (group by + values in field names not matching the regex). +--grfx {regex} Shorthand for --gr {regex} --fx {that same regex}. +-i Use interpolated percentiles, like R's type=7; default like + type=1. Not sensical for string-valued fields. +-s Print iterative stats. Useful in tail -f contexts, in which + case please avoid pprint-format output since end of input + stream will never be seen. Likewise, if input is coming from + `tail -f` be sure to use `--records-per-batch 1`. +-S No-op flag for backward compatibility with Miller 5. +-F No-op flag for backward compatibility with Miller 5. +-h|--help Show this message. +Names of accumulators for -a, one or more of: median This is the same as p50 p10 p25.2 p50 p98 p100 etc. count Count instances of fields @@ -2543,28 +2650,6 @@ Options: max Compute maximum values of specified fields minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields - --f {a,b,c} Value-field names on which to compute statistics ---fr {regex} Regex for value-field names on which to compute statistics - (compute statistics on values in all field names matching regex ---fx {regex} Inverted regex for value-field names on which to compute statistics - (compute statistics on values in all field names not matching regex) - --g {d,e,f} Optional group-by-field names ---gr {regex} Regex for optional group-by-field names - (group by values in field names matching regex) ---gx {regex} Inverted regex for optional group-by-field names - (group by values in field names not matching regex) - ---grfx {regex} Shorthand for --gr {regex} --fx {that same regex} - --i Use interpolated percentiles, like R's type=7; default like type=1. - Not sensical for string-valued fields.\en"); --s Print iterative stats. Useful in tail -f contexts, in which - case please avoid pprint-format output since end of input - stream will never be seen. Likewise, if input is coming from `tail -f` - be sure to use `--records-per-batch 1`. --h|--help Show this message. Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape Example: mlr stats1 -a count,mode -f size Example: mlr stats1 -a count,mode -f size -g shape @@ -2593,7 +2678,27 @@ Notes: Usage: mlr stats2 [options] Computes bivariate statistics for one or more given field-name pairs, accumulated across the input record stream. --a {linreg-ols,corr,...} Names of accumulators: one or more of: +Options: +-a {linreg-ols,corr,...} Names of accumulators: one or more of the listed + values. +-f {a,b,c,d} Value-field name-pairs on which to compute statistics. + There must be an even number of names. +-g {e,f,g} Optional group-by-field names. +-v Print additional output for linreg-pca. +-s Print iterative stats. Useful in tail -f contexts, in + which case please avoid pprint-format output since end + of input stream will never be seen. Likewise, if input + is coming from `tail -f`, be sure to use + `--records-per-batch 1`. +--fit Rather than printing regression parameters, applies + them to the input data to compute new fit fields. All + input records are held in memory until end of input + stream. Has effect only for linreg-ols, linreg-pca, and + logireg. +-S No-op flag for backward compatibility with Miller 5. +-F No-op flag for backward compatibility with Miller 5. +-h|--help Show this message. +Names of accumulators for -a, one or more of: linreg-ols Linear regression using ordinary least squares linreg-pca Linear regression using principal component analysis r2 Quality metric for linreg-ols (linreg-pca emits its own) @@ -2601,18 +2706,6 @@ accumulated across the input record stream. corr Sample correlation cov Sample covariance covx Sample-covariance matrix --f {a,b,c,d} Value-field name-pairs on which to compute statistics. - There must be an even number of names. --g {e,f,g} Optional group-by-field names. --v Print additional output for linreg-pca. --s Print iterative stats. Useful in tail -f contexts, in which - case please avoid pprint-format output since end of input - stream will never be seen. Likewise, if input is coming from - `tail -f`, be sure to use `--records-per-batch 1`. ---fit Rather than printing regression parameters, applies them to - the input data to compute new fit fields. All input records are - held in memory until end of input stream. Has effect only for - linreg-ols, linreg-pca, and logireg. Only one of -s or --fit may be used. Example: mlr stats2 -a linreg-pca -f x,y Example: mlr stats2 -a linreg-ols,r2 -f x,y -g size,shape @@ -2628,7 +2721,25 @@ Example: mlr stats2 -a corr -f x,y Usage: mlr step [options] Computes values dependent on earlier/later records, optionally grouped by category. Options: --a {delta,rsum,...} Names of steppers: comma-separated, one or more of: +-a {delta,rsum,...} Names of steppers: comma-separated, one or more of the + listed values. +-f {a,b,c} Value-field names on which to compute statistics. +-g {d,e,f} Optional group-by-field names. +-F Computes integerable things (e.g. counter) in floating + point. As of Miller 6 this happens automatically, but the + flag is accepted as a no-op for backward compatibility with + Miller 5 and below. +-d {x,y,z} Weights for EWMA. 1 means current sample gets all weight (no + smoothing), near under 1 is light smoothing, near over 0 is + heavy smoothing. Multiple weights may be specified, e.g. + "mlr step -a ewma -f sys_load -d 0.01,0.1,0.9". Default if + omitted is "-d 0.5". +-o {a,b,c} Custom suffixes for EWMA output fields. If omitted, these + default to the -d values. If supplied, the number of -o + values must be the same as the number of -d values. +-h|--help Show this message. + +Names of steppers for -a, comma-separated, one or more of: counter Count instances of field(s) between successive records delta Compute differences in field(s) between successive records ewma Exponentially weighted moving average over successive records @@ -2641,21 +2752,6 @@ Options: shift_lead Include value(s) in field(s) from the next record, if any slwin Sliding-window averages over m records back and n forward. E.g. slwin_7_2 for 7 back and 2 forward. --f {a,b,c} Value-field names on which to compute statistics --g {d,e,f} Optional group-by-field names --F Computes integerable things (e.g. counter) in floating point. - As of Miller 6 this happens automatically, but the flag is accepted - as a no-op for backward compatibility with Miller 5 and below. --d {x,y,z} Weights for EWMA. 1 means current sample gets all weight (no - smoothing), near under 1 is light smoothing, near over 0 is - heavy smoothing. Multiple weights may be specified, e.g. - "mlr step -a ewma -f sys_load -d 0.01,0.1,0.9". Default if omitted - is "-d 0.5". --o {a,b,c} Custom suffixes for EWMA output fields. If omitted, these default to - the -d values. If supplied, the number of -o values must be the same - as the number of -d values. --h|--help Show this message. - Examples: mlr step -a rsum -f request_size mlr step -a delta -f request_size -g hostname @@ -2682,10 +2778,10 @@ The replacement string supports C-style backslash escapes such as \en, \et, and \ex1f. Write \e\e to get a literal backslash. See also the `gsub` and `ssub` verbs. Options: --f {a,b,c} Field names to convert. --r {regex} Regular expression for field names to convert. --a Convert all fields. --h|--help Show this message. +-f {a,b,c} Field names to apply substitution to. +-r {regex} Regular expression for field names to apply substitution to. +-a Apply substitution to all fields. +-h|--help Show this message. .fi .if n \{\ .RE @@ -2731,10 +2827,10 @@ Notes: Options: -a {mean,sum,etc.} Use only the specified summarizers. --x {mean,sum,etc.} Use all summarizers, except the specified ones. +-x {mean,sum,etc.} Use all summarizers except the specified ones. --all Use all available summarizers. ---transpose Show output with field names as column names.. --h|--help Show this message. +--transpose Show output with field names as column names. +-h|--help Show this message. .fi .if n \{\ .RE @@ -2747,9 +2843,9 @@ Usage: mlr surv -d {duration-field} -s {status-field} Estimate Kaplan-Meier survival curve (right-censored). Options: - -d {field} Name of duration field (time-to-event or censoring). - -s {field} Name of status field (0=censored, 1=event). - -h, --help Show this message. +-d {field} Name of duration field (time-to-event or censoring). +-s {field} Name of status field (0=censored, 1=event). +-h|--help Show this message. .fi .if n \{\ .RE @@ -2774,11 +2870,10 @@ Usage: mlr tail [options] Passes through the last n records, optionally by category. Options: -g {a,b,c} Optional group-by-field names for tail counts, e.g. a,b,c. --n {n} Tail-count to print. Default 10. - A leading '+' means start at the nth record rather than print - the last n: e.g. -n +3 passes through all but the first 2 - records, optionally by category. --h|--help Show this message. +-n {n} Tail-count to print. Default 10. A leading '+' means start at the nth + record rather than print the last n: e.g. -n +3 passes through all + but the first 2 records, optionally by category. +-h|--help Show this message. .fi .if n \{\ .RE @@ -2789,14 +2884,13 @@ Options: .nf Usage: mlr tee [options] {filename} Options: --a Append to existing file, if any, rather than overwriting. --p Treat filename as a pipe-to command. +-a Append to existing file, if any, rather than overwriting. +-p Treat filename as a pipe-to command. +-h|--help Show this message. Any of the output-format command-line flags (see mlr -h). Example: using mlr --icsv --opprint put '...' then tee --ojson ./mytap.dat then stats1 ... the input is CSV, the output is pretty-print tabular, but the tee-file output is written in JSON format. - --h|--help Show this message. .fi .if n \{\ .RE @@ -2810,10 +2904,13 @@ Places input-record fields in the order specified by list of column names. If the input record is missing a specified field, it will be filled with the fill-with. If the input record possesses an unspecified field, it will be discarded. Options: - -f {a,b,c} Comma-separated field names for template, e.g. a,b,c. - -t {filename} CSV file whose header line will be used for template. ---fill-with {filler string} What to fill absent fields with. Defaults to the empty string. --h|--help Show this message. +-f {a,b,c} Comma-separated field names for template, e.g. + a,b,c. +-t {filename} CSV file whose header line will be used for + template. +--fill-with {filler string} What to fill absent fields with. Defaults to the + empty string. +-h|--help Show this message. Example: * Specified fields are a,b,c. * Input record is c=3,a=1,f=6. @@ -2827,16 +2924,19 @@ Example: .\} .nf Usage: mlr top [options] --f {a,b,c} Value-field names for top counts. --g {d,e,f} Optional group-by-field names for top counts. --n {count} How many records to print per category; default 1. --a Print all fields for top-value records; default is - to print only value and group-by fields. Requires a single - value-field name only. ---min Print top smallest values; default is top largest values. --F Keep top values as floats even if they look like integers. --o {name} Field name for output indices. Default "top_idx". - This is ignored if -a is used. +Options: +-f {a,b,c} Value-field names for top counts. +-g {d,e,f} Optional group-by-field names for top counts. +-n {count} How many records to print per category; default 1. +-a Print all fields for top-value records; default is to print only + value and group-by fields. Requires a single value-field name only. +--max Print top largest values. This is the default. +--min Print top smallest values; default is top largest values. +-F Keep top values as floats even if they look like integers (ignored in + Miller 6, kept for backward compatibility). +-o {name} Field name for output indices. Default "top_idx". Ignored if -a is + used. +-h|--help Show this message. Prints the n records with smallest/largest values at specified fields, optionally by category. If -a is given, then the top records are emitted with the same fields as they appeared in the input. Without -a, only fields @@ -2851,7 +2951,7 @@ please see https://miller.readthedocs.io/en/latest/reference-verbs#top .\} .nf Usage: mlr utf8-to-latin1, with no options. -Recursively converts record strings from Latin-1 to UTF-8. +Recursively converts record strings from UTF-8 to Latin-1. For field-level control, please see the utf8_to_latin1 DSL function. Options: -h|--help Show this message. @@ -2867,9 +2967,9 @@ Usage: mlr unflatten [options] Reverses flatten. Example: field with name 'a.b.c' and value 4 becomes name 'a' and value '{"b": { "c": 4 }}'. Options: --f {a,b,c} Comma-separated list of field names to unflatten (default all). +-f {a,b,c} Comma-separated list of field names to unflatten (default all). -s {string} Separator, defaulting to mlr --flatsep value. --h|--help Show this message. +-h|--help Show this message. .fi .if n \{\ .RE @@ -2883,15 +2983,18 @@ Prints distinct values for specified field names. With -c, same as count-distinct. For uniq, -f is a synonym for -g. Options: --g {d,e,f} Group-by-field names for uniq counts. --x {a,b,c} Field names to exclude for uniq: use each record's others instead. --c Show repeat counts in addition to unique values. --n Show only the number of distinct values. --o {name} Field name for output count. Default "count". --a Output each unique record only once. Incompatible with -g. - With -c, produces unique records, with repeat counts for each. - With -n, produces only one record which is the unique-record count. - With neither -c nor -n, produces unique records. +-g {d,e,f} Group-by field names for uniq counts. +-f {d,e,f} Synonym for -g. +-x {a,b,c} Field names to exclude for uniq; use each record's other fields + instead. +-c Show repeat counts in addition to unique values. +-n Show only the number of distinct values. +-o {name} Field name for output count. Default "count". +-a Output each unique record only once. Incompatible with -g. With -c, + produces unique records, with repeat counts for each. With -n, + produces only one record which is the unique-record count. With + neither -c nor -n, produces unique records. +-h|--help Show this message. .fi .if n \{\ .RE @@ -2903,7 +3006,7 @@ Options: Usage: mlr unspace [options] Replaces spaces in record keys and/or values with _. This is helpful for PPRINT output. Options: --f {x} Replace spaces with specified filler character. +-f {x} Replace spaces with specified filler character. Default "_". -k Unspace only keys, not keys and values. -v Unspace only values, not keys and values. -h|--help Show this message. @@ -2920,11 +3023,11 @@ Prints records with the union of field names over all input records. For field names absent in a given record but present in others, fills in a value. This verb retains all input before producing any output. Options: ---fill-with {filler string} What to fill absent fields with. Defaults to - the empty string. --f {a,b,c} Specify field names to be operated on. Any other fields won't be - modified, and operation will be streaming. --h|--help Show this message. +--fill-with {filler string} What to fill absent fields with. Defaults to the + empty string. +-f {a,b,c} Specify field names to be operated on; others are + not modified and operation is streaming. +-h|--help Show this message. Example: if the input is two records, one being 'a=1,b=2' and the other being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and \(cqa=,b=3,c=4'. diff --git a/pkg/climain/errors_json.go b/pkg/climain/errors_json.go index a5324ed8a..886f5313f 100644 --- a/pkg/climain/errors_json.go +++ b/pkg/climain/errors_json.go @@ -142,8 +142,13 @@ func categorize(err error) StructuredError { } } - // DSL parse errors - if strings.Contains(msg, "cannot parse DSL") || strings.Contains(msg, "DSL expression") { + // DSL parse errors. The "parse error:" prefix comes from the DSL parser + // (pkg/parsing/parser); "cannot parse DSL"/"DSL expression" are the wrapper + // messages. (The CSV reader's "parse error on line ..." is a stream-time + // error and never reaches this command-line-parse categorizer.) + if strings.Contains(msg, "cannot parse DSL") || + strings.Contains(msg, "DSL expression") || + strings.Contains(msg, "parse error:") { return StructuredError{ Error: msg, Kind: "dsl-parse-error", diff --git a/pkg/climain/errors_json_test.go b/pkg/climain/errors_json_test.go index bf3de4cab..86b84ef2c 100644 --- a/pkg/climain/errors_json_test.go +++ b/pkg/climain/errors_json_test.go @@ -175,3 +175,27 @@ func TestCategorizeGenericFallback(t *testing.T) { t.Errorf("kind: got %q, want generic", se.Kind) } } + +func TestCategorizeDSLParseError(t *testing.T) { + // The DSL parser (pkg/parsing/parser) emits bare "parse error: ..." + // messages; these should categorize as dsl-parse-error, e.g. for + // `mlr put --explain` with --errors-json. + err := fmt.Errorf("parse error: unexpected equals (\"=\")") + se := categorize(err) + if se.Kind != "dsl-parse-error" { + t.Errorf("kind: got %q, want dsl-parse-error", se.Kind) + } + if se.Hint == "" { + t.Error("hint should be non-empty for dsl-parse-error") + } +} + +func TestCategorizeDSLParseErrorNotCSV(t *testing.T) { + // The CSV reader's "parse error on line ..." is a stream-time error and + // should not be mistaken for a DSL parse error by the substring match. + err := fmt.Errorf("parse error on line 3, column 5: bare \" in non-quoted-field") + se := categorize(err) + if se.Kind == "dsl-parse-error" { + t.Errorf("kind: got dsl-parse-error, want non-DSL categorization for a CSV parse error") + } +} diff --git a/pkg/transformers/put_or_filter.go b/pkg/transformers/put_or_filter.go index 3915491ab..5351c3352 100644 --- a/pkg/transformers/put_or_filter.go +++ b/pkg/transformers/put_or_filter.go @@ -32,6 +32,7 @@ var putOptions = []OptionSpec{ {Flag: "-E", Type: "bool", Desc: "Echo DSL expression before printing parse-tree."}, {Flag: "-v", Type: "bool", Desc: "Same as -E -p."}, {Flag: "-X", Type: "bool", Desc: "Exit after parsing but before stream-processing. Useful with -v/-d/-D, if you only want to look at parser information."}, + {Flag: "--explain", Type: "bool", Desc: "Parse and type-check the DSL expression, report whether it is valid, and exit without reading the input stream. Exit status is 0 if the expression is valid and non-zero otherwise; combine with --errors-json for a machine-readable error."}, } var PutSetup = TransformerSetup{ @@ -60,6 +61,7 @@ var filterOptions = []OptionSpec{ {Flag: "-E", Type: "bool", Desc: "Echo DSL expression before printing parse-tree."}, {Flag: "-v", Type: "bool", Desc: "Same as -E -p."}, {Flag: "-X", Type: "bool", Desc: "Exit after parsing but before stream-processing. Useful with -v/-d/-D, if you only want to look at parser information."}, + {Flag: "--explain", Type: "bool", Desc: "Parse and type-check the DSL expression, report whether it is valid, and exit without reading the input stream. Exit status is 0 if the expression is valid and non-zero otherwise; combine with --errors-json for a machine-readable error."}, } var FilterSetup = TransformerSetup{ @@ -195,6 +197,7 @@ func transformerPutOrFilterParseCLI( printASTMultiLine := false printASTSingleLine := false exitAfterParse := false + doExplain := false doWarnings := false warningsAreFatal := false strictMode := false @@ -301,6 +304,8 @@ func transformerPutOrFilterParseCLI( printASTSingleLine = true case "-X": exitAfterParse = true + case "--explain": + doExplain = true case "-w": doWarnings = true warningsAreFatal = false @@ -375,6 +380,7 @@ func transformerPutOrFilterParseCLI( printASTMultiLine, printASTSingleLine, exitAfterParse, + doExplain, doWarnings, warningsAreFatal, strictMode, @@ -409,6 +415,7 @@ func NewTransformerPut( printASTMultiLine bool, printASTSingleLine bool, exitAfterParse bool, + doExplain bool, doWarnings bool, warningsAreFatal bool, strictMode bool, @@ -449,6 +456,26 @@ func NewTransformerPut( }, ) + // --explain is a validate/dry-run: report whether the DSL parsed and + // type-checked, then exit without reading the input stream. A parse/build + // error is returned so it flows through the normal error path (including + // --errors-json); a valid expression prints a confirmation and exits 0. + if doExplain { + if err != nil { + return nil, err + } + verbName := "put" + if doFilter { + verbName = "filter" + } + if warningsAreFatal && hadWarnings { + fmt.Fprintf(os.Stderr, "mlr %s: DSL expression has warnings treated as fatal.\n", verbName) + os.Exit(1) + } + fmt.Printf("mlr %s: DSL expression is valid.\n", verbName) + os.Exit(0) + } + if warningsAreFatal && hadWarnings { fmt.Printf( "%s: Exiting due to warnings treated as fatal.\n", diff --git a/plans/plan-2098-llm.md b/plans/plan-2098-llm.md index f024e828e..b5aa8413c 100644 --- a/plans/plan-2098-llm.md +++ b/plans/plan-2098-llm.md @@ -226,7 +226,7 @@ per the issue.) --- -## PR 5 — DSL `--explain` / validate dry-run +## PR 5 — DSL `--explain` / validate dry-run *(landed)* **Goal.** Validate/type-check a DSL expression *before* spending a full input pass (a big context saver for agents). @@ -236,6 +236,19 @@ pass (a big context saver for agents). **without consuming the full input stream**. - Reuse the existing DSL parse/CST build path; gate it before the record loop. +**Landed.** `--explain` added to put/filter (`put_or_filter.go`): after the +existing `cstRootNode.Build` (which already does parse → ValidateAST → CST build +→ Resolve), a valid expression prints `mlr {put,filter}: DSL expression is +valid.` and exits 0; an invalid one returns the build error up the normal path, +so `--errors-json` yields a structured document. The gate sits in the pass-two +constructor, before any input file is opened, so no input is read. DSL parser +messages (`parse error: ...`) now categorize as `dsl-parse-error` rather than +`generic` (`climain/errors_json.go`). Tests: `dsl-explain/0001-0004` regression +cases (valid put/filter, invalid plain, invalid `--errors-json`) plus categorize +unit tests. Note: the older `-X` ("exit after parsing") still exits 0 even on a +parse error — a pre-existing quirk left as-is since `--explain` is the correct +validation path. + --- ## PR 6 — `mlr describe` schema/shape introspection diff --git a/test/cases/cli-help/0001/expout b/test/cases/cli-help/0001/expout index 0de6e045b..955288af6 100644 --- a/test/cases/cli-help/0001/expout +++ b/test/cases/cli-help/0001/expout @@ -231,6 +231,10 @@ Options: -v Same as -E -p. -X Exit after parsing but before stream-processing. Useful with -v/-d/-D, if you only want to look at parser information. +--explain Parse and type-check the DSL expression, report whether it is + valid, and exit without reading the input stream. Exit status is + 0 if the expression is valid and non-zero otherwise; combine + with --errors-json for a machine-readable error. -h|--help Show this message. If you mix -e and -f then the expressions are evaluated in the order encountered. @@ -759,6 +763,10 @@ Options: -v Same as -E -p. -X Exit after parsing but before stream-processing. Useful with -v/-d/-D, if you only want to look at parser information. +--explain Parse and type-check the DSL expression, report whether it is + valid, and exit without reading the input stream. Exit status is + 0 if the expression is valid and non-zero otherwise; combine + with --errors-json for a machine-readable error. -h|--help Show this message. If you mix -e and -f then the expressions are evaluated in the order encountered. diff --git a/test/cases/dsl-explain/0001/cmd b/test/cases/dsl-explain/0001/cmd new file mode 100644 index 000000000..f1db63ada --- /dev/null +++ b/test/cases/dsl-explain/0001/cmd @@ -0,0 +1 @@ +mlr -n put --explain -f ${CASEDIR}/mlr diff --git a/test/cases/dsl-explain/0001/experr b/test/cases/dsl-explain/0001/experr new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/dsl-explain/0001/expout b/test/cases/dsl-explain/0001/expout new file mode 100644 index 000000000..cdb166c7d --- /dev/null +++ b/test/cases/dsl-explain/0001/expout @@ -0,0 +1 @@ +mlr put: DSL expression is valid. diff --git a/test/cases/dsl-explain/0001/mlr b/test/cases/dsl-explain/0001/mlr new file mode 100644 index 000000000..7ca615a8e --- /dev/null +++ b/test/cases/dsl-explain/0001/mlr @@ -0,0 +1 @@ +$y = $x + 1 diff --git a/test/cases/dsl-explain/0002/cmd b/test/cases/dsl-explain/0002/cmd new file mode 100644 index 000000000..da83ed909 --- /dev/null +++ b/test/cases/dsl-explain/0002/cmd @@ -0,0 +1 @@ +mlr -n filter --explain -f ${CASEDIR}/mlr diff --git a/test/cases/dsl-explain/0002/experr b/test/cases/dsl-explain/0002/experr new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/dsl-explain/0002/expout b/test/cases/dsl-explain/0002/expout new file mode 100644 index 000000000..03508c32e --- /dev/null +++ b/test/cases/dsl-explain/0002/expout @@ -0,0 +1 @@ +mlr filter: DSL expression is valid. diff --git a/test/cases/dsl-explain/0002/mlr b/test/cases/dsl-explain/0002/mlr new file mode 100644 index 000000000..847749527 --- /dev/null +++ b/test/cases/dsl-explain/0002/mlr @@ -0,0 +1 @@ +$x > 0 diff --git a/test/cases/dsl-explain/0003/cmd b/test/cases/dsl-explain/0003/cmd new file mode 100644 index 000000000..f1db63ada --- /dev/null +++ b/test/cases/dsl-explain/0003/cmd @@ -0,0 +1 @@ +mlr -n put --explain -f ${CASEDIR}/mlr diff --git a/test/cases/dsl-explain/0003/experr b/test/cases/dsl-explain/0003/experr new file mode 100644 index 000000000..30e632a0c --- /dev/null +++ b/test/cases/dsl-explain/0003/experr @@ -0,0 +1,2 @@ +mlr: cannot parse DSL expression. +mlr: parse error: unexpected equals ("=") diff --git a/test/cases/dsl-explain/0003/expout b/test/cases/dsl-explain/0003/expout new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/dsl-explain/0003/mlr b/test/cases/dsl-explain/0003/mlr new file mode 100644 index 000000000..90db8987d --- /dev/null +++ b/test/cases/dsl-explain/0003/mlr @@ -0,0 +1 @@ +$y = = $x diff --git a/test/cases/dsl-explain/0003/should-fail b/test/cases/dsl-explain/0003/should-fail new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/dsl-explain/0004/cmd b/test/cases/dsl-explain/0004/cmd new file mode 100644 index 000000000..e9aed3d8b --- /dev/null +++ b/test/cases/dsl-explain/0004/cmd @@ -0,0 +1 @@ +mlr --errors-json -n put --explain -f ${CASEDIR}/mlr diff --git a/test/cases/dsl-explain/0004/experr b/test/cases/dsl-explain/0004/experr new file mode 100644 index 000000000..ec134000e --- /dev/null +++ b/test/cases/dsl-explain/0004/experr @@ -0,0 +1,6 @@ +mlr: cannot parse DSL expression. +{ + "error": "parse error: unexpected equals (\"=\")", + "kind": "dsl-parse-error", + "hint": "Run 'mlr put --help' for DSL syntax reference." +} diff --git a/test/cases/dsl-explain/0004/expout b/test/cases/dsl-explain/0004/expout new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/dsl-explain/0004/mlr b/test/cases/dsl-explain/0004/mlr new file mode 100644 index 000000000..90db8987d --- /dev/null +++ b/test/cases/dsl-explain/0004/mlr @@ -0,0 +1 @@ +$y = = $x diff --git a/test/cases/dsl-explain/0004/should-fail b/test/cases/dsl-explain/0004/should-fail new file mode 100644 index 000000000..e69de29bb