diff --git a/docs6/docs/10min.md b/docs6/docs/10min.md
index d75a63584..db6900811 100644
--- a/docs6/docs/10min.md
+++ b/docs6/docs/10min.md
@@ -87,7 +87,7 @@ purple,square,false,10,91,72.3735,8.2430
{
"color": "yellow",
"shape": "circle",
- "flag": true,
+ "flag": "true",
"k": 9,
"index": 87,
"quantity": 63.5058,
@@ -96,7 +96,7 @@ purple,square,false,10,91,72.3735,8.2430
{
"color": "purple",
"shape": "square",
- "flag": false,
+ "flag": "false",
"k": 10,
"index": 91,
"quantity": 72.3735,
@@ -212,14 +212,9 @@ red square false 4 48 77.5542 7.4670
red square false 6 64 77.1991 9.5310
-
+
mlr --icsv --opprint filter '$color == "red" && $flag == true' example.csv
-
-color shape flag k index quantity rate
-red square true 2 15 79.2778 0.0130
-red circle true 3 16 13.8103 2.9010
-
You can use `put` to create new fields which are computed from other fields:
diff --git a/docs6/docs/cpu.md b/docs6/docs/cpu.md
index 3934cebe1..8f82d6cf2 100644
--- a/docs6/docs/cpu.md
+++ b/docs6/docs/cpu.md
@@ -39,6 +39,13 @@ In practice, profiling has shown that the input-reader uses the most CPU of all
the above. This means CPUs running verbs may not be 100% utilized, since they
are likely to be spending some of their time waiting for input data.
+Running Miller on a machine with more CPUs than active channels (as listed
+above) won't speed up a given invocation of Miller. However, of course, you'll
+be able to run more invocations of Miller at the same time if you like.
+
You can set the Go-standard environment variable `GOMAXPROCS` if you like. If
you don't, Miller will (as is standard for Go programs in Go 1.16 and above) up
to all available CPUs.
+
+If you set `$GOMAXPROCS=1` in the environment, that's fine -- the Go runtime
+will multiplex different channel-handling goroutines onto the same CPU.
diff --git a/docs6/docs/cpu.md.in b/docs6/docs/cpu.md.in
index 520cd54b0..6248d5ad3 100644
--- a/docs6/docs/cpu.md.in
+++ b/docs6/docs/cpu.md.in
@@ -25,6 +25,13 @@ In practice, profiling has shown that the input-reader uses the most CPU of all
the above. This means CPUs running verbs may not be 100% utilized, since they
are likely to be spending some of their time waiting for input data.
+Running Miller on a machine with more CPUs than active channels (as listed
+above) won't speed up a given invocation of Miller. However, of course, you'll
+be able to run more invocations of Miller at the same time if you like.
+
You can set the Go-standard environment variable `GOMAXPROCS` if you like. If
you don't, Miller will (as is standard for Go programs in Go 1.16 and above) up
to all available CPUs.
+
+If you set `$GOMAXPROCS=1` in the environment, that's fine -- the Go runtime
+will multiplex different channel-handling goroutines onto the same CPU.
diff --git a/docs6/docs/data-cleaning-examples.md b/docs6/docs/data-cleaning-examples.md
index b03c96b23..5c06ecf30 100644
--- a/docs6/docs/data-cleaning-examples.md
+++ b/docs6/docs/data-cleaning-examples.md
@@ -58,9 +58,9 @@ A second option is to flag badly formatted data within the output stream:
name reachable format_ok
-barney false false
-betty true false
-fred true false
+barney false true
+betty true true
+fred true true
wilma 1 false
@@ -72,9 +72,6 @@ Or perhaps to flag badly formatted data outside the output stream:
' data/het-bool.csv
-Malformed at NR=1
-Malformed at NR=2
-Malformed at NR=3
Malformed at NR=4
name reachable
barney false
@@ -89,5 +86,5 @@ A third way is to abort the process on first instance of bad data:
mlr --csv put '$reachable = asserting_string($reachable)' data/het-bool.csv
-Miller: is_string type-assertion failed at NR=1 FNR=1 FILENAME=data/het-bool.csv
+Miller: is_string type-assertion failed at NR=4 FNR=4 FILENAME=data/het-bool.csv
diff --git a/docs6/docs/data/json-in-csv.csv b/docs6/docs/data/json-in-csv.csv
new file mode 100644
index 000000000..99a9c9519
--- /dev/null
+++ b/docs6/docs/data/json-in-csv.csv
@@ -0,0 +1,3 @@
+id,blob
+100,"{""a"":1,""b"":[2,3,4]}"
+105,"{""a"":6,""b"":[7,8,9]}"
diff --git a/docs6/docs/data/type-infer.csv b/docs6/docs/data/type-infer.csv
new file mode 100644
index 000000000..2a061ee83
--- /dev/null
+++ b/docs6/docs/data/type-infer.csv
@@ -0,0 +1,3 @@
+a,b,c
+1.2,3,true
+4,5.6,buongiorno
diff --git a/docs6/docs/file-formats.md b/docs6/docs/file-formats.md
index 21d2e314e..7abdb8822 100644
--- a/docs6/docs/file-formats.md
+++ b/docs6/docs/file-formats.md
@@ -271,6 +271,13 @@ TODO: probably remove entirely
* Use `--jflatsep yourseparatorhere` to specify the string used for key concatenation: this defaults to a single dot.
+### JSON-in-CSV
+
+It's quite common to have CSV data which contains stringified JSON as a column.
+See the [JSON parse and stringify
+section](reference-main-data-types.md#json-parse-and-stringify) for ways to
+decode these in Miller.
+
## PPRINT: Pretty-printed tabular
Miller's pretty-print format is like CSV, but column-aligned. For example, compare
diff --git a/docs6/docs/file-formats.md.in b/docs6/docs/file-formats.md.in
index 26f978b76..64fb1e383 100644
--- a/docs6/docs/file-formats.md.in
+++ b/docs6/docs/file-formats.md.in
@@ -118,6 +118,13 @@ TODO: probably remove entirely
* Use `--jflatsep yourseparatorhere` to specify the string used for key concatenation: this defaults to a single dot.
+### JSON-in-CSV
+
+It's quite common to have CSV data which contains stringified JSON as a column.
+See the [JSON parse and stringify
+section](reference-main-data-types.md#json-parse-and-stringify) for ways to
+decode these in Miller.
+
## PPRINT: Pretty-printed tabular
Miller's pretty-print format is like CSV, but column-aligned. For example, compare
diff --git a/docs6/docs/javascripts/tables.js b/docs6/docs/javascripts/tables.js
new file mode 100644
index 000000000..e848f07dd
--- /dev/null
+++ b/docs6/docs/javascripts/tables.js
@@ -0,0 +1,6 @@
+document$.subscribe(function() {
+ var tables = document.querySelectorAll("article table")
+ tables.forEach(function(table) {
+ new Tablesort(table)
+ })
+})
diff --git a/docs6/docs/keystroke-savers.md b/docs6/docs/keystroke-savers.md
index fa48095e1..df88f3885 100644
--- a/docs6/docs/keystroke-savers.md
+++ b/docs6/docs/keystroke-savers.md
@@ -34,7 +34,7 @@ red square true 2 15 79.2778 0.0130
{
"color": "yellow",
"shape": "triangle",
- "flag": true,
+ "flag": "true",
"k": 1,
"index": 11,
"quantity": 43.6498,
@@ -43,7 +43,7 @@ red square true 2 15 79.2778 0.0130
{
"color": "red",
"shape": "square",
- "flag": true,
+ "flag": "true",
"k": 2,
"index": 15,
"quantity": 79.2778,
diff --git a/docs6/docs/mk-func-info.rb b/docs6/docs/mk-func-info.rb
new file mode 100755
index 000000000..2e72121c6
--- /dev/null
+++ b/docs6/docs/mk-func-info.rb
@@ -0,0 +1,114 @@
+#!/usr/bin/env ruby
+
+# ================================================================
+# Markdown autogen for built-in functions: table, or full details page.
+# Invoked by files in *.md.in.
+# ================================================================
+
+# ----------------------------------------------------------------
+# Displayname is for HTML rendered in the document body. Linkname is for
+# #-style links which don't do so well with special characters.
+
+LOOKUP = {
+ # name display_name link_name
+'!' => [ '\!', 'exclamation-point' ],
+'!=' => [ '\!=', 'exclamation-point-equals' ],
+'!=~' => [ '!=~', 'regnotmatch' ],
+'%' => [ '%', 'percent' ],
+'&&' => [ '&&', 'logical-and' ],
+'&' => [ '&', 'bitwise-and' ],
+'*' => [ '\*', 'times' ],
+'**' => [ '\**', 'exponentiation' ],
+'+' => [ '\+', 'plus' ],
+'-' => [ '\-', 'minus' ],
+'.' => [ '\.', 'dot' ],
+'.*' => [ '\.\*', 'dot-times' ],
+'.+' => [ '\.\+', 'dot-plus' ],
+'.-' => [ '\.\-', 'dot-minus' ],
+'./' => [ '\./', 'dot-slash' ],
+'/' => [ '/', 'slash' ],
+'//' => [ '//', 'slash-slash' ],
+':' => [ '\:', 'colon' ],
+'<' => [ '<', 'less-than' ],
+'<<' => [ '<<', 'lsh' ],
+'<=' => [ '<=', 'less-than-or-equals' ],
+'=' => [ '=', 'equals' ],
+'==' => [ '==', 'double-equals' ],
+'=~' => [ '=~', 'regmatch' ],
+'>' => [ '>', 'greater-than' ],
+'>=' => [ '>=', 'greater-than-or-equals' ],
+'>>' => [ '>>', 'srsh' ],
+'>>>' => [ '>>>', 'ursh' ],
+'>>>=' => [ '>>>=', 'ursheq' ],
+'?' => [ '?', 'question-mark' ],
+'?.:' => [ '?:', 'question-mark-colon' ],
+'?:' => [ '?:', 'question-mark-colon' ],
+'??' => [ '??', 'absent-coalesce' ],
+'???' => [ '???', 'absent-empty-coalesce' ],
+'^' => [ '^', 'bitwise-xor' ],
+'^^' => [ '^^', 'logical-xor' ],
+'|' => [ '\|', 'bitwise-or' ],
+'||' => [ '\|\|', 'logical-or' ],
+'~' => [ '~', 'bitwise-not' ],
+}
+
+def name_to_display_name_and_link_name(name)
+ entry = LOOKUP[name]
+ if entry.nil?
+ return [name, name]
+ else
+ display_name, link_name = entry
+ return [display_name, link_name]
+ end
+end
+
+# ----------------------------------------------------------------
+def make_func_details
+ function_classes = `mlr help list-function-classes`.split
+
+ puts
+ puts "## Summary"
+ puts
+ for function_class in function_classes
+ class_link_name = "#"+"#{function_class}-functions"
+ class_display_name = "#{function_class.capitalize} functions"
+ #puts "* [#{display_name}](#{link_name})"
+
+ functions_in_class = `mlr help list-functions-in-class #{function_class}`.split
+ foo = []
+ for function_name in functions_in_class
+ display_name, link_name = name_to_display_name_and_link_name(function_name)
+ foo.append " [#{display_name}]("+"#"+"#{link_name})"
+ end
+ #puts " * #{foo.join(',')}"
+ puts "* [**#{class_display_name}**](#{class_link_name}): #{foo.join(', ')}."
+
+ end
+ puts
+
+ for function_class in function_classes
+ display_name = "#{function_class.capitalize} functions"
+ puts "## #{display_name}"
+ puts
+
+ functions_in_class = `mlr help list-functions-in-class #{function_class}`.split
+
+ for function_name in functions_in_class
+ display_name, link_name = name_to_display_name_and_link_name(function_name)
+
+ puts
+ if display_name != link_name
+ puts ""
+ end
+ puts "### #{display_name}"
+
+ puts ''
+ system("mlr help function '#{function_name}'")
+ puts ''
+ puts
+ end
+ end
+end
+
+# ----------------------------------------------------------------
+make_func_details
diff --git a/docs6/docs/mk-func-table.rb b/docs6/docs/mk-func-table.rb
deleted file mode 100755
index aeab061af..000000000
--- a/docs6/docs/mk-func-table.rb
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/usr/bin/env ruby
-
-# Sphinx tables need to be precisely lined up, unlike Markdown tables which
-# are more flexible. For example this is fine Markdown:
-#
-# | a | b | i | x | y |
-# | --- | --- | --- | --- | --- |
-# | pan | pan | 1 | 0.3467901443380824 | 0.7268028627434533 |
-# | eks | pan | 2 | 0.7586799647899636 | 0.5221511083334797 |
-# | wye | wye | 3 | 0.20460330576630303 | 0.33831852551664776 |
-# | eks | wye | 4 | 0.38139939387114097 | 0.13418874328430463 |
-#
-# but Sphinx wants
-#
-# +-----+-----+---+---------------------+---------------------+
-# | a | b | i | x | y |
-# +=====+=====+===+=====================+=====================+
-# | pan | pan | 1 | 0.3467901443380824 | 0.7268028627434533 |
-# +-----+-----+---+---------------------+---------------------+
-# | eks | pan | 2 | 0.7586799647899636 | 0.5221511083334797 |
-# +-----+-----+---+---------------------+---------------------+
-# | wye | wye | 3 | 0.20460330576630303 | 0.33831852551664776 |
-# +-----+-----+---+---------------------+---------------------+
-# | eks | wye | 4 | 0.38139939387114097 | 0.13418874328430463 |
-# +-----+-----+---+---------------------+---------------------+
-#
-# So we need to build up a list of tuples, then compute the max length down
-# each column, then write a fixed number of -/= per row.
-
-lines = `mlr --list-all-functions-as-table`
-
-# ----------------------------------------------------------------
-# Pass 1
-
-max_name_length = 1
-max_function_class_length = 1
-max_nargs_length = 1
-
-rows = []
-lines.split("\n").each do |line|
- if line =~ /^? :.*/ # has an extra whitespace which throws off the naive split
- ig, nore, function_class, nargs = line.split(/\s+/)
- name = '? :'
- else
- name, function_class, nargs = line.split(/\s+/)
- end
-
- name = '``' + name + '``'
-
- if max_name_length < name.length
- max_name_length = name.length
- end
- if max_function_class_length < function_class.length
- max_function_class_length = function_class.length
- end
- if max_nargs_length < nargs.length
- max_nargs_length = nargs.length
- end
-
- # TODO: format name with page-internal link
- row = [name, function_class, nargs]
- rows.append(row)
-end
-
-# ----------------------------------------------------------------
-# Pass 2
-
-name_dashes = '-' * max_name_length
-function_class_dashes = '-' * max_function_class_length
-nargs_dashes = '-' * max_nargs_length
-
-name_equals = '=' * max_name_length
-function_class_equals = '=' * max_function_class_length
-nargs_equals = '=' * max_nargs_length
-
-dashes_line = "+-#{name_dashes}-+-#{function_class_dashes}-+-#{nargs_dashes}-+"
-equals_line = "+=#{name_equals}=+=#{function_class_equals}=+=#{nargs_equals}=+"
-
-counter = 0
-rows.each do |row|
- name, function_class, nargs = row
-
- counter = counter + 1
- if counter == 1
- puts dashes_line
- puts "| #{name.ljust(max_name_length)} | #{function_class.ljust(max_function_class_length)} | #{nargs.ljust(max_nargs_length)} |"
- puts equals_line
- else
- puts "| #{name.ljust(max_name_length)} | #{function_class.ljust(max_function_class_length)} | #{nargs.ljust(max_nargs_length)} |"
- puts dashes_line
- end
-end
-
diff --git a/docs6/docs/online-help.md b/docs6/docs/online-help.md
index e77d28999..537dfcf80 100644
--- a/docs6/docs/online-help.md
+++ b/docs6/docs/online-help.md
@@ -49,7 +49,10 @@ Type 'mlr help {topic}' for any of the following:
mlr help function
mlr help keyword
mlr help list-functions
+ mlr help list-function-classes
+ mlr help list-functions-in-class
mlr help list-functions-as-paragraph
+ mlr help list-functions-as-table
mlr help list-keywords
mlr help list-keywords-as-paragraph
mlr help list-verbs
@@ -61,6 +64,7 @@ Type 'mlr help {topic}' for any of the following:
mlr help separator-options
mlr help type-arithmetic-info
mlr help usage-functions
+ mlr help usage-functions-by-class
mlr help usage-keywords
mlr help usage-verbs
mlr help verb
@@ -91,7 +95,10 @@ Type 'mlr help {topic}' for any of the following:
mlr help function
mlr help keyword
mlr help list-functions
+ mlr help list-function-classes
+ mlr help list-functions-in-class
mlr help list-functions-as-paragraph
+ mlr help list-functions-as-table
mlr help list-keywords
mlr help list-keywords-as-paragraph
mlr help list-verbs
@@ -103,6 +110,7 @@ Type 'mlr help {topic}' for any of the following:
mlr help separator-options
mlr help type-arithmetic-info
mlr help usage-functions
+ mlr help usage-functions-by-class
mlr help usage-keywords
mlr help usage-verbs
mlr help verb
@@ -179,7 +187,7 @@ Given the name of a DSL function (from `mlr -f`) you can use `mlr help function`
mlr help function append
-append (class=maps/arrays #args=2) Appends second argument to end of first argument, which must be an array.
+append (class=collections #args=2) Appends second argument to end of first argument, which must be an array.
diff --git a/docs6/docs/proofreads.txt b/docs6/docs/proofreads.txt
index a710de1a2..c6ae1c8e6 100644
--- a/docs6/docs/proofreads.txt
+++ b/docs6/docs/proofreads.txt
@@ -1,14 +1,21 @@
-----------------------------------------------------------------
-! merge 3 new 4 !
+TOP:
+c! seps \001 etc !
+ mlrc --iusv --oxtab cat regtest/input/example.usv
+ mlr --iusv --oxtab cat regtest/input/example.usv
+C! repifs !! https://pkg.go.dev/regexp#Regexp.Split 2-for-1 -- get regexp as well ?
+
+E flatten/unflatten page
+? twi-dm re all-contribs: all-contributors.org
+C flags LUTs
+
+----------------------------------------------------------------
ALL:
* csv to csv,tsv throughout
-* rid of explicitly passing around os.Stdout in all various help functions, annoying
* het.dkvp > het.json in more places
* check each page for adequate h2 coverage
* deduping pass!
-* 1-up/0-up -> one-up/zero-up everywhere
* flags-doc
C refactor cli-parser w/ LUT & help strings
@@ -19,15 +26,21 @@ ALL:
https://squidfunk.github.io/mkdocs-material/customization/#extending-the-theme
----------------------------------------------------------------
-c! repifs !!
-c! seps \001 etc !
-C flags LUTs
-? twi-dm re all-contribs: all-contributors.org
e fzf-ish w/ head -n 4, --from, up-arrow & append verb, then cat -- find & update the existing section
+? figure out flatsep vs iflatsep/oflatsep & have a story around the choice either way
+ - ctx ptr-split for invars
E fill out flatten/unflatten page
-E fill out data-types page
+ echo a.b.c=3 | mlr --ojson cat
+ echo a.b.c=3 | mlr --ojson --no-jvstack cat | mlr --j2c --oflatsep : cat
+ echo a:b:c=3 | mlr --ojson --no-jvstack cat
+ echo a:b:c=3 | mlr --ojson --no-jvstack --oflatsep : cat
+ -> call it --flatsep
+
+E fill out strings page
+ e substr0/substr1/substr. slices & string-indexing new in m6.
+ c "abcde"[2:3] works but "abcde"[2] does not :(
E fill out arrays page
- xref to fla/unfla page
- slices including semantics for out-of-range slices
@@ -37,8 +50,6 @@ E fill out maps page
E make flags page
C show-all-flags help -- fully alpha, or alpha-by-type; LUT refactor
-* functions: somewhere organize by type. olh/man6/docs6 ...
-
* regex: more about what is / is not
https://github.com/johnkerl/miller/issues/77#issuecomment-538553828
@@ -94,24 +105,12 @@ reference-verbs:
E data/colored-shapes.dkvp (this page & throughout) a CSV file ...
* ... I ONLY READ UP TO CUT & PAUSED ...
-reference-main-data-types:
-E field values are usually strings -> update
-E Field values are treated as numeric for the following: -> update
-? Miller's types for function processing are -> add JSON null; & check for others (array/map)
-E true/false -> add info about NaN and Inf et al.
-!! very much missing stuff!! where's the listing of mlrval types?!?
- - reference-dsl-variables.md.in should link to it
- - ditto programming-language.md.in
-c UTs for r"a" . r"b" and so on and so on
-
-reference-dsl-variables.md:
-? IFLATSEP & OFLATSEP -- ?
+reference-dsl-variables.md: FLATSEP
reference-main-null-data:
? more variants of is_... ?
? Records with one or more empty sort-field values sort after records with all sort-field values present -> apparently not true for sort -nr
e split out h2's
-! spell out JSON-null
reference-main-arithmetic:
? test stats1/step -F flag
@@ -132,7 +131,6 @@ E User-defined subroutines -> non-factorial example -- maybe some useful aggrega
l link to --load and --mload
reference-dsl-builtin-functions:
-c mlr: option "--list-all-functions-as-table" not recognized. Please run "mlr --help" for usage information -> fix
! ... need to proofread entire list ...
reference-dsl-output-statements:
@@ -157,5 +155,3 @@ E write up which file formats support which flags
manpage:
? [NEEDS READ-THROUGH]
-
-mk-func-table.rb: port comments from sphinx to mkdocs
diff --git a/docs6/docs/reference-dsl-builtin-functions.md b/docs6/docs/reference-dsl-builtin-functions.md
index 245371f8a..c879ac18a 100644
--- a/docs6/docs/reference-dsl-builtin-functions.md
+++ b/docs6/docs/reference-dsl-builtin-functions.md
@@ -14,357 +14,335 @@ Quick links:
# DSL built-in functions
+Each function takes a specific number of arguments, as shown below, except for
+functions marked as variadic such as `min` and `max`. (The latter compute min
+and max of any number of arguments.) There is no notion of optional or
+default-on-absent arguments. All argument-passing is positional rather than by
+name; arguments are passed by value, not by reference.
+
+At the command line, you can get a list of all functions using `mlr -f`, with
+details using `mlr -F`. (Or, `mlr help usage-functions-by-class` to get
+details in the order shown on this page.) You can get detail for a given
+function using `mlr help function namegoeshere`, e.g. `mlr help function
+gsub`.
+
+Operators are listed here along with functions. In this case, the
+argument-count is the number of items involved in the infix operator, e.g. we
+say `x+y` so the details for the `+` operator say that its number of arguments
+is 2. Unary operators such as `!` and `~` show argument-count of 1; the ternary
+`? :` operator shows an argument-count of 3.
+
+
## Summary
-mlr: option "--list-all-functions-as-table" not recognized.
-Please run "mlr --help" for usage information.
+* [**Arithmetic functions**](#arithmetic-functions): [bitcount](#bitcount), [madd](#madd), [mexp](#mexp), [mmul](#mmul), [msub](#msub), [pow](#pow), [%](#percent), [&](#bitwise-and), [\*](#times), [\**](#exponentiation), [\+](#plus), [\-](#minus), [\.\*](#dot-times), [\.\+](#dot-plus), [\.\-](#dot-minus), [\./](#dot-slash), [/](#slash), [//](#slash-slash), [<<](#lsh), [>>](#srsh), [>>>](#ursh), [^](#bitwise-xor), [\|](#bitwise-or), [~](#bitwise-not).
+* [**Boolean functions**](#boolean-functions): [\!](#exclamation-point), [\!=](#exclamation-point-equals), [!=~](#regnotmatch), [&&](#logical-and), [<](#less-than), [<=](#less-than-or-equals), [==](#double-equals), [=~](#regmatch), [>](#greater-than), [>=](#greater-than-or-equals), [?:](#question-mark-colon), [??](#absent-coalesce), [???](#absent-empty-coalesce), [^^](#logical-xor), [\|\|](#logical-or).
+* [**Collections functions**](#collections-functions): [append](#append), [arrayify](#arrayify), [depth](#depth), [flatten](#flatten), [get_keys](#get_keys), [get_values](#get_values), [haskey](#haskey), [json_parse](#json_parse), [json_stringify](#json_stringify), [leafcount](#leafcount), [length](#length), [mapdiff](#mapdiff), [mapexcept](#mapexcept), [mapselect](#mapselect), [mapsum](#mapsum), [unflatten](#unflatten).
+* [**Conversion functions**](#conversion-functions): [boolean](#boolean), [float](#float), [fmtnum](#fmtnum), [hexfmt](#hexfmt), [int](#int), [joink](#joink), [joinkv](#joinkv), [joinv](#joinv), [splita](#splita), [splitax](#splitax), [splitkv](#splitkv), [splitkvx](#splitkvx), [splitnv](#splitnv), [splitnvx](#splitnvx), [string](#string).
+* [**Hashing functions**](#hashing-functions): [md5](#md5), [sha1](#sha1), [sha256](#sha256), [sha512](#sha512).
+* [**Math functions**](#math-functions): [abs](#abs), [acos](#acos), [acosh](#acosh), [asin](#asin), [asinh](#asinh), [atan](#atan), [atan2](#atan2), [atanh](#atanh), [cbrt](#cbrt), [ceil](#ceil), [cos](#cos), [cosh](#cosh), [erf](#erf), [erfc](#erfc), [exp](#exp), [expm1](#expm1), [floor](#floor), [invqnorm](#invqnorm), [log](#log), [log10](#log10), [log1p](#log1p), [logifit](#logifit), [max](#max), [min](#min), [qnorm](#qnorm), [round](#round), [roundm](#roundm), [sgn](#sgn), [sin](#sin), [sinh](#sinh), [sqrt](#sqrt), [tan](#tan), [tanh](#tanh), [urand](#urand), [urand32](#urand32), [urandint](#urandint), [urandrange](#urandrange).
+* [**String functions**](#string-functions): [capitalize](#capitalize), [clean_whitespace](#clean_whitespace), [collapse_whitespace](#collapse_whitespace), [gsub](#gsub), [lstrip](#lstrip), [regextract](#regextract), [regextract_or_else](#regextract_or_else), [rstrip](#rstrip), [ssub](#ssub), [strip](#strip), [strlen](#strlen), [sub](#sub), [substr](#substr), [substr0](#substr0), [substr1](#substr1), [tolower](#tolower), [toupper](#toupper), [truncate](#truncate), [\.](#dot).
+* [**System functions**](#system-functions): [hostname](#hostname), [os](#os), [system](#system), [version](#version).
+* [**Time functions**](#time-functions): [dhms2fsec](#dhms2fsec), [dhms2sec](#dhms2sec), [fsec2dhms](#fsec2dhms), [fsec2hms](#fsec2hms), [gmt2sec](#gmt2sec), [hms2fsec](#hms2fsec), [hms2sec](#hms2sec), [sec2dhms](#sec2dhms), [sec2gmt](#sec2gmt), [sec2gmtdate](#sec2gmtdate), [sec2hms](#sec2hms), [strftime](#strftime), [strptime](#strptime), [systime](#systime), [systimeint](#systimeint), [uptime](#uptime).
+* [**Typing functions**](#typing-functions): [asserting_absent](#asserting_absent), [asserting_array](#asserting_array), [asserting_bool](#asserting_bool), [asserting_boolean](#asserting_boolean), [asserting_empty](#asserting_empty), [asserting_empty_map](#asserting_empty_map), [asserting_error](#asserting_error), [asserting_float](#asserting_float), [asserting_int](#asserting_int), [asserting_map](#asserting_map), [asserting_nonempty_map](#asserting_nonempty_map), [asserting_not_array](#asserting_not_array), [asserting_not_empty](#asserting_not_empty), [asserting_not_map](#asserting_not_map), [asserting_not_null](#asserting_not_null), [asserting_null](#asserting_null), [asserting_numeric](#asserting_numeric), [asserting_present](#asserting_present), [asserting_string](#asserting_string), [is_absent](#is_absent), [is_array](#is_array), [is_bool](#is_bool), [is_boolean](#is_boolean), [is_empty](#is_empty), [is_empty_map](#is_empty_map), [is_error](#is_error), [is_float](#is_float), [is_int](#is_int), [is_map](#is_map), [is_nonempty_map](#is_nonempty_map), [is_not_array](#is_not_array), [is_not_empty](#is_not_empty), [is_not_map](#is_not_map), [is_not_null](#is_not_null), [is_null](#is_null), [is_numeric](#is_numeric), [is_present](#is_present), [is_string](#is_string), [typeof](#typeof).
-## List of functions
+## Arithmetic functions
-Each function takes a specific number of arguments, as shown below, except for functions marked as variadic such as `min` and `max`. (The latter compute min and max of any number of numerical arguments.) There is no notion of optional or default-on-absent arguments. All argument-passing is positional rather than by name; arguments are passed by value, not by reference.
-
-You can get a list of all functions using **mlr -f**, with details using **mlr -F**.
-
-
-## abs
-
-
-abs (class=math #args=1) Absolute value.
-
-
-
-## acos
-
-
-acos (class=math #args=1) Inverse trigonometric cosine.
-
-
-
-## acosh
-
-
-acosh (class=math #args=1) Inverse hyperbolic cosine.
-
-
-
-## append
-
-
-append (class=maps/arrays #args=2) Appends second argument to end of first argument, which must be an array.
-
-
-
-## arrayify
-
-
-arrayify (class=maps/arrays #args=1) Walks through a nested map/array, converting any map with consecutive keys
-"1", "2", ... into an array. Useful to wrap the output of unflatten.
-
-
-
-## asin
-
-
-asin (class=math #args=1) Inverse trigonometric sine.
-
-
-
-## asinh
-
-
-asinh (class=math #args=1) Inverse hyperbolic sine.
-
-
-
-## asserting_absent
-
-
-asserting_absent (class=typing #args=1) Aborts with an error if is_absent on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_array
-
-
-asserting_array (class=typing #args=1) Aborts with an error if is_array on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_bool
-
-
-asserting_bool (class=typing #args=1) Aborts with an error if is_bool on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_boolean
-
-
-asserting_boolean (class=typing #args=1) Aborts with an error if is_boolean on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_empty
-
-
-asserting_empty (class=typing #args=1) Aborts with an error if is_empty on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_empty_map
-
-
-asserting_empty_map (class=typing #args=1) Aborts with an error if is_empty_map on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_error
-
-
-asserting_error (class=typing #args=1) Aborts with an error if is_error on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_float
-
-
-asserting_float (class=typing #args=1) Aborts with an error if is_float on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_int
-
-
-asserting_int (class=typing #args=1) Aborts with an error if is_int on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_map
-
-
-asserting_map (class=typing #args=1) Aborts with an error if is_map on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_nonempty_map
-
-
-asserting_nonempty_map (class=typing #args=1) Aborts with an error if is_nonempty_map on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_not_array
-
-
-asserting_not_array (class=typing #args=1) Aborts with an error if is_not_array on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_not_empty
-
-
-asserting_not_empty (class=typing #args=1) Aborts with an error if is_not_empty on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_not_map
-
-
-asserting_not_map (class=typing #args=1) Aborts with an error if is_not_map on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_not_null
-
-
-asserting_not_null (class=typing #args=1) Aborts with an error if is_not_null on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_null
-
-
-asserting_null (class=typing #args=1) Aborts with an error if is_null on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_numeric
-
-
-asserting_numeric (class=typing #args=1) Aborts with an error if is_numeric on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_present
-
-
-asserting_present (class=typing #args=1) Aborts with an error if is_present on the argument returns false,
-else returns its argument.
-
-
-
-## asserting_string
-
-
-asserting_string (class=typing #args=1) Aborts with an error if is_string on the argument returns false,
-else returns its argument.
-
-
-
-## atan
-
-
-atan (class=math #args=1) One-argument arctangent.
-
-
-
-## atan2
-
-
-atan2 (class=math #args=2) Two-argument arctangent.
-
-
-
-## atanh
-
-
-atanh (class=math #args=1) Inverse hyperbolic tangent.
-
-
-
-## bitcount
+### bitcount
bitcount (class=arithmetic #args=1) Count of 1-bits.
-## boolean
-
+### madd
-boolean (class=conversion #args=1) Convert int/float/bool/string to boolean.
+madd (class=arithmetic #args=3) a + b mod m (integers)
-## capitalize
-
+### mexp
-capitalize (class=string #args=1) Convert string's first character to uppercase.
+mexp (class=arithmetic #args=3) a ** b mod m (integers)
-## cbrt
-
+### mmul
-cbrt (class=math #args=1) Cube root.
+mmul (class=arithmetic #args=3) a * b mod m (integers)
-## ceil
-
+### msub
-ceil (class=math #args=1) Ceiling: nearest integer at or above.
+msub (class=arithmetic #args=3) a - b mod m (integers)
-## clean_whitespace
-
+### pow
-clean_whitespace (class=string #args=1) Same as collapse_whitespace and strip.
+pow (class=arithmetic #args=2) Exponentiation. Same as **, but as a function.
-## collapse_whitespace
-
+
+### %
-collapse_whitespace (class=string #args=1) Strip repeated whitespace from string.
+% (class=arithmetic #args=2) Remainder; never negative-valued (pythonic).
-## cos
-
+
+### &
-cos (class=math #args=1) Trigonometric cosine.
+& (class=arithmetic #args=2) Bitwise AND.
-## cosh
-
+
+### \*
-cosh (class=math #args=1) Hyperbolic cosine.
+* (class=arithmetic #args=2) Multiplication, with integer*integer overflow to float.
-## depth
-
+
+### \**
-depth (class=maps/arrays #args=1) Prints maximum depth of map/array. Scalars have depth 0.
+** (class=arithmetic #args=2) Exponentiation. Same as pow, but as an infix operator.
-## dhms2fsec
-
+
+### \+
-dhms2fsec (class=time #args=1) Recovers floating-point seconds as in dhms2fsec("5d18h53m20.250000s") = 500000.250000
++ (class=arithmetic #args=1,2) Addition as binary operator; unary plus operator.
-## dhms2sec
-
+
+### \-
-dhms2sec (class=time #args=1) Recovers integer seconds as in dhms2sec("5d18h53m20s") = 500000
+- (class=arithmetic #args=1,2) Subtraction as binary operator; unary negation operator.
-## erf
-
+
+### \.\*
-erf (class=math #args=1) Error function.
+.* (class=arithmetic #args=2) Multiplication, with integer-to-integer overflow.
-## erfc
-
+
+### \.\+
-erfc (class=math #args=1) Complementary error function.
+.+ (class=arithmetic #args=2) Addition, with integer-to-integer overflow.
-## exp
-
+
+### \.\-
-exp (class=math #args=1) Exponential function e**x.
+.- (class=arithmetic #args=2) Subtraction, with integer-to-integer overflow.
-## expm1
-
+
+### \./
-expm1 (class=math #args=1) e**x - 1.
+./ (class=arithmetic #args=2) Integer division; not pythonic.
-## flatten
-
+
+### /
-flatten (class=maps/arrays #args=3) Flattens multi-level maps to single-level ones. Examples:
+/ (class=arithmetic #args=2) Division. Integer / integer is floating-point.
+
+
+
+
+### //
+
+// (class=arithmetic #args=2) Pythonic integer division, rounding toward negative.
+
+
+
+
+### <<
+
+<< (class=arithmetic #args=2) Bitwise left-shift.
+
+
+
+
+### >>
+
+>> (class=arithmetic #args=2) Bitwise signed right-shift.
+
+
+
+
+### >>>
+
+>>> (class=arithmetic #args=2) Bitwise unsigned right-shift.
+
+
+
+
+### ^
+
+^ (class=arithmetic #args=2) Bitwise XOR.
+
+
+
+
+### \|
+
+| (class=arithmetic #args=2) Bitwise OR.
+
+
+
+
+### ~
+
+~ (class=arithmetic #args=1) Bitwise NOT. Beware '$y=~$x' since =~ is the
+regex-match operator: try '$y = ~$x'.
+
+
+## Boolean functions
+
+
+
+### \!
+
+! (class=boolean #args=1) Logical negation.
+
+
+
+
+### \!=
+
+!= (class=boolean #args=2) String/numeric inequality. Mixing number and string results in string compare.
+
+
+
+
+### !=~
+
+!=~ (class=boolean #args=2) String (left-hand side) does not match regex (right-hand side), e.g. '$name !=~ "^a.*b$"'.
+
+
+
+
+### &&
+
+&& (class=boolean #args=2) Logical AND.
+
+
+
+
+### <
+
+< (class=boolean #args=2) String/numeric less-than. Mixing number and string results in string compare.
+
+
+
+
+### <=
+
+<= (class=boolean #args=2) String/numeric less-than-or-equals. Mixing number and string results in string compare.
+
+
+
+
+### ==
+
+== (class=boolean #args=2) String/numeric equality. Mixing number and string results in string compare.
+
+
+
+
+### =~
+
+=~ (class=boolean #args=2) String (left-hand side) matches regex (right-hand side), e.g. '$name =~ "^a.*b$"'.
+
+
+
+
+### >
+
+> (class=boolean #args=2) String/numeric greater-than. Mixing number and string results in string compare.
+
+
+
+
+### >=
+
+>= (class=boolean #args=2) String/numeric greater-than-or-equals. Mixing number and string results in string compare.
+
+
+
+
+### ?:
+
+?: (class=boolean #args=3) Standard ternary operator.
+
+
+
+
+### ??
+
+?? (class=boolean #args=2) Absent-coalesce operator. $a ?? 1 evaluates to 1 if $a isn't defined in the current record.
+
+
+
+
+### ???
+
+??? (class=boolean #args=2) Absent-coalesce operator. $a ?? 1 evaluates to 1 if $a isn't defined in the current record, or has empty value.
+
+
+
+
+### ^^
+
+^^ (class=boolean #args=2) Logical XOR.
+
+
+
+
+### \|\|
+
+|| (class=boolean #args=2) Logical OR.
+
+
+## Collections functions
+
+
+### append
+
+append (class=collections #args=2) Appends second argument to end of first argument, which must be an array.
+
+
+
+### arrayify
+
+arrayify (class=collections #args=1) Walks through a nested map/array, converting any map with consecutive keys
+"1", "2", ... into an array. Useful to wrap the output of unflatten.
+
+
+
+### depth
+
+depth (class=collections #args=1) Prints maximum depth of map/array. Scalars have depth 0.
+
+
+
+### flatten
+
+flatten (class=collections #args=3) Flattens multi-level maps to single-level ones. Examples:
flatten("a", ".", {"b": { "c": 4 }}) is {"a.b.c" : 4}.
flatten("", ".", {"a": { "b": 3 }}) is {"a.b" : 3}.
Two-argument version: flatten($*, ".") is the same as flatten("", ".", $*).
@@ -372,257 +350,129 @@ Useful for nested JSON-like structures for non-JSON file formats like CSV.
-## float
+### get_keys
+
+get_keys (class=collections #args=1) Returns array of keys of map or array
+
+
+### get_values
+
+get_values (class=collections #args=1) Returns array of keys of map or array -- in the latter case, returns a copy of the array
+
+
+
+### haskey
+
+haskey (class=collections #args=2) True/false if map has/hasn't key, e.g. 'haskey($*, "a")' or
+'haskey(mymap, mykey)', or true/false if array index is in bounds / out of bounds.
+Error if 1st argument is not a map or array. Note -n..-1 alias to 1..n in Miller arrays.
+
+
+
+### json_parse
+
+json_parse (class=collections #args=1) Converts value from JSON-formatted string.
+
+
+
+### json_stringify
+
+json_stringify (class=collections #args=1,2) Converts value to JSON-formatted string. Default output is single-line.
+With optional second boolean argument set to true, produces multiline output.
+
+
+
+### leafcount
+
+leafcount (class=collections #args=1) Counts total number of terminal values in map/array. For single-level
+map/array, same as length.
+
+
+
+### length
+
+length (class=collections #args=1) Counts number of top-level entries in array/map. Scalars have length 1.
+
+
+
+### mapdiff
+
+mapdiff (class=collections #args=variadic) With 0 args, returns empty map. With 1 arg, returns copy of arg.
+With 2 or more, returns copy of arg 1 with all keys from any of remaining
+argument maps removed.
+
+
+
+### mapexcept
+
+mapexcept (class=collections #args=variadic) Returns a map with keys from remaining arguments, if any, unset.
+Remaining arguments can be strings or arrays of string.
+E.g. 'mapexcept({1:2,3:4,5:6}, 1, 5, 7)' is '{3:4}'
+and 'mapexcept({1:2,3:4,5:6}, [1, 5, 7])' is '{3:4}'.
+
+
+
+### mapselect
+
+mapselect (class=collections #args=variadic) Returns a map with only keys from remaining arguments set.
+Remaining arguments can be strings or arrays of string.
+E.g. 'mapselect({1:2,3:4,5:6}, 1, 5, 7)' is '{1:2,5:6}'
+and 'mapselect({1:2,3:4,5:6}, [1, 5, 7])' is '{1:2,5:6}'.
+
+
+
+### mapsum
+
+mapsum (class=collections #args=variadic) With 0 args, returns empty map. With >= 1 arg, returns a map with
+key-value pairs from all arguments. Rightmost collisions win, e.g.
+'mapsum({1:2,3:4},{1:5})' is '{1:5,3:4}'.
+
+
+
+### unflatten
+
+unflatten (class=collections #args=2) Reverses flatten. Example:
+unflatten({"a.b.c" : 4}, ".") is {"a": "b": { "c": 4 }}.
+Useful for nested JSON-like structures for non-JSON file formats like CSV.
+See also arrayify.
+
+
+## Conversion functions
+
+
+### boolean
+
+boolean (class=conversion #args=1) Convert int/float/bool/string to boolean.
+
+
+
+### float
float (class=conversion #args=1) Convert int/float/bool/string to float.
-## floor
-
-
-floor (class=math #args=1) Floor: nearest integer at or below.
-
-
-
-## fmtnum
-
+### fmtnum
fmtnum (class=conversion #args=2) Convert int/float/bool to string using
printf-style format string, e.g. '$s = fmtnum($n, "%06lld")'.
-## fsec2dhms
-
-
-fsec2dhms (class=time #args=1) Formats floating-point seconds as in fsec2dhms(500000.25) = "5d18h53m20.250000s"
-
-
-
-## fsec2hms
-
-
-fsec2hms (class=time #args=1) Formats floating-point seconds as in fsec2hms(5000.25) = "01:23:20.250000"
-
-
-
-## get_keys
-
-
-get_keys (class=maps/arrays #args=1) Returns array of keys of map or array
-
-
-
-## get_values
-
-
-get_values (class=maps/arrays #args=1) Returns array of keys of map or array -- in the latter case, returns a copy of the array
-
-
-
-## gmt2sec
-
-
-gmt2sec (class=time #args=1) Parses GMT timestamp as integer seconds since the epoch.
-
-
-
-## gsub
-
-
-gsub (class=string #args=3) Example: '$name=gsub($name, "old", "new")' (replace all).
-
-
-
-## haskey
-
-
-haskey (class=maps/arrays #args=2) True/false if map has/hasn't key, e.g. 'haskey($*, "a")' or
-'haskey(mymap, mykey)', or true/false if array index is in bounds / out of bounds.
-Error if 1st argument is not a map or array. Note -n..-1 alias to 1..n in Miller arrays.
-
-
-
-## hexfmt
-
+### hexfmt
hexfmt (class=conversion #args=1) Convert int to hex string, e.g. 255 to "0xff".
-## hms2fsec
-
-
-hms2fsec (class=time #args=1) Recovers floating-point seconds as in hms2fsec("01:23:20.250000") = 5000.250000
-
-
-
-## hms2sec
-
-
-hms2sec (class=time #args=1) Recovers integer seconds as in hms2sec("01:23:20") = 5000
-
-
-
-## hostname
-
-
-hostname (class=system #args=0) Returns the hostname as a string.
-
-
-
-## int
-
+### int
int (class=conversion #args=1) Convert int/float/bool/string to int.
-## invqnorm
-
-
-invqnorm (class=math #args=1) Inverse of normal cumulative distribution function.
-Note that invqorm(urand()) is normally distributed.
-
-
-
-## is_absent
-
-
-is_absent (class=typing #args=1) False if field is present in input, true otherwise
-
-
-
-## is_array
-
-
-is_array (class=typing #args=1) True if argument is an array.
-
-
-
-## is_bool
-
-
-is_bool (class=typing #args=1) True if field is present with boolean value. Synonymous with is_boolean.
-
-
-
-## is_boolean
-
-
-is_boolean (class=typing #args=1) True if field is present with boolean value. Synonymous with is_bool.
-
-
-
-## is_empty
-
-
-is_empty (class=typing #args=1) True if field is present in input with empty string value, false otherwise.
-
-
-
-## is_empty_map
-
-
-is_empty_map (class=typing #args=1) True if argument is a map which is empty.
-
-
-
-## is_error
-
-
-is_error (class=typing #args=1) True if if argument is an error, such as taking string length of an integer.
-
-
-
-## is_float
-
-
-is_float (class=typing #args=1) True if field is present with value inferred to be float
-
-
-
-## is_int
-
-
-is_int (class=typing #args=1) True if field is present with value inferred to be int
-
-
-
-## is_map
-
-
-is_map (class=typing #args=1) True if argument is a map.
-
-
-
-## is_nonempty_map
-
-
-is_nonempty_map (class=typing #args=1) True if argument is a map which is non-empty.
-
-
-
-## is_not_array
-
-
-is_not_array (class=typing #args=1) True if argument is not an array.
-
-
-
-## is_not_empty
-
-
-is_not_empty (class=typing #args=1) False if field is present in input with empty value, true otherwise
-
-
-
-## is_not_map
-
-
-is_not_map (class=typing #args=1) True if argument is not a map.
-
-
-
-## is_not_null
-
-
-is_not_null (class=typing #args=1) False if argument is null (empty or absent), true otherwise.
-
-
-
-## is_null
-
-
-is_null (class=typing #args=1) True if argument is null (empty or absent), false otherwise.
-
-
-
-## is_numeric
-
-
-is_numeric (class=typing #args=1) True if field is present with value inferred to be int or float
-
-
-
-## is_present
-
-
-is_present (class=typing #args=1) True if field is present in input, false otherwise.
-
-
-
-## is_string
-
-
-is_string (class=typing #args=1) True if field is present with string (including empty-string) value
-
-
-
-## joink
-
+### joink
joink (class=conversion #args=2) Makes string from map/array keys. Examples:
joink({"a":3,"b":4,"c":5}, ",") = "a,b,c"
@@ -630,8 +480,7 @@ joink([1,2,3], ",") = "1,2,3".
-## joinkv
-
+### joinkv
joinkv (class=conversion #args=3) Makes string from map/array key-value pairs. Examples:
joinkv([3,4,5], "=", ",") = "1=3,2=4,3=5"
@@ -639,8 +488,7 @@ joinkv({"a":3,"b":4,"c":5}, "=", ",") = "a=3,b=4,c=5"
-## joinv
-
+### joinv
joinv (class=conversion #args=2) Makes string from map/array values.
joinv([3,4,5], ",") = "3,4,5"
@@ -648,225 +496,508 @@ joinv({"a":3,"b":4,"c":5}, ",") = "3,4,5"
-## json_parse
-
+### splita
-json_parse (class=maps/arrays #args=1) Converts value from JSON-formatted string.
+splita (class=conversion #args=2) Splits string into array with type inference. Example:
+splita("3,4,5", ",") = [3,4,5]
-## json_stringify
-
+### splitax
-json_stringify (class=maps/arrays #args=1,2) Converts value to JSON-formatted string. Default output is single-line.
-With optional second boolean argument set to true, produces multiline output.
+splitax (class=conversion #args=2) Splits string into array without type inference. Example:
+splita("3,4,5", ",") = ["3","4","5"]
-## leafcount
-
+### splitkv
-leafcount (class=maps/arrays #args=1) Counts total number of terminal values in map/array. For single-level
-map/array, same as length.
+splitkv (class=conversion #args=3) Splits string by separators into map with type inference. Example:
+splitkv("a=3,b=4,c=5", "=", ",") = {"a":3,"b":4,"c":5}
-## length
-
+### splitkvx
-length (class=maps/arrays #args=1) Counts number of top-level entries in array/map. Scalars have length 1.
+splitkvx (class=conversion #args=3) Splits string by separators into map without type inference (keys and
+values are strings). Example:
+splitkvx("a=3,b=4,c=5", "=", ",") = {"a":"3","b":"4","c":"5"}
-## log
+### splitnv
+
+splitnv (class=conversion #args=2) Splits string by separator into integer-indexed map with type inference. Example:
+splitnv("a,b,c", ",") = {"1":"a","2":"b","3":"c"}
+
+
+### splitnvx
+
+splitnvx (class=conversion #args=2) Splits string by separator into integer-indexed map without type
+inference (values are strings). Example:
+splitnvx("3,4,5", ",") = {"1":"3","2":"4","3":"5"}
+
+
+
+### string
+
+string (class=conversion #args=1) Convert int/float/bool/string/array/map to string.
+
+
+## Hashing functions
+
+
+### md5
+
+md5 (class=hashing #args=1) MD5 hash.
+
+
+
+### sha1
+
+sha1 (class=hashing #args=1) SHA1 hash.
+
+
+
+### sha256
+
+sha256 (class=hashing #args=1) SHA256 hash.
+
+
+
+### sha512
+
+sha512 (class=hashing #args=1) SHA512 hash.
+
+
+## Math functions
+
+
+### abs
+
+abs (class=math #args=1) Absolute value.
+
+
+
+### acos
+
+acos (class=math #args=1) Inverse trigonometric cosine.
+
+
+
+### acosh
+
+acosh (class=math #args=1) Inverse hyperbolic cosine.
+
+
+
+### asin
+
+asin (class=math #args=1) Inverse trigonometric sine.
+
+
+
+### asinh
+
+asinh (class=math #args=1) Inverse hyperbolic sine.
+
+
+
+### atan
+
+atan (class=math #args=1) One-argument arctangent.
+
+
+
+### atan2
+
+atan2 (class=math #args=2) Two-argument arctangent.
+
+
+
+### atanh
+
+atanh (class=math #args=1) Inverse hyperbolic tangent.
+
+
+
+### cbrt
+
+cbrt (class=math #args=1) Cube root.
+
+
+
+### ceil
+
+ceil (class=math #args=1) Ceiling: nearest integer at or above.
+
+
+
+### cos
+
+cos (class=math #args=1) Trigonometric cosine.
+
+
+
+### cosh
+
+cosh (class=math #args=1) Hyperbolic cosine.
+
+
+
+### erf
+
+erf (class=math #args=1) Error function.
+
+
+
+### erfc
+
+erfc (class=math #args=1) Complementary error function.
+
+
+
+### exp
+
+exp (class=math #args=1) Exponential function e**x.
+
+
+
+### expm1
+
+expm1 (class=math #args=1) e**x - 1.
+
+
+
+### floor
+
+floor (class=math #args=1) Floor: nearest integer at or below.
+
+
+
+### invqnorm
+
+invqnorm (class=math #args=1) Inverse of normal cumulative distribution function.
+Note that invqorm(urand()) is normally distributed.
+
+
+
+### log
log (class=math #args=1) Natural (base-e) logarithm.
-## log10
-
+### log10
log10 (class=math #args=1) Base-10 logarithm.
-## log1p
-
+### log1p
log1p (class=math #args=1) log(1-x).
-## logifit
-
+### logifit
logifit (class=math #args=3) Given m and b from logistic regression, compute fit:
$yhat=logifit($x,$m,$b).
-## lstrip
-
-
-lstrip (class=string #args=1) Strip leading whitespace from string.
-
-
-
-## madd
-
-
-madd (class=arithmetic #args=3) a + b mod m (integers)
-
-
-
-## mapdiff
-
-
-mapdiff (class=maps/arrays #args=variadic) With 0 args, returns empty map. With 1 arg, returns copy of arg.
-With 2 or more, returns copy of arg 1 with all keys from any of remaining
-argument maps removed.
-
-
-
-## mapexcept
-
-
-mapexcept (class=maps/arrays #args=variadic) Returns a map with keys from remaining arguments, if any, unset.
-Remaining arguments can be strings or arrays of string.
-E.g. 'mapexcept({1:2,3:4,5:6}, 1, 5, 7)' is '{3:4}'
-and 'mapexcept({1:2,3:4,5:6}, [1, 5, 7])' is '{3:4}'.
-
-
-
-## mapselect
-
-
-mapselect (class=maps/arrays #args=variadic) Returns a map with only keys from remaining arguments set.
-Remaining arguments can be strings or arrays of string.
-E.g. 'mapselect({1:2,3:4,5:6}, 1, 5, 7)' is '{1:2,5:6}'
-and 'mapselect({1:2,3:4,5:6}, [1, 5, 7])' is '{1:2,5:6}'.
-
-
-
-## mapsum
-
-
-mapsum (class=maps/arrays #args=variadic) With 0 args, returns empty map. With >= 1 arg, returns a map with
-key-value pairs from all arguments. Rightmost collisions win, e.g.
-'mapsum({1:2,3:4},{1:5})' is '{1:5,3:4}'.
-
-
-
-## max
-
+### max
max (class=math #args=variadic) Max of n numbers; null loses.
-## md5
-
-
-md5 (class=hashing #args=1) MD5 hash.
-
-
-
-## mexp
-
-
-mexp (class=arithmetic #args=3) a ** b mod m (integers)
-
-
-
-## min
-
+### min
min (class=math #args=variadic) Min of n numbers; null loses.
-## mmul
-
-
-mmul (class=arithmetic #args=3) a * b mod m (integers)
-
-
-
-## msub
-
-
-msub (class=arithmetic #args=3) a - b mod m (integers)
-
-
-
-## os
-
-
-os (class=system #args=0) Returns the operating-system name as a string.
-
-
-
-## pow
-
-
-pow (class=arithmetic #args=2) Exponentiation. Same as **, but as a function.
-
-
-
-## qnorm
-
+### qnorm
qnorm (class=math #args=1) Normal cumulative distribution function.
-## regextract
-
-
-regextract (class=string #args=2) Example: '$name=regextract($name, "[A-Z]{3}[0-9]{2}")'
-
-
-
-## regextract_or_else
-
-
-regextract_or_else (class=string #args=3) Example: '$name=regextract_or_else($name, "[A-Z]{3}[0-9]{2}", "default")'
-
-
-
-## round
-
+### round
round (class=math #args=1) Round to nearest integer.
-## roundm
-
+### roundm
roundm (class=math #args=2) Round to nearest multiple of m: roundm($x,$m) is
the same as round($x/$m)*$m.
-## rstrip
+### sgn
+
+sgn (class=math #args=1) +1, 0, -1 for positive, zero, negative input respectively.
+
+
+### sin
+
+sin (class=math #args=1) Trigonometric sine.
+
+
+
+### sinh
+
+sinh (class=math #args=1) Hyperbolic sine.
+
+
+
+### sqrt
+
+sqrt (class=math #args=1) Square root.
+
+
+
+### tan
+
+tan (class=math #args=1) Trigonometric tangent.
+
+
+
+### tanh
+
+tanh (class=math #args=1) Hyperbolic tangent.
+
+
+
+### urand
+
+urand (class=math #args=0) Floating-point numbers uniformly distributed on the unit interval.
+Int-valued example: '$n=floor(20+urand()*11)'.
+
+
+
+### urand32
+
+urand32 (class=math #args=0) Integer uniformly distributed 0 and 2**32-1 inclusive.
+
+
+
+### urandint
+
+urandint (class=math #args=2) Integer uniformly distributed between inclusive integer endpoints.
+
+
+
+### urandrange
+
+urandrange (class=math #args=2) Floating-point numbers uniformly distributed on the interval [a, b).
+
+
+## String functions
+
+
+### capitalize
+
+capitalize (class=string #args=1) Convert string's first character to uppercase.
+
+
+
+### clean_whitespace
+
+clean_whitespace (class=string #args=1) Same as collapse_whitespace and strip.
+
+
+
+### collapse_whitespace
+
+collapse_whitespace (class=string #args=1) Strip repeated whitespace from string.
+
+
+
+### gsub
+
+gsub (class=string #args=3) Example: '$name=gsub($name, "old", "new")' (replace all).
+
+
+
+### lstrip
+
+lstrip (class=string #args=1) Strip leading whitespace from string.
+
+
+
+### regextract
+
+regextract (class=string #args=2) Example: '$name=regextract($name, "[A-Z]{3}[0-9]{2}")'
+
+
+
+### regextract_or_else
+
+regextract_or_else (class=string #args=3) Example: '$name=regextract_or_else($name, "[A-Z]{3}[0-9]{2}", "default")'
+
+
+
+### rstrip
rstrip (class=string #args=1) Strip trailing whitespace from string.
-## sec2dhms
+### ssub
+
+ssub (class=string #args=3) Like sub but does no regexing. No characters are special.
+
+
+### strip
+
+strip (class=string #args=1) Strip leading and trailing whitespace from string.
+
+
+
+### strlen
+
+strlen (class=string #args=1) String length.
+
+
+
+### sub
+
+sub (class=string #args=3) Example: '$name=sub($name, "old", "new")' (replace once).
+
+
+
+### substr
+
+substr (class=string #args=3) substr is an alias for substr0. See also substr1. Miller is generally 1-up
+with all array and string indices, but, this is a backward-compatibility issue with Miller 5
+and below. Arrays are new in Miller 6; the substr function is older.
+
+
+
+### substr0
+
+substr0 (class=string #args=3) substr0(s,m,n) gives substring of s from 0-up position m to n
+inclusive. Negative indices -len .. -1 alias to 0 .. len-1. See also substr and substr1.
+
+
+
+### substr1
+
+substr1 (class=string #args=3) substr1(s,m,n) gives substring of s from 1-up position m to n
+inclusive. Negative indices -len .. -1 alias to 1 .. len. See also substr and substr0.
+
+
+
+### tolower
+
+tolower (class=string #args=1) Convert string to lowercase.
+
+
+
+### toupper
+
+toupper (class=string #args=1) Convert string to uppercase.
+
+
+
+### truncate
+
+truncate (class=string #args=2) Truncates string first argument to max length of int second argument.
+
+
+
+
+### \.
+
+. (class=string #args=2) String concatenation.
+
+
+## System functions
+
+
+### hostname
+
+hostname (class=system #args=0) Returns the hostname as a string.
+
+
+
+### os
+
+os (class=system #args=0) Returns the operating-system name as a string.
+
+
+
+### system
+
+system (class=system #args=1) Run command string, yielding its stdout minus final carriage return.
+
+
+
+### version
+
+version (class=system #args=0) Returns the Miller version as a string.
+
+
+## Time functions
+
+
+### dhms2fsec
+
+dhms2fsec (class=time #args=1) Recovers floating-point seconds as in dhms2fsec("5d18h53m20.250000s") = 500000.250000
+
+
+
+### dhms2sec
+
+dhms2sec (class=time #args=1) Recovers integer seconds as in dhms2sec("5d18h53m20s") = 500000
+
+
+
+### fsec2dhms
+
+fsec2dhms (class=time #args=1) Formats floating-point seconds as in fsec2dhms(500000.25) = "5d18h53m20.250000s"
+
+
+
+### fsec2hms
+
+fsec2hms (class=time #args=1) Formats floating-point seconds as in fsec2hms(5000.25) = "01:23:20.250000"
+
+
+
+### gmt2sec
+
+gmt2sec (class=time #args=1) Parses GMT timestamp as integer seconds since the epoch.
+
+
+
+### hms2fsec
+
+hms2fsec (class=time #args=1) Recovers floating-point seconds as in hms2fsec("01:23:20.250000") = 5000.250000
+
+
+
+### hms2sec
+
+hms2sec (class=time #args=1) Recovers integer seconds as in hms2sec("01:23:20") = 5000
+
+
+
+### sec2dhms
sec2dhms (class=time #args=1) Formats integer seconds as in sec2dhms(500000) = "5d18h53m20s"
-## sec2gmt
-
+### sec2gmt
sec2gmt (class=time #args=1,2) Formats seconds since epoch (integer part)
as GMT timestamp, e.g. sec2gmt(1440768801.7) = "2015-08-28T13:33:21Z".
@@ -875,8 +1006,7 @@ for the seconds part
-## sec2gmtdate
-
+### sec2gmtdate
sec2gmtdate (class=time #args=1) Formats seconds since epoch (integer part)
as GMT timestamp with year-month-date, e.g. sec2gmtdate(1440768801.7) = "2015-08-28".
@@ -884,121 +1014,13 @@ Leaves non-numbers as-is.
-## sec2hms
-
+### sec2hms
sec2hms (class=time #args=1) Formats integer seconds as in sec2hms(5000) = "01:23:20"
-## sgn
-
-
-sgn (class=math #args=1) +1, 0, -1 for positive, zero, negative input respectively.
-
-
-
-## sha1
-
-
-sha1 (class=hashing #args=1) SHA1 hash.
-
-
-
-## sha256
-
-
-sha256 (class=hashing #args=1) SHA256 hash.
-
-
-
-## sha512
-
-
-sha512 (class=hashing #args=1) SHA512 hash.
-
-
-
-## sin
-
-
-sin (class=math #args=1) Trigonometric sine.
-
-
-
-## sinh
-
-
-sinh (class=math #args=1) Hyperbolic sine.
-
-
-
-## splita
-
-
-splita (class=conversion #args=2) Splits string into array with type inference. Example:
-splita("3,4,5", ",") = [3,4,5]
-
-
-
-## splitax
-
-
-splitax (class=conversion #args=2) Splits string into array without type inference. Example:
-splita("3,4,5", ",") = ["3","4","5"]
-
-
-
-## splitkv
-
-
-splitkv (class=conversion #args=3) Splits string by separators into map with type inference. Example:
-splitkv("a=3,b=4,c=5", "=", ",") = {"a":3,"b":4,"c":5}
-
-
-
-## splitkvx
-
-
-splitkvx (class=conversion #args=3) Splits string by separators into map without type inference (keys and
-values are strings). Example:
-splitkvx("a=3,b=4,c=5", "=", ",") = {"a":"3","b":"4","c":"5"}
-
-
-
-## splitnv
-
-
-splitnv (class=conversion #args=2) Splits string by separator into integer-indexed map with type inference. Example:
-splitnv("a,b,c", ",") = {"1":"a","2":"b","3":"c"}
-
-
-
-## splitnvx
-
-
-splitnvx (class=conversion #args=2) Splits string by separator into integer-indexed map without type
-inference (values are strings). Example:
-splitnvx("3,4,5", ",") = {"1":"3","2":"4","3":"5"}
-
-
-
-## sqrt
-
-
-sqrt (class=math #args=1) Square root.
-
-
-
-## ssub
-
-
-ssub (class=string #args=3) Like sub but does no regexing. No characters are special.
-
-
-
-## strftime
-
+### strftime
strftime (class=time #args=2) Formats seconds since the epoch as timestamp, e.g.
strftime(1440768801.7,"%Y-%m-%dT%H:%M:%SZ") = "2015-08-28T13:33:21Z", and
@@ -1010,29 +1032,7 @@ strftime (class=time #args=2) Formats seconds since the epoch as timestamp, e.
-## string
-
-
-string (class=conversion #args=1) Convert int/float/bool/string/array/map to string.
-
-
-
-## strip
-
-
-strip (class=string #args=1) Strip leading and trailing whitespace from string.
-
-
-
-## strlen
-
-
-strlen (class=string #args=1) String length.
-
-
-
-## strptime
-
+### strptime
strptime (class=time #args=2) strptime: Parses timestamp as floating-point seconds since the epoch,
e.g. strptime("2015-08-28T13:33:21Z","%Y-%m-%dT%H:%M:%SZ") = 1440768801.000000,
@@ -1041,407 +1041,275 @@ strptime (class=time #args=2) strptime: Parses timestamp as floating-point seco
-## sub
-
-
-sub (class=string #args=3) Example: '$name=sub($name, "old", "new")' (replace once).
-
-
-
-## substr
-
-
-substr (class=string #args=3) substr is an alias for substr0. See also substr1. Miller is generally 1-up
-with all array indices, but, this is a backward-compatibility issue with Miller 5 and below.
-Arrays are new in Miller 6; the substr function is older.
-
-
-
-## substr0
-
-
-substr0 (class=string #args=3) substr0(s,m,n) gives substring of s from 0-up position m to n
-inclusive. Negative indices -len .. -1 alias to 0 .. len-1.
-
-
-
-## substr1
-
-
-substr1 (class=string #args=3) substr1(s,m,n) gives substring of s from 1-up position m to n
-inclusive. Negative indices -len .. -1 alias to 1 .. len.
-
-
-
-## system
-
-
-system (class=system #args=1) Run command string, yielding its stdout minus final carriage return.
-
-
-
-## systime
-
+### systime
systime (class=time #args=0) help string will go here
-## systimeint
-
+### systimeint
systimeint (class=time #args=0) help string will go here
-## tan
-
-
-tan (class=math #args=1) Trigonometric tangent.
-
-
-
-## tanh
-
-
-tanh (class=math #args=1) Hyperbolic tangent.
-
-
-
-## tolower
-
-
-tolower (class=string #args=1) Convert string to lowercase.
-
-
-
-## toupper
-
-
-toupper (class=string #args=1) Convert string to uppercase.
-
-
-
-## truncate
-
-
-truncate (class=string #args=2) Truncates string first argument to max length of int second argument.
-
-
-
-## typeof
-
-
-typeof (class=typing #args=1) Convert argument to type of argument (e.g. "str"). For debug.
-
-
-
-## unflatten
-
-
-unflatten (class=maps/arrays #args=2) Reverses flatten. Example:
-unflatten({"a.b.c" : 4}, ".") is {"a": "b": { "c": 4 }}.
-Useful for nested JSON-like structures for non-JSON file formats like CSV.
-See also arrayify.
-
-
-
-## uptime
-
+### uptime
uptime (class=time #args=0) help string will go here
+## Typing functions
-## urand
+### asserting_absent
-urand (class=math #args=0) Floating-point numbers uniformly distributed on the unit interval.
-Int-valued example: '$n=floor(20+urand()*11)'.
+asserting_absent (class=typing #args=1) Aborts with an error if is_absent on the argument returns false,
+else returns its argument.
-## urand32
-
+### asserting_array
-urand32 (class=math #args=0) Integer uniformly distributed 0 and 2**32-1 inclusive.
+asserting_array (class=typing #args=1) Aborts with an error if is_array on the argument returns false,
+else returns its argument.
-## urandint
-
+### asserting_bool
-urandint (class=math #args=2) Integer uniformly distributed between inclusive integer endpoints.
+asserting_bool (class=typing #args=1) Aborts with an error if is_bool on the argument returns false,
+else returns its argument.
-## urandrange
-
+### asserting_boolean
-urandrange (class=math #args=2) Floating-point numbers uniformly distributed on the interval [a, b).
+asserting_boolean (class=typing #args=1) Aborts with an error if is_boolean on the argument returns false,
+else returns its argument.
-## version
-
+### asserting_empty
-version (class=system #args=0) Returns the Miller version as a string.
+asserting_empty (class=typing #args=1) Aborts with an error if is_empty on the argument returns false,
+else returns its argument.
-
-## \!
-
+### asserting_empty_map
-! (class=boolean #args=1) Logical negation.
+asserting_empty_map (class=typing #args=1) Aborts with an error if is_empty_map on the argument returns false,
+else returns its argument.
-## !=
-
+### asserting_error
-!= (class=boolean #args=2) String/numeric inequality. Mixing number and string results in string compare.
+asserting_error (class=typing #args=1) Aborts with an error if is_error on the argument returns false,
+else returns its argument.
-
-## !=~
-
+### asserting_float
-!=~ (class=boolean #args=2) String (left-hand side) does not match regex (right-hand side), e.g. '$name !=~ "^a.*b$"'.
+asserting_float (class=typing #args=1) Aborts with an error if is_float on the argument returns false,
+else returns its argument.
-## %
-
+### asserting_int
-% (class=arithmetic #args=2) Remainder; never negative-valued (pythonic).
+asserting_int (class=typing #args=1) Aborts with an error if is_int on the argument returns false,
+else returns its argument.
-
-## &
-
+### asserting_map
-& (class=arithmetic #args=2) Bitwise AND.
+asserting_map (class=typing #args=1) Aborts with an error if is_map on the argument returns false,
+else returns its argument.
-
-## &&
-
+### asserting_nonempty_map
-&& (class=boolean #args=2) Logical AND.
+asserting_nonempty_map (class=typing #args=1) Aborts with an error if is_nonempty_map on the argument returns false,
+else returns its argument.
-
-## \*
-
+### asserting_not_array
-* (class=arithmetic #args=2) Multiplication, with integer*integer overflow to float.
+asserting_not_array (class=typing #args=1) Aborts with an error if is_not_array on the argument returns false,
+else returns its argument.
-
-## \**
-
+### asserting_not_empty
-** (class=arithmetic #args=2) Exponentiation. Same as pow, but as an infix operator.
+asserting_not_empty (class=typing #args=1) Aborts with an error if is_not_empty on the argument returns false,
+else returns its argument.
-
-## \+
-
+### asserting_not_map
-+ (class=arithmetic #args=1,2) Addition as binary operator; unary plus operator.
+asserting_not_map (class=typing #args=1) Aborts with an error if is_not_map on the argument returns false,
+else returns its argument.
-
-## \-
-
+### asserting_not_null
-- (class=arithmetic #args=1,2) Subtraction as binary operator; unary negation operator.
+asserting_not_null (class=typing #args=1) Aborts with an error if is_not_null on the argument returns false,
+else returns its argument.
-## .
-
+### asserting_null
-. (class=string #args=2) String concatenation.
+asserting_null (class=typing #args=1) Aborts with an error if is_null on the argument returns false,
+else returns its argument.
-## .*
-
+### asserting_numeric
-.* (class=arithmetic #args=2) Multiplication, with integer-to-integer overflow.
+asserting_numeric (class=typing #args=1) Aborts with an error if is_numeric on the argument returns false,
+else returns its argument.
-## .+
-
+### asserting_present
-.+ (class=arithmetic #args=2) Addition, with integer-to-integer overflow.
+asserting_present (class=typing #args=1) Aborts with an error if is_present on the argument returns false,
+else returns its argument.
-## .-
-
+### asserting_string
-.- (class=arithmetic #args=2) Subtraction, with integer-to-integer overflow.
+asserting_string (class=typing #args=1) Aborts with an error if is_string on the argument returns false,
+else returns its argument.
-## ./
-
+### is_absent
-./ (class=arithmetic #args=2) Integer division; not pythonic.
+is_absent (class=typing #args=1) False if field is present in input, true otherwise
-## /
-
+### is_array
-/ (class=arithmetic #args=2) Division. Integer / integer is floating-point.
+is_array (class=typing #args=1) True if argument is an array.
-## //
-
+### is_bool
-// (class=arithmetic #args=2) Pythonic integer division, rounding toward negative.
+is_bool (class=typing #args=1) True if field is present with boolean value. Synonymous with is_boolean.
-## <
-
+### is_boolean
-< (class=boolean #args=2) String/numeric less-than. Mixing number and string results in string compare.
+is_boolean (class=typing #args=1) True if field is present with boolean value. Synonymous with is_bool.
-## <<
-
+### is_empty
-<< (class=arithmetic #args=2) Bitwise left-shift.
+is_empty (class=typing #args=1) True if field is present in input with empty string value, false otherwise.
-## <=
-
+### is_empty_map
-<= (class=boolean #args=2) String/numeric less-than-or-equals. Mixing number and string results in string compare.
+is_empty_map (class=typing #args=1) True if argument is a map which is empty.
-## ==
-
+### is_error
-== (class=boolean #args=2) String/numeric equality. Mixing number and string results in string compare.
+is_error (class=typing #args=1) True if if argument is an error, such as taking string length of an integer.
-
-## =~
-
+### is_float
-=~ (class=boolean #args=2) String (left-hand side) matches regex (right-hand side), e.g. '$name =~ "^a.*b$"'.
+is_float (class=typing #args=1) True if field is present with value inferred to be float
-## >
-
+### is_int
-> (class=boolean #args=2) String/numeric greater-than. Mixing number and string results in string compare.
+is_int (class=typing #args=1) True if field is present with value inferred to be int
-## >=
-
+### is_map
->= (class=boolean #args=2) String/numeric greater-than-or-equals. Mixing number and string results in string compare.
+is_map (class=typing #args=1) True if argument is a map.
-
-## \>\>
-
+### is_nonempty_map
->> (class=arithmetic #args=2) Bitwise signed right-shift.
+is_nonempty_map (class=typing #args=1) True if argument is a map which is non-empty.
-
-## \>\>\>
-
+### is_not_array
->>> (class=arithmetic #args=2) Bitwise unsigned right-shift.
+is_not_array (class=typing #args=1) True if argument is not an array.
-
-## ?:
-
+### is_not_empty
-?: (class=boolean #args=3) Standard ternary operator.
+is_not_empty (class=typing #args=1) False if field is present in input with empty value, true otherwise
-
-## ??
-
+### is_not_map
-?? (class=boolean #args=2) Absent-coalesce operator. $a ?? 1 evaluates to 1 if $a isn't defined in the current record.
+is_not_map (class=typing #args=1) True if argument is not a map.
-
-## ???
-
+### is_not_null
-??? (class=boolean #args=2) Absent-coalesce operator. $a ?? 1 evaluates to 1 if $a isn't defined in the current record, or has empty value.
+is_not_null (class=typing #args=1) False if argument is null (empty or absent), true otherwise.
-
-## ^
-
+### is_null
-^ (class=arithmetic #args=2) Bitwise XOR.
+is_null (class=typing #args=1) True if argument is null (empty or absent), false otherwise.
-
-## ^^
-
+### is_numeric
-^^ (class=boolean #args=2) Logical XOR.
+is_numeric (class=typing #args=1) True if field is present with value inferred to be int or float
-
-## \|
-
+### is_present
-| (class=arithmetic #args=2) Bitwise OR.
+is_present (class=typing #args=1) True if field is present in input, false otherwise.
-
-## \|\|
-
+### is_string
-|| (class=boolean #args=2) Logical OR.
+is_string (class=typing #args=1) True if field is present with string (including empty-string) value
-## ~
-
+### typeof
-~ (class=arithmetic #args=1) Bitwise NOT. Beware '$y=~$x' since =~ is the
-regex-match operator: try '$y = ~$x'.
+typeof (class=typing #args=1) Convert argument to type of argument (e.g. "str"). For debug.
diff --git a/docs6/docs/reference-dsl-builtin-functions.md.in b/docs6/docs/reference-dsl-builtin-functions.md.in
index 5079c8c98..0cf7e516c 100644
--- a/docs6/docs/reference-dsl-builtin-functions.md.in
+++ b/docs6/docs/reference-dsl-builtin-functions.md.in
@@ -1,13 +1,21 @@
# DSL built-in functions
-## Summary
+Each function takes a specific number of arguments, as shown below, except for
+functions marked as variadic such as `min` and `max`. (The latter compute min
+and max of any number of arguments.) There is no notion of optional or
+default-on-absent arguments. All argument-passing is positional rather than by
+name; arguments are passed by value, not by reference.
-GENMD_RUN_CONTENT_GENERATOR(./mk-func-table.rb)
+At the command line, you can get a list of all functions using `mlr -f`, with
+details using `mlr -F`. (Or, `mlr help usage-functions-by-class` to get
+details in the order shown on this page.) You can get detail for a given
+function using `mlr help function namegoeshere`, e.g. `mlr help function
+gsub`.
-## List of functions
+Operators are listed here along with functions. In this case, the
+argument-count is the number of items involved in the infix operator, e.g. we
+say `x+y` so the details for the `+` operator say that its number of arguments
+is 2. Unary operators such as `!` and `~` show argument-count of 1; the ternary
+`? :` operator shows an argument-count of 3.
-Each function takes a specific number of arguments, as shown below, except for functions marked as variadic such as `min` and `max`. (The latter compute min and max of any number of numerical arguments.) There is no notion of optional or default-on-absent arguments. All argument-passing is positional rather than by name; arguments are passed by value, not by reference.
-
-You can get a list of all functions using **mlr -f**, with details using **mlr -F**.
-
-GENMD_RUN_CONTENT_GENERATOR(./mk-func-h2s.sh)
+GENMD_RUN_CONTENT_GENERATOR(./mk-func-info.rb)
diff --git a/docs6/docs/reference-dsl-differences.md b/docs6/docs/reference-dsl-differences.md
index 2d583bbd4..d43df8397 100644
--- a/docs6/docs/reference-dsl-differences.md
+++ b/docs6/docs/reference-dsl-differences.md
@@ -144,6 +144,28 @@ avoid this, use the dot operator for string-concatenation instead.
Similarly, a final newline is printed for you; use [`printn`](reference-dsl-output-statements.md#print-statements) to avoid this.
+## String literals with double quotes only
+
+In some languages, like Ruby and Bash, string literals can be in single quotes or double quotes,
+where single quotes suppress `\n` converting to a newline character and double quotes allowing it:
+`'a\nb'` prints as the four characters `a`, `\`, `n`, and `b` on one line; `"a\nb"` prints as an
+`a` on one line and a `b` on another.
+
+In others, like Python and JavaScript, string literals can be in single quotes or double quotes,
+interchangeably -- so you can have `"don't"` or `'the "right" thing'` as you wish.
+
+In yet others, such as C/C++ and Java, string literals are in double auotes, like `"abc"`,
+while single quotes are for character literals like `'a'` or `'\n'`. In these, if `s` is a non-empty string,
+then `s[0]` is its first character.
+
+In the [Miller programming language](programming-language.md):
+
+* String literals are always in double quotes, like `"abc"`.
+* String-indexing/slicing always results in strings (even of length 1): `"abc"[1:1]` is the string `"a"`, and there is no notion in the Miller programming language of a character type.
+* The single-quote character plays no role whatsoever in the grammar of the Miller programming language.
+* Single quotes are reserved for wrapping expressions at the system command line. For example, in `mlr put '$message = "hello"' ...`, the [`put` verb](reference-dsl.md) gets the string `$message = "hello"`; the shell has consumed the outer single quotes by the time the Miller parser receives it.
+* Things are a little different on Windows, where `"""` sequences are sometimes necessary: see the [Miller on Windows page](miller-on-windows.md).
+
## Absent-null
Miller has a somewhat novel flavor of null data called _absent_: if a record
@@ -194,3 +216,12 @@ Miller has a [key-value loop flavor](reference-dsl-control-structures.md#key-val
## Semantics for one-variable for-loops
Miller also has a [single-variable loop flavor](reference-dsl-control-structures.md#single-variable-for-loops). If `x` is a map then `for (e in x) { ... }` binds `e` to successive map _keys_ (not values as in PHP). But if `x` is an array then `for e in x) { ... }` binds `e` to successive array _values_ (not indices).
+
+## JSON parse, stringify, decode, and encode
+
+Miller has the verbs
+[`json-parse`](reference-verbs.md#json-parse) and
+[`json-stringify`](reference-verbs.md#json-stringify), and the DSL functions
+[`json_parse`](reference-dsl-builtin-functions.md#json_parse) and
+[`json_stringify`](reference-dsl-builtin-functions.md#json_stringify).
+In some other lannguages these are called `json_decode` and `json_encode`.
diff --git a/docs6/docs/reference-dsl-differences.md.in b/docs6/docs/reference-dsl-differences.md.in
index 0d5f5bc12..21619c63a 100644
--- a/docs6/docs/reference-dsl-differences.md.in
+++ b/docs6/docs/reference-dsl-differences.md.in
@@ -116,6 +116,28 @@ GENMD_EOF
Similarly, a final newline is printed for you; use [`printn`](reference-dsl-output-statements.md#print-statements) to avoid this.
+## String literals with double quotes only
+
+In some languages, like Ruby and Bash, string literals can be in single quotes or double quotes,
+where single quotes suppress `\n` converting to a newline character and double quotes allowing it:
+`'a\nb'` prints as the four characters `a`, `\`, `n`, and `b` on one line; `"a\nb"` prints as an
+`a` on one line and a `b` on another.
+
+In others, like Python and JavaScript, string literals can be in single quotes or double quotes,
+interchangeably -- so you can have `"don't"` or `'the "right" thing'` as you wish.
+
+In yet others, such as C/C++ and Java, string literals are in double auotes, like `"abc"`,
+while single quotes are for character literals like `'a'` or `'\n'`. In these, if `s` is a non-empty string,
+then `s[0]` is its first character.
+
+In the [Miller programming language](programming-language.md):
+
+* String literals are always in double quotes, like `"abc"`.
+* String-indexing/slicing always results in strings (even of length 1): `"abc"[1:1]` is the string `"a"`, and there is no notion in the Miller programming language of a character type.
+* The single-quote character plays no role whatsoever in the grammar of the Miller programming language.
+* Single quotes are reserved for wrapping expressions at the system command line. For example, in `mlr put '$message = "hello"' ...`, the [`put` verb](reference-dsl.md) gets the string `$message = "hello"`; the shell has consumed the outer single quotes by the time the Miller parser receives it.
+* Things are a little different on Windows, where `"""` sequences are sometimes necessary: see the [Miller on Windows page](miller-on-windows.md).
+
## Absent-null
Miller has a somewhat novel flavor of null data called _absent_: if a record
@@ -155,3 +177,12 @@ Miller has a [key-value loop flavor](reference-dsl-control-structures.md#key-val
## Semantics for one-variable for-loops
Miller also has a [single-variable loop flavor](reference-dsl-control-structures.md#single-variable-for-loops). If `x` is a map then `for (e in x) { ... }` binds `e` to successive map _keys_ (not values as in PHP). But if `x` is an array then `for e in x) { ... }` binds `e` to successive array _values_ (not indices).
+
+## JSON parse, stringify, decode, and encode
+
+Miller has the verbs
+[`json-parse`](reference-verbs.md#json-parse) and
+[`json-stringify`](reference-verbs.md#json-stringify), and the DSL functions
+[`json_parse`](reference-dsl-builtin-functions.md#json_parse) and
+[`json_stringify`](reference-dsl-builtin-functions.md#json_stringify).
+In some other lannguages these are called `json_decode` and `json_encode`.
diff --git a/docs6/docs/reference-main-arrays.md b/docs6/docs/reference-main-arrays.md
index 2129fb538..22ceae93e 100644
--- a/docs6/docs/reference-main-arrays.md
+++ b/docs6/docs/reference-main-arrays.md
@@ -162,9 +162,9 @@ If you want to auto-extend an [array](reference-main-arrays.md), initialize it e
Once an array is initialized, it can be extended by assigning to indices beyond
its length. If each write is one past the end of the array, the array will
-grow by one. (Memory management, handled for you, is a little more careful: not
-to worry, capacity is doubled so performance doesn't suffer a rellocate on
-every single extend.)
+grow by one. (Memory management, handled for you, is careful handled here in
+Miller: not to worry, capacity is doubled so performance doesn't suffer a
+rellocate on every single extend.)
This is important in Miller so you can do things like `@records[NR] = $*` with
a minimum of keystrokes without worrying about explicitly resizing arrays. In
diff --git a/docs6/docs/reference-main-arrays.md.in b/docs6/docs/reference-main-arrays.md.in
index 5a94a6fab..be3d44d04 100644
--- a/docs6/docs/reference-main-arrays.md.in
+++ b/docs6/docs/reference-main-arrays.md.in
@@ -108,9 +108,9 @@ GENMD_EOF
Once an array is initialized, it can be extended by assigning to indices beyond
its length. If each write is one past the end of the array, the array will
-grow by one. (Memory management, handled for you, is a little more careful: not
-to worry, capacity is doubled so performance doesn't suffer a rellocate on
-every single extend.)
+grow by one. (Memory management, handled for you, is careful handled here in
+Miller: not to worry, capacity is doubled so performance doesn't suffer a
+rellocate on every single extend.)
This is important in Miller so you can do things like `@records[NR] = $*` with
a minimum of keystrokes without worrying about explicitly resizing arrays. In
diff --git a/docs6/docs/reference-main-data-types.md b/docs6/docs/reference-main-data-types.md
index 87820ad28..a1d6e4af0 100644
--- a/docs6/docs/reference-main-data-types.md
+++ b/docs6/docs/reference-main-data-types.md
@@ -16,62 +16,193 @@ Quick links:
## List of types
-Miller's types for function processing are:
+Miller's types are:
* Scalars:
- * **string**
- * **float** (double-precision) and **int** (64-bit signed): see the [Arithmetic](reference-main-arithmetic.md) page
- * **boolean**
+ * **string**: such as `"abcdefg"`, supporting concatenation, one-up indexing and slicing, and [library functions](reference-dsl-builtin-functions.md#string-functions). See the pages on [strings](reference-main-strings.md) and [regular expressions](reference-main-regular-expressions.md).
+ * **float** and **int**: such as `1.2` and `3`: double-precision and 64-bit signed, respectively. See the section on [arithmetic operators and math-related library functions](reference-dsl-builtin-functions.md#math-functions) as well as the [Arithmetic](reference-main-arithmetic.md) page.
+ * **boolean**: literals `true` and `false`; results of `==`, `<`, `>`, etc. See the section on [boolean operators](reference-dsl-builtin-functions.md#boolean-functions).
* Collections:
- * **map**: see the [Maps](reference-main-maps.md) page
- * **array**: see the [Arrays](reference-main-arrays.md) page
+ * **map**: such as `{"a":1,"b":[2,3,4]}`, supporting key-indexing, preservation of insertion order, [library functions](reference-dsl-builtin-functions.md#collections-functions), etc. See the [Maps](reference-main-maps.md) page.
+ * **array**: such as `["a", 2, true]`, supporting one-up indexing and slicing, [library functions](reference-dsl-builtin-functions.md#collections-functions), etc. See the [Arrays](reference-main-arrays.md) page.
* Nulls and error:
- * **absent-null** (reads of unset right-hand sides, or fall-through non-explicit return values from user-defined functions) and **JSON-null**: see the [null-data page](reference-main-null-data.md)
- * **error** (TODO: give an example)
+ * **absent-null**: Such as on reads of unset right-hand sides, or fall-through non-explicit return values from user-defined functions. See the [null-data page](reference-main-null-data.md).
+ * **JSON-null**: For `null` in JSON files; also used in [gapped auto-extend of arrays](reference-main-arrays.md#auto-extend-and-null-gaps). See the [null-data page](reference-main-null-data.md).
+ * **error** -- for various results which cannot be computed, often when the input to a [built-in function](reference-dsl-builtin-functions.md) is of the wrong type. For example, doing [strlen](reference-dsl-builtin-functions.md#strlen) or [substr](reference-dsl-builtin-functions.md#substr) on a non-string, [sec2gmt](reference-dsl-builtin-functions.md#sec2gmt) on a non-integer, etc.
-TODO: point to null-data page; arrays page; arithmetic page; what else?
+See also the list of [type-checking
+functions](reference-dsl-builtin-functions.md#type-checkin -functions) for the
+[Miller programming language](programming-language.md).
-## To be ported
+See also [Differences from other programming languages](reference-dsl-differences.md).
-Miller's input and output are all string-oriented: there is (as of August 2015 anyway) no support for binary record packing. In this sense, everything is a string in and out of Miller. During processing, field names are always strings, even if they have names like "3"; field values can have various data types. Field values' ability to be interpreted as a non-string type only has meaning when comparison or function operations are done on them. And it is an error condition if Miller encounters non-numeric (or otherwise mistyped) data in a field in which it has been asked to do numeric (or otherwise type-specific) operations.
+## Type inference for literal and record data
-Field values are treated as numeric for the following:
+Miller's input and output are all text-oriented: all the
+[file formats supported by Miller](file-formats.md) are human-readable text,
+such as CSV, TSV, and JSON; binary formats such as
+[BSON](https://bsonspec.org/) and [Parquet](https://parquet.apache.org/) are
+not supported (as of mid-2021). In this sense, everything is a string in and out of
+Miller -- be it in data files, or in DSL expressions you key in.
-* Numeric sort: `mlr sort -n`, `mlr sort -nr`.
-* Statistics: `mlr histogram`, `mlr stats1`, `mlr stats2`.
-* Cross-record arithmetic: `mlr step`.
+In the [DSL](programming-language.md), `7` is an `int` and `8.9` is a float, as
+one would expect. Likewise, on input from [data files](file-formats.md),
+string values representable as numbers, e.g. `1.2` or `3`, are treated as int
+or float, respectively. If a record has `x=1,y=2` then `mlr put '$z=$x+$y'`
+will produce `x=1,y=2,z=3`.
-For `mlr put` and `mlr filter`:
+Numbers retain their original string representation, so if `x` is `1.2` on one
+record and `1.200` on another, they'll print out that way on output (unless of
+course they've been modified during processing, e.g. `mlr put '$x = $x + 10`).
-* Miller's types for function processing are **empty-null** (empty string), **absent-null** (reads of unset right-hand sides, or fall-through non-explicit return values from user-defined functions), **error**, **string**, **float** (double-precision), **int** (64-bit signed), and **boolean**.
+Generally strings, numbers, and booleans don't mix; use type-casting like
+`string($x)` to convert. However, the dot (string-concatenation) operator has
+been special-cased: `mlr put '$z=$x.$y'` does not give an error, because the
+dot operator has been generalized to stringify non-strings
-* On input, string values representable as numbers, e.g. "3" or "3.1", are treated as int or float, respectively. If a record has `x=1,y=2` then `mlr put '$z=$x+$y'` will produce `x=1,y=2,z=3`, and `mlr put '$z=$x.$y'` does not give an error, but only because the dot operator has been generalized to stringify non-strings. To coerce back to string for processing, use the `string` function: `mlr put '$z=string($x).string($y)'` will produce `x=1,y=2,z=12`.
+Examples:
-* On input, string values representable as boolean (e.g. `"true"`, `"false"`) are *not* automatically treated as boolean. (This is because `"true"` and `"false"` are ordinary words, and auto string-to-boolean on a column consisting of words would result in some strings mixed with some booleans.) Use the `boolean` function to coerce: e.g. giving the record `x=1,y=2,w=false` to `mlr put '$z=($x<$y) || boolean($w)'`.
+
+mlr --csv cat data/type-infer.csv
+
+
+a,b,c
+1.2,3,true
+4,5.6,buongiorno
+
-* Functions take types as described in `mlr --help-all-functions`: for example, `log10` takes float input and produces float output, `gmt2sec` maps string to int, and `sec2gmt` maps int to string.
+
+mlr --icsv --oxtab --from data/type-infer.csv put '
+ $d = $a . $c;
+ $e = 7;
+ $f = 8.9;
+ $g = $e + $f;
+ $ta = typeof($a);
+ $tb = typeof($b);
+ $tc = typeof($c);
+ $td = typeof($d);
+ $te = typeof($e);
+ $tf = typeof($f);
+ $tg = typeof($g);
+' then reorder -f a,ta,b,tb,c,tc,d,td,e,te,f,tf,g,tg
+
+
+a 1.2
+ta float
+b 3
+tb int
+c true
+tc string
+d 1.2true
+td string
+e 7
+te int
+f 8.9
+tf float
+g 15.9
+tg float
-* All math functions described in `mlr --help-all-functions` take integer as well as float input.
+a 4
+ta int
+b 5.6
+tb float
+c buongiorno
+tc string
+d 4buongiorno
+td string
+e 7
+te int
+f 8.9
+tf float
+g 15.9
+tg float
+
-## Escape sequences for string literals
+On input, string values representable as boolean (e.g. `"true"`, `"false"`)
+are *not* automatically treated as boolean. This is because `"true"` and
+`"false"` are ordinary words, and auto string-to-boolean on a column consisting
+of words would result in some strings mixed with some booleans. Use the
+`boolean` function to coerce: e.g. giving the record `x=1,y=2,w=false` to `mlr
+filter '$z=($x<$y) || boolean($w)'`.
-You can use the following backslash escapes for strings such as between the double quotes in contexts such as `mlr filter '$name =~ "..."'`, `mlr put '$name = $othername . "..."'`, `mlr put '$name = sub($name, "...", "...")`, etc.:
+The same is true for `inf`, `+inf`, `-inf`, `infinity`, `+infinity`,
+`-infinity`, `NaN`, and all upper-cased/lower-cased/mixed-case variants of
+those. These are valid IEEE floating-point numbers, but Miller treats these as
+strings. You can explicit force conversion: if `x=infinity` in a data file,
+then `typeof($x)` is `string` but `typeof(float($x))` is `float`.
-* `\a`: ASCII code 0x07 (alarm/bell)
-* `\b`: ASCII code 0x08 (backspace)
-* `\f`: ASCII code 0x0c (formfeed)
-* `\n`: ASCII code 0x0a (LF/linefeed/newline)
-* `\r`: ASCII code 0x0d (CR/carriage return)
-* `\t`: ASCII code 0x09 (tab)
-* `\v`: ASCII code 0x0b (vertical tab)
-* `\\`: backslash
-* `\"`: double quote
-* `\123`: Octal 123, etc. for `\000` up to `\377`
-* `\x7f`: Hexadecimal 7f, etc. for `\x00` up to `\xff`
+## JSON parse and stringify
-See also [https://en.wikipedia.org/wiki/Escape_sequences_in_C](https://en.wikipedia.org/wiki/Escape_sequences_in_C).
+If you have, say, a CSV file whose columns contain strings which are well-formatted JSON,
+they will not be auto-converted, but you can use the
+[`json-parse` verb](reference-verbs.md#json-parse)
+or the
+[`json_parse` DSL function](reference-dsl-builtin-functions.md#json_parse):
-These replacements apply only to strings you key in for the DSL expressions for `filter` and `put`: that is, if you type `\t` in a string literal for a `filter`/`put` expression, it will be turned into a tab character. If you want a backslash followed by a `t`, then please type `\\t`.
+
+mlr --csv --from data/json-in-csv.csv cat
+
+
+id,blob
+100,"{""a"":1,""b"":[2,3,4]}"
+105,"{""a"":6,""b"":[7,8,9]}"
+
-However, these replacements are done automatically only for string literals within DSL expressions -- they are not done automatically to fields within your data stream. If you wish to make these replacements, you can do (for example) `mlr put '$field = gsub($field, "\\t", "\t")'`. If you need to make such a replacement for all fields in your data, you should probably use the system `sed` command instead.
+
+mlr --icsv --ojson --from data/json-in-csv.csv cat
+
+
+{
+ "id": 100,
+ "blob": "{\"a\":1,\"b\":[2,3,4]}"
+}
+{
+ "id": 105,
+ "blob": "{\"a\":6,\"b\":[7,8,9]}"
+}
+
+
+mlr --icsv --ojson --from data/json-in-csv.csv json-parse -f blob
+
+
+{
+ "id": 100,
+ "blob": {
+ "a": 1,
+ "b": [2, 3, 4]
+ }
+}
+{
+ "id": 105,
+ "blob": {
+ "a": 6,
+ "b": [7, 8, 9]
+ }
+}
+
+
+
+mlr --icsv --ojson --from data/json-in-csv.csv put '$blob = json_parse($blob)'
+
+
+{
+ "id": 100,
+ "blob": {
+ "a": 1,
+ "b": [2, 3, 4]
+ }
+}
+{
+ "id": 105,
+ "blob": {
+ "a": 6,
+ "b": [7, 8, 9]
+ }
+}
+
+
+These have their respective operations to convert back to string: the
+[`json-stringify` verb](reference-verbs.md#json-stringify)
+and
+[`json_stringify` DSL function](reference-dsl-builtin-functions.md#json_stringify).
diff --git a/docs6/docs/reference-main-data-types.md.in b/docs6/docs/reference-main-data-types.md.in
index 388a6888a..7e7c530c2 100644
--- a/docs6/docs/reference-main-data-types.md.in
+++ b/docs6/docs/reference-main-data-types.md.in
@@ -2,62 +2,110 @@
## List of types
-Miller's types for function processing are:
+Miller's types are:
* Scalars:
- * **string**
- * **float** (double-precision) and **int** (64-bit signed): see the [Arithmetic](reference-main-arithmetic.md) page
- * **boolean**
+ * **string**: such as `"abcdefg"`, supporting concatenation, one-up indexing and slicing, and [library functions](reference-dsl-builtin-functions.md#string-functions). See the pages on [strings](reference-main-strings.md) and [regular expressions](reference-main-regular-expressions.md).
+ * **float** and **int**: such as `1.2` and `3`: double-precision and 64-bit signed, respectively. See the section on [arithmetic operators and math-related library functions](reference-dsl-builtin-functions.md#math-functions) as well as the [Arithmetic](reference-main-arithmetic.md) page.
+ * **boolean**: literals `true` and `false`; results of `==`, `<`, `>`, etc. See the section on [boolean operators](reference-dsl-builtin-functions.md#boolean-functions).
* Collections:
- * **map**: see the [Maps](reference-main-maps.md) page
- * **array**: see the [Arrays](reference-main-arrays.md) page
+ * **map**: such as `{"a":1,"b":[2,3,4]}`, supporting key-indexing, preservation of insertion order, [library functions](reference-dsl-builtin-functions.md#collections-functions), etc. See the [Maps](reference-main-maps.md) page.
+ * **array**: such as `["a", 2, true]`, supporting one-up indexing and slicing, [library functions](reference-dsl-builtin-functions.md#collections-functions), etc. See the [Arrays](reference-main-arrays.md) page.
* Nulls and error:
- * **absent-null** (reads of unset right-hand sides, or fall-through non-explicit return values from user-defined functions) and **JSON-null**: see the [null-data page](reference-main-null-data.md)
- * **error** (TODO: give an example)
+ * **absent-null**: Such as on reads of unset right-hand sides, or fall-through non-explicit return values from user-defined functions. See the [null-data page](reference-main-null-data.md).
+ * **JSON-null**: For `null` in JSON files; also used in [gapped auto-extend of arrays](reference-main-arrays.md#auto-extend-and-null-gaps). See the [null-data page](reference-main-null-data.md).
+ * **error** -- for various results which cannot be computed, often when the input to a [built-in function](reference-dsl-builtin-functions.md) is of the wrong type. For example, doing [strlen](reference-dsl-builtin-functions.md#strlen) or [substr](reference-dsl-builtin-functions.md#substr) on a non-string, [sec2gmt](reference-dsl-builtin-functions.md#sec2gmt) on a non-integer, etc.
-TODO: point to null-data page; arrays page; arithmetic page; what else?
+See also the list of [type-checking
+functions](reference-dsl-builtin-functions.md#type-checkin -functions) for the
+[Miller programming language](programming-language.md).
-## To be ported
+See also [Differences from other programming languages](reference-dsl-differences.md).
-Miller's input and output are all string-oriented: there is (as of August 2015 anyway) no support for binary record packing. In this sense, everything is a string in and out of Miller. During processing, field names are always strings, even if they have names like "3"; field values can have various data types. Field values' ability to be interpreted as a non-string type only has meaning when comparison or function operations are done on them. And it is an error condition if Miller encounters non-numeric (or otherwise mistyped) data in a field in which it has been asked to do numeric (or otherwise type-specific) operations.
+## Type inference for literal and record data
-Field values are treated as numeric for the following:
+Miller's input and output are all text-oriented: all the
+[file formats supported by Miller](file-formats.md) are human-readable text,
+such as CSV, TSV, and JSON; binary formats such as
+[BSON](https://bsonspec.org/) and [Parquet](https://parquet.apache.org/) are
+not supported (as of mid-2021). In this sense, everything is a string in and out of
+Miller -- be it in data files, or in DSL expressions you key in.
-* Numeric sort: `mlr sort -n`, `mlr sort -nr`.
-* Statistics: `mlr histogram`, `mlr stats1`, `mlr stats2`.
-* Cross-record arithmetic: `mlr step`.
+In the [DSL](programming-language.md), `7` is an `int` and `8.9` is a float, as
+one would expect. Likewise, on input from [data files](file-formats.md),
+string values representable as numbers, e.g. `1.2` or `3`, are treated as int
+or float, respectively. If a record has `x=1,y=2` then `mlr put '$z=$x+$y'`
+will produce `x=1,y=2,z=3`.
-For `mlr put` and `mlr filter`:
+Numbers retain their original string representation, so if `x` is `1.2` on one
+record and `1.200` on another, they'll print out that way on output (unless of
+course they've been modified during processing, e.g. `mlr put '$x = $x + 10`).
-* Miller's types for function processing are **empty-null** (empty string), **absent-null** (reads of unset right-hand sides, or fall-through non-explicit return values from user-defined functions), **error**, **string**, **float** (double-precision), **int** (64-bit signed), and **boolean**.
+Generally strings, numbers, and booleans don't mix; use type-casting like
+`string($x)` to convert. However, the dot (string-concatenation) operator has
+been special-cased: `mlr put '$z=$x.$y'` does not give an error, because the
+dot operator has been generalized to stringify non-strings
-* On input, string values representable as numbers, e.g. "3" or "3.1", are treated as int or float, respectively. If a record has `x=1,y=2` then `mlr put '$z=$x+$y'` will produce `x=1,y=2,z=3`, and `mlr put '$z=$x.$y'` does not give an error, but only because the dot operator has been generalized to stringify non-strings. To coerce back to string for processing, use the `string` function: `mlr put '$z=string($x).string($y)'` will produce `x=1,y=2,z=12`.
+Examples:
-* On input, string values representable as boolean (e.g. `"true"`, `"false"`) are *not* automatically treated as boolean. (This is because `"true"` and `"false"` are ordinary words, and auto string-to-boolean on a column consisting of words would result in some strings mixed with some booleans.) Use the `boolean` function to coerce: e.g. giving the record `x=1,y=2,w=false` to `mlr put '$z=($x<$y) || boolean($w)'`.
+GENMD_RUN_COMMAND
+mlr --csv cat data/type-infer.csv
+GENMD_EOF
-* Functions take types as described in `mlr --help-all-functions`: for example, `log10` takes float input and produces float output, `gmt2sec` maps string to int, and `sec2gmt` maps int to string.
+GENMD_RUN_COMMAND
+mlr --icsv --oxtab --from data/type-infer.csv put '
+ $d = $a . $c;
+ $e = 7;
+ $f = 8.9;
+ $g = $e + $f;
+ $ta = typeof($a);
+ $tb = typeof($b);
+ $tc = typeof($c);
+ $td = typeof($d);
+ $te = typeof($e);
+ $tf = typeof($f);
+ $tg = typeof($g);
+' then reorder -f a,ta,b,tb,c,tc,d,td,e,te,f,tf,g,tg
+GENMD_EOF
-* All math functions described in `mlr --help-all-functions` take integer as well as float input.
+On input, string values representable as boolean (e.g. `"true"`, `"false"`)
+are *not* automatically treated as boolean. This is because `"true"` and
+`"false"` are ordinary words, and auto string-to-boolean on a column consisting
+of words would result in some strings mixed with some booleans. Use the
+`boolean` function to coerce: e.g. giving the record `x=1,y=2,w=false` to `mlr
+filter '$z=($x<$y) || boolean($w)'`.
-## Escape sequences for string literals
+The same is true for `inf`, `+inf`, `-inf`, `infinity`, `+infinity`,
+`-infinity`, `NaN`, and all upper-cased/lower-cased/mixed-case variants of
+those. These are valid IEEE floating-point numbers, but Miller treats these as
+strings. You can explicit force conversion: if `x=infinity` in a data file,
+then `typeof($x)` is `string` but `typeof(float($x))` is `float`.
-You can use the following backslash escapes for strings such as between the double quotes in contexts such as `mlr filter '$name =~ "..."'`, `mlr put '$name = $othername . "..."'`, `mlr put '$name = sub($name, "...", "...")`, etc.:
+## JSON parse and stringify
-* `\a`: ASCII code 0x07 (alarm/bell)
-* `\b`: ASCII code 0x08 (backspace)
-* `\f`: ASCII code 0x0c (formfeed)
-* `\n`: ASCII code 0x0a (LF/linefeed/newline)
-* `\r`: ASCII code 0x0d (CR/carriage return)
-* `\t`: ASCII code 0x09 (tab)
-* `\v`: ASCII code 0x0b (vertical tab)
-* `\\`: backslash
-* `\"`: double quote
-* `\123`: Octal 123, etc. for `\000` up to `\377`
-* `\x7f`: Hexadecimal 7f, etc. for `\x00` up to `\xff`
+If you have, say, a CSV file whose columns contain strings which are well-formatted JSON,
+they will not be auto-converted, but you can use the
+[`json-parse` verb](reference-verbs.md#json-parse)
+or the
+[`json_parse` DSL function](reference-dsl-builtin-functions.md#json_parse):
-See also [https://en.wikipedia.org/wiki/Escape_sequences_in_C](https://en.wikipedia.org/wiki/Escape_sequences_in_C).
+GENMD_RUN_COMMAND
+mlr --csv --from data/json-in-csv.csv cat
+GENMD_EOF
-These replacements apply only to strings you key in for the DSL expressions for `filter` and `put`: that is, if you type `\t` in a string literal for a `filter`/`put` expression, it will be turned into a tab character. If you want a backslash followed by a `t`, then please type `\\t`.
+GENMD_RUN_COMMAND
+mlr --icsv --ojson --from data/json-in-csv.csv cat
+GENMD_EOF
-However, these replacements are done automatically only for string literals within DSL expressions -- they are not done automatically to fields within your data stream. If you wish to make these replacements, you can do (for example) `mlr put '$field = gsub($field, "\\t", "\t")'`. If you need to make such a replacement for all fields in your data, you should probably use the system `sed` command instead.
+GENMD_RUN_COMMAND
+mlr --icsv --ojson --from data/json-in-csv.csv json-parse -f blob
+GENMD_EOF
+GENMD_RUN_COMMAND
+mlr --icsv --ojson --from data/json-in-csv.csv put '$blob = json_parse($blob)'
+GENMD_EOF
+
+These have their respective operations to convert back to string: the
+[`json-stringify` verb](reference-verbs.md#json-stringify)
+and
+[`json_stringify` DSL function](reference-dsl-builtin-functions.md#json_stringify).
diff --git a/docs6/docs/reference-main-maps.md b/docs6/docs/reference-main-maps.md
index ad23b1ae4..81debf235 100644
--- a/docs6/docs/reference-main-maps.md
+++ b/docs6/docs/reference-main-maps.md
@@ -82,7 +82,7 @@ The current record, accessible using `$*`, is a map.
{
"color": "yellow",
"shape": "triangle",
- "flag": true,
+ "flag": "true",
"k": 1,
"index": 11,
"quantity": 43.6498,
@@ -92,7 +92,7 @@ Color is yellow
{
"color": "red",
"shape": "square",
- "flag": true,
+ "flag": "true",
"k": 2,
"index": 15,
"quantity": 79.2778,
diff --git a/docs6/docs/reference-main-null-data.md b/docs6/docs/reference-main-null-data.md
index ae645de8c..4e9a24470 100644
--- a/docs6/docs/reference-main-null-data.md
+++ b/docs6/docs/reference-main-null-data.md
@@ -22,7 +22,7 @@ Miller has three kinds of null data:
* **Absent (key not present)**: a field name is not present, e.g. input record is `x=1,y=2` and a `put` or `filter` expression refers to `$z`. Or, reading an out-of-stream variable which hasn't been assigned a value yet, e.g. `mlr put -q '@sum += $x; end{emit @sum}'` or `mlr put -q '@sum[$a][$b] += $x; end{emit @sum, "a", "b"}'`.
-* **JSON null**: (TODO: describe)
+* **JSON null**: The main purpose of this is to support reading the `null` type in JSON files. The [Miller programming language](programming-language.md) has a `null` keyword as well, so you can also write the null type using `$x = null`. Addtionally, though, when you write past the end of an array, leaving gaps -- e.g. writing `a[12]` when the array `a` has length 10 -- JSON-null is used to fill the gaps. See also the [arrays page](reference-main-arrays.md#auto-extend-and-null-gaps).
You can test these programatically using the functions `is_empty`/`is_not_empty`, `is_absent`/`is_present`, and `is_null`/`is_not_null`. For the last pair, note that null means either empty or absent.
diff --git a/docs6/docs/reference-main-null-data.md.in b/docs6/docs/reference-main-null-data.md.in
index 3235e6e5d..47ac6c2af 100644
--- a/docs6/docs/reference-main-null-data.md.in
+++ b/docs6/docs/reference-main-null-data.md.in
@@ -8,7 +8,7 @@ Miller has three kinds of null data:
* **Absent (key not present)**: a field name is not present, e.g. input record is `x=1,y=2` and a `put` or `filter` expression refers to `$z`. Or, reading an out-of-stream variable which hasn't been assigned a value yet, e.g. `mlr put -q '@sum += $x; end{emit @sum}'` or `mlr put -q '@sum[$a][$b] += $x; end{emit @sum, "a", "b"}'`.
-* **JSON null**: (TODO: describe)
+* **JSON null**: The main purpose of this is to support reading the `null` type in JSON files. The [Miller programming language](programming-language.md) has a `null` keyword as well, so you can also write the null type using `$x = null`. Addtionally, though, when you write past the end of an array, leaving gaps -- e.g. writing `a[12]` when the array `a` has length 10 -- JSON-null is used to fill the gaps. See also the [arrays page](reference-main-arrays.md#auto-extend-and-null-gaps).
You can test these programatically using the functions `is_empty`/`is_not_empty`, `is_absent`/`is_present`, and `is_null`/`is_not_null`. For the last pair, note that null means either empty or absent.
diff --git a/docs6/docs/reference-main-strings.md b/docs6/docs/reference-main-strings.md
new file mode 100644
index 000000000..ec54fc11a
--- /dev/null
+++ b/docs6/docs/reference-main-strings.md
@@ -0,0 +1,52 @@
+
+
+# Strings
+
+TODO
+
+xxx concat
+
+xxx index and slice 1-up
+
+xxx lib functions
+
+always double-quote
+
+single-quote for shell; see windows page
+
+dot operator ...
+
+## Escape sequences for string literals
+
+You can use the following backslash escapes for strings such as between the double quotes in contexts such as `mlr filter '$name =~ "..."'`, `mlr put '$name = $othername . "..."'`, `mlr put '$name = sub($name, "...", "...")`, etc.:
+
+* `\a`: ASCII code 0x07 (alarm/bell)
+* `\b`: ASCII code 0x08 (backspace)
+* `\f`: ASCII code 0x0c (formfeed)
+* `\n`: ASCII code 0x0a (LF/linefeed/newline)
+* `\r`: ASCII code 0x0d (CR/carriage return)
+* `\t`: ASCII code 0x09 (tab)
+* `\v`: ASCII code 0x0b (vertical tab)
+* `\\`: backslash
+* `\"`: double quote
+* `\123`: Octal 123, etc. for `\000` up to `\377`
+* `\x7f`: Hexadecimal 7f, etc. for `\x00` up to `\xff`
+
+See also [https://en.wikipedia.org/wiki/Escape_sequences_in_C](https://en.wikipedia.org/wiki/Escape_sequences_in_C).
+
+These replacements apply only to strings you key in for the DSL expressions for `filter` and `put`: that is, if you type `\t` in a string literal for a `filter`/`put` expression, it will be turned into a tab character. If you want a backslash followed by a `t`, then please type `\\t`.
+
+However, these replacements are done automatically only for string literals within DSL expressions -- they are not done automatically to fields within your data stream. If you wish to make these replacements, you can do (for example) `mlr put '$field = gsub($field, "\\t", "\t")'`. If you need to make such a replacement for all fields in your data, you should probably use the system `sed` command instead.
+
diff --git a/docs6/docs/reference-main-strings.md.in b/docs6/docs/reference-main-strings.md.in
new file mode 100644
index 000000000..453983b2b
--- /dev/null
+++ b/docs6/docs/reference-main-strings.md.in
@@ -0,0 +1,38 @@
+# Strings
+
+TODO
+
+xxx concat
+
+xxx index and slice 1-up
+
+xxx lib functions
+
+always double-quote
+
+single-quote for shell; see windows page
+
+dot operator ...
+
+## Escape sequences for string literals
+
+You can use the following backslash escapes for strings such as between the double quotes in contexts such as `mlr filter '$name =~ "..."'`, `mlr put '$name = $othername . "..."'`, `mlr put '$name = sub($name, "...", "...")`, etc.:
+
+* `\a`: ASCII code 0x07 (alarm/bell)
+* `\b`: ASCII code 0x08 (backspace)
+* `\f`: ASCII code 0x0c (formfeed)
+* `\n`: ASCII code 0x0a (LF/linefeed/newline)
+* `\r`: ASCII code 0x0d (CR/carriage return)
+* `\t`: ASCII code 0x09 (tab)
+* `\v`: ASCII code 0x0b (vertical tab)
+* `\\`: backslash
+* `\"`: double quote
+* `\123`: Octal 123, etc. for `\000` up to `\377`
+* `\x7f`: Hexadecimal 7f, etc. for `\x00` up to `\xff`
+
+See also [https://en.wikipedia.org/wiki/Escape_sequences_in_C](https://en.wikipedia.org/wiki/Escape_sequences_in_C).
+
+These replacements apply only to strings you key in for the DSL expressions for `filter` and `put`: that is, if you type `\t` in a string literal for a `filter`/`put` expression, it will be turned into a tab character. If you want a backslash followed by a `t`, then please type `\\t`.
+
+However, these replacements are done automatically only for string literals within DSL expressions -- they are not done automatically to fields within your data stream. If you wish to make these replacements, you can do (for example) `mlr put '$field = gsub($field, "\\t", "\t")'`. If you need to make such a replacement for all fields in your data, you should probably use the system `sed` command instead.
+
diff --git a/docs6/docs/reference-verbs.md b/docs6/docs/reference-verbs.md
index 6734d0fcb..2102b8fd7 100644
--- a/docs6/docs/reference-verbs.md
+++ b/docs6/docs/reference-verbs.md
@@ -1639,6 +1639,34 @@ left_a left_b left_c right_a right_b right_c
1 4 5 1 4 5
+## json-parse
+
+
+mlr json-parse --help
+
+
+Usage: mlr json-parse [options]
+Tries to convert string field values to parsed JSON, e.g. "[1,2,3]" -> [1,2,3].
+Options:
+-f {...} Comma-separated list of field names to json-parse (default all).
+-h|--help Show this message.
+
+
+## json-stringify
+
+
+mlr json-stringify --help
+
+
+Usage: mlr json-stringify [options]
+Produces string field values from field-value data, e.g. [1,2,3] -> "[1,2,3]".
+Options:
+-f {...} Comma-separated list of field names to json-parse (default all).
+--jvstack Produce multi-line JSON output.
+--no-jvstack Produce single-line JSON output per record (default).
+-h|--help Show this message.
+
+
## label
diff --git a/docs6/docs/reference-verbs.md.in b/docs6/docs/reference-verbs.md.in
index f24c67737..cadb0fa8b 100644
--- a/docs6/docs/reference-verbs.md.in
+++ b/docs6/docs/reference-verbs.md.in
@@ -548,6 +548,18 @@ GENMD_RUN_COMMAND
mlr --csvlite --opprint join -j "" --lp left_ --rp right_ -f data/self-join.csv data/self-join.csv
GENMD_EOF
+## json-parse
+
+GENMD_RUN_COMMAND
+mlr json-parse --help
+GENMD_EOF
+
+## json-stringify
+
+GENMD_RUN_COMMAND
+mlr json-stringify --help
+GENMD_EOF
+
## label
GENMD_RUN_COMMAND
diff --git a/docs6/docs/repl.md b/docs6/docs/repl.md
index 3cb2e90a4..61bf3c42e 100644
--- a/docs6/docs/repl.md
+++ b/docs6/docs/repl.md
@@ -48,7 +48,7 @@ HELLO
{
"color": "yellow",
"shape": "triangle",
- "flag": true,
+ "flag": "true",
"k": 1,
"index": 11,
"quantity": 43.6498,
@@ -58,7 +58,7 @@ HELLO
{
"color": "red",
"shape": "square",
- "flag": true,
+ "flag": "true",
"k": 2,
"index": 15,
"quantity": 79.2778,
diff --git a/docs6/docs/special-symbols-and-formatting.md b/docs6/docs/special-symbols-and-formatting.md
index 44b2bad10..ea50e6ecb 100644
--- a/docs6/docs/special-symbols-and-formatting.md
+++ b/docs6/docs/special-symbols-and-formatting.md
@@ -83,8 +83,8 @@ characters as delimiters -- here, control-A:
mlr --icsv --odkvp --ofs '\001' cat commas.csv | cat -v
-Name=Xiao, Lin\001Role=administrator
-Name=Khavari, Darius\001Role=tester
+Name=Xiao, Lin^ARole=administrator
+Name=Khavari, Darius^ARole=tester
## How can I handle field names with special symbols in them?
diff --git a/docs6/mkdocs.yml b/docs6/mkdocs.yml
index ccab3e8ac..c9cdcf3a0 100644
--- a/docs6/mkdocs.yml
+++ b/docs6/mkdocs.yml
@@ -7,6 +7,9 @@ theme:
text: Lato
code: Lato Mono
extra_css: ['extra.css']
+extra_javascript:
+ - https://cdnjs.cloudflare.com/ajax/libs/tablesort/5.2.1/tablesort.min.js
+ - javascripts/tables.js
nav:
- "Introduction": "index.md"
- 'Getting started':
@@ -65,13 +68,16 @@ nav:
- "Streaming processing, and memory usage": "streaming-and-memory.md"
- "CPU/multicore usage": "cpu.md"
- "Data types": "reference-main-data-types.md"
+ - "Strings": "reference-main-strings.md"
+ - "Regular expressions": "reference-main-regular-expressions.md"
- "Arithmetic": "reference-main-arithmetic.md"
- "Maps": "reference-main-maps.md"
- "Arrays": "reference-main-arrays.md"
- "Null/empty/absent data": "reference-main-null-data.md"
- - "Regular expressions": "reference-main-regular-expressions.md"
- "Flatten/unflatten: JSON vs. tabular formats": "flatten-unflatten.md"
- "Compressed data": "reference-main-compressed-data.md"
+ - "Streaming processing, and memory usage": "streaming-and-memory.md"
+ - "CPU/multicore usage": "cpu.md"
- "Miller environment variables": "reference-main-env-vars.md"
- 'DSL reference':
- "DSL overview": "reference-dsl.md"
diff --git a/go/regtest/cases/dsl-min-max-types/0001/expout b/go/regtest/cases/dsl-min-max-types/0001/expout
index 6e03b4091..e021b5638 100644
--- a/go/regtest/cases/dsl-min-max-types/0001/expout
+++ b/go/regtest/cases/dsl-min-max-types/0001/expout
@@ -1,6 +1,6 @@
{
"n": 1,
- "b": true,
+ "b": "true",
"v": "",
"s": "abc",
"min": {
@@ -12,19 +12,19 @@
},
"b": {
"n": 1,
- "b": true,
- "v": true,
- "s": true
+ "b": "true",
+ "v": "",
+ "s": "abc"
},
"v": {
"n": 1,
- "b": true,
+ "b": "",
"v": "",
"s": ""
},
"s": {
"n": 1,
- "b": true,
+ "b": "abc",
"v": "",
"s": "abc"
}
diff --git a/go/regtest/cases/dsl-min-max-types/0002/expout b/go/regtest/cases/dsl-min-max-types/0002/expout
index 34f50b766..4c4057af9 100644
--- a/go/regtest/cases/dsl-min-max-types/0002/expout
+++ b/go/regtest/cases/dsl-min-max-types/0002/expout
@@ -1,30 +1,30 @@
{
"n": 1,
- "b": true,
+ "b": "true",
"v": "",
"s": "abc",
"max": {
"n": {
"n": 1,
- "b": true,
+ "b": "true",
"v": "",
"s": "abc"
},
"b": {
- "n": true,
- "b": true,
- "v": "",
- "s": "abc"
+ "n": "true",
+ "b": "true",
+ "v": "true",
+ "s": "true"
},
"v": {
"n": "",
- "b": "",
+ "b": "true",
"v": "",
"s": "abc"
},
"s": {
"n": "abc",
- "b": "abc",
+ "b": "true",
"v": "abc",
"s": "abc"
}
diff --git a/go/regtest/cases/dsl-min-max-types/0003/expout b/go/regtest/cases/dsl-min-max-types/0003/expout
index 714c9de22..bd0cd7c8f 100644
--- a/go/regtest/cases/dsl-min-max-types/0003/expout
+++ b/go/regtest/cases/dsl-min-max-types/0003/expout
@@ -1,12 +1,12 @@
{
"n": 1,
- "b": true,
+ "b": "true",
"v": "",
"s": "abc",
"le": {
"n": {
"n": true,
- "b": false,
+ "b": true,
"v": false,
"s": true
},
@@ -18,13 +18,13 @@
},
"v": {
"n": true,
- "b": false,
+ "b": true,
"v": true,
"s": true
},
"s": {
"n": false,
- "b": false,
+ "b": true,
"v": false,
"s": true
}
diff --git a/go/regtest/cases/dsl-min-max-types/0004/expout b/go/regtest/cases/dsl-min-max-types/0004/expout
index 1575d57b5..2e62305c0 100644
--- a/go/regtest/cases/dsl-min-max-types/0004/expout
+++ b/go/regtest/cases/dsl-min-max-types/0004/expout
@@ -1,6 +1,6 @@
{
"n": 1,
- "b": true,
+ "b": "true",
"v": "",
"s": "abc",
"ge": {
@@ -11,10 +11,10 @@
"s": false
},
"b": {
- "n": false,
+ "n": true,
"b": true,
- "v": false,
- "s": false
+ "v": true,
+ "s": true
},
"v": {
"n": false,
diff --git a/go/regtest/cases/dsl-type-inference/0001/cmd b/go/regtest/cases/dsl-type-inference/0001/cmd
index 9ace894e0..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0001/cmd
+++ b/go/regtest/cases/dsl-type-inference/0001/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0002/cmd b/go/regtest/cases/dsl-type-inference/0002/cmd
index 954d105b9..88c4ca46a 100644
--- a/go/regtest/cases/dsl-type-inference/0002/cmd
+++ b/go/regtest/cases/dsl-type-inference/0002/cmd
@@ -1 +1 @@
-mlr --xtab filter -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab filter -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0003/cmd b/go/regtest/cases/dsl-type-inference/0003/cmd
index 637fd26d9..f1e9713c5 100644
--- a/go/regtest/cases/dsl-type-inference/0003/cmd
+++ b/go/regtest/cases/dsl-type-inference/0003/cmd
@@ -1 +1 @@
-mlr --oxtab put -f ${CASEDIR}/mlr ./${CASEDIR}/input
+mlr --oxtab put -f ${CASEDIR}/mlr ./${CASEDIR}/input
diff --git a/go/regtest/cases/dsl-type-inference/0004/cmd b/go/regtest/cases/dsl-type-inference/0004/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0004/cmd
+++ b/go/regtest/cases/dsl-type-inference/0004/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0005/cmd b/go/regtest/cases/dsl-type-inference/0005/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0005/cmd
+++ b/go/regtest/cases/dsl-type-inference/0005/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0006/cmd b/go/regtest/cases/dsl-type-inference/0006/cmd
index 2872eeeb8..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0006/cmd
+++ b/go/regtest/cases/dsl-type-inference/0006/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0007/cmd b/go/regtest/cases/dsl-type-inference/0007/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0007/cmd
+++ b/go/regtest/cases/dsl-type-inference/0007/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0008/cmd b/go/regtest/cases/dsl-type-inference/0008/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0008/cmd
+++ b/go/regtest/cases/dsl-type-inference/0008/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0009/cmd b/go/regtest/cases/dsl-type-inference/0009/cmd
index 2872eeeb8..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0009/cmd
+++ b/go/regtest/cases/dsl-type-inference/0009/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0012/cmd b/go/regtest/cases/dsl-type-inference/0012/cmd
index da804817c..cef492ace 100644
--- a/go/regtest/cases/dsl-type-inference/0012/cmd
+++ b/go/regtest/cases/dsl-type-inference/0012/cmd
@@ -1 +1 @@
-mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0015/cmd b/go/regtest/cases/dsl-type-inference/0015/cmd
index da804817c..cef492ace 100644
--- a/go/regtest/cases/dsl-type-inference/0015/cmd
+++ b/go/regtest/cases/dsl-type-inference/0015/cmd
@@ -1 +1 @@
-mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0016/cmd b/go/regtest/cases/dsl-type-inference/0016/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0016/cmd
+++ b/go/regtest/cases/dsl-type-inference/0016/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0017/cmd b/go/regtest/cases/dsl-type-inference/0017/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0017/cmd
+++ b/go/regtest/cases/dsl-type-inference/0017/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0018/cmd b/go/regtest/cases/dsl-type-inference/0018/cmd
index 2872eeeb8..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0018/cmd
+++ b/go/regtest/cases/dsl-type-inference/0018/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0019/cmd b/go/regtest/cases/dsl-type-inference/0019/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0019/cmd
+++ b/go/regtest/cases/dsl-type-inference/0019/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0020/cmd b/go/regtest/cases/dsl-type-inference/0020/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0020/cmd
+++ b/go/regtest/cases/dsl-type-inference/0020/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0021/cmd b/go/regtest/cases/dsl-type-inference/0021/cmd
index 2872eeeb8..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0021/cmd
+++ b/go/regtest/cases/dsl-type-inference/0021/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0024/cmd b/go/regtest/cases/dsl-type-inference/0024/cmd
index da804817c..cef492ace 100644
--- a/go/regtest/cases/dsl-type-inference/0024/cmd
+++ b/go/regtest/cases/dsl-type-inference/0024/cmd
@@ -1 +1 @@
-mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0027/cmd b/go/regtest/cases/dsl-type-inference/0027/cmd
index da804817c..cef492ace 100644
--- a/go/regtest/cases/dsl-type-inference/0027/cmd
+++ b/go/regtest/cases/dsl-type-inference/0027/cmd
@@ -1 +1 @@
-mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0028/cmd b/go/regtest/cases/dsl-type-inference/0028/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0028/cmd
+++ b/go/regtest/cases/dsl-type-inference/0028/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0029/cmd b/go/regtest/cases/dsl-type-inference/0029/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0029/cmd
+++ b/go/regtest/cases/dsl-type-inference/0029/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0030/cmd b/go/regtest/cases/dsl-type-inference/0030/cmd
index 2872eeeb8..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0030/cmd
+++ b/go/regtest/cases/dsl-type-inference/0030/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0031/cmd b/go/regtest/cases/dsl-type-inference/0031/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0031/cmd
+++ b/go/regtest/cases/dsl-type-inference/0031/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0032/cmd b/go/regtest/cases/dsl-type-inference/0032/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0032/cmd
+++ b/go/regtest/cases/dsl-type-inference/0032/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0033/cmd b/go/regtest/cases/dsl-type-inference/0033/cmd
index 2872eeeb8..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0033/cmd
+++ b/go/regtest/cases/dsl-type-inference/0033/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0036/cmd b/go/regtest/cases/dsl-type-inference/0036/cmd
index da804817c..cef492ace 100644
--- a/go/regtest/cases/dsl-type-inference/0036/cmd
+++ b/go/regtest/cases/dsl-type-inference/0036/cmd
@@ -1 +1 @@
-mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0039/cmd b/go/regtest/cases/dsl-type-inference/0039/cmd
index da804817c..cef492ace 100644
--- a/go/regtest/cases/dsl-type-inference/0039/cmd
+++ b/go/regtest/cases/dsl-type-inference/0039/cmd
@@ -1 +1 @@
-mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0040/cmd b/go/regtest/cases/dsl-type-inference/0040/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0040/cmd
+++ b/go/regtest/cases/dsl-type-inference/0040/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0041/cmd b/go/regtest/cases/dsl-type-inference/0041/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0041/cmd
+++ b/go/regtest/cases/dsl-type-inference/0041/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0042/cmd b/go/regtest/cases/dsl-type-inference/0042/cmd
index 2872eeeb8..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0042/cmd
+++ b/go/regtest/cases/dsl-type-inference/0042/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0043/cmd b/go/regtest/cases/dsl-type-inference/0043/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0043/cmd
+++ b/go/regtest/cases/dsl-type-inference/0043/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0044/cmd b/go/regtest/cases/dsl-type-inference/0044/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0044/cmd
+++ b/go/regtest/cases/dsl-type-inference/0044/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0045/cmd b/go/regtest/cases/dsl-type-inference/0045/cmd
index 2872eeeb8..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0045/cmd
+++ b/go/regtest/cases/dsl-type-inference/0045/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0048/cmd b/go/regtest/cases/dsl-type-inference/0048/cmd
index da804817c..cef492ace 100644
--- a/go/regtest/cases/dsl-type-inference/0048/cmd
+++ b/go/regtest/cases/dsl-type-inference/0048/cmd
@@ -1 +1 @@
-mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0051/cmd b/go/regtest/cases/dsl-type-inference/0051/cmd
index da804817c..cef492ace 100644
--- a/go/regtest/cases/dsl-type-inference/0051/cmd
+++ b/go/regtest/cases/dsl-type-inference/0051/cmd
@@ -1 +1 @@
-mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0052/cmd b/go/regtest/cases/dsl-type-inference/0052/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0052/cmd
+++ b/go/regtest/cases/dsl-type-inference/0052/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0053/cmd b/go/regtest/cases/dsl-type-inference/0053/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0053/cmd
+++ b/go/regtest/cases/dsl-type-inference/0053/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0054/cmd b/go/regtest/cases/dsl-type-inference/0054/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0054/cmd
+++ b/go/regtest/cases/dsl-type-inference/0054/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0055/cmd b/go/regtest/cases/dsl-type-inference/0055/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0055/cmd
+++ b/go/regtest/cases/dsl-type-inference/0055/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0060/cmd b/go/regtest/cases/dsl-type-inference/0060/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0060/cmd
+++ b/go/regtest/cases/dsl-type-inference/0060/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0061/cmd b/go/regtest/cases/dsl-type-inference/0061/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0061/cmd
+++ b/go/regtest/cases/dsl-type-inference/0061/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0062/cmd b/go/regtest/cases/dsl-type-inference/0062/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0062/cmd
+++ b/go/regtest/cases/dsl-type-inference/0062/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0063/cmd b/go/regtest/cases/dsl-type-inference/0063/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0063/cmd
+++ b/go/regtest/cases/dsl-type-inference/0063/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0068/cmd b/go/regtest/cases/dsl-type-inference/0068/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0068/cmd
+++ b/go/regtest/cases/dsl-type-inference/0068/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0069/cmd b/go/regtest/cases/dsl-type-inference/0069/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0069/cmd
+++ b/go/regtest/cases/dsl-type-inference/0069/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0070/cmd b/go/regtest/cases/dsl-type-inference/0070/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0070/cmd
+++ b/go/regtest/cases/dsl-type-inference/0070/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0071/cmd b/go/regtest/cases/dsl-type-inference/0071/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0071/cmd
+++ b/go/regtest/cases/dsl-type-inference/0071/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0076/cmd b/go/regtest/cases/dsl-type-inference/0076/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0076/cmd
+++ b/go/regtest/cases/dsl-type-inference/0076/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0077/cmd b/go/regtest/cases/dsl-type-inference/0077/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0077/cmd
+++ b/go/regtest/cases/dsl-type-inference/0077/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0078/cmd b/go/regtest/cases/dsl-type-inference/0078/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0078/cmd
+++ b/go/regtest/cases/dsl-type-inference/0078/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0079/cmd b/go/regtest/cases/dsl-type-inference/0079/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0079/cmd
+++ b/go/regtest/cases/dsl-type-inference/0079/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0084/cmd b/go/regtest/cases/dsl-type-inference/0084/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0084/cmd
+++ b/go/regtest/cases/dsl-type-inference/0084/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0085/cmd b/go/regtest/cases/dsl-type-inference/0085/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0085/cmd
+++ b/go/regtest/cases/dsl-type-inference/0085/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0086/cmd b/go/regtest/cases/dsl-type-inference/0086/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0086/cmd
+++ b/go/regtest/cases/dsl-type-inference/0086/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0087/cmd b/go/regtest/cases/dsl-type-inference/0087/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0087/cmd
+++ b/go/regtest/cases/dsl-type-inference/0087/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0092/cmd b/go/regtest/cases/dsl-type-inference/0092/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0092/cmd
+++ b/go/regtest/cases/dsl-type-inference/0092/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0093/cmd b/go/regtest/cases/dsl-type-inference/0093/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0093/cmd
+++ b/go/regtest/cases/dsl-type-inference/0093/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0094/cmd b/go/regtest/cases/dsl-type-inference/0094/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0094/cmd
+++ b/go/regtest/cases/dsl-type-inference/0094/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0095/cmd b/go/regtest/cases/dsl-type-inference/0095/cmd
index 2f51026ca..6c932f0bf 100644
--- a/go/regtest/cases/dsl-type-inference/0095/cmd
+++ b/go/regtest/cases/dsl-type-inference/0095/cmd
@@ -1 +1 @@
-mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
+mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab
diff --git a/go/regtest/cases/dsl-type-inference/0097/cmd b/go/regtest/cases/dsl-type-inference/0097/cmd
new file mode 100644
index 000000000..77144583a
--- /dev/null
+++ b/go/regtest/cases/dsl-type-inference/0097/cmd
@@ -0,0 +1 @@
+mlr --idkvp --opprint put -f ${CASEDIR}/mlr ./${CASEDIR}/input
diff --git a/go/regtest/cases/dsl-type-inference/0097/experr b/go/regtest/cases/dsl-type-inference/0097/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/go/regtest/cases/dsl-type-inference/0097/expout b/go/regtest/cases/dsl-type-inference/0097/expout
new file mode 100644
index 000000000..9c0def3cf
--- /dev/null
+++ b/go/regtest/cases/dsl-type-inference/0097/expout
@@ -0,0 +1,33 @@
+a ta
+12 int
+1.2 float
+inf string
++inf string
+-inf string
+Inf string
++Inf string
+-Inf string
+INF string
++INF string
+-INF string
+infinity string
++infinity string
+-infinity string
+Infinity string
++Infinity string
+-Infinity string
+INFINITY string
++INFINITY string
+-INFINITY string
+InFiNiTy string
++InFiNiTy string
+-InFiNiTy string
+infiniti string
++infiniti string
+-infiniti string
+true string
+false string
+True string
+False string
+TRUE string
+FALSE string
diff --git a/go/regtest/cases/dsl-type-inference/0097/input b/go/regtest/cases/dsl-type-inference/0097/input
new file mode 100644
index 000000000..b4e84a1b5
--- /dev/null
+++ b/go/regtest/cases/dsl-type-inference/0097/input
@@ -0,0 +1,32 @@
+a=12
+a=1.2
+a=inf
+a=+inf
+a=-inf
+a=Inf
+a=+Inf
+a=-Inf
+a=INF
+a=+INF
+a=-INF
+a=infinity
+a=+infinity
+a=-infinity
+a=Infinity
+a=+Infinity
+a=-Infinity
+a=INFINITY
+a=+INFINITY
+a=-INFINITY
+a=InFiNiTy
+a=+InFiNiTy
+a=-InFiNiTy
+a=infiniti
+a=+infiniti
+a=-infiniti
+a=true
+a=false
+a=True
+a=False
+a=TRUE
+a=FALSE
diff --git a/go/regtest/cases/dsl-type-inference/0097/mlr b/go/regtest/cases/dsl-type-inference/0097/mlr
new file mode 100644
index 000000000..7f1766894
--- /dev/null
+++ b/go/regtest/cases/dsl-type-inference/0097/mlr
@@ -0,0 +1 @@
+$ta = typeof($a)
diff --git a/go/src/auxents/help/entry.go b/go/src/auxents/help/entry.go
index 7d32cfa16..f3c024195 100644
--- a/go/src/auxents/help/entry.go
+++ b/go/src/auxents/help/entry.go
@@ -65,7 +65,10 @@ func init() {
{name: "function", unaryHandlerFunc: helpForFunction},
{name: "keyword", unaryHandlerFunc: helpForKeyword},
{name: "list-functions", zaryHandlerFunc: listFunctions},
+ {name: "list-function-classes", zaryHandlerFunc: listFunctionClasses},
+ {name: "list-functions-in-class", unaryHandlerFunc: listFunctionsInClass},
{name: "list-functions-as-paragraph", zaryHandlerFunc: listFunctionsAsParagraph},
+ {name: "list-functions-as-table", zaryHandlerFunc: listFunctionsAsTable},
{name: "list-keywords", zaryHandlerFunc: listKeywords},
{name: "list-keywords-as-paragraph", zaryHandlerFunc: listKeywordsAsParagraph},
{name: "list-verbs", zaryHandlerFunc: listVerbs},
@@ -77,6 +80,7 @@ func init() {
{name: "separator-options", zaryHandlerFunc: helpSeparatorOptions},
{name: "type-arithmetic-info", zaryHandlerFunc: helpTypeArithmeticInfo},
{name: "usage-functions", zaryHandlerFunc: usageFunctions},
+ {name: "usage-functions-by-class", zaryHandlerFunc: usageFunctionsByClass},
{name: "usage-keywords", zaryHandlerFunc: usageKeywords},
{name: "usage-verbs", zaryHandlerFunc: usageVerbs},
{name: "verb", unaryHandlerFunc: helpForVerb},
@@ -693,22 +697,38 @@ func helpTypeArithmeticInfo() {
// ----------------------------------------------------------------
func listFunctions() {
if isatty.IsTerminal(os.Stdout.Fd()) {
- cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesAsParagraph(os.Stdout)
+ cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesAsParagraph()
} else {
- cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesVertically(os.Stdout)
+ cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesVertically()
}
}
+func listFunctionClasses() {
+ cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionClasses()
+}
+
+func listFunctionsInClass(class string) {
+ cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionsInClass(class)
+}
+
func listFunctionsAsParagraph() {
- cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesAsParagraph(os.Stdout)
+ cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesAsParagraph()
+}
+
+func listFunctionsAsTable() {
+ cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionsAsTable()
}
func usageFunctions() {
cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionUsages()
}
+func usageFunctionsByClass() {
+ cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionUsagesByClass()
+}
+
func helpForFunction(arg string) {
- cst.BuiltinFunctionManagerInstance.TryListBuiltinFunctionUsage(arg, os.Stdout)
+ cst.BuiltinFunctionManagerInstance.TryListBuiltinFunctionUsage(arg)
}
// ----------------------------------------------------------------
diff --git a/go/src/auxents/repl/verbs.go b/go/src/auxents/repl/verbs.go
index e79e0a7c9..74cd483ab 100644
--- a/go/src/auxents/repl/verbs.go
+++ b/go/src/auxents/repl/verbs.go
@@ -855,7 +855,7 @@ func handleHelpSingle(repl *Repl, arg string) {
}
if arg == "function-names" {
- cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesAsParagraph(os.Stdout)
+ cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesAsParagraph()
return
}
@@ -868,7 +868,7 @@ func handleHelpSingle(repl *Repl, arg string) {
return
}
- if cst.BuiltinFunctionManagerInstance.TryListBuiltinFunctionUsage(arg, os.Stdout) {
+ if cst.BuiltinFunctionManagerInstance.TryListBuiltinFunctionUsage(arg) {
return
}
diff --git a/go/src/cliutil/mlrcli_util.go b/go/src/cliutil/mlrcli_util.go
index 4e126eb3c..89ad87f17 100644
--- a/go/src/cliutil/mlrcli_util.go
+++ b/go/src/cliutil/mlrcli_util.go
@@ -3,6 +3,8 @@ package cliutil
import (
"fmt"
"os"
+
+ "mlr/src/lib"
)
// For flags with values, e.g. ["-n" "10"], while we're looking at the "-n"
@@ -39,16 +41,8 @@ var SEPARATOR_NAMES_TO_VALUES = map[string]string{
func SeparatorFromArg(name string) string {
sep, ok := SEPARATOR_NAMES_TO_VALUES[name]
if !ok {
- // xxx temp
- //fmt.Fprintf(os.Stderr, "Miller: could not handle separator \"%s\".\n", name)
- //os.Exit(1)
- // It's OK if they do '--ifs ,' -- just pass it back.
- return name
+ // "\001" -> control-A, etc.
+ return lib.UnbackslashStringLiteral(name)
}
return sep
- // char* chars = lhmss_get(get_desc_to_chars_map(), arg);
- // if (chars != nil) // E.g. crlf
- // return mlr_strdup_or_die(chars);
- // else // E.g. '\r\n'
- // return mlr_alloc_unbackslash(arg);
}
diff --git a/go/src/dsl/cst/builtin_function_manager.go b/go/src/dsl/cst/builtin_function_manager.go
index 635ae84e5..e0adf57ca 100644
--- a/go/src/dsl/cst/builtin_function_manager.go
+++ b/go/src/dsl/cst/builtin_function_manager.go
@@ -27,7 +27,7 @@ const (
FUNC_CLASS_HASHING = "hashing"
FUNC_CLASS_CONVERSION = "conversion"
FUNC_CLASS_TYPING = "typing"
- FUNC_CLASS_COLLECTIONS = "maps/arrays"
+ FUNC_CLASS_COLLECTIONS = "collections"
FUNC_CLASS_SYSTEM = "system"
FUNC_CLASS_TIME = "time"
)
@@ -470,22 +470,22 @@ regex-match operator: try '$y = ~$x'.`,
name: "substr0",
class: FUNC_CLASS_STRING,
help: `substr0(s,m,n) gives substring of s from 0-up position m to n
-inclusive. Negative indices -len .. -1 alias to 0 .. len-1.`,
+inclusive. Negative indices -len .. -1 alias to 0 .. len-1. See also substr and substr1.`,
ternaryFunc: types.MlrvalSubstr0Up,
},
{
name: "substr1",
class: FUNC_CLASS_STRING,
help: `substr1(s,m,n) gives substring of s from 1-up position m to n
-inclusive. Negative indices -len .. -1 alias to 1 .. len.`,
+inclusive. Negative indices -len .. -1 alias to 1 .. len. See also substr and substr0.`,
ternaryFunc: types.MlrvalSubstr1Up,
},
{
name: "substr",
class: FUNC_CLASS_STRING,
help: `substr is an alias for substr0. See also substr1. Miller is generally 1-up
-with all array indices, but, this is a backward-compatibility issue with Miller 5 and below.
-Arrays are new in Miller 6; the substr function is older.`,
+with all array and string indices, but, this is a backward-compatibility issue with Miller 5
+and below. Arrays are new in Miller 6; the substr function is older.`,
ternaryFunc: types.MlrvalSubstr0Up,
},
@@ -1591,19 +1591,62 @@ func hashifyLookupTable(lookupTable *[]BuiltinFunctionInfo) map[string]*BuiltinF
}
// ----------------------------------------------------------------
-func (manager *BuiltinFunctionManager) ListBuiltinFunctionNamesVertically(o *os.File) {
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionClasses() {
+ classesList := manager.getBuiltinFunctionClasses()
+ for _, class := range classesList {
+ fmt.Println(class)
+ }
+}
+
+func (manager *BuiltinFunctionManager) getBuiltinFunctionClasses() []string {
+ classesSeen := make(map[string]bool)
+ classesList := make([]string, 0)
for _, builtinFunctionInfo := range *manager.lookupTable {
- fmt.Fprintln(o, builtinFunctionInfo.name)
+ class := string(builtinFunctionInfo.class)
+ if classesSeen[class] == false {
+ classesList = append(classesList, class)
+ classesSeen[class] = true
+ }
+ }
+ sort.Strings(classesList)
+ return classesList
+}
+
+// ----------------------------------------------------------------
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionsInClass(class string) {
+ for _, builtinFunctionInfo := range *manager.lookupTable {
+ if string(builtinFunctionInfo.class) == class {
+ fmt.Println(builtinFunctionInfo.name)
+ }
}
}
// ----------------------------------------------------------------
-func (manager *BuiltinFunctionManager) ListBuiltinFunctionNamesAsParagraph(o *os.File) {
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionNamesVertically() {
+ for _, builtinFunctionInfo := range *manager.lookupTable {
+ fmt.Println(builtinFunctionInfo.name)
+ }
+}
+
+// ----------------------------------------------------------------
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionNamesAsParagraph() {
functionNames := make([]string, len(*manager.lookupTable))
for i, builtinFunctionInfo := range *manager.lookupTable {
functionNames[i] = builtinFunctionInfo.name
}
- lib.PrintWordsAsParagraph(functionNames, o)
+ lib.PrintWordsAsParagraph(functionNames)
+}
+
+// ----------------------------------------------------------------
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionsAsTable() {
+ fmt.Printf("%-30s %-12s %s\n", "Name", "Class", "Args")
+ for _, builtinFunctionInfo := range *manager.lookupTable {
+ fmt.Printf("%-30s %-12s %s\n",
+ builtinFunctionInfo.name,
+ builtinFunctionInfo.class,
+ describeNargs(&builtinFunctionInfo),
+ )
+ }
}
// ----------------------------------------------------------------
@@ -1622,13 +1665,37 @@ func (manager *BuiltinFunctionManager) ListBuiltinFunctionUsages() {
}
}
-func (manager *BuiltinFunctionManager) ListBuiltinFunctionUsage(functionName string, o *os.File) {
- if !manager.TryListBuiltinFunctionUsage(functionName, o) {
+// ----------------------------------------------------------------
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionUsagesByClass() {
+ classesList := manager.getBuiltinFunctionClasses()
+
+ for _, class := range classesList {
+ fmt.Println()
+ fmt.Println(colorizer.MaybeColorizeHelp(strings.ToUpper(class), true))
+ fmt.Println()
+ for _, builtinFunctionInfo := range *manager.lookupTable {
+ if string(builtinFunctionInfo.class) != class {
+ continue
+ }
+ lib.InternalCodingErrorIf(builtinFunctionInfo.help == "")
+ fmt.Print(colorizer.MaybeColorizeHelp(builtinFunctionInfo.name, true))
+ fmt.Printf(" (class=%s #args=%s) %s\n",
+ builtinFunctionInfo.class,
+ describeNargs(&builtinFunctionInfo),
+ builtinFunctionInfo.help,
+ )
+ fmt.Println()
+ }
+ }
+}
+
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionUsage(functionName string) {
+ if !manager.TryListBuiltinFunctionUsage(functionName) {
fmt.Fprintf(os.Stderr, "Function \"%s\" not found.\n", functionName)
}
}
-func (manager *BuiltinFunctionManager) TryListBuiltinFunctionUsage(functionName string, o *os.File) bool {
+func (manager *BuiltinFunctionManager) TryListBuiltinFunctionUsage(functionName string) bool {
builtinFunctionInfo := manager.LookUp(functionName)
if builtinFunctionInfo == nil {
manager.listBuiltinFunctionUsageApproximate(functionName)
diff --git a/go/src/dsl/cst/keyword_usage.go b/go/src/dsl/cst/keyword_usage.go
index e9fc107c5..77ca6a453 100644
--- a/go/src/dsl/cst/keyword_usage.go
+++ b/go/src/dsl/cst/keyword_usage.go
@@ -2,7 +2,6 @@ package cst
import (
"fmt"
- "os"
"mlr/src/colorizer"
"mlr/src/lib"
@@ -114,7 +113,7 @@ func ListKeywordsAsParagraph() {
for i, entry := range KEYWORD_USAGE_TABLE {
keywords[i] = entry.name
}
- lib.PrintWordsAsParagraph(keywords, os.Stdout)
+ lib.PrintWordsAsParagraph(keywords)
}
// ----------------------------------------------------------------
diff --git a/go/src/input/record_reader_csv.go b/go/src/input/record_reader_csv.go
index 8e3809fdb..0cc3a719b 100644
--- a/go/src/input/record_reader_csv.go
+++ b/go/src/input/record_reader_csv.go
@@ -140,7 +140,7 @@ func (reader *RecordReaderCSV) processHandle(
if nh == nd {
for i := 0; i < nh; i++ {
key := header[i]
- value := types.MlrvalPointerFromInferredType(csvRecord[i])
+ value := types.MlrvalPointerFromInferredTypeForDataFiles(csvRecord[i])
record.PutReference(key, value)
}
@@ -160,13 +160,13 @@ func (reader *RecordReaderCSV) processHandle(
n := lib.IntMin2(nh, nd)
for i = 0; i < n; i++ {
key := header[i]
- value := types.MlrvalPointerFromInferredType(csvRecord[i])
+ value := types.MlrvalPointerFromInferredTypeForDataFiles(csvRecord[i])
record.PutReference(key, value)
}
if nh < nd {
// if header shorter than data: use 1-up itoa keys
key := strconv.Itoa(i + 1)
- value := types.MlrvalPointerFromInferredType(csvRecord[i])
+ value := types.MlrvalPointerFromInferredTypeForDataFiles(csvRecord[i])
record.PutCopy(key, value)
}
if nh > nd {
diff --git a/go/src/input/record_reader_csvlite.go b/go/src/input/record_reader_csvlite.go
index 308d39401..03335c0f3 100644
--- a/go/src/input/record_reader_csvlite.go
+++ b/go/src/input/record_reader_csvlite.go
@@ -193,7 +193,7 @@ func (reader *RecordReaderCSVLite) processHandleExplicitCSVHeader(
record := types.NewMlrmap()
if !reader.readerOptions.AllowRaggedCSVInput {
for i, field := range fields {
- value := types.MlrvalPointerFromInferredType(field)
+ value := types.MlrvalPointerFromInferredTypeForDataFiles(field)
record.PutCopy(headerStrings[i], value)
}
} else {
@@ -202,14 +202,14 @@ func (reader *RecordReaderCSVLite) processHandleExplicitCSVHeader(
n := lib.IntMin2(nh, nd)
var i int
for i = 0; i < n; i++ {
- value := types.MlrvalPointerFromInferredType(fields[i])
+ value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i])
record.PutCopy(headerStrings[i], value)
}
if nh < nd {
// if header shorter than data: use 1-up itoa keys
for i = nh; i < nd; i++ {
key := strconv.Itoa(i + 1)
- value := types.MlrvalPointerFromInferredType(fields[i])
+ value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i])
record.PutCopy(key, value)
}
}
@@ -306,7 +306,7 @@ func (reader *RecordReaderCSVLite) processHandleImplicitCSVHeader(
record := types.NewMlrmap()
if !reader.readerOptions.AllowRaggedCSVInput {
for i, field := range fields {
- value := types.MlrvalPointerFromInferredType(field)
+ value := types.MlrvalPointerFromInferredTypeForDataFiles(field)
record.PutCopy(headerStrings[i], value)
}
} else {
@@ -315,13 +315,13 @@ func (reader *RecordReaderCSVLite) processHandleImplicitCSVHeader(
n := lib.IntMin2(nh, nd)
var i int
for i = 0; i < n; i++ {
- value := types.MlrvalPointerFromInferredType(fields[i])
+ value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i])
record.PutCopy(headerStrings[i], value)
}
if nh < nd {
// if header shorter than data: use 1-up itoa keys
key := strconv.Itoa(i + 1)
- value := types.MlrvalPointerFromInferredType(fields[i])
+ value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i])
record.PutCopy(key, value)
}
if nh > nd {
diff --git a/go/src/input/record_reader_dkvp.go b/go/src/input/record_reader_dkvp.go
index 44fdd9786..8a094d8d1 100644
--- a/go/src/input/record_reader_dkvp.go
+++ b/go/src/input/record_reader_dkvp.go
@@ -122,11 +122,11 @@ func (reader *RecordReaderDKVP) recordFromDKVPLine(
// "a=". Here we use the positional index as the key. This way
// DKVP is a generalization of NIDX.
key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up
- value := types.MlrvalPointerFromInferredType(kv[0])
+ value := types.MlrvalPointerFromInferredTypeForDataFiles(kv[0])
record.PutReference(key, value)
} else {
key := kv[0]
- value := types.MlrvalPointerFromInferredType(kv[1])
+ value := types.MlrvalPointerFromInferredTypeForDataFiles(kv[1])
record.PutReference(key, value)
}
}
diff --git a/go/src/input/record_reader_nidx.go b/go/src/input/record_reader_nidx.go
index 0ebdc4d13..6eed7d769 100644
--- a/go/src/input/record_reader_nidx.go
+++ b/go/src/input/record_reader_nidx.go
@@ -121,7 +121,7 @@ func recordFromNIDXLine(
for _, value := range values {
i++
key := strconv.Itoa(i)
- mval := types.MlrvalPointerFromInferredType(value)
+ mval := types.MlrvalPointerFromInferredTypeForDataFiles(value)
record.PutReference(key, mval)
}
return record
diff --git a/go/src/input/record_reader_xtab.go b/go/src/input/record_reader_xtab.go
index f1385d97e..276b7cf64 100644
--- a/go/src/input/record_reader_xtab.go
+++ b/go/src/input/record_reader_xtab.go
@@ -165,7 +165,7 @@ func (reader *RecordReaderXTAB) recordFromXTABLines(
value := types.MLRVAL_VOID
record.PutReference(key, value)
} else {
- value := types.MlrvalPointerFromInferredType(kv[1])
+ value := types.MlrvalPointerFromInferredTypeForDataFiles(kv[1])
record.PutReference(key, value)
}
}
diff --git a/go/src/lib/paragraph.go b/go/src/lib/paragraph.go
index 289f59486..4ca9f7409 100644
--- a/go/src/lib/paragraph.go
+++ b/go/src/lib/paragraph.go
@@ -2,12 +2,11 @@ package lib
import (
"fmt"
- "os"
)
// For online help contexts like printing all the built-in DSL functions, or
// the list of all verbs.
-func PrintWordsAsParagraph(words []string, o *os.File) {
+func PrintWordsAsParagraph(words []string) {
separator := " "
maxlen := 80
@@ -19,14 +18,14 @@ func PrintWordsAsParagraph(words []string, o *os.File) {
wordlen := len(word)
linelen += separatorlen + wordlen
if linelen >= maxlen {
- fmt.Fprintf(o, "\n")
+ fmt.Printf("\n")
linelen = separatorlen + wordlen
j = 0
}
if j > 0 {
- fmt.Fprint(o, separator)
+ fmt.Print(separator)
}
- fmt.Fprint(o, word)
+ fmt.Print(word)
j++
}
diff --git a/go/src/transformers/aaa_transformer_table.go b/go/src/transformers/aaa_transformer_table.go
index 53ee4de5b..9b395e969 100644
--- a/go/src/transformers/aaa_transformer_table.go
+++ b/go/src/transformers/aaa_transformer_table.go
@@ -96,7 +96,7 @@ func ListVerbNamesAsParagraph() {
verbNames[i] = transformerSetup.Verb
}
- lib.PrintWordsAsParagraph(verbNames, os.Stdout)
+ lib.PrintWordsAsParagraph(verbNames)
}
// ----------------------------------------------------------------
diff --git a/go/src/types/mlrval_new.go b/go/src/types/mlrval_new.go
index 68c70c3d7..d7e550650 100644
--- a/go/src/types/mlrval_new.go
+++ b/go/src/types/mlrval_new.go
@@ -6,6 +6,7 @@ package types
import (
"errors"
+ "strings"
"mlr/src/lib"
)
@@ -101,10 +102,41 @@ func MlrvalPointerFromBoolString(input string) *Mlrval {
}
}
+var downcasedFloatNamesToNotInfer = map[string]bool{
+ "inf": true,
+ "+inf": true,
+ "-inf": true,
+ "infinity": true,
+ "+infinity": true,
+ "-infinity": true,
+ "nan": true,
+}
+
+// MlrvalPointerFromInferredTypeForDataFiles is for parsing field values from
+// data files (except JSON, which is typed -- "true" and true are distinct).
+// Mostly the same as MlrvalPointerFromInferredType, except it doesn't
+// auto-infer true/false to bool; don't auto-infer NaN/Inf to float; etc.
+func MlrvalPointerFromInferredTypeForDataFiles(input string) *Mlrval {
+ if input == "" {
+ return MLRVAL_VOID
+ }
+
+ _, iok := lib.TryIntFromString(input)
+ if iok {
+ return MlrvalPointerFromIntString(input)
+ }
+
+ if downcasedFloatNamesToNotInfer[strings.ToLower(input)] == false {
+ _, fok := lib.TryFloat64FromString(input)
+ if fok {
+ return MlrvalPointerFromFloat64String(input)
+ }
+ }
+
+ return MlrvalPointerFromString(input)
+}
+
func MlrvalPointerFromInferredType(input string) *Mlrval {
- // xxx the parsing has happened so stash it ...
- // xxx emphasize the invariant that a non-invalid printrep always
- // matches the nval ...
if input == "" {
return MLRVAL_VOID
}
diff --git a/go/todo.txt b/go/todo.txt
index 9d5f6b95f..e65ca6402 100644
--- a/go/todo.txt
+++ b/go/todo.txt
@@ -8,6 +8,7 @@ TOP OF LIST:
* shell-commands.html; while-read
+* repifs * "\001" et al. *
* I/O redirects * fmtnum * localtime * stats1 -r/--fr *
* --nr-progress-mod * mlr -k * json comment-handling *
* cli-audits * doc6 * survey * check issues * regtest pendings *
@@ -112,15 +113,25 @@ w survey
----------------------------------------------------------------
BLOCKERS
-* bug:
+* UTs for r"a" . r"b" and so on and so on
+
+* document precedence issue:
+! probably remove this feature !
+
+ μεταμόρφωσις: z = [{"x":3}, {"x":4}]
μεταμόρφωσις: z[1].x
- 0.3467901443380824
+ 3
μεταμόρφωσις: z[2].x
- 0.7586799647899636
+ 4
μεταμόρφωσις: z[1]["x"] + z[2]["x"]
- 1.105470109128046
+ 7
μεταμόρφωσις: z[1].x + z[2].x
(absent)
+ μεταμόρφωσις: (z[1].x) + (z[2].x)
+ 7
+
+! repifs
+! seps "\001" etc
! repifs
! seps "\001" etc