From 85f3e5ecf7d9388d74c75a0ab30d41f97084cccb Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sat, 4 Sep 2021 00:20:47 -0400 Subject: [PATCH 1/5] misc to-do items --- docs6/docs/cpu.md | 44 +++++++++++++++ docs6/docs/cpu.md.in | 30 ++++++++++ docs6/docs/installation.md | 2 + docs6/docs/installation.md.in | 2 + docs6/docs/installing-miller.md | 2 +- docs6/docs/installing-miller.md.in | 2 +- docs6/docs/proofreads.txt | 37 ++++++------- docs6/docs/reference-dsl-differences.md | 4 ++ docs6/docs/reference-dsl-differences.md.in | 4 ++ docs6/docs/reference-verbs.md | 3 +- docs6/mkdocs.yml | 1 + go/mlr.go | 19 ++++++- go/todo.txt | 64 ++++------------------ 13 files changed, 139 insertions(+), 75 deletions(-) create mode 100644 docs6/docs/cpu.md create mode 100644 docs6/docs/cpu.md.in diff --git a/docs6/docs/cpu.md b/docs6/docs/cpu.md new file mode 100644 index 000000000..3934cebe1 --- /dev/null +++ b/docs6/docs/cpu.md @@ -0,0 +1,44 @@ + +
+ +Quick links: +  +Verb list +  +Function list +  +Glossary +  +Repository ↗ + +
+# CPU/multicore usage + +Miller 6 is written in [Go](https://golang.org/) which supports multicore programming. + +Miller uses Go's _channel_ concept. The following are all separate goroutines: + +* One channel for input-record reader: parsing input file(s) to record objects +* One channel for each [verb](reference-verbs.md) in [then-chains](reference-main-then-chaining.md) +* One channel for output-record writer: formatting output records as text +* One controller channel which coordinates all these, without much work to do. + +For example, `mlr --csv cut -f somefield then sort -f otherfield then put '$z = +$x + $y' a.csv b.csv c.csv` will have 6 goroutines running: input-reader, +`cut`, `sort`, `put`, output-writer, controller. + +If all the verbs in the chain are [streaming](streaming-and-memory.md) -- +operating on each record as it arrives, then passing it on -- then all verbs in +the chain will be active at once. On the other hand, if there is a +non-streaming verb in the chain, which produces output only after receiving all +input -- for example, `sort` -- then we would expect verbs after that in the +chain to sit idle until the end of the input stream is reached, the `sort` does +its computation, then sends its output to downstream verbs. + +In practice, profiling has shown that the input-reader uses the most CPU of all +the above. This means CPUs running verbs may not be 100% utilized, since they +are likely to be spending some of their time waiting for input data. + +You can set the Go-standard environment variable `GOMAXPROCS` if you like. If +you don't, Miller will (as is standard for Go programs in Go 1.16 and above) up +to all available CPUs. diff --git a/docs6/docs/cpu.md.in b/docs6/docs/cpu.md.in new file mode 100644 index 000000000..520cd54b0 --- /dev/null +++ b/docs6/docs/cpu.md.in @@ -0,0 +1,30 @@ +# CPU/multicore usage + +Miller 6 is written in [Go](https://golang.org/) which supports multicore programming. + +Miller uses Go's _channel_ concept. The following are all separate goroutines: + +* One channel for input-record reader: parsing input file(s) to record objects +* One channel for each [verb](reference-verbs.md) in [then-chains](reference-main-then-chaining.md) +* One channel for output-record writer: formatting output records as text +* One controller channel which coordinates all these, without much work to do. + +For example, `mlr --csv cut -f somefield then sort -f otherfield then put '$z = +$x + $y' a.csv b.csv c.csv` will have 6 goroutines running: input-reader, +`cut`, `sort`, `put`, output-writer, controller. + +If all the verbs in the chain are [streaming](streaming-and-memory.md) -- +operating on each record as it arrives, then passing it on -- then all verbs in +the chain will be active at once. On the other hand, if there is a +non-streaming verb in the chain, which produces output only after receiving all +input -- for example, `sort` -- then we would expect verbs after that in the +chain to sit idle until the end of the input stream is reached, the `sort` does +its computation, then sends its output to downstream verbs. + +In practice, profiling has shown that the input-reader uses the most CPU of all +the above. This means CPUs running verbs may not be 100% utilized, since they +are likely to be spending some of their time waiting for input data. + +You can set the Go-standard environment variable `GOMAXPROCS` if you like. If +you don't, Miller will (as is standard for Go programs in Go 1.16 and above) up +to all available CPUs. diff --git a/docs6/docs/installation.md b/docs6/docs/installation.md index a8d600a37..0d65f7eb8 100644 --- a/docs6/docs/installation.md +++ b/docs6/docs/installation.md @@ -33,6 +33,8 @@ Note: sudo port selfupdate && sudo port install miller +Note that Homebrew is available for Linux as well: [https://docs.brew.sh/linux](https://docs.brew.sh/linux). + You may already have the `mlr` executable available in your platform's package manager on NetBSD, Debian Linux, Ubuntu Xenial and upward, Arch Linux, or perhaps other distributions. For example, on various Linux distributions you might do one of the following: 
diff --git a/docs6/docs/installation.md.in b/docs6/docs/installation.md.in
index 6ec720c8d..1fdf5435b 100644
--- a/docs6/docs/installation.md.in
+++ b/docs6/docs/installation.md.in
@@ -19,6 +19,8 @@ GENMD_CARDIFY_HIGHLIGHT_ONE
 sudo port selfupdate && sudo port install miller
 GENMD_EOF
 
+Note that Homebrew is available for Linux as well: [https://docs.brew.sh/linux](https://docs.brew.sh/linux).
+
 You may already have the `mlr` executable available in your platform's package manager on NetBSD, Debian Linux, Ubuntu Xenial and upward, Arch Linux, or perhaps other distributions. For example, on various Linux distributions you might do one of the following:
 
 GENMD_CARDIFY_HIGHLIGHT_ONE
diff --git a/docs6/docs/installing-miller.md b/docs6/docs/installing-miller.md
index 8e72275fe..435093174 100644
--- a/docs6/docs/installing-miller.md
+++ b/docs6/docs/installing-miller.md
@@ -20,7 +20,7 @@ You can install Miller for various platforms as follows.
     * You can get latest Miller 6 builds for Linux, MacOS, and Windows by visiting [https://github.com/johnkerl/miller/actions](https://github.com/johnkerl/miller/actions), selecting the latest build, and clicking _Artifacts_. (These are retained for 5 days after each commit.)
     * See also the [build page](build.md) if you prefer -- in particular, if your platform's package manager doesn't have the latest release.
 * Miller 5 is released, and is described by [https://miller.readthedocs.io](https://miller.readthedocs.io).
-    * Linux: `yum install miller` or `apt-get install miller` depending on your flavor of Linux.
+    * Linux: `yum install miller` or `apt-get install miller` depending on your flavor of Linux, or [Homebrew](https://docs.brew.sh/linux).
     * MacOS: `brew install miller` or `port install miller` depending on your preference of [Homebrew](https://brew.sh) or [MacPorts](https://macports.org).
     * Windows: `choco install miller` using [Chocolatey](https://chocolatey.org).
 
diff --git a/docs6/docs/installing-miller.md.in b/docs6/docs/installing-miller.md.in
index c656a179d..2899ddcd5 100644
--- a/docs6/docs/installing-miller.md.in
+++ b/docs6/docs/installing-miller.md.in
@@ -6,7 +6,7 @@ You can install Miller for various platforms as follows.
     * You can get latest Miller 6 builds for Linux, MacOS, and Windows by visiting [https://github.com/johnkerl/miller/actions](https://github.com/johnkerl/miller/actions), selecting the latest build, and clicking _Artifacts_. (These are retained for 5 days after each commit.)
     * See also the [build page](build.md) if you prefer -- in particular, if your platform's package manager doesn't have the latest release.
 * Miller 5 is released, and is described by [https://miller.readthedocs.io](https://miller.readthedocs.io).
-    * Linux: `yum install miller` or `apt-get install miller` depending on your flavor of Linux.
+    * Linux: `yum install miller` or `apt-get install miller` depending on your flavor of Linux, or [Homebrew](https://docs.brew.sh/linux).
     * MacOS: `brew install miller` or `port install miller` depending on your preference of [Homebrew](https://brew.sh) or [MacPorts](https://macports.org).
     * Windows: `choco install miller` using [Chocolatey](https://chocolatey.org).
 
diff --git a/docs6/docs/proofreads.txt b/docs6/docs/proofreads.txt
index 5ac200f47..d102f7f6f 100644
--- a/docs6/docs/proofreads.txt
+++ b/docs6/docs/proofreads.txt
@@ -1,6 +1,13 @@
-----------------------------------------------------------------
-! merge 3 new 4 !
+TOP:
 
+E data-types page
+E flatten/unflatten page
+c! repifs !!
+c! seps \001 etc !
+C flags LUTs
+? twi-dm re all-contribs: all-contributors.org
+
+----------------------------------------------------------------
 ALL:
 * csv to csv,tsv throughout
 * rid of explicitly passing around os.Stdout in all various help functions, annoying
@@ -19,14 +26,18 @@ ALL:
   https://squidfunk.github.io/mkdocs-material/customization/#extending-the-theme
 
 ----------------------------------------------------------------
-c! repifs !!
-c! seps \001 etc !
-C flags LUTs
-? twi-dm re all-contribs: all-contributors.org
 
 e fzf-ish w/ head -n 4, --from, up-arrow & append verb, then cat -- find & update the existing section
 
+? figure out flatsep vs iflatsep/oflatsep & have a story around the choice either way
+  - ctx ptr-split for invars
 E fill out flatten/unflatten page
+  echo a.b.c=3 | mlr --ojson cat
+  echo a.b.c=3 | mlr --ojson --no-jvstack cat | mlr --j2c --oflatsep : cat
+  echo a:b:c=3 | mlr --ojson --no-jvstack cat
+  echo a:b:c=3 | mlr --ojson --no-jvstack --oflatsep : cat
+  -> call it --flatsep
+
 E fill out data-types page
 E fill out arrays page
   - xref to fla/unfla page
@@ -37,11 +48,6 @@ E fill out maps page
 E make flags page
   C show-all-flags help -- fully alpha, or alpha-by-type; LUT refactor
 
-c GOMAXPROCS -- up it? separate page maybe -- ?
-  - note one goroutine for in, out, & each verb
-  - check and respect env-var
-  - add note on threading and how it scales on multicore arches
-
 * functions: somewhere organize by type. olh/man6/docs6 ...
 
 * regex: more about what is / is not
@@ -154,12 +160,8 @@ E Keep in mind that out-of-stream variables are a nested, multi-level hashmap (d
   o why not '--oflatsep /' respected?
 
 diffs other langs:
-e slice incl/excl; split-in-half
+e slice incl/excl; split-in-half: x[:2] x[2:] vs x[:2] x[3:]
 e localvars in if/else
-e elif
-
-glossary:
-! after it's done, link-check *all* ........ :^/
 
 separators:
 E write up which file formats support which flags
@@ -167,7 +169,4 @@ E write up which file formats support which flags
 manpage:
 ? [NEEDS READ-THROUGH]
 
-install:
-* n.b. homebrew linux https://github.com/johnkerl/miller/issues/77#issuecomment-536611087
-
 mk-func-table.rb: port comments from sphinx to mkdocs
diff --git a/docs6/docs/reference-dsl-differences.md b/docs6/docs/reference-dsl-differences.md
index 1dbc6627d..2d583bbd4 100644
--- a/docs6/docs/reference-dsl-differences.md
+++ b/docs6/docs/reference-dsl-differences.md
@@ -69,6 +69,10 @@ Expected one of:
   * / // % .* ./ .// ** [ [[ [[[
+## elif + +Miller has [`elif`](reference-dsl-control-structures.md#if-statements), not `else if` or `elsif`. + ## Required curly braces Bodies for all compound statements must be enclosed in curly braces, even if the body is a single statement: diff --git a/docs6/docs/reference-dsl-differences.md.in b/docs6/docs/reference-dsl-differences.md.in index 7632652fa..0d5f5bc12 100644 --- a/docs6/docs/reference-dsl-differences.md.in +++ b/docs6/docs/reference-dsl-differences.md.in @@ -47,6 +47,10 @@ mlr --csv --from example.csv put -q ' ' GENMD_EOF +## elif + +Miller has [`elif`](reference-dsl-control-structures.md#if-statements), not `else if` or `elsif`. + ## Required curly braces Bodies for all compound statements must be enclosed in curly braces, even if the body is a single statement: diff --git a/docs6/docs/reference-verbs.md b/docs6/docs/reference-verbs.md index 388f4f520..6734d0fcb 100644 --- a/docs6/docs/reference-verbs.md +++ b/docs6/docs/reference-verbs.md @@ -2072,7 +2072,8 @@ Please see the [DSL reference](reference-dsl.md) for more information about the 
 Usage: mlr regularize [options]
-Outputs records sorted lexically ascending by keys.Options:
+Outputs records sorted lexically ascending by keys.
+Options:
 -h|--help Show this message.
diff --git a/docs6/mkdocs.yml b/docs6/mkdocs.yml index 003aa0eaf..ccab3e8ac 100644 --- a/docs6/mkdocs.yml +++ b/docs6/mkdocs.yml @@ -63,6 +63,7 @@ nav: - "Separators": "reference-main-separators.md" - "List of verbs": "reference-verbs.md" - "Streaming processing, and memory usage": "streaming-and-memory.md" + - "CPU/multicore usage": "cpu.md" - "Data types": "reference-main-data-types.md" - "Arithmetic": "reference-main-arithmetic.md" - "Maps": "reference-main-maps.md" diff --git a/go/mlr.go b/go/mlr.go index 93686c671..9b2894bd4 100644 --- a/go/mlr.go +++ b/go/mlr.go @@ -6,13 +6,30 @@ import ( "runtime" "runtime/debug" "runtime/pprof" + "strconv" "mlr/src/entrypoint" ) // ---------------------------------------------------------------- func main() { - runtime.GOMAXPROCS(4) // Seems reasonable these days + + // Respect env $GOMAXPROCS, if provided, else set default. + haveSetGoMaxProcs := false + goMaxProcsString := os.Getenv("GOMAXPROCS") + if goMaxProcsString != "" { + goMaxProcs, err := strconv.Atoi(goMaxProcsString) + if err != nil { + runtime.GOMAXPROCS(goMaxProcs) + haveSetGoMaxProcs = true + } + } + if !haveSetGoMaxProcs { + // As of Go 1.16 this is the default anyway. For 1.15 and below we need + // to explicitly set this. + runtime.GOMAXPROCS(runtime.NumCPU()) + } + debug.SetGCPercent(500) // Empirical: See README-profiling.md // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/go/todo.txt b/go/todo.txt index d9ab1008e..84a5fab54 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -1,39 +1,23 @@ ================================================================ TOP OF LIST: -* pre-release: - o explicit punchdown list - o draft release w/ m6 binaries YYYYMMDD -- ? - o gha download steps clearer - ! allcontribs -* doc6 proofreads +! doc6 proofreads.txt * 'explicit r' issue 297: code, UT, doc * shell-commands.html; while-read -* memory/streaming page +* repifs * "\001" et al. * * I/O redirects * fmtnum * localtime * stats1 -r/--fr * * --nr-progress-mod * mlr -k * json comment-handling * * cli-audits * doc6 * survey * check issues * regtest pendings * * survey followups/discussion-post * -* PD wtf ! - -* GOMAXPROCS env-override, else default 8 with comment - ----------------------------------------------------------------- -* mlr repl (w/o mrpl) doesn't print final newline on EOF -* also: feature/shorthand for repl newline before prompt - -* slice-indexing bug: - - μεταμόρφωσις: x=[1,2,3,4,5] - μεταμόρφωσις: x[:2] - [1, 2] - μεταμόρφωσις: x[2:] - [2, 3, 4, 5] +d pre-release: + o explicit punchdown list + o draft release w/ m6 binaries YYYYMMDD -- ? + o gha download steps clearer ---------------------------------------------------------------- implicit r: @@ -47,10 +31,8 @@ docs: * mlr --j2c --no-auto-flatten cat $mlg/regtest/input/flatten-input-2.json - code: this is ok ... maybe prefer single-line -- ? -* src/man/doc hygiene @ build script - -w discussion re docs6 ... * single cheatsheet page -- put out RFH? + https://twitter.com/icymi_py/status/1426622817785765898/photo/1 * note somewhere (maybe 'what's new in miller6' and/or to-be-written survey review): o miller was first of all a tool for myself, and anyone else who might find it useful @@ -71,46 +53,19 @@ w discussion re docs6 ... * link to SE table ... https://github.com/johnkerl/miller/discussions/609#discussioncomment-1115715 -* csv -> csv/tsv throughout - -* cheatsheet someday? example: - https://twitter.com/icymi_py/status/1426622817785765898/photo/1 - ----------------------------------------------------------------- -memory/streaming page - -* https://github.com/johnkerl/miller/issues/587 - ---------------------------------------------------------------- DOC6 * colored-shapes.dkvp -> csv; also mkdat2 * data/small -> csv throughout. and/or just use example.csv -* quicklinks -- ? -* vim syntax highlighting for .mlr files -- highlight this somewhere * shell-commands.html while-read example issue -* memory page: - o open with "out of memory" -- what next? - o streaming vs non-streaming - o unsparsify -f - o no out-of-core sort yet ... - o older versions - * number-formatting page -* list-of-verbs shortlist @ top - * check <-> manpage/olh sections & doc6 sections -* fill out: - o new-in-miller-6: flatten/unflatten - o JSON flatten/unflatten section - o Arrays section including 1-up and why - o differences from other languages (cf. juliadocs) - * doc notes about --jknquotein & --jvquoteall. make sure they're ignore-accepted in the CLIP. * file-formats: @@ -168,6 +123,9 @@ BLOCKERS μεταμόρφωσις: z[1].x + z[2].x (absent) +! repifs +! seps "\001" etc + * finish stats1 -r * what about -I mixed with .gzin -- ? @@ -226,6 +184,8 @@ NON-BLOCKERS * non-blocker: commenting passes ... +* non-blocker: feature/shorthand for repl newline before prompt + * non-blocker: new functions: o new columns-to-arrays and arrays-to-columns for stan format From 54b415d4e2508bb1323647ace1ef41175f0566a3 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sat, 4 Sep 2021 13:43:15 -0400 Subject: [PATCH 2/5] Rework data-types page and function-table presentation --- docs6/docs/cpu.md | 10 +- docs6/docs/cpu.md.in | 10 +- docs6/docs/data/json-in-csv.csv | 3 + docs6/docs/data/type-infer.csv | 3 + docs6/docs/file-formats.md | 7 + docs6/docs/file-formats.md.in | 7 + docs6/docs/javascripts/tables.js | 6 + docs6/docs/mk-func-info.rb | 114 + docs6/docs/mk-func-table.rb | 93 - docs6/docs/online-help.md | 10 +- docs6/docs/proofreads.txt | 20 +- docs6/docs/reference-dsl-builtin-functions.md | 1916 ++++++++--------- .../reference-dsl-builtin-functions.md.in | 24 +- docs6/docs/reference-dsl-differences.md | 9 + docs6/docs/reference-dsl-differences.md.in | 9 + docs6/docs/reference-main-data-types.md | 199 +- docs6/docs/reference-main-data-types.md.in | 116 +- docs6/docs/reference-main-strings.md | 52 + docs6/docs/reference-main-strings.md.in | 38 + docs6/docs/reference-verbs.md | 28 + docs6/docs/reference-verbs.md.in | 12 + docs6/mkdocs.yml | 10 +- go/src/auxents/help/entry.go | 20 + go/src/dsl/cst/builtin_function_manager.go | 77 +- go/todo.txt | 13 +- 25 files changed, 1582 insertions(+), 1224 deletions(-) create mode 100644 docs6/docs/data/json-in-csv.csv create mode 100644 docs6/docs/data/type-infer.csv create mode 100644 docs6/docs/javascripts/tables.js create mode 100755 docs6/docs/mk-func-info.rb delete mode 100755 docs6/docs/mk-func-table.rb create mode 100644 docs6/docs/reference-main-strings.md create mode 100644 docs6/docs/reference-main-strings.md.in diff --git a/docs6/docs/cpu.md b/docs6/docs/cpu.md index 3934cebe1..fe5c1459e 100644 --- a/docs6/docs/cpu.md +++ b/docs6/docs/cpu.md @@ -14,7 +14,8 @@ Quick links: # CPU/multicore usage -Miller 6 is written in [Go](https://golang.org/) which supports multicore programming. +Miller 6 is written in [Go](https://golang.org/) which supports multicore +programming. Miller uses Go's _channel_ concept. The following are all separate goroutines: @@ -39,6 +40,13 @@ In practice, profiling has shown that the input-reader uses the most CPU of all the above. This means CPUs running verbs may not be 100% utilized, since they are likely to be spending some of their time waiting for input data. +Running Miller on a machine with more CPUs than active channels (as listed +above) won't speed up a given invocation of Miller. However, of course, you'll +be able to run more invocations of Miller at the same time if you like. + +If you set `$GOMAXPROCS=1` in the environment, that's fine -- the Go runtime +will multiplex different channel-handling goroutines onto the same CPU. + You can set the Go-standard environment variable `GOMAXPROCS` if you like. If you don't, Miller will (as is standard for Go programs in Go 1.16 and above) up to all available CPUs. diff --git a/docs6/docs/cpu.md.in b/docs6/docs/cpu.md.in index 520cd54b0..6fe07e9d3 100644 --- a/docs6/docs/cpu.md.in +++ b/docs6/docs/cpu.md.in @@ -1,6 +1,7 @@ # CPU/multicore usage -Miller 6 is written in [Go](https://golang.org/) which supports multicore programming. +Miller 6 is written in [Go](https://golang.org/) which supports multicore +programming. Miller uses Go's _channel_ concept. The following are all separate goroutines: @@ -25,6 +26,13 @@ In practice, profiling has shown that the input-reader uses the most CPU of all the above. This means CPUs running verbs may not be 100% utilized, since they are likely to be spending some of their time waiting for input data. +Running Miller on a machine with more CPUs than active channels (as listed +above) won't speed up a given invocation of Miller. However, of course, you'll +be able to run more invocations of Miller at the same time if you like. + +If you set `$GOMAXPROCS=1` in the environment, that's fine -- the Go runtime +will multiplex different channel-handling goroutines onto the same CPU. + You can set the Go-standard environment variable `GOMAXPROCS` if you like. If you don't, Miller will (as is standard for Go programs in Go 1.16 and above) up to all available CPUs. diff --git a/docs6/docs/data/json-in-csv.csv b/docs6/docs/data/json-in-csv.csv new file mode 100644 index 000000000..99a9c9519 --- /dev/null +++ b/docs6/docs/data/json-in-csv.csv @@ -0,0 +1,3 @@ +id,blob +100,"{""a"":1,""b"":[2,3,4]}" +105,"{""a"":6,""b"":[7,8,9]}" diff --git a/docs6/docs/data/type-infer.csv b/docs6/docs/data/type-infer.csv new file mode 100644 index 000000000..2a061ee83 --- /dev/null +++ b/docs6/docs/data/type-infer.csv @@ -0,0 +1,3 @@ +a,b,c +1.2,3,true +4,5.6,buongiorno diff --git a/docs6/docs/file-formats.md b/docs6/docs/file-formats.md index 21d2e314e..7abdb8822 100644 --- a/docs6/docs/file-formats.md +++ b/docs6/docs/file-formats.md @@ -271,6 +271,13 @@ TODO: probably remove entirely * Use `--jflatsep yourseparatorhere` to specify the string used for key concatenation: this defaults to a single dot. +### JSON-in-CSV + +It's quite common to have CSV data which contains stringified JSON as a column. +See the [JSON parse and stringify +section](reference-main-data-types.md#json-parse-and-stringify) for ways to +decode these in Miller. + ## PPRINT: Pretty-printed tabular Miller's pretty-print format is like CSV, but column-aligned. For example, compare diff --git a/docs6/docs/file-formats.md.in b/docs6/docs/file-formats.md.in index 26f978b76..64fb1e383 100644 --- a/docs6/docs/file-formats.md.in +++ b/docs6/docs/file-formats.md.in @@ -118,6 +118,13 @@ TODO: probably remove entirely * Use `--jflatsep yourseparatorhere` to specify the string used for key concatenation: this defaults to a single dot. +### JSON-in-CSV + +It's quite common to have CSV data which contains stringified JSON as a column. +See the [JSON parse and stringify +section](reference-main-data-types.md#json-parse-and-stringify) for ways to +decode these in Miller. + ## PPRINT: Pretty-printed tabular Miller's pretty-print format is like CSV, but column-aligned. For example, compare diff --git a/docs6/docs/javascripts/tables.js b/docs6/docs/javascripts/tables.js new file mode 100644 index 000000000..e848f07dd --- /dev/null +++ b/docs6/docs/javascripts/tables.js @@ -0,0 +1,6 @@ +document$.subscribe(function() { + var tables = document.querySelectorAll("article table") + tables.forEach(function(table) { + new Tablesort(table) + }) +}) diff --git a/docs6/docs/mk-func-info.rb b/docs6/docs/mk-func-info.rb new file mode 100755 index 000000000..2e72121c6 --- /dev/null +++ b/docs6/docs/mk-func-info.rb @@ -0,0 +1,114 @@ +#!/usr/bin/env ruby + +# ================================================================ +# Markdown autogen for built-in functions: table, or full details page. +# Invoked by files in *.md.in. +# ================================================================ + +# ---------------------------------------------------------------- +# Displayname is for HTML rendered in the document body. Linkname is for +# #-style links which don't do so well with special characters. + +LOOKUP = { + # name display_name link_name +'!' => [ '\!', 'exclamation-point' ], +'!=' => [ '\!=', 'exclamation-point-equals' ], +'!=~' => [ '!=~', 'regnotmatch' ], +'%' => [ '%', 'percent' ], +'&&' => [ '&&', 'logical-and' ], +'&' => [ '&', 'bitwise-and' ], +'*' => [ '\*', 'times' ], +'**' => [ '\**', 'exponentiation' ], +'+' => [ '\+', 'plus' ], +'-' => [ '\-', 'minus' ], +'.' => [ '\.', 'dot' ], +'.*' => [ '\.\*', 'dot-times' ], +'.+' => [ '\.\+', 'dot-plus' ], +'.-' => [ '\.\-', 'dot-minus' ], +'./' => [ '\./', 'dot-slash' ], +'/' => [ '/', 'slash' ], +'//' => [ '//', 'slash-slash' ], +':' => [ '\:', 'colon' ], +'<' => [ '<', 'less-than' ], +'<<' => [ '<<', 'lsh' ], +'<=' => [ '<=', 'less-than-or-equals' ], +'=' => [ '=', 'equals' ], +'==' => [ '==', 'double-equals' ], +'=~' => [ '=~', 'regmatch' ], +'>' => [ '>', 'greater-than' ], +'>=' => [ '>=', 'greater-than-or-equals' ], +'>>' => [ '>>', 'srsh' ], +'>>>' => [ '>>>', 'ursh' ], +'>>>=' => [ '>>>=', 'ursheq' ], +'?' => [ '?', 'question-mark' ], +'?.:' => [ '?:', 'question-mark-colon' ], +'?:' => [ '?:', 'question-mark-colon' ], +'??' => [ '??', 'absent-coalesce' ], +'???' => [ '???', 'absent-empty-coalesce' ], +'^' => [ '^', 'bitwise-xor' ], +'^^' => [ '^^', 'logical-xor' ], +'|' => [ '\|', 'bitwise-or' ], +'||' => [ '\|\|', 'logical-or' ], +'~' => [ '~', 'bitwise-not' ], +} + +def name_to_display_name_and_link_name(name) + entry = LOOKUP[name] + if entry.nil? + return [name, name] + else + display_name, link_name = entry + return [display_name, link_name] + end +end + +# ---------------------------------------------------------------- +def make_func_details + function_classes = `mlr help list-function-classes`.split + + puts + puts "## Summary" + puts + for function_class in function_classes + class_link_name = "#"+"#{function_class}-functions" + class_display_name = "#{function_class.capitalize} functions" + #puts "* [#{display_name}](#{link_name})" + + functions_in_class = `mlr help list-functions-in-class #{function_class}`.split + foo = [] + for function_name in functions_in_class + display_name, link_name = name_to_display_name_and_link_name(function_name) + foo.append " [#{display_name}]("+"#"+"#{link_name})" + end + #puts " * #{foo.join(',')}" + puts "* [**#{class_display_name}**](#{class_link_name}): #{foo.join(', ')}." + + end + puts + + for function_class in function_classes + display_name = "#{function_class.capitalize} functions" + puts "## #{display_name}" + puts + + functions_in_class = `mlr help list-functions-in-class #{function_class}`.split + + for function_name in functions_in_class + display_name, link_name = name_to_display_name_and_link_name(function_name) + + puts + if display_name != link_name + puts "" + end + puts "### #{display_name}" + + puts '
'
+      system("mlr help function '#{function_name}'")
+      puts '
' + puts + end + end +end + +# ---------------------------------------------------------------- +make_func_details diff --git a/docs6/docs/mk-func-table.rb b/docs6/docs/mk-func-table.rb deleted file mode 100755 index aeab061af..000000000 --- a/docs6/docs/mk-func-table.rb +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env ruby - -# Sphinx tables need to be precisely lined up, unlike Markdown tables which -# are more flexible. For example this is fine Markdown: -# -# | a | b | i | x | y | -# | --- | --- | --- | --- | --- | -# | pan | pan | 1 | 0.3467901443380824 | 0.7268028627434533 | -# | eks | pan | 2 | 0.7586799647899636 | 0.5221511083334797 | -# | wye | wye | 3 | 0.20460330576630303 | 0.33831852551664776 | -# | eks | wye | 4 | 0.38139939387114097 | 0.13418874328430463 | -# -# but Sphinx wants -# -# +-----+-----+---+---------------------+---------------------+ -# | a | b | i | x | y | -# +=====+=====+===+=====================+=====================+ -# | pan | pan | 1 | 0.3467901443380824 | 0.7268028627434533 | -# +-----+-----+---+---------------------+---------------------+ -# | eks | pan | 2 | 0.7586799647899636 | 0.5221511083334797 | -# +-----+-----+---+---------------------+---------------------+ -# | wye | wye | 3 | 0.20460330576630303 | 0.33831852551664776 | -# +-----+-----+---+---------------------+---------------------+ -# | eks | wye | 4 | 0.38139939387114097 | 0.13418874328430463 | -# +-----+-----+---+---------------------+---------------------+ -# -# So we need to build up a list of tuples, then compute the max length down -# each column, then write a fixed number of -/= per row. - -lines = `mlr --list-all-functions-as-table` - -# ---------------------------------------------------------------- -# Pass 1 - -max_name_length = 1 -max_function_class_length = 1 -max_nargs_length = 1 - -rows = [] -lines.split("\n").each do |line| - if line =~ /^? :.*/ # has an extra whitespace which throws off the naive split - ig, nore, function_class, nargs = line.split(/\s+/) - name = '? :' - else - name, function_class, nargs = line.split(/\s+/) - end - - name = '``' + name + '``' - - if max_name_length < name.length - max_name_length = name.length - end - if max_function_class_length < function_class.length - max_function_class_length = function_class.length - end - if max_nargs_length < nargs.length - max_nargs_length = nargs.length - end - - # TODO: format name with page-internal link - row = [name, function_class, nargs] - rows.append(row) -end - -# ---------------------------------------------------------------- -# Pass 2 - -name_dashes = '-' * max_name_length -function_class_dashes = '-' * max_function_class_length -nargs_dashes = '-' * max_nargs_length - -name_equals = '=' * max_name_length -function_class_equals = '=' * max_function_class_length -nargs_equals = '=' * max_nargs_length - -dashes_line = "+-#{name_dashes}-+-#{function_class_dashes}-+-#{nargs_dashes}-+" -equals_line = "+=#{name_equals}=+=#{function_class_equals}=+=#{nargs_equals}=+" - -counter = 0 -rows.each do |row| - name, function_class, nargs = row - - counter = counter + 1 - if counter == 1 - puts dashes_line - puts "| #{name.ljust(max_name_length)} | #{function_class.ljust(max_function_class_length)} | #{nargs.ljust(max_nargs_length)} |" - puts equals_line - else - puts "| #{name.ljust(max_name_length)} | #{function_class.ljust(max_function_class_length)} | #{nargs.ljust(max_nargs_length)} |" - puts dashes_line - end -end - diff --git a/docs6/docs/online-help.md b/docs6/docs/online-help.md index e77d28999..537dfcf80 100644 --- a/docs6/docs/online-help.md +++ b/docs6/docs/online-help.md @@ -49,7 +49,10 @@ Type 'mlr help {topic}' for any of the following: mlr help function mlr help keyword mlr help list-functions + mlr help list-function-classes + mlr help list-functions-in-class mlr help list-functions-as-paragraph + mlr help list-functions-as-table mlr help list-keywords mlr help list-keywords-as-paragraph mlr help list-verbs @@ -61,6 +64,7 @@ Type 'mlr help {topic}' for any of the following: mlr help separator-options mlr help type-arithmetic-info mlr help usage-functions + mlr help usage-functions-by-class mlr help usage-keywords mlr help usage-verbs mlr help verb @@ -91,7 +95,10 @@ Type 'mlr help {topic}' for any of the following: mlr help function mlr help keyword mlr help list-functions + mlr help list-function-classes + mlr help list-functions-in-class mlr help list-functions-as-paragraph + mlr help list-functions-as-table mlr help list-keywords mlr help list-keywords-as-paragraph mlr help list-verbs @@ -103,6 +110,7 @@ Type 'mlr help {topic}' for any of the following: mlr help separator-options mlr help type-arithmetic-info mlr help usage-functions + mlr help usage-functions-by-class mlr help usage-keywords mlr help usage-verbs mlr help verb @@ -179,7 +187,7 @@ Given the name of a DSL function (from `mlr -f`) you can use `mlr help function` mlr help function append 
-append  (class=maps/arrays #args=2) Appends second argument to end of first argument, which must be an array.
+append  (class=collections #args=2) Appends second argument to end of first argument, which must be an array.
 
diff --git a/docs6/docs/proofreads.txt b/docs6/docs/proofreads.txt
index d102f7f6f..51384d902 100644
--- a/docs6/docs/proofreads.txt
+++ b/docs6/docs/proofreads.txt
@@ -1,12 +1,15 @@
 TOP:
 
-E data-types page
+c data-types:
+  ! don't auto-infer bools; then, data-types page
+  ! don't auto-infer Infinity, etc -- ?!? then, data-types page
 E flatten/unflatten page
-c! repifs !!
 c! seps \001 etc !
-C flags LUTs
+C! repifs !! https://pkg.go.dev/regexp#Regexp.Split 2-for-1 -- get regexp as well ?
 ? twi-dm re all-contribs: all-contributors.org
 
+C flags LUTs
+
 ----------------------------------------------------------------
 ALL:
 * csv to csv,tsv throughout
@@ -38,7 +41,9 @@ E fill out flatten/unflatten page
   echo a:b:c=3 | mlr --ojson --no-jvstack --oflatsep : cat
   -> call it --flatsep
 
-E fill out data-types page
+E fill out strings page
+  e substr0/substr1/substr. slices & string-indexing new in m6.
+  c "abcde"[2:3] works but "abcde"[2] does not :(
 E fill out arrays page
   - xref to fla/unfla page
   - slices including semantics for out-of-range slices
@@ -48,8 +53,6 @@ E fill out maps page
 E make flags page
   C show-all-flags help -- fully alpha, or alpha-by-type; LUT refactor
 
-* functions: somewhere organize by type. olh/man6/docs6 ...
-
 * regex: more about what is / is not
   https://github.com/johnkerl/miller/issues/77#issuecomment-538553828
 
@@ -106,9 +109,9 @@ E data/colored-shapes.dkvp (this page & throughout) a CSV file ...
 * ... I ONLY READ UP TO CUT & PAUSED ...
 
 reference-main-data-types:
+! don't auto-infer bool from input data :(
 E field values are usually strings -> update
 E Field values are treated as numeric for the following: -> update
-? Miller's types for function processing are -> add JSON null; & check for others (array/map)
 E true/false -> add info about NaN and Inf et al.
 !! very much missing stuff!! where's the listing of mlrval types?!?
   - reference-dsl-variables.md.in should link to it
@@ -143,7 +146,6 @@ E User-defined subroutines -> non-factorial example -- maybe some useful aggrega
 l link to --load and --mload
 
 reference-dsl-builtin-functions:
-c mlr: option "--list-all-functions-as-table" not recognized. Please run "mlr --help" for usage information -> fix
 ! ... need to proofread entire list ...
 
 reference-dsl-output-statements:
@@ -168,5 +170,3 @@ E write up which file formats support which flags
 
 manpage:
 ? [NEEDS READ-THROUGH]
-
-mk-func-table.rb: port comments from sphinx to mkdocs
diff --git a/docs6/docs/reference-dsl-builtin-functions.md b/docs6/docs/reference-dsl-builtin-functions.md
index 245371f8a..c879ac18a 100644
--- a/docs6/docs/reference-dsl-builtin-functions.md
+++ b/docs6/docs/reference-dsl-builtin-functions.md
@@ -14,357 +14,335 @@ Quick links:
 
 # DSL built-in functions
 
+Each function takes a specific number of arguments, as shown below, except for
+functions marked as variadic such as `min` and `max`. (The latter compute min
+and max of any number of arguments.) There is no notion of optional or
+default-on-absent arguments. All argument-passing is positional rather than by
+name; arguments are passed by value, not by reference.
+
+At the command line, you can get a list of all functions using `mlr -f`, with
+details using `mlr -F`.  (Or, `mlr help usage-functions-by-class` to get
+details in the order shown on this page.) You can get detail for a given
+function using `mlr help function namegoeshere`, e.g.  `mlr help function
+gsub`.
+
+Operators are listed here along with functions. In this case, the
+argument-count is the number of items involved in the infix operator, e.g. we
+say `x+y` so the details for the `+` operator say that its number of arguments
+is 2. Unary operators such as `!` and `~` show argument-count of 1; the ternary
+`? :` operator shows an argument-count of 3.
+
+
 ## Summary
 
-mlr: option "--list-all-functions-as-table" not recognized.
-Please run "mlr --help" for usage information.
+* [**Arithmetic functions**](#arithmetic-functions):  [bitcount](#bitcount),  [madd](#madd),  [mexp](#mexp),  [mmul](#mmul),  [msub](#msub),  [pow](#pow),  [%](#percent),  [&](#bitwise-and),  [\*](#times),  [\**](#exponentiation),  [\+](#plus),  [\-](#minus),  [\.\*](#dot-times),  [\.\+](#dot-plus),  [\.\-](#dot-minus),  [\./](#dot-slash),  [/](#slash),  [//](#slash-slash),  [<<](#lsh),  [>>](#srsh),  [>>>](#ursh),  [^](#bitwise-xor),  [\|](#bitwise-or),  [~](#bitwise-not).
+* [**Boolean functions**](#boolean-functions):  [\!](#exclamation-point),  [\!=](#exclamation-point-equals),  [!=~](#regnotmatch),  [&&](#logical-and),  [<](#less-than),  [<=](#less-than-or-equals),  [==](#double-equals),  [=~](#regmatch),  [>](#greater-than),  [>=](#greater-than-or-equals),  [?:](#question-mark-colon),  [??](#absent-coalesce),  [???](#absent-empty-coalesce),  [^^](#logical-xor),  [\|\|](#logical-or).
+* [**Collections functions**](#collections-functions):  [append](#append),  [arrayify](#arrayify),  [depth](#depth),  [flatten](#flatten),  [get_keys](#get_keys),  [get_values](#get_values),  [haskey](#haskey),  [json_parse](#json_parse),  [json_stringify](#json_stringify),  [leafcount](#leafcount),  [length](#length),  [mapdiff](#mapdiff),  [mapexcept](#mapexcept),  [mapselect](#mapselect),  [mapsum](#mapsum),  [unflatten](#unflatten).
+* [**Conversion functions**](#conversion-functions):  [boolean](#boolean),  [float](#float),  [fmtnum](#fmtnum),  [hexfmt](#hexfmt),  [int](#int),  [joink](#joink),  [joinkv](#joinkv),  [joinv](#joinv),  [splita](#splita),  [splitax](#splitax),  [splitkv](#splitkv),  [splitkvx](#splitkvx),  [splitnv](#splitnv),  [splitnvx](#splitnvx),  [string](#string).
+* [**Hashing functions**](#hashing-functions):  [md5](#md5),  [sha1](#sha1),  [sha256](#sha256),  [sha512](#sha512).
+* [**Math functions**](#math-functions):  [abs](#abs),  [acos](#acos),  [acosh](#acosh),  [asin](#asin),  [asinh](#asinh),  [atan](#atan),  [atan2](#atan2),  [atanh](#atanh),  [cbrt](#cbrt),  [ceil](#ceil),  [cos](#cos),  [cosh](#cosh),  [erf](#erf),  [erfc](#erfc),  [exp](#exp),  [expm1](#expm1),  [floor](#floor),  [invqnorm](#invqnorm),  [log](#log),  [log10](#log10),  [log1p](#log1p),  [logifit](#logifit),  [max](#max),  [min](#min),  [qnorm](#qnorm),  [round](#round),  [roundm](#roundm),  [sgn](#sgn),  [sin](#sin),  [sinh](#sinh),  [sqrt](#sqrt),  [tan](#tan),  [tanh](#tanh),  [urand](#urand),  [urand32](#urand32),  [urandint](#urandint),  [urandrange](#urandrange).
+* [**String functions**](#string-functions):  [capitalize](#capitalize),  [clean_whitespace](#clean_whitespace),  [collapse_whitespace](#collapse_whitespace),  [gsub](#gsub),  [lstrip](#lstrip),  [regextract](#regextract),  [regextract_or_else](#regextract_or_else),  [rstrip](#rstrip),  [ssub](#ssub),  [strip](#strip),  [strlen](#strlen),  [sub](#sub),  [substr](#substr),  [substr0](#substr0),  [substr1](#substr1),  [tolower](#tolower),  [toupper](#toupper),  [truncate](#truncate),  [\.](#dot).
+* [**System functions**](#system-functions):  [hostname](#hostname),  [os](#os),  [system](#system),  [version](#version).
+* [**Time functions**](#time-functions):  [dhms2fsec](#dhms2fsec),  [dhms2sec](#dhms2sec),  [fsec2dhms](#fsec2dhms),  [fsec2hms](#fsec2hms),  [gmt2sec](#gmt2sec),  [hms2fsec](#hms2fsec),  [hms2sec](#hms2sec),  [sec2dhms](#sec2dhms),  [sec2gmt](#sec2gmt),  [sec2gmtdate](#sec2gmtdate),  [sec2hms](#sec2hms),  [strftime](#strftime),  [strptime](#strptime),  [systime](#systime),  [systimeint](#systimeint),  [uptime](#uptime).
+* [**Typing functions**](#typing-functions):  [asserting_absent](#asserting_absent),  [asserting_array](#asserting_array),  [asserting_bool](#asserting_bool),  [asserting_boolean](#asserting_boolean),  [asserting_empty](#asserting_empty),  [asserting_empty_map](#asserting_empty_map),  [asserting_error](#asserting_error),  [asserting_float](#asserting_float),  [asserting_int](#asserting_int),  [asserting_map](#asserting_map),  [asserting_nonempty_map](#asserting_nonempty_map),  [asserting_not_array](#asserting_not_array),  [asserting_not_empty](#asserting_not_empty),  [asserting_not_map](#asserting_not_map),  [asserting_not_null](#asserting_not_null),  [asserting_null](#asserting_null),  [asserting_numeric](#asserting_numeric),  [asserting_present](#asserting_present),  [asserting_string](#asserting_string),  [is_absent](#is_absent),  [is_array](#is_array),  [is_bool](#is_bool),  [is_boolean](#is_boolean),  [is_empty](#is_empty),  [is_empty_map](#is_empty_map),  [is_error](#is_error),  [is_float](#is_float),  [is_int](#is_int),  [is_map](#is_map),  [is_nonempty_map](#is_nonempty_map),  [is_not_array](#is_not_array),  [is_not_empty](#is_not_empty),  [is_not_map](#is_not_map),  [is_not_null](#is_not_null),  [is_null](#is_null),  [is_numeric](#is_numeric),  [is_present](#is_present),  [is_string](#is_string),  [typeof](#typeof).
 
-## List of functions
+## Arithmetic functions
 
-Each function takes a specific number of arguments, as shown below, except for functions marked as variadic such as `min` and `max`. (The latter compute min and max of any number of numerical arguments.) There is no notion of optional or default-on-absent arguments. All argument-passing is positional rather than by name; arguments are passed by value, not by reference.
-
-You can get a list of all functions using **mlr -f**, with details using **mlr -F**.
-
-
-## abs
-
-
-abs  (class=math #args=1) Absolute value.
-

-
-
-## acos
-
-
-acos  (class=math #args=1) Inverse trigonometric cosine.
-

-
-
-## acosh
-
-
-acosh  (class=math #args=1) Inverse hyperbolic cosine.
-

-
-
-## append
-
-
-append  (class=maps/arrays #args=2) Appends second argument to end of first argument, which must be an array.
-

-
-
-## arrayify
-
-
-arrayify  (class=maps/arrays #args=1) Walks through a nested map/array, converting any map with consecutive keys
-"1", "2", ... into an array. Useful to wrap the output of unflatten.
-

-
-
-## asin
-
-
-asin  (class=math #args=1) Inverse trigonometric sine.
-

-
-
-## asinh
-
-
-asinh  (class=math #args=1) Inverse hyperbolic sine.
-

-
-
-## asserting_absent
-
-
-asserting_absent  (class=typing #args=1) Aborts with an error if is_absent on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_array
-
-
-asserting_array  (class=typing #args=1) Aborts with an error if is_array on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_bool
-
-
-asserting_bool  (class=typing #args=1) Aborts with an error if is_bool on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_boolean
-
-
-asserting_boolean  (class=typing #args=1) Aborts with an error if is_boolean on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_empty
-
-
-asserting_empty  (class=typing #args=1) Aborts with an error if is_empty on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_empty_map
-
-
-asserting_empty_map  (class=typing #args=1) Aborts with an error if is_empty_map on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_error
-
-
-asserting_error  (class=typing #args=1) Aborts with an error if is_error on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_float
-
-
-asserting_float  (class=typing #args=1) Aborts with an error if is_float on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_int
-
-
-asserting_int  (class=typing #args=1) Aborts with an error if is_int on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_map
-
-
-asserting_map  (class=typing #args=1) Aborts with an error if is_map on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_nonempty_map
-
-
-asserting_nonempty_map  (class=typing #args=1) Aborts with an error if is_nonempty_map on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_not_array
-
-
-asserting_not_array  (class=typing #args=1) Aborts with an error if is_not_array on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_not_empty
-
-
-asserting_not_empty  (class=typing #args=1) Aborts with an error if is_not_empty on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_not_map
-
-
-asserting_not_map  (class=typing #args=1) Aborts with an error if is_not_map on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_not_null
-
-
-asserting_not_null  (class=typing #args=1) Aborts with an error if is_not_null on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_null
-
-
-asserting_null  (class=typing #args=1) Aborts with an error if is_null on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_numeric
-
-
-asserting_numeric  (class=typing #args=1) Aborts with an error if is_numeric on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_present
-
-
-asserting_present  (class=typing #args=1) Aborts with an error if is_present on the argument returns false,
-else returns its argument.
-

-
-
-## asserting_string
-
-
-asserting_string  (class=typing #args=1) Aborts with an error if is_string on the argument returns false,
-else returns its argument.
-

-
-
-## atan
-
-
-atan  (class=math #args=1) One-argument arctangent.
-

-
-
-## atan2
-
-
-atan2  (class=math #args=2) Two-argument arctangent.
-

-
-
-## atanh
-
-
-atanh  (class=math #args=1) Inverse hyperbolic tangent.
-

-
-
-## bitcount
 
+### bitcount
 
 bitcount  (class=arithmetic #args=1) Count of 1-bits.
 

 
 
-## boolean
-
+### madd
 
-boolean  (class=conversion #args=1) Convert int/float/bool/string to boolean.
+madd  (class=arithmetic #args=3) a + b mod m (integers)
 

 
 
-## capitalize
-
+### mexp
 
-capitalize  (class=string #args=1) Convert string's first character to uppercase.
+mexp  (class=arithmetic #args=3) a ** b mod m (integers)
 

 
 
-## cbrt
-
+### mmul
 
-cbrt  (class=math #args=1) Cube root.
+mmul  (class=arithmetic #args=3) a * b mod m (integers)
 

 
 
-## ceil
-
+### msub
 
-ceil  (class=math #args=1) Ceiling: nearest integer at or above.
+msub  (class=arithmetic #args=3) a - b mod m (integers)
 

 
 
-## clean_whitespace
-
+### pow
 
-clean_whitespace  (class=string #args=1) Same as collapse_whitespace and strip.
+pow  (class=arithmetic #args=2) Exponentiation. Same as **, but as a function.
 

 
 
-## collapse_whitespace
-
+
+### %
 
-collapse_whitespace  (class=string #args=1) Strip repeated whitespace from string.
+%  (class=arithmetic #args=2) Remainder; never negative-valued (pythonic).
 

 
 
-## cos
-
+
+### &
 
-cos  (class=math #args=1) Trigonometric cosine.
+&  (class=arithmetic #args=2) Bitwise AND.
 

 
 
-## cosh
-
+
+### \*
 
-cosh  (class=math #args=1) Hyperbolic cosine.
+*  (class=arithmetic #args=2) Multiplication, with integer*integer overflow to float.
 

 
 
-## depth
-
+
+### \**
 
-depth  (class=maps/arrays #args=1) Prints maximum depth of map/array. Scalars have depth 0.
+**  (class=arithmetic #args=2) Exponentiation. Same as pow, but as an infix operator.
 

 
 
-## dhms2fsec
-
+
+### \+
 
-dhms2fsec  (class=time #args=1) Recovers floating-point seconds as in dhms2fsec("5d18h53m20.250000s") = 500000.250000
++  (class=arithmetic #args=1,2) Addition as binary operator; unary plus operator.
 

 
 
-## dhms2sec
-
+
+### \-
 
-dhms2sec  (class=time #args=1) Recovers integer seconds as in dhms2sec("5d18h53m20s") = 500000
+-  (class=arithmetic #args=1,2) Subtraction as binary operator; unary negation operator.
 

 
 
-## erf
-
+
+### \.\*
 
-erf  (class=math #args=1) Error function.
+.*  (class=arithmetic #args=2) Multiplication, with integer-to-integer overflow.
 

 
 
-## erfc
-
+
+### \.\+
 
-erfc  (class=math #args=1) Complementary error function.
+.+  (class=arithmetic #args=2) Addition, with integer-to-integer overflow.
 

 
 
-## exp
-
+
+### \.\-
 
-exp  (class=math #args=1) Exponential function e**x.
+.-  (class=arithmetic #args=2) Subtraction, with integer-to-integer overflow.
 

 
 
-## expm1
-
+
+### \./
 
-expm1  (class=math #args=1) e**x - 1.
+./  (class=arithmetic #args=2) Integer division; not pythonic.
 

 
 
-## flatten
-
+
+### /
 
-flatten  (class=maps/arrays #args=3) Flattens multi-level maps to single-level ones. Examples:
+/  (class=arithmetic #args=2) Division. Integer / integer is floating-point.
+

+
+
+
+### //
+
+//  (class=arithmetic #args=2) Pythonic integer division, rounding toward negative.
+

+
+
+
+### <<
+
+<<  (class=arithmetic #args=2) Bitwise left-shift.
+

+
+
+
+### >>
+
+>>  (class=arithmetic #args=2) Bitwise signed right-shift.
+

+
+
+
+### >>>
+
+>>>  (class=arithmetic #args=2) Bitwise unsigned right-shift.
+

+
+
+
+### ^
+
+^  (class=arithmetic #args=2) Bitwise XOR.
+

+
+
+
+### \|
+
+|  (class=arithmetic #args=2) Bitwise OR.
+

+
+
+
+### ~
+
+~  (class=arithmetic #args=1) Bitwise NOT. Beware '$y=~$x' since =~ is the
+regex-match operator: try '$y = ~$x'.
+

+
+## Boolean functions
+
+
+
+### \!
+
+!  (class=boolean #args=1) Logical negation.
+

+
+
+
+### \!=
+
+!=  (class=boolean #args=2) String/numeric inequality. Mixing number and string results in string compare.
+

+
+
+
+### !=~
+
+!=~  (class=boolean #args=2) String (left-hand side) does not match regex (right-hand side), e.g. '$name !=~ "^a.*b$"'.
+

+
+
+
+### &&
+
+&&  (class=boolean #args=2) Logical AND.
+

+
+
+
+### <
+
+<  (class=boolean #args=2) String/numeric less-than. Mixing number and string results in string compare.
+

+
+
+
+### <=
+
+<=  (class=boolean #args=2) String/numeric less-than-or-equals. Mixing number and string results in string compare.
+

+
+
+
+### ==
+
+==  (class=boolean #args=2) String/numeric equality. Mixing number and string results in string compare.
+

+
+
+
+### =~
+
+=~  (class=boolean #args=2) String (left-hand side) matches regex (right-hand side), e.g. '$name =~ "^a.*b$"'.
+

+
+
+
+### >
+
+>  (class=boolean #args=2) String/numeric greater-than. Mixing number and string results in string compare.
+

+
+
+
+### >=
+
+>=  (class=boolean #args=2) String/numeric greater-than-or-equals. Mixing number and string results in string compare.
+

+
+
+
+### ?:
+
+?:  (class=boolean #args=3) Standard ternary operator.
+

+
+
+
+### ??
+
+??  (class=boolean #args=2) Absent-coalesce operator. $a ?? 1 evaluates to 1 if $a isn't defined in the current record.
+

+
+
+
+### ???
+
+???  (class=boolean #args=2) Absent-coalesce operator. $a ?? 1 evaluates to 1 if $a isn't defined in the current record, or has empty value.
+

+
+
+
+### ^^
+
+^^  (class=boolean #args=2) Logical XOR.
+

+
+
+
+### \|\|
+
+||  (class=boolean #args=2) Logical OR.
+

+
+## Collections functions
+
+
+### append
+
+append  (class=collections #args=2) Appends second argument to end of first argument, which must be an array.
+

+
+
+### arrayify
+
+arrayify  (class=collections #args=1) Walks through a nested map/array, converting any map with consecutive keys
+"1", "2", ... into an array. Useful to wrap the output of unflatten.
+

+
+
+### depth
+
+depth  (class=collections #args=1) Prints maximum depth of map/array. Scalars have depth 0.
+

+
+
+### flatten
+
+flatten  (class=collections #args=3) Flattens multi-level maps to single-level ones. Examples:
 flatten("a", ".", {"b": { "c": 4 }}) is {"a.b.c" : 4}.
 flatten("", ".", {"a": { "b": 3 }}) is {"a.b" : 3}.
 Two-argument version: flatten($*, ".") is the same as flatten("", ".", $*).
@@ -372,257 +350,129 @@ Useful for nested JSON-like structures for non-JSON file formats like CSV.
 

 
 
-## float
+### get_keys
+
+get_keys  (class=collections #args=1) Returns array of keys of map or array
+

 
+
+### get_values
+
+get_values  (class=collections #args=1) Returns array of keys of map or array -- in the latter case, returns a copy of the array
+

+
+
+### haskey
+
+haskey  (class=collections #args=2) True/false if map has/hasn't key, e.g. 'haskey($*, "a")' or
+'haskey(mymap, mykey)', or true/false if array index is in bounds / out of bounds.
+Error if 1st argument is not a map or array. Note -n..-1 alias to 1..n in Miller arrays.
+

+
+
+### json_parse
+
+json_parse  (class=collections #args=1) Converts value from JSON-formatted string.
+

+
+
+### json_stringify
+
+json_stringify  (class=collections #args=1,2) Converts value to JSON-formatted string. Default output is single-line.
+With optional second boolean argument set to true, produces multiline output.
+

+
+
+### leafcount
+
+leafcount  (class=collections #args=1) Counts total number of terminal values in map/array. For single-level
+map/array, same as length.
+

+
+
+### length
+
+length  (class=collections #args=1) Counts number of top-level entries in array/map. Scalars have length 1.
+

+
+
+### mapdiff
+
+mapdiff  (class=collections #args=variadic) With 0 args, returns empty map. With 1 arg, returns copy of arg.
+With 2 or more, returns copy of arg 1 with all keys from any of remaining
+argument maps removed.
+

+
+
+### mapexcept
+
+mapexcept  (class=collections #args=variadic) Returns a map with keys from remaining arguments, if any, unset.
+Remaining arguments can be strings or arrays of string.
+E.g. 'mapexcept({1:2,3:4,5:6}, 1, 5, 7)' is '{3:4}'
+and  'mapexcept({1:2,3:4,5:6}, [1, 5, 7])' is '{3:4}'.
+

+
+
+### mapselect
+
+mapselect  (class=collections #args=variadic) Returns a map with only keys from remaining arguments set.
+Remaining arguments can be strings or arrays of string.
+E.g. 'mapselect({1:2,3:4,5:6}, 1, 5, 7)' is '{1:2,5:6}'
+and  'mapselect({1:2,3:4,5:6}, [1, 5, 7])' is '{1:2,5:6}'.
+

+
+
+### mapsum
+
+mapsum  (class=collections #args=variadic) With 0 args, returns empty map. With >= 1 arg, returns a map with
+key-value pairs from all arguments. Rightmost collisions win, e.g.
+'mapsum({1:2,3:4},{1:5})' is '{1:5,3:4}'.
+

+
+
+### unflatten
+
+unflatten  (class=collections #args=2) Reverses flatten. Example:
+unflatten({"a.b.c" : 4}, ".") is {"a": "b": { "c": 4 }}.
+Useful for nested JSON-like structures for non-JSON file formats like CSV.
+See also arrayify.
+

+
+## Conversion functions
+
+
+### boolean
+
+boolean  (class=conversion #args=1) Convert int/float/bool/string to boolean.
+

+
+
+### float
 
 float  (class=conversion #args=1) Convert int/float/bool/string to float.
 

 
 
-## floor
-
-
-floor  (class=math #args=1) Floor: nearest integer at or below.
-

-
-
-## fmtnum
-
+### fmtnum
 
 fmtnum  (class=conversion #args=2) Convert int/float/bool to string using
 printf-style format string, e.g. '$s = fmtnum($n, "%06lld")'.
 

 
 
-## fsec2dhms
-
-
-fsec2dhms  (class=time #args=1) Formats floating-point seconds as in fsec2dhms(500000.25) = "5d18h53m20.250000s"
-

-
-
-## fsec2hms
-
-
-fsec2hms  (class=time #args=1) Formats floating-point seconds as in fsec2hms(5000.25) = "01:23:20.250000"
-

-
-
-## get_keys
-
-
-get_keys  (class=maps/arrays #args=1) Returns array of keys of map or array
-

-
-
-## get_values
-
-
-get_values  (class=maps/arrays #args=1) Returns array of keys of map or array -- in the latter case, returns a copy of the array
-

-
-
-## gmt2sec
-
-
-gmt2sec  (class=time #args=1) Parses GMT timestamp as integer seconds since the epoch.
-

-
-
-## gsub
-
-
-gsub  (class=string #args=3) Example: '$name=gsub($name, "old", "new")' (replace all).
-

-
-
-## haskey
-
-
-haskey  (class=maps/arrays #args=2) True/false if map has/hasn't key, e.g. 'haskey($*, "a")' or
-'haskey(mymap, mykey)', or true/false if array index is in bounds / out of bounds.
-Error if 1st argument is not a map or array. Note -n..-1 alias to 1..n in Miller arrays.
-

-
-
-## hexfmt
-
+### hexfmt
 
 hexfmt  (class=conversion #args=1) Convert int to hex string, e.g. 255 to "0xff".
 

 
 
-## hms2fsec
-
-
-hms2fsec  (class=time #args=1) Recovers floating-point seconds as in hms2fsec("01:23:20.250000") = 5000.250000
-

-
-
-## hms2sec
-
-
-hms2sec  (class=time #args=1) Recovers integer seconds as in hms2sec("01:23:20") = 5000
-

-
-
-## hostname
-
-
-hostname  (class=system #args=0) Returns the hostname as a string.
-

-
-
-## int
-
+### int
 
 int  (class=conversion #args=1) Convert int/float/bool/string to int.
 

 
 
-## invqnorm
-
-
-invqnorm  (class=math #args=1) Inverse of normal cumulative distribution function.
-Note that invqorm(urand()) is normally distributed.
-

-
-
-## is_absent
-
-
-is_absent  (class=typing #args=1) False if field is present in input, true otherwise
-

-
-
-## is_array
-
-
-is_array  (class=typing #args=1) True if argument is an array.
-

-
-
-## is_bool
-
-
-is_bool  (class=typing #args=1) True if field is present with boolean value. Synonymous with is_boolean.
-

-
-
-## is_boolean
-
-
-is_boolean  (class=typing #args=1) True if field is present with boolean value. Synonymous with is_bool.
-

-
-
-## is_empty
-
-
-is_empty  (class=typing #args=1) True if field is present in input with empty string value, false otherwise.
-

-
-
-## is_empty_map
-
-
-is_empty_map  (class=typing #args=1) True if argument is a map which is empty.
-

-
-
-## is_error
-
-
-is_error  (class=typing #args=1) True if if argument is an error, such as taking string length of an integer.
-

-
-
-## is_float
-
-
-is_float  (class=typing #args=1) True if field is present with value inferred to be float
-

-
-
-## is_int
-
-
-is_int  (class=typing #args=1) True if field is present with value inferred to be int
-

-
-
-## is_map
-
-
-is_map  (class=typing #args=1) True if argument is a map.
-

-
-
-## is_nonempty_map
-
-
-is_nonempty_map  (class=typing #args=1) True if argument is a map which is non-empty.
-

-
-
-## is_not_array
-
-
-is_not_array  (class=typing #args=1) True if argument is not an array.
-

-
-
-## is_not_empty
-
-
-is_not_empty  (class=typing #args=1) False if field is present in input with empty value, true otherwise
-

-
-
-## is_not_map
-
-
-is_not_map  (class=typing #args=1) True if argument is not a map.
-

-
-
-## is_not_null
-
-
-is_not_null  (class=typing #args=1) False if argument is null (empty or absent), true otherwise.
-

-
-
-## is_null
-
-
-is_null  (class=typing #args=1) True if argument is null (empty or absent), false otherwise.
-

-
-
-## is_numeric
-
-
-is_numeric  (class=typing #args=1) True if field is present with value inferred to be int or float
-

-
-
-## is_present
-
-
-is_present  (class=typing #args=1) True if field is present in input, false otherwise.
-

-
-
-## is_string
-
-
-is_string  (class=typing #args=1) True if field is present with string (including empty-string) value
-

-
-
-## joink
-
+### joink
 
 joink  (class=conversion #args=2) Makes string from map/array keys. Examples:
 joink({"a":3,"b":4,"c":5}, ",") = "a,b,c"
@@ -630,8 +480,7 @@ joink([1,2,3], ",") = "1,2,3".
 

 
 
-## joinkv
-
+### joinkv
 
 joinkv  (class=conversion #args=3) Makes string from map/array key-value pairs. Examples:
 joinkv([3,4,5], "=", ",") = "1=3,2=4,3=5"
@@ -639,8 +488,7 @@ joinkv({"a":3,"b":4,"c":5}, "=", ",") = "a=3,b=4,c=5"
 

 
 
-## joinv
-
+### joinv
 
 joinv  (class=conversion #args=2) Makes string from map/array values.
 joinv([3,4,5], ",") = "3,4,5"
@@ -648,225 +496,508 @@ joinv({"a":3,"b":4,"c":5}, ",") = "3,4,5"
 

 
 
-## json_parse
-
+### splita
 
-json_parse  (class=maps/arrays #args=1) Converts value from JSON-formatted string.
+splita  (class=conversion #args=2) Splits string into array with type inference. Example:
+splita("3,4,5", ",") = [3,4,5]
 

 
 
-## json_stringify
-
+### splitax
 
-json_stringify  (class=maps/arrays #args=1,2) Converts value to JSON-formatted string. Default output is single-line.
-With optional second boolean argument set to true, produces multiline output.
+splitax  (class=conversion #args=2) Splits string into array without type inference. Example:
+splita("3,4,5", ",") = ["3","4","5"]
 

 
 
-## leafcount
-
+### splitkv
 
-leafcount  (class=maps/arrays #args=1) Counts total number of terminal values in map/array. For single-level
-map/array, same as length.
+splitkv  (class=conversion #args=3) Splits string by separators into map with type inference. Example:
+splitkv("a=3,b=4,c=5", "=", ",") = {"a":3,"b":4,"c":5}
 

 
 
-## length
-
+### splitkvx
 
-length  (class=maps/arrays #args=1) Counts number of top-level entries in array/map. Scalars have length 1.
+splitkvx  (class=conversion #args=3) Splits string by separators into map without type inference (keys and
+values are strings). Example:
+splitkvx("a=3,b=4,c=5", "=", ",") = {"a":"3","b":"4","c":"5"}
 

 
 
-## log
+### splitnv
+
+splitnv  (class=conversion #args=2) Splits string by separator into integer-indexed map with type inference. Example:
+splitnv("a,b,c", ",") = {"1":"a","2":"b","3":"c"}
+

 
+
+### splitnvx
+
+splitnvx  (class=conversion #args=2) Splits string by separator into integer-indexed map without type
+inference (values are strings). Example:
+splitnvx("3,4,5", ",") = {"1":"3","2":"4","3":"5"}
+

+
+
+### string
+
+string  (class=conversion #args=1) Convert int/float/bool/string/array/map to string.
+

+
+## Hashing functions
+
+
+### md5
+
+md5  (class=hashing #args=1) MD5 hash.
+

+
+
+### sha1
+
+sha1  (class=hashing #args=1) SHA1 hash.
+

+
+
+### sha256
+
+sha256  (class=hashing #args=1) SHA256 hash.
+

+
+
+### sha512
+
+sha512  (class=hashing #args=1) SHA512 hash.
+

+
+## Math functions
+
+
+### abs
+
+abs  (class=math #args=1) Absolute value.
+

+
+
+### acos
+
+acos  (class=math #args=1) Inverse trigonometric cosine.
+

+
+
+### acosh
+
+acosh  (class=math #args=1) Inverse hyperbolic cosine.
+

+
+
+### asin
+
+asin  (class=math #args=1) Inverse trigonometric sine.
+

+
+
+### asinh
+
+asinh  (class=math #args=1) Inverse hyperbolic sine.
+

+
+
+### atan
+
+atan  (class=math #args=1) One-argument arctangent.
+

+
+
+### atan2
+
+atan2  (class=math #args=2) Two-argument arctangent.
+

+
+
+### atanh
+
+atanh  (class=math #args=1) Inverse hyperbolic tangent.
+

+
+
+### cbrt
+
+cbrt  (class=math #args=1) Cube root.
+

+
+
+### ceil
+
+ceil  (class=math #args=1) Ceiling: nearest integer at or above.
+

+
+
+### cos
+
+cos  (class=math #args=1) Trigonometric cosine.
+

+
+
+### cosh
+
+cosh  (class=math #args=1) Hyperbolic cosine.
+

+
+
+### erf
+
+erf  (class=math #args=1) Error function.
+

+
+
+### erfc
+
+erfc  (class=math #args=1) Complementary error function.
+

+
+
+### exp
+
+exp  (class=math #args=1) Exponential function e**x.
+

+
+
+### expm1
+
+expm1  (class=math #args=1) e**x - 1.
+

+
+
+### floor
+
+floor  (class=math #args=1) Floor: nearest integer at or below.
+

+
+
+### invqnorm
+
+invqnorm  (class=math #args=1) Inverse of normal cumulative distribution function.
+Note that invqorm(urand()) is normally distributed.
+

+
+
+### log
 
 log  (class=math #args=1) Natural (base-e) logarithm.
 

 
 
-## log10
-
+### log10
 
 log10  (class=math #args=1) Base-10 logarithm.
 

 
 
-## log1p
-
+### log1p
 
 log1p  (class=math #args=1) log(1-x).
 

 
 
-## logifit
-
+### logifit
 
 logifit  (class=math #args=3)  Given m and b from logistic regression, compute fit:
 $yhat=logifit($x,$m,$b).
 

 
 
-## lstrip
-
-
-lstrip  (class=string #args=1) Strip leading whitespace from string.
-

-
-
-## madd
-
-
-madd  (class=arithmetic #args=3) a + b mod m (integers)
-

-
-
-## mapdiff
-
-
-mapdiff  (class=maps/arrays #args=variadic) With 0 args, returns empty map. With 1 arg, returns copy of arg.
-With 2 or more, returns copy of arg 1 with all keys from any of remaining
-argument maps removed.
-

-
-
-## mapexcept
-
-
-mapexcept  (class=maps/arrays #args=variadic) Returns a map with keys from remaining arguments, if any, unset.
-Remaining arguments can be strings or arrays of string.
-E.g. 'mapexcept({1:2,3:4,5:6}, 1, 5, 7)' is '{3:4}'
-and  'mapexcept({1:2,3:4,5:6}, [1, 5, 7])' is '{3:4}'.
-

-
-
-## mapselect
-
-
-mapselect  (class=maps/arrays #args=variadic) Returns a map with only keys from remaining arguments set.
-Remaining arguments can be strings or arrays of string.
-E.g. 'mapselect({1:2,3:4,5:6}, 1, 5, 7)' is '{1:2,5:6}'
-and  'mapselect({1:2,3:4,5:6}, [1, 5, 7])' is '{1:2,5:6}'.
-

-
-
-## mapsum
-
-
-mapsum  (class=maps/arrays #args=variadic) With 0 args, returns empty map. With >= 1 arg, returns a map with
-key-value pairs from all arguments. Rightmost collisions win, e.g.
-'mapsum({1:2,3:4},{1:5})' is '{1:5,3:4}'.
-

-
-
-## max
-
+### max
 
 max  (class=math #args=variadic) Max of n numbers; null loses.
 

 
 
-## md5
-
-
-md5  (class=hashing #args=1) MD5 hash.
-

-
-
-## mexp
-
-
-mexp  (class=arithmetic #args=3) a ** b mod m (integers)
-

-
-
-## min
-
+### min
 
 min  (class=math #args=variadic) Min of n numbers; null loses.
 

 
 
-## mmul
-
-
-mmul  (class=arithmetic #args=3) a * b mod m (integers)
-

-
-
-## msub
-
-
-msub  (class=arithmetic #args=3) a - b mod m (integers)
-

-
-
-## os
-
-
-os  (class=system #args=0) Returns the operating-system name as a string.
-

-
-
-## pow
-
-
-pow  (class=arithmetic #args=2) Exponentiation. Same as **, but as a function.
-

-
-
-## qnorm
-
+### qnorm
 
 qnorm  (class=math #args=1) Normal cumulative distribution function.
 

 
 
-## regextract
-
-
-regextract  (class=string #args=2) Example: '$name=regextract($name, "[A-Z]{3}[0-9]{2}")'
-

-
-
-## regextract_or_else
-
-
-regextract_or_else  (class=string #args=3) Example: '$name=regextract_or_else($name, "[A-Z]{3}[0-9]{2}", "default")'
-

-
-
-## round
-
+### round
 
 round  (class=math #args=1) Round to nearest integer.
 

 
 
-## roundm
-
+### roundm
 
 roundm  (class=math #args=2) Round to nearest multiple of m: roundm($x,$m) is
 the same as round($x/$m)*$m.
 

 
 
-## rstrip
+### sgn
+
+sgn  (class=math #args=1)  +1, 0, -1 for positive, zero, negative input respectively.
+

 
+
+### sin
+
+sin  (class=math #args=1) Trigonometric sine.
+

+
+
+### sinh
+
+sinh  (class=math #args=1) Hyperbolic sine.
+

+
+
+### sqrt
+
+sqrt  (class=math #args=1) Square root.
+

+
+
+### tan
+
+tan  (class=math #args=1) Trigonometric tangent.
+

+
+
+### tanh
+
+tanh  (class=math #args=1) Hyperbolic tangent.
+

+
+
+### urand
+
+urand  (class=math #args=0) Floating-point numbers uniformly distributed on the unit interval.
+Int-valued example: '$n=floor(20+urand()*11)'.
+

+
+
+### urand32
+
+urand32  (class=math #args=0) Integer uniformly distributed 0 and 2**32-1 inclusive.
+

+
+
+### urandint
+
+urandint  (class=math #args=2) Integer uniformly distributed between inclusive integer endpoints.
+

+
+
+### urandrange
+
+urandrange  (class=math #args=2) Floating-point numbers uniformly distributed on the interval [a, b).
+

+
+## String functions
+
+
+### capitalize
+
+capitalize  (class=string #args=1) Convert string's first character to uppercase.
+

+
+
+### clean_whitespace
+
+clean_whitespace  (class=string #args=1) Same as collapse_whitespace and strip.
+

+
+
+### collapse_whitespace
+
+collapse_whitespace  (class=string #args=1) Strip repeated whitespace from string.
+

+
+
+### gsub
+
+gsub  (class=string #args=3) Example: '$name=gsub($name, "old", "new")' (replace all).
+

+
+
+### lstrip
+
+lstrip  (class=string #args=1) Strip leading whitespace from string.
+

+
+
+### regextract
+
+regextract  (class=string #args=2) Example: '$name=regextract($name, "[A-Z]{3}[0-9]{2}")'
+

+
+
+### regextract_or_else
+
+regextract_or_else  (class=string #args=3) Example: '$name=regextract_or_else($name, "[A-Z]{3}[0-9]{2}", "default")'
+

+
+
+### rstrip
 
 rstrip  (class=string #args=1) Strip trailing whitespace from string.
 

 
 
-## sec2dhms
+### ssub
+
+ssub  (class=string #args=3) Like sub but does no regexing. No characters are special.
+

 
+
+### strip
+
+strip  (class=string #args=1) Strip leading and trailing whitespace from string.
+

+
+
+### strlen
+
+strlen  (class=string #args=1) String length.
+

+
+
+### sub
+
+sub  (class=string #args=3) Example: '$name=sub($name, "old", "new")' (replace once).
+

+
+
+### substr
+
+substr  (class=string #args=3) substr is an alias for substr0. See also substr1. Miller is generally 1-up
+with all array and string indices, but, this is a backward-compatibility issue with Miller 5
+and below. Arrays are new in Miller 6; the substr function is older.
+

+
+
+### substr0
+
+substr0  (class=string #args=3) substr0(s,m,n) gives substring of s from 0-up position m to n
+inclusive. Negative indices -len .. -1 alias to 0 .. len-1. See also substr and substr1.
+

+
+
+### substr1
+
+substr1  (class=string #args=3) substr1(s,m,n) gives substring of s from 1-up position m to n
+inclusive. Negative indices -len .. -1 alias to 1 .. len. See also substr and substr0.
+

+
+
+### tolower
+
+tolower  (class=string #args=1) Convert string to lowercase.
+

+
+
+### toupper
+
+toupper  (class=string #args=1) Convert string to uppercase.
+

+
+
+### truncate
+
+truncate  (class=string #args=2) Truncates string first argument to max length of int second argument.
+

+
+
+
+### \.
+
+.  (class=string #args=2) String concatenation.
+

+
+## System functions
+
+
+### hostname
+
+hostname  (class=system #args=0) Returns the hostname as a string.
+

+
+
+### os
+
+os  (class=system #args=0) Returns the operating-system name as a string.
+

+
+
+### system
+
+system  (class=system #args=1) Run command string, yielding its stdout minus final carriage return.
+

+
+
+### version
+
+version  (class=system #args=0) Returns the Miller version as a string.
+

+
+## Time functions
+
+
+### dhms2fsec
+
+dhms2fsec  (class=time #args=1) Recovers floating-point seconds as in dhms2fsec("5d18h53m20.250000s") = 500000.250000
+

+
+
+### dhms2sec
+
+dhms2sec  (class=time #args=1) Recovers integer seconds as in dhms2sec("5d18h53m20s") = 500000
+

+
+
+### fsec2dhms
+
+fsec2dhms  (class=time #args=1) Formats floating-point seconds as in fsec2dhms(500000.25) = "5d18h53m20.250000s"
+

+
+
+### fsec2hms
+
+fsec2hms  (class=time #args=1) Formats floating-point seconds as in fsec2hms(5000.25) = "01:23:20.250000"
+

+
+
+### gmt2sec
+
+gmt2sec  (class=time #args=1) Parses GMT timestamp as integer seconds since the epoch.
+

+
+
+### hms2fsec
+
+hms2fsec  (class=time #args=1) Recovers floating-point seconds as in hms2fsec("01:23:20.250000") = 5000.250000
+

+
+
+### hms2sec
+
+hms2sec  (class=time #args=1) Recovers integer seconds as in hms2sec("01:23:20") = 5000
+

+
+
+### sec2dhms
 
 sec2dhms  (class=time #args=1) Formats integer seconds as in sec2dhms(500000) = "5d18h53m20s"
 

 
 
-## sec2gmt
-
+### sec2gmt
 
 sec2gmt  (class=time #args=1,2) Formats seconds since epoch (integer part)
 as GMT timestamp, e.g. sec2gmt(1440768801.7) = "2015-08-28T13:33:21Z".
@@ -875,8 +1006,7 @@ for the seconds part
 

 
 
-## sec2gmtdate
-
+### sec2gmtdate
 
 sec2gmtdate  (class=time #args=1) Formats seconds since epoch (integer part)
 as GMT timestamp with year-month-date, e.g. sec2gmtdate(1440768801.7) = "2015-08-28".
@@ -884,121 +1014,13 @@ Leaves non-numbers as-is.
 

 
 
-## sec2hms
-
+### sec2hms
 
 sec2hms  (class=time #args=1) Formats integer seconds as in sec2hms(5000) = "01:23:20"
 

 
 
-## sgn
-
-
-sgn  (class=math #args=1)  +1, 0, -1 for positive, zero, negative input respectively.
-

-
-
-## sha1
-
-
-sha1  (class=hashing #args=1) SHA1 hash.
-

-
-
-## sha256
-
-
-sha256  (class=hashing #args=1) SHA256 hash.
-

-
-
-## sha512
-
-
-sha512  (class=hashing #args=1) SHA512 hash.
-

-
-
-## sin
-
-
-sin  (class=math #args=1) Trigonometric sine.
-

-
-
-## sinh
-
-
-sinh  (class=math #args=1) Hyperbolic sine.
-

-
-
-## splita
-
-
-splita  (class=conversion #args=2) Splits string into array with type inference. Example:
-splita("3,4,5", ",") = [3,4,5]
-

-
-
-## splitax
-
-
-splitax  (class=conversion #args=2) Splits string into array without type inference. Example:
-splita("3,4,5", ",") = ["3","4","5"]
-

-
-
-## splitkv
-
-
-splitkv  (class=conversion #args=3) Splits string by separators into map with type inference. Example:
-splitkv("a=3,b=4,c=5", "=", ",") = {"a":3,"b":4,"c":5}
-

-
-
-## splitkvx
-
-
-splitkvx  (class=conversion #args=3) Splits string by separators into map without type inference (keys and
-values are strings). Example:
-splitkvx("a=3,b=4,c=5", "=", ",") = {"a":"3","b":"4","c":"5"}
-

-
-
-## splitnv
-
-
-splitnv  (class=conversion #args=2) Splits string by separator into integer-indexed map with type inference. Example:
-splitnv("a,b,c", ",") = {"1":"a","2":"b","3":"c"}
-

-
-
-## splitnvx
-
-
-splitnvx  (class=conversion #args=2) Splits string by separator into integer-indexed map without type
-inference (values are strings). Example:
-splitnvx("3,4,5", ",") = {"1":"3","2":"4","3":"5"}
-

-
-
-## sqrt
-
-
-sqrt  (class=math #args=1) Square root.
-

-
-
-## ssub
-
-
-ssub  (class=string #args=3) Like sub but does no regexing. No characters are special.
-

-
-
-## strftime
-
+### strftime
 
 strftime  (class=time #args=2)  Formats seconds since the epoch as timestamp, e.g.
 	strftime(1440768801.7,"%Y-%m-%dT%H:%M:%SZ") = "2015-08-28T13:33:21Z", and
@@ -1010,29 +1032,7 @@ strftime  (class=time #args=2)  Formats seconds since the epoch as timestamp, e.
 

 
 
-## string
-
-
-string  (class=conversion #args=1) Convert int/float/bool/string/array/map to string.
-

-
-
-## strip
-
-
-strip  (class=string #args=1) Strip leading and trailing whitespace from string.
-

-
-
-## strlen
-
-
-strlen  (class=string #args=1) String length.
-

-
-
-## strptime
-
+### strptime
 
 strptime  (class=time #args=2) strptime: Parses timestamp as floating-point seconds since the epoch,
 	e.g. strptime("2015-08-28T13:33:21Z","%Y-%m-%dT%H:%M:%SZ") = 1440768801.000000,
@@ -1041,407 +1041,275 @@ strptime  (class=time #args=2) strptime: Parses timestamp as floating-point seco
 

 
 
-## sub
-
-
-sub  (class=string #args=3) Example: '$name=sub($name, "old", "new")' (replace once).
-

-
-
-## substr
-
-
-substr  (class=string #args=3) substr is an alias for substr0. See also substr1. Miller is generally 1-up
-with all array indices, but, this is a backward-compatibility issue with Miller 5 and below.
-Arrays are new in Miller 6; the substr function is older.
-

-
-
-## substr0
-
-
-substr0  (class=string #args=3) substr0(s,m,n) gives substring of s from 0-up position m to n
-inclusive. Negative indices -len .. -1 alias to 0 .. len-1.
-

-
-
-## substr1
-
-
-substr1  (class=string #args=3) substr1(s,m,n) gives substring of s from 1-up position m to n
-inclusive. Negative indices -len .. -1 alias to 1 .. len.
-

-
-
-## system
-
-
-system  (class=system #args=1) Run command string, yielding its stdout minus final carriage return.
-

-
-
-## systime
-
+### systime
 
 systime  (class=time #args=0) help string will go here
 

 
 
-## systimeint
-
+### systimeint
 
 systimeint  (class=time #args=0) help string will go here
 

 
 
-## tan
-
-
-tan  (class=math #args=1) Trigonometric tangent.
-

-
-
-## tanh
-
-
-tanh  (class=math #args=1) Hyperbolic tangent.
-

-
-
-## tolower
-
-
-tolower  (class=string #args=1) Convert string to lowercase.
-

-
-
-## toupper
-
-
-toupper  (class=string #args=1) Convert string to uppercase.
-

-
-
-## truncate
-
-
-truncate  (class=string #args=2) Truncates string first argument to max length of int second argument.
-

-
-
-## typeof
-
-
-typeof  (class=typing #args=1) Convert argument to type of argument (e.g. "str"). For debug.
-

-
-
-## unflatten
-
-
-unflatten  (class=maps/arrays #args=2) Reverses flatten. Example:
-unflatten({"a.b.c" : 4}, ".") is {"a": "b": { "c": 4 }}.
-Useful for nested JSON-like structures for non-JSON file formats like CSV.
-See also arrayify.
-

-
-
-## uptime
-
+### uptime
 
 uptime  (class=time #args=0) help string will go here
 

 
+## Typing functions
 
-## urand
 
+### asserting_absent
 
-urand  (class=math #args=0) Floating-point numbers uniformly distributed on the unit interval.
-Int-valued example: '$n=floor(20+urand()*11)'.
+asserting_absent  (class=typing #args=1) Aborts with an error if is_absent on the argument returns false,
+else returns its argument.
 

 
 
-## urand32
-
+### asserting_array
 
-urand32  (class=math #args=0) Integer uniformly distributed 0 and 2**32-1 inclusive.
+asserting_array  (class=typing #args=1) Aborts with an error if is_array on the argument returns false,
+else returns its argument.
 

 
 
-## urandint
-
+### asserting_bool
 
-urandint  (class=math #args=2) Integer uniformly distributed between inclusive integer endpoints.
+asserting_bool  (class=typing #args=1) Aborts with an error if is_bool on the argument returns false,
+else returns its argument.
 

 
 
-## urandrange
-
+### asserting_boolean
 
-urandrange  (class=math #args=2) Floating-point numbers uniformly distributed on the interval [a, b).
+asserting_boolean  (class=typing #args=1) Aborts with an error if is_boolean on the argument returns false,
+else returns its argument.
 

 
 
-## version
-
+### asserting_empty
 
-version  (class=system #args=0) Returns the Miller version as a string.
+asserting_empty  (class=typing #args=1) Aborts with an error if is_empty on the argument returns false,
+else returns its argument.
 

 
 
-
-## \!
-
+### asserting_empty_map
 
-!  (class=boolean #args=1) Logical negation.
+asserting_empty_map  (class=typing #args=1) Aborts with an error if is_empty_map on the argument returns false,
+else returns its argument.
 

 
 
-## !=
-
+### asserting_error
 
-!=  (class=boolean #args=2) String/numeric inequality. Mixing number and string results in string compare.
+asserting_error  (class=typing #args=1) Aborts with an error if is_error on the argument returns false,
+else returns its argument.
 

 
 
-
-## !=~
-
+### asserting_float
 
-!=~  (class=boolean #args=2) String (left-hand side) does not match regex (right-hand side), e.g. '$name !=~ "^a.*b$"'.
+asserting_float  (class=typing #args=1) Aborts with an error if is_float on the argument returns false,
+else returns its argument.
 

 
 
-## %
-
+### asserting_int
 
-%  (class=arithmetic #args=2) Remainder; never negative-valued (pythonic).
+asserting_int  (class=typing #args=1) Aborts with an error if is_int on the argument returns false,
+else returns its argument.
 

 
 
-
-## &
-
+### asserting_map
 
-&  (class=arithmetic #args=2) Bitwise AND.
+asserting_map  (class=typing #args=1) Aborts with an error if is_map on the argument returns false,
+else returns its argument.
 

 
 
-
-## &&
-
+### asserting_nonempty_map
 
-&&  (class=boolean #args=2) Logical AND.
+asserting_nonempty_map  (class=typing #args=1) Aborts with an error if is_nonempty_map on the argument returns false,
+else returns its argument.
 

 
 
-
-## \*
-
+### asserting_not_array
 
-*  (class=arithmetic #args=2) Multiplication, with integer*integer overflow to float.
+asserting_not_array  (class=typing #args=1) Aborts with an error if is_not_array on the argument returns false,
+else returns its argument.
 

 
 
-
-## \**
-
+### asserting_not_empty
 
-**  (class=arithmetic #args=2) Exponentiation. Same as pow, but as an infix operator.
+asserting_not_empty  (class=typing #args=1) Aborts with an error if is_not_empty on the argument returns false,
+else returns its argument.
 

 
 
-
-## \+
-
+### asserting_not_map
 
-+  (class=arithmetic #args=1,2) Addition as binary operator; unary plus operator.
+asserting_not_map  (class=typing #args=1) Aborts with an error if is_not_map on the argument returns false,
+else returns its argument.
 

 
 
-
-## \-
-
+### asserting_not_null
 
--  (class=arithmetic #args=1,2) Subtraction as binary operator; unary negation operator.
+asserting_not_null  (class=typing #args=1) Aborts with an error if is_not_null on the argument returns false,
+else returns its argument.
 

 
 
-## .
-
+### asserting_null
 
-.  (class=string #args=2) String concatenation.
+asserting_null  (class=typing #args=1) Aborts with an error if is_null on the argument returns false,
+else returns its argument.
 

 
 
-## .*
-
+### asserting_numeric
 
-.*  (class=arithmetic #args=2) Multiplication, with integer-to-integer overflow.
+asserting_numeric  (class=typing #args=1) Aborts with an error if is_numeric on the argument returns false,
+else returns its argument.
 

 
 
-## .+
-
+### asserting_present
 
-.+  (class=arithmetic #args=2) Addition, with integer-to-integer overflow.
+asserting_present  (class=typing #args=1) Aborts with an error if is_present on the argument returns false,
+else returns its argument.
 

 
 
-## .-
-
+### asserting_string
 
-.-  (class=arithmetic #args=2) Subtraction, with integer-to-integer overflow.
+asserting_string  (class=typing #args=1) Aborts with an error if is_string on the argument returns false,
+else returns its argument.
 

 
 
-## ./
-
+### is_absent
 
-./  (class=arithmetic #args=2) Integer division; not pythonic.
+is_absent  (class=typing #args=1) False if field is present in input, true otherwise
 

 
 
-## /
-
+### is_array
 
-/  (class=arithmetic #args=2) Division. Integer / integer is floating-point.
+is_array  (class=typing #args=1) True if argument is an array.
 

 
 
-## //
-
+### is_bool
 
-//  (class=arithmetic #args=2) Pythonic integer division, rounding toward negative.
+is_bool  (class=typing #args=1) True if field is present with boolean value. Synonymous with is_boolean.
 

 
 
-## <
-
+### is_boolean
 
-<  (class=boolean #args=2) String/numeric less-than. Mixing number and string results in string compare.
+is_boolean  (class=typing #args=1) True if field is present with boolean value. Synonymous with is_bool.
 

 
 
-## <<
-
+### is_empty
 
-<<  (class=arithmetic #args=2) Bitwise left-shift.
+is_empty  (class=typing #args=1) True if field is present in input with empty string value, false otherwise.
 

 
 
-## <=
-
+### is_empty_map
 
-<=  (class=boolean #args=2) String/numeric less-than-or-equals. Mixing number and string results in string compare.
+is_empty_map  (class=typing #args=1) True if argument is a map which is empty.
 

 
 
-## ==
-
+### is_error
 
-==  (class=boolean #args=2) String/numeric equality. Mixing number and string results in string compare.
+is_error  (class=typing #args=1) True if if argument is an error, such as taking string length of an integer.
 

 
 
-
-## =~
-
+### is_float
 
-=~  (class=boolean #args=2) String (left-hand side) matches regex (right-hand side), e.g. '$name =~ "^a.*b$"'.
+is_float  (class=typing #args=1) True if field is present with value inferred to be float
 

 
 
-## >
-
+### is_int
 
->  (class=boolean #args=2) String/numeric greater-than. Mixing number and string results in string compare.
+is_int  (class=typing #args=1) True if field is present with value inferred to be int
 

 
 
-## >=
-
+### is_map
 
->=  (class=boolean #args=2) String/numeric greater-than-or-equals. Mixing number and string results in string compare.
+is_map  (class=typing #args=1) True if argument is a map.
 

 
 
-
-## \>\>
-
+### is_nonempty_map
 
->>  (class=arithmetic #args=2) Bitwise signed right-shift.
+is_nonempty_map  (class=typing #args=1) True if argument is a map which is non-empty.
 

 
 
-
-## \>\>\>
-
+### is_not_array
 
->>>  (class=arithmetic #args=2) Bitwise unsigned right-shift.
+is_not_array  (class=typing #args=1) True if argument is not an array.
 

 
 
-
-## ?:
-
+### is_not_empty
 
-?:  (class=boolean #args=3) Standard ternary operator.
+is_not_empty  (class=typing #args=1) False if field is present in input with empty value, true otherwise
 

 
 
-
-## ??
-
+### is_not_map
 
-??  (class=boolean #args=2) Absent-coalesce operator. $a ?? 1 evaluates to 1 if $a isn't defined in the current record.
+is_not_map  (class=typing #args=1) True if argument is not a map.
 

 
 
-
-## ???
-
+### is_not_null
 
-???  (class=boolean #args=2) Absent-coalesce operator. $a ?? 1 evaluates to 1 if $a isn't defined in the current record, or has empty value.
+is_not_null  (class=typing #args=1) False if argument is null (empty or absent), true otherwise.
 

 
 
-
-## ^
-
+### is_null
 
-^  (class=arithmetic #args=2) Bitwise XOR.
+is_null  (class=typing #args=1) True if argument is null (empty or absent), false otherwise.
 

 
 
-
-## ^^
-
+### is_numeric
 
-^^  (class=boolean #args=2) Logical XOR.
+is_numeric  (class=typing #args=1) True if field is present with value inferred to be int or float
 

 
 
-
-## \|
-
+### is_present
 
-|  (class=arithmetic #args=2) Bitwise OR.
+is_present  (class=typing #args=1) True if field is present in input, false otherwise.
 

 
 
-
-## \|\|
-
+### is_string
 
-||  (class=boolean #args=2) Logical OR.
+is_string  (class=typing #args=1) True if field is present with string (including empty-string) value
 

 
 
-## ~
-
+### typeof
 
-~  (class=arithmetic #args=1) Bitwise NOT. Beware '$y=~$x' since =~ is the
-regex-match operator: try '$y = ~$x'.
+typeof  (class=typing #args=1) Convert argument to type of argument (e.g. "str"). For debug.
 

 
diff --git a/docs6/docs/reference-dsl-builtin-functions.md.in b/docs6/docs/reference-dsl-builtin-functions.md.in
index 5079c8c98..0cf7e516c 100644
--- a/docs6/docs/reference-dsl-builtin-functions.md.in
+++ b/docs6/docs/reference-dsl-builtin-functions.md.in
@@ -1,13 +1,21 @@
 # DSL built-in functions
 
-## Summary
+Each function takes a specific number of arguments, as shown below, except for
+functions marked as variadic such as `min` and `max`. (The latter compute min
+and max of any number of arguments.) There is no notion of optional or
+default-on-absent arguments. All argument-passing is positional rather than by
+name; arguments are passed by value, not by reference.
 
-GENMD_RUN_CONTENT_GENERATOR(./mk-func-table.rb)
+At the command line, you can get a list of all functions using `mlr -f`, with
+details using `mlr -F`.  (Or, `mlr help usage-functions-by-class` to get
+details in the order shown on this page.) You can get detail for a given
+function using `mlr help function namegoeshere`, e.g.  `mlr help function
+gsub`.
 
-## List of functions
+Operators are listed here along with functions. In this case, the
+argument-count is the number of items involved in the infix operator, e.g. we
+say `x+y` so the details for the `+` operator say that its number of arguments
+is 2. Unary operators such as `!` and `~` show argument-count of 1; the ternary
+`? :` operator shows an argument-count of 3.
 
-Each function takes a specific number of arguments, as shown below, except for functions marked as variadic such as `min` and `max`. (The latter compute min and max of any number of numerical arguments.) There is no notion of optional or default-on-absent arguments. All argument-passing is positional rather than by name; arguments are passed by value, not by reference.
-
-You can get a list of all functions using **mlr -f**, with details using **mlr -F**.
-
-GENMD_RUN_CONTENT_GENERATOR(./mk-func-h2s.sh)
+GENMD_RUN_CONTENT_GENERATOR(./mk-func-info.rb)
diff --git a/docs6/docs/reference-dsl-differences.md b/docs6/docs/reference-dsl-differences.md
index 2d583bbd4..a3249219a 100644
--- a/docs6/docs/reference-dsl-differences.md
+++ b/docs6/docs/reference-dsl-differences.md
@@ -194,3 +194,12 @@ Miller has a [key-value loop flavor](reference-dsl-control-structures.md#key-val
 ## Semantics for one-variable for-loops
 
 Miller also has a [single-variable loop flavor](reference-dsl-control-structures.md#single-variable-for-loops). If `x` is a map then `for (e in x) { ... }` binds `e` to successive map _keys_ (not values as in PHP). But if `x` is an array then `for e in x) { ... }` binds `e` to successive array _values_ (not indices).
+
+## JSON parse, stringify, decode, encode
+
+Miller has the verbs
+[`json-parse`](reference-verbs.md#json-parse) and
+[`json-stringify`](reference-verbs.md#json-stringify), and the DSL functions
+[`json_parse`](reference-dsl-builtin-functions.md#json_parse) and
+[`json_stringify`](reference-dsl-builtin-functions.md#json_stringify).
+In some other lannguages these are called `json_decode` and `json_encode`.
diff --git a/docs6/docs/reference-dsl-differences.md.in b/docs6/docs/reference-dsl-differences.md.in
index 0d5f5bc12..dcb211144 100644
--- a/docs6/docs/reference-dsl-differences.md.in
+++ b/docs6/docs/reference-dsl-differences.md.in
@@ -155,3 +155,12 @@ Miller has a [key-value loop flavor](reference-dsl-control-structures.md#key-val
 ## Semantics for one-variable for-loops
 
 Miller also has a [single-variable loop flavor](reference-dsl-control-structures.md#single-variable-for-loops). If `x` is a map then `for (e in x) { ... }` binds `e` to successive map _keys_ (not values as in PHP). But if `x` is an array then `for e in x) { ... }` binds `e` to successive array _values_ (not indices).
+
+## JSON parse, stringify, decode, encode
+
+Miller has the verbs
+[`json-parse`](reference-verbs.md#json-parse) and
+[`json-stringify`](reference-verbs.md#json-stringify), and the DSL functions
+[`json_parse`](reference-dsl-builtin-functions.md#json_parse) and
+[`json_stringify`](reference-dsl-builtin-functions.md#json_stringify).
+In some other lannguages these are called `json_decode` and `json_encode`.
diff --git a/docs6/docs/reference-main-data-types.md b/docs6/docs/reference-main-data-types.md
index 87820ad28..ed33a0350 100644
--- a/docs6/docs/reference-main-data-types.md
+++ b/docs6/docs/reference-main-data-types.md
@@ -16,62 +16,187 @@ Quick links:
 
 ## List of types
 
-Miller's types for function processing are:
+Miller's types are:
 
 * Scalars:
-    * **string**
-    * **float** (double-precision) and **int** (64-bit signed): see the [Arithmetic](reference-main-arithmetic.md) page
-    * **boolean**
+    * **string**: such as `"abcdefg"`, supporting concatenation, one-up indexing and slicing, and [library functions](reference-dsl-builtin-functions.md#string-functions). See the pages on [strings](reference-main-strings.md) and [regular expressions](reference-main-regular-expressions.md).
+    * **float** and **int**: such as `1.2` and `3`: double-precision and 64-bit signed, respectively. See the section on [arithmetic operators and math-related library functions](reference-dsl-builtin-functions.md#math-functions) as well as the [Arithmetic](reference-main-arithmetic.md) page.
+    * **boolean**: literals `true` and `false`; results of `==`, `<`, `>`, etc. See the section on [boolean operators](reference-dsl-builtin-functions.md#boolean-functions).
 * Collections:
-    * **map**: see the [Maps](reference-main-maps.md) page
-    * **array**: see the [Arrays](reference-main-arrays.md) page
+    * **map**: such as `{"a":1,"b":[2,3,4]}`, supporting key-indexing, preservation of insertion order, [library functions](reference-dsl-builtin-functions.md#collections-functions), etc. See the [Maps](reference-main-maps.md) page.
+    * **array**: such as `["a", 2, true]`, supporting one-up indexing and slicing, [library functions](reference-dsl-builtin-functions.md#collections-functions), etc. See the [Arrays](reference-main-arrays.md) page.
 * Nulls and error:
-    * **absent-null** (reads of unset right-hand sides, or fall-through non-explicit return values from user-defined functions) and **JSON-null**: see the [null-data page](reference-main-null-data.md)
-    * **error** (TODO: give an example)
+    * **absent-null**: Such as on reads of unset right-hand sides, or fall-through non-explicit return values from user-defined functions. See the [null-data page](reference-main-null-data.md).
+    * **JSON-null**: For `null` in JSON files; also used in [gapped auto-extend of arrays](reference-main-arrays.md#auto-extend-and-null-gaps). See the [null-data page](reference-main-null-data.md).
+    * **error** -- for various results which cannot be computed, often when the input to a [built-in function](reference-dsl-builtin-functions.md) is of the wrong type. For example, doing [strlen](reference-dsl-builtin-functions.md#strlen) or [substr](reference-dsl-builtin-functions.md#substr) on a non-string, [sec2gmt](reference-dsl-builtin-functions.md#sec2gmt) on a non-integer, etc.
 
-TODO: point to null-data page; arrays page; arithmetic page; what else?
+See also the list of [type-checking
+functions](reference-dsl-builtin-functions.md#type-checkin -functions) for the
+[Miller programming language](programming-language.md).
 
-## To be ported
+See also [Differences from other programming languages](reference-dsl-differences.md).
 
-Miller's input and output are all string-oriented: there is (as of August 2015 anyway) no support for binary record packing. In this sense, everything is a string in and out of Miller.  During processing, field names are always strings, even if they have names like "3"; field values can have various data types.  Field values' ability to be interpreted as a non-string type only has meaning when comparison or function operations are done on them.  And it is an error condition if Miller encounters non-numeric (or otherwise mistyped) data in a field in which it has been asked to do numeric (or otherwise type-specific) operations.
+## Type inference for literal and record data
 
-Field values are treated as numeric for the following:
+Miller's input and output are all text-oriented: all the
+[file formats supported by Miller](file-formats.md) are human-readable text,
+such as CSV, TSV, and JSON; binary formats such as
+[BSON](https://bsonspec.org/) and [Parquet](https://parquet.apache.org/) are
+not supported (as of mid-2021). In this sense, everything is a string in and out of
+Miller -- be it in data files, or in DSL expressions you key in.
 
-* Numeric sort: `mlr sort -n`, `mlr sort -nr`.
-* Statistics: `mlr histogram`, `mlr stats1`, `mlr stats2`.
-* Cross-record arithmetic: `mlr step`.
+In the [DSL](programming-language.md), `7` is an `int` and `8.9` is a float, as
+one would expect.  Likewise, on input from [data files](file-formats.md),
+string values representable as numbers, e.g. `1.2` or `3`, are treated as int
+or float, respectively. If a record has `x=1,y=2` then `mlr put '$z=$x+$y'`
+will produce `x=1,y=2,z=3`.
 
-For `mlr put` and `mlr filter`:
+Numbers retain their original string representation, so if `x` is `1.2` on one
+record and `1.200` on another, they'll print out that way on output (unless of
+course they've been modified during processing, e.g. `mlr put '$x = $x + 10`).
 
-* Miller's types for function processing are **empty-null** (empty string), **absent-null** (reads of unset right-hand sides, or fall-through non-explicit return values from user-defined functions), **error**, **string**, **float** (double-precision), **int** (64-bit signed), and **boolean**.
+Generally strings, numbers, and booleans don't mix; use type-casting like
+`string($x)` to convert. However, the dot (string-concatenation) operator has
+been special-cased: `mlr put '$z=$x.$y'` does not give an error, because the
+dot operator has been generalized to stringify non-strings
 
-* On input, string values representable as numbers, e.g. "3" or "3.1", are treated as int or float, respectively. If a record has `x=1,y=2` then `mlr put '$z=$x+$y'` will produce `x=1,y=2,z=3`, and `mlr put '$z=$x.$y'` does not give an error, but only because the dot operator has been generalized to stringify non-strings.  To coerce back to string for processing, use the `string` function: `mlr put '$z=string($x).string($y)'` will produce `x=1,y=2,z=12`.
+Examples:
 
-* On input, string values representable as boolean  (e.g. `"true"`, `"false"`) are *not* automatically treated as boolean.  (This is because `"true"` and `"false"` are ordinary words, and auto string-to-boolean on a column consisting of words would result in some strings mixed with some booleans.) Use the `boolean` function to coerce: e.g. giving the record `x=1,y=2,w=false` to `mlr put '$z=($x<$y) || boolean($w)'`.
+
+mlr --csv cat data/type-infer.csv
+

+
+a,b,c
+1.2,3,true
+4,5.6,buongiorno
+

 
-* Functions take types as described in `mlr --help-all-functions`: for example, `log10` takes float input and produces float output, `gmt2sec` maps string to int, and `sec2gmt` maps int to string.
+
+mlr --icsv --oxtab --from data/type-infer.csv put '
+  $d = $a . $c;
+  $e = 7;
+  $f = 8.9;
+  $g = $e + $f;
+  $ta = typeof($a);
+  $tb = typeof($b);
+  $tc = typeof($c);
+  $td = typeof($d);
+  $te = typeof($e);
+  $tf = typeof($f);
+  $tg = typeof($g);
+' then reorder -f a,ta,b,tb,c,tc,d,td,e,te,f,tf,g,tg
+

+
+a  1.2
+ta float
+b  3
+tb int
+c  true
+tc bool
+d  1.2true
+td string
+e  7
+te int
+f  8.9
+tf float
+g  15.9
+tg float
 
-* All math functions described in `mlr --help-all-functions` take integer as well as float input.
+a  4
+ta int
+b  5.6
+tb float
+c  buongiorno
+tc string
+d  4buongiorno
+td string
+e  7
+te int
+f  8.9
+tf float
+g  15.9
+tg float
+

 
-## Escape sequences for string literals
+On input, string values representable as boolean  (e.g. `"true"`, `"false"`)
+are *not* automatically treated as boolean.  This is because `"true"` and
+`"false"` are ordinary words, and auto string-to-boolean on a column consisting
+of words would result in some strings mixed with some booleans. Use the
+`boolean` function to coerce: e.g. giving the record `x=1,y=2,w=false` to `mlr
+filter '$z=($x<$y) || boolean($w)'`.
 
-You can use the following backslash escapes for strings such as between the double quotes in contexts such as `mlr filter '$name =~ "..."'`, `mlr put '$name = $othername . "..."'`, `mlr put '$name = sub($name, "...", "...")`, etc.:
+## JSON parse and stringify
 
-* `\a`: ASCII code 0x07 (alarm/bell)
-* `\b`: ASCII code 0x08 (backspace)
-* `\f`: ASCII code 0x0c (formfeed)
-* `\n`: ASCII code 0x0a (LF/linefeed/newline)
-* `\r`: ASCII code 0x0d (CR/carriage return)
-* `\t`: ASCII code 0x09 (tab)
-* `\v`: ASCII code 0x0b (vertical tab)
-* `\\`: backslash
-* `\"`: double quote
-* `\123`: Octal 123, etc. for `\000` up to `\377`
-* `\x7f`: Hexadecimal 7f, etc. for `\x00` up to `\xff`
+If you have, say, a CSV file whose columns contain strings which are well-formatted JSON,
+they will not be auto-converted, but you can use the
+[`json-parse` verb](reference-verbs.md#json-parse)
+or the
+[`json_parse` DSL function](reference-dsl-builtin-functions.md#json_parse):
 
-See also [https://en.wikipedia.org/wiki/Escape_sequences_in_C](https://en.wikipedia.org/wiki/Escape_sequences_in_C).
+
+mlr --csv --from data/json-in-csv.csv cat
+

+
+id,blob
+100,"{""a"":1,""b"":[2,3,4]}"
+105,"{""a"":6,""b"":[7,8,9]}"
+

 
-These replacements apply only to strings you key in for the DSL expressions for `filter` and `put`: that is, if you type `\t` in a string literal for a `filter`/`put` expression, it will be turned into a tab character. If you want a backslash followed by a `t`, then please type `\\t`.
+
+mlr --icsv --ojson --from data/json-in-csv.csv cat
+

+
+{
+  "id": 100,
+  "blob": "{\"a\":1,\"b\":[2,3,4]}"
+}
+{
+  "id": 105,
+  "blob": "{\"a\":6,\"b\":[7,8,9]}"
+}
+

 
-However, these replacements are done automatically only for string literals within DSL expressions -- they are not done automatically to fields within your data stream.  If you wish to make these replacements, you can do (for example) `mlr put '$field = gsub($field, "\\t", "\t")'`. If you need to make such a replacement for all fields in your data, you should probably use the system `sed` command instead. 
+
+mlr --icsv --ojson --from data/json-in-csv.csv json-parse -f blob
+

+
+{
+  "id": 100,
+  "blob": {
+    "a": 1,
+    "b": [2, 3, 4]
+  }
+}
+{
+  "id": 105,
+  "blob": {
+    "a": 6,
+    "b": [7, 8, 9]
+  }
+}
+

 
+
+mlr --icsv --ojson --from data/json-in-csv.csv put '$blob = json_parse($blob)'
+

+
+{
+  "id": 100,
+  "blob": {
+    "a": 1,
+    "b": [2, 3, 4]
+  }
+}
+{
+  "id": 105,
+  "blob": {
+    "a": 6,
+    "b": [7, 8, 9]
+  }
+}
+

+
+These have their respective operations to convert back to string: the
+[`json-stringify` verb](reference-verbs.md#json-stringify)
+and
+[`json_stringify` DSL function](reference-dsl-builtin-functions.md#json_stringify).
diff --git a/docs6/docs/reference-main-data-types.md.in b/docs6/docs/reference-main-data-types.md.in
index 388a6888a..2fdfd5b82 100644
--- a/docs6/docs/reference-main-data-types.md.in
+++ b/docs6/docs/reference-main-data-types.md.in
@@ -2,62 +2,104 @@
 
 ## List of types
 
-Miller's types for function processing are:
+Miller's types are:
 
 * Scalars:
-    * **string**
-    * **float** (double-precision) and **int** (64-bit signed): see the [Arithmetic](reference-main-arithmetic.md) page
-    * **boolean**
+    * **string**: such as `"abcdefg"`, supporting concatenation, one-up indexing and slicing, and [library functions](reference-dsl-builtin-functions.md#string-functions). See the pages on [strings](reference-main-strings.md) and [regular expressions](reference-main-regular-expressions.md).
+    * **float** and **int**: such as `1.2` and `3`: double-precision and 64-bit signed, respectively. See the section on [arithmetic operators and math-related library functions](reference-dsl-builtin-functions.md#math-functions) as well as the [Arithmetic](reference-main-arithmetic.md) page.
+    * **boolean**: literals `true` and `false`; results of `==`, `<`, `>`, etc. See the section on [boolean operators](reference-dsl-builtin-functions.md#boolean-functions).
 * Collections:
-    * **map**: see the [Maps](reference-main-maps.md) page
-    * **array**: see the [Arrays](reference-main-arrays.md) page
+    * **map**: such as `{"a":1,"b":[2,3,4]}`, supporting key-indexing, preservation of insertion order, [library functions](reference-dsl-builtin-functions.md#collections-functions), etc. See the [Maps](reference-main-maps.md) page.
+    * **array**: such as `["a", 2, true]`, supporting one-up indexing and slicing, [library functions](reference-dsl-builtin-functions.md#collections-functions), etc. See the [Arrays](reference-main-arrays.md) page.
 * Nulls and error:
-    * **absent-null** (reads of unset right-hand sides, or fall-through non-explicit return values from user-defined functions) and **JSON-null**: see the [null-data page](reference-main-null-data.md)
-    * **error** (TODO: give an example)
+    * **absent-null**: Such as on reads of unset right-hand sides, or fall-through non-explicit return values from user-defined functions. See the [null-data page](reference-main-null-data.md).
+    * **JSON-null**: For `null` in JSON files; also used in [gapped auto-extend of arrays](reference-main-arrays.md#auto-extend-and-null-gaps). See the [null-data page](reference-main-null-data.md).
+    * **error** -- for various results which cannot be computed, often when the input to a [built-in function](reference-dsl-builtin-functions.md) is of the wrong type. For example, doing [strlen](reference-dsl-builtin-functions.md#strlen) or [substr](reference-dsl-builtin-functions.md#substr) on a non-string, [sec2gmt](reference-dsl-builtin-functions.md#sec2gmt) on a non-integer, etc.
 
-TODO: point to null-data page; arrays page; arithmetic page; what else?
+See also the list of [type-checking
+functions](reference-dsl-builtin-functions.md#type-checkin -functions) for the
+[Miller programming language](programming-language.md).
 
-## To be ported
+See also [Differences from other programming languages](reference-dsl-differences.md).
 
-Miller's input and output are all string-oriented: there is (as of August 2015 anyway) no support for binary record packing. In this sense, everything is a string in and out of Miller.  During processing, field names are always strings, even if they have names like "3"; field values can have various data types.  Field values' ability to be interpreted as a non-string type only has meaning when comparison or function operations are done on them.  And it is an error condition if Miller encounters non-numeric (or otherwise mistyped) data in a field in which it has been asked to do numeric (or otherwise type-specific) operations.
+## Type inference for literal and record data
 
-Field values are treated as numeric for the following:
+Miller's input and output are all text-oriented: all the
+[file formats supported by Miller](file-formats.md) are human-readable text,
+such as CSV, TSV, and JSON; binary formats such as
+[BSON](https://bsonspec.org/) and [Parquet](https://parquet.apache.org/) are
+not supported (as of mid-2021). In this sense, everything is a string in and out of
+Miller -- be it in data files, or in DSL expressions you key in.
 
-* Numeric sort: `mlr sort -n`, `mlr sort -nr`.
-* Statistics: `mlr histogram`, `mlr stats1`, `mlr stats2`.
-* Cross-record arithmetic: `mlr step`.
+In the [DSL](programming-language.md), `7` is an `int` and `8.9` is a float, as
+one would expect.  Likewise, on input from [data files](file-formats.md),
+string values representable as numbers, e.g. `1.2` or `3`, are treated as int
+or float, respectively. If a record has `x=1,y=2` then `mlr put '$z=$x+$y'`
+will produce `x=1,y=2,z=3`.
 
-For `mlr put` and `mlr filter`:
+Numbers retain their original string representation, so if `x` is `1.2` on one
+record and `1.200` on another, they'll print out that way on output (unless of
+course they've been modified during processing, e.g. `mlr put '$x = $x + 10`).
 
-* Miller's types for function processing are **empty-null** (empty string), **absent-null** (reads of unset right-hand sides, or fall-through non-explicit return values from user-defined functions), **error**, **string**, **float** (double-precision), **int** (64-bit signed), and **boolean**.
+Generally strings, numbers, and booleans don't mix; use type-casting like
+`string($x)` to convert. However, the dot (string-concatenation) operator has
+been special-cased: `mlr put '$z=$x.$y'` does not give an error, because the
+dot operator has been generalized to stringify non-strings
 
-* On input, string values representable as numbers, e.g. "3" or "3.1", are treated as int or float, respectively. If a record has `x=1,y=2` then `mlr put '$z=$x+$y'` will produce `x=1,y=2,z=3`, and `mlr put '$z=$x.$y'` does not give an error, but only because the dot operator has been generalized to stringify non-strings.  To coerce back to string for processing, use the `string` function: `mlr put '$z=string($x).string($y)'` will produce `x=1,y=2,z=12`.
+Examples:
 
-* On input, string values representable as boolean  (e.g. `"true"`, `"false"`) are *not* automatically treated as boolean.  (This is because `"true"` and `"false"` are ordinary words, and auto string-to-boolean on a column consisting of words would result in some strings mixed with some booleans.) Use the `boolean` function to coerce: e.g. giving the record `x=1,y=2,w=false` to `mlr put '$z=($x<$y) || boolean($w)'`.
+GENMD_RUN_COMMAND
+mlr --csv cat data/type-infer.csv
+GENMD_EOF
 
-* Functions take types as described in `mlr --help-all-functions`: for example, `log10` takes float input and produces float output, `gmt2sec` maps string to int, and `sec2gmt` maps int to string.
+GENMD_RUN_COMMAND
+mlr --icsv --oxtab --from data/type-infer.csv put '
+  $d = $a . $c;
+  $e = 7;
+  $f = 8.9;
+  $g = $e + $f;
+  $ta = typeof($a);
+  $tb = typeof($b);
+  $tc = typeof($c);
+  $td = typeof($d);
+  $te = typeof($e);
+  $tf = typeof($f);
+  $tg = typeof($g);
+' then reorder -f a,ta,b,tb,c,tc,d,td,e,te,f,tf,g,tg
+GENMD_EOF
 
-* All math functions described in `mlr --help-all-functions` take integer as well as float input.
+On input, string values representable as boolean  (e.g. `"true"`, `"false"`)
+are *not* automatically treated as boolean.  This is because `"true"` and
+`"false"` are ordinary words, and auto string-to-boolean on a column consisting
+of words would result in some strings mixed with some booleans. Use the
+`boolean` function to coerce: e.g. giving the record `x=1,y=2,w=false` to `mlr
+filter '$z=($x<$y) || boolean($w)'`.
 
-## Escape sequences for string literals
+## JSON parse and stringify
 
-You can use the following backslash escapes for strings such as between the double quotes in contexts such as `mlr filter '$name =~ "..."'`, `mlr put '$name = $othername . "..."'`, `mlr put '$name = sub($name, "...", "...")`, etc.:
+If you have, say, a CSV file whose columns contain strings which are well-formatted JSON,
+they will not be auto-converted, but you can use the
+[`json-parse` verb](reference-verbs.md#json-parse)
+or the
+[`json_parse` DSL function](reference-dsl-builtin-functions.md#json_parse):
 
-* `\a`: ASCII code 0x07 (alarm/bell)
-* `\b`: ASCII code 0x08 (backspace)
-* `\f`: ASCII code 0x0c (formfeed)
-* `\n`: ASCII code 0x0a (LF/linefeed/newline)
-* `\r`: ASCII code 0x0d (CR/carriage return)
-* `\t`: ASCII code 0x09 (tab)
-* `\v`: ASCII code 0x0b (vertical tab)
-* `\\`: backslash
-* `\"`: double quote
-* `\123`: Octal 123, etc. for `\000` up to `\377`
-* `\x7f`: Hexadecimal 7f, etc. for `\x00` up to `\xff`
+GENMD_RUN_COMMAND
+mlr --csv --from data/json-in-csv.csv cat
+GENMD_EOF
 
-See also [https://en.wikipedia.org/wiki/Escape_sequences_in_C](https://en.wikipedia.org/wiki/Escape_sequences_in_C).
+GENMD_RUN_COMMAND
+mlr --icsv --ojson --from data/json-in-csv.csv cat
+GENMD_EOF
 
-These replacements apply only to strings you key in for the DSL expressions for `filter` and `put`: that is, if you type `\t` in a string literal for a `filter`/`put` expression, it will be turned into a tab character. If you want a backslash followed by a `t`, then please type `\\t`.
+GENMD_RUN_COMMAND
+mlr --icsv --ojson --from data/json-in-csv.csv json-parse -f blob
+GENMD_EOF
 
-However, these replacements are done automatically only for string literals within DSL expressions -- they are not done automatically to fields within your data stream.  If you wish to make these replacements, you can do (for example) `mlr put '$field = gsub($field, "\\t", "\t")'`. If you need to make such a replacement for all fields in your data, you should probably use the system `sed` command instead. 
+GENMD_RUN_COMMAND
+mlr --icsv --ojson --from data/json-in-csv.csv put '$blob = json_parse($blob)'
+GENMD_EOF
 
+These have their respective operations to convert back to string: the
+[`json-stringify` verb](reference-verbs.md#json-stringify)
+and
+[`json_stringify` DSL function](reference-dsl-builtin-functions.md#json_stringify).
diff --git a/docs6/docs/reference-main-strings.md b/docs6/docs/reference-main-strings.md
new file mode 100644
index 000000000..ec54fc11a
--- /dev/null
+++ b/docs6/docs/reference-main-strings.md
@@ -0,0 +1,52 @@
+
+

+
+Quick links:
+ 
+Verb list
+ 
+Function list
+ 
+Glossary
+ 
+Repository ↗
+
+

+# Strings
+
+TODO
+
+xxx concat
+
+xxx index and slice 1-up
+
+xxx lib functions
+
+always double-quote
+
+single-quote for shell; see windows page
+
+dot operator ...
+
+## Escape sequences for string literals
+
+You can use the following backslash escapes for strings such as between the double quotes in contexts such as `mlr filter '$name =~ "..."'`, `mlr put '$name = $othername . "..."'`, `mlr put '$name = sub($name, "...", "...")`, etc.:
+
+* `\a`: ASCII code 0x07 (alarm/bell)
+* `\b`: ASCII code 0x08 (backspace)
+* `\f`: ASCII code 0x0c (formfeed)
+* `\n`: ASCII code 0x0a (LF/linefeed/newline)
+* `\r`: ASCII code 0x0d (CR/carriage return)
+* `\t`: ASCII code 0x09 (tab)
+* `\v`: ASCII code 0x0b (vertical tab)
+* `\\`: backslash
+* `\"`: double quote
+* `\123`: Octal 123, etc. for `\000` up to `\377`
+* `\x7f`: Hexadecimal 7f, etc. for `\x00` up to `\xff`
+
+See also [https://en.wikipedia.org/wiki/Escape_sequences_in_C](https://en.wikipedia.org/wiki/Escape_sequences_in_C).
+
+These replacements apply only to strings you key in for the DSL expressions for `filter` and `put`: that is, if you type `\t` in a string literal for a `filter`/`put` expression, it will be turned into a tab character. If you want a backslash followed by a `t`, then please type `\\t`.
+
+However, these replacements are done automatically only for string literals within DSL expressions -- they are not done automatically to fields within your data stream.  If you wish to make these replacements, you can do (for example) `mlr put '$field = gsub($field, "\\t", "\t")'`. If you need to make such a replacement for all fields in your data, you should probably use the system `sed` command instead. 
+
diff --git a/docs6/docs/reference-main-strings.md.in b/docs6/docs/reference-main-strings.md.in
new file mode 100644
index 000000000..453983b2b
--- /dev/null
+++ b/docs6/docs/reference-main-strings.md.in
@@ -0,0 +1,38 @@
+# Strings
+
+TODO
+
+xxx concat
+
+xxx index and slice 1-up
+
+xxx lib functions
+
+always double-quote
+
+single-quote for shell; see windows page
+
+dot operator ...
+
+## Escape sequences for string literals
+
+You can use the following backslash escapes for strings such as between the double quotes in contexts such as `mlr filter '$name =~ "..."'`, `mlr put '$name = $othername . "..."'`, `mlr put '$name = sub($name, "...", "...")`, etc.:
+
+* `\a`: ASCII code 0x07 (alarm/bell)
+* `\b`: ASCII code 0x08 (backspace)
+* `\f`: ASCII code 0x0c (formfeed)
+* `\n`: ASCII code 0x0a (LF/linefeed/newline)
+* `\r`: ASCII code 0x0d (CR/carriage return)
+* `\t`: ASCII code 0x09 (tab)
+* `\v`: ASCII code 0x0b (vertical tab)
+* `\\`: backslash
+* `\"`: double quote
+* `\123`: Octal 123, etc. for `\000` up to `\377`
+* `\x7f`: Hexadecimal 7f, etc. for `\x00` up to `\xff`
+
+See also [https://en.wikipedia.org/wiki/Escape_sequences_in_C](https://en.wikipedia.org/wiki/Escape_sequences_in_C).
+
+These replacements apply only to strings you key in for the DSL expressions for `filter` and `put`: that is, if you type `\t` in a string literal for a `filter`/`put` expression, it will be turned into a tab character. If you want a backslash followed by a `t`, then please type `\\t`.
+
+However, these replacements are done automatically only for string literals within DSL expressions -- they are not done automatically to fields within your data stream.  If you wish to make these replacements, you can do (for example) `mlr put '$field = gsub($field, "\\t", "\t")'`. If you need to make such a replacement for all fields in your data, you should probably use the system `sed` command instead. 
+
diff --git a/docs6/docs/reference-verbs.md b/docs6/docs/reference-verbs.md
index 6734d0fcb..2102b8fd7 100644
--- a/docs6/docs/reference-verbs.md
+++ b/docs6/docs/reference-verbs.md
@@ -1639,6 +1639,34 @@ left_a left_b left_c right_a right_b right_c
 1      4      5      1       4       5
+## json-parse + +
+mlr json-parse --help
+
+
+Usage: mlr json-parse [options]
+Tries to convert string field values to parsed JSON, e.g. "[1,2,3]" -> [1,2,3].
+Options:
+-f {...} Comma-separated list of field names to json-parse (default all).
+-h|--help Show this message.
+
+ +## json-stringify + +
+mlr json-stringify --help
+
+
+Usage: mlr json-stringify [options]
+Produces string field values from field-value data, e.g. [1,2,3] -> "[1,2,3]".
+Options:
+-f {...} Comma-separated list of field names to json-parse (default all).
+--jvstack Produce multi-line JSON output.
+--no-jvstack Produce single-line JSON output per record (default).
+-h|--help Show this message.
+
+ ## label 
diff --git a/docs6/docs/reference-verbs.md.in b/docs6/docs/reference-verbs.md.in
index f24c67737..cadb0fa8b 100644
--- a/docs6/docs/reference-verbs.md.in
+++ b/docs6/docs/reference-verbs.md.in
@@ -548,6 +548,18 @@ GENMD_RUN_COMMAND
 mlr --csvlite --opprint join -j "" --lp left_ --rp right_ -f data/self-join.csv data/self-join.csv
 GENMD_EOF
 
+## json-parse
+
+GENMD_RUN_COMMAND
+mlr json-parse --help
+GENMD_EOF
+
+## json-stringify
+
+GENMD_RUN_COMMAND
+mlr json-stringify --help
+GENMD_EOF
+
 ## label
 
 GENMD_RUN_COMMAND
diff --git a/docs6/mkdocs.yml b/docs6/mkdocs.yml
index ccab3e8ac..65c9ff262 100644
--- a/docs6/mkdocs.yml
+++ b/docs6/mkdocs.yml
@@ -7,6 +7,9 @@ theme:
     text: Lato
     code: Lato Mono
 extra_css: ['extra.css']
+extra_javascript:
+  - https://cdnjs.cloudflare.com/ajax/libs/tablesort/5.2.1/tablesort.min.js
+  - javascripts/tables.js
 nav:
   - "Introduction": "index.md"
   - 'Getting started':
@@ -62,16 +65,17 @@ nav:
       - "I/O options": "reference-main-io-options.md"
       - "Separators": "reference-main-separators.md"
       - "List of verbs": "reference-verbs.md"
-      - "Streaming processing, and memory usage": "streaming-and-memory.md"
-      - "CPU/multicore usage": "cpu.md"
       - "Data types": "reference-main-data-types.md"
+      - "Strings": "reference-main-strings.md"
+      - "Regular expressions": "reference-main-regular-expressions.md"
       - "Arithmetic": "reference-main-arithmetic.md"
       - "Maps": "reference-main-maps.md"
       - "Arrays": "reference-main-arrays.md"
       - "Null/empty/absent data": "reference-main-null-data.md"
-      - "Regular expressions": "reference-main-regular-expressions.md"
       - "Flatten/unflatten: JSON vs. tabular formats": "flatten-unflatten.md"
       - "Compressed data": "reference-main-compressed-data.md"
+      - "Streaming processing, and memory usage": "streaming-and-memory.md"
+      - "CPU/multicore usage": "cpu.md"
       - "Miller environment variables": "reference-main-env-vars.md"
     - 'DSL reference':
       - "DSL overview": "reference-dsl.md"
diff --git a/go/src/auxents/help/entry.go b/go/src/auxents/help/entry.go
index 7d32cfa16..20d1d3402 100644
--- a/go/src/auxents/help/entry.go
+++ b/go/src/auxents/help/entry.go
@@ -65,7 +65,10 @@ func init() {
 		{name: "function", unaryHandlerFunc: helpForFunction},
 		{name: "keyword", unaryHandlerFunc: helpForKeyword},
 		{name: "list-functions", zaryHandlerFunc: listFunctions},
+		{name: "list-function-classes", zaryHandlerFunc: listFunctionClasses},
+		{name: "list-functions-in-class", unaryHandlerFunc: listFunctionsInClass},
 		{name: "list-functions-as-paragraph", zaryHandlerFunc: listFunctionsAsParagraph},
+		{name: "list-functions-as-table", zaryHandlerFunc: listFunctionsAsTable},
 		{name: "list-keywords", zaryHandlerFunc: listKeywords},
 		{name: "list-keywords-as-paragraph", zaryHandlerFunc: listKeywordsAsParagraph},
 		{name: "list-verbs", zaryHandlerFunc: listVerbs},
@@ -77,6 +80,7 @@ func init() {
 		{name: "separator-options", zaryHandlerFunc: helpSeparatorOptions},
 		{name: "type-arithmetic-info", zaryHandlerFunc: helpTypeArithmeticInfo},
 		{name: "usage-functions", zaryHandlerFunc: usageFunctions},
+		{name: "usage-functions-by-class", zaryHandlerFunc: usageFunctionsByClass},
 		{name: "usage-keywords", zaryHandlerFunc: usageKeywords},
 		{name: "usage-verbs", zaryHandlerFunc: usageVerbs},
 		{name: "verb", unaryHandlerFunc: helpForVerb},
@@ -699,14 +703,30 @@ func listFunctions() {
 	}
 }
 
+func listFunctionClasses() {
+	cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionClasses(os.Stdout)
+}
+
+func listFunctionsInClass(class string) {
+	cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionsInClass(class, os.Stdout)
+}
+
 func listFunctionsAsParagraph() {
 	cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesAsParagraph(os.Stdout)
 }
 
+func listFunctionsAsTable() {
+	cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionsAsTable(os.Stdout)
+}
+
 func usageFunctions() {
 	cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionUsages()
 }
 
+func usageFunctionsByClass() {
+	cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionUsagesByClass()
+}
+
 func helpForFunction(arg string) {
 	cst.BuiltinFunctionManagerInstance.TryListBuiltinFunctionUsage(arg, os.Stdout)
 }
diff --git a/go/src/dsl/cst/builtin_function_manager.go b/go/src/dsl/cst/builtin_function_manager.go
index 635ae84e5..95d9c2089 100644
--- a/go/src/dsl/cst/builtin_function_manager.go
+++ b/go/src/dsl/cst/builtin_function_manager.go
@@ -27,7 +27,7 @@ const (
 	FUNC_CLASS_HASHING                    = "hashing"
 	FUNC_CLASS_CONVERSION                 = "conversion"
 	FUNC_CLASS_TYPING                     = "typing"
-	FUNC_CLASS_COLLECTIONS                = "maps/arrays"
+	FUNC_CLASS_COLLECTIONS                = "collections"
 	FUNC_CLASS_SYSTEM                     = "system"
 	FUNC_CLASS_TIME                       = "time"
 )
@@ -470,22 +470,22 @@ regex-match operator: try '$y = ~$x'.`,
 		name:  "substr0",
 		class: FUNC_CLASS_STRING,
 		help: `substr0(s,m,n) gives substring of s from 0-up position m to n
-inclusive. Negative indices -len .. -1 alias to 0 .. len-1.`,
+inclusive. Negative indices -len .. -1 alias to 0 .. len-1. See also substr and substr1.`,
 		ternaryFunc: types.MlrvalSubstr0Up,
 	},
 	{
 		name:  "substr1",
 		class: FUNC_CLASS_STRING,
 		help: `substr1(s,m,n) gives substring of s from 1-up position m to n
-inclusive. Negative indices -len .. -1 alias to 1 .. len.`,
+inclusive. Negative indices -len .. -1 alias to 1 .. len. See also substr and substr0.`,
 		ternaryFunc: types.MlrvalSubstr1Up,
 	},
 	{
 		name:  "substr",
 		class: FUNC_CLASS_STRING,
 		help: `substr is an alias for substr0. See also substr1. Miller is generally 1-up
-with all array indices, but, this is a backward-compatibility issue with Miller 5 and below.
-Arrays are new in Miller 6; the substr function is older.`,
+with all array and string indices, but, this is a backward-compatibility issue with Miller 5
+and below. Arrays are new in Miller 6; the substr function is older.`,
 		ternaryFunc: types.MlrvalSubstr0Up,
 	},
 
@@ -1590,6 +1590,37 @@ func hashifyLookupTable(lookupTable *[]BuiltinFunctionInfo) map[string]*BuiltinF
 	return hashTable
 }
 
+// ----------------------------------------------------------------
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionClasses(o *os.File) {
+	classesList := manager.getBuiltinFunctionClasses()
+	for _, class := range classesList {
+		fmt.Println(class)
+	}
+}
+
+func (manager *BuiltinFunctionManager) getBuiltinFunctionClasses() []string {
+	classesSeen := make(map[string]bool)
+	classesList := make([]string, 0)
+	for _, builtinFunctionInfo := range *manager.lookupTable {
+		class := string(builtinFunctionInfo.class)
+		if classesSeen[class] == false {
+			classesList = append(classesList, class)
+			classesSeen[class] = true
+		}
+	}
+	sort.Strings(classesList)
+	return classesList
+}
+
+// ----------------------------------------------------------------
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionsInClass(class string, o *os.File) {
+	for _, builtinFunctionInfo := range *manager.lookupTable {
+		if string(builtinFunctionInfo.class) == class {
+			fmt.Fprintln(o, builtinFunctionInfo.name)
+		}
+	}
+}
+
 // ----------------------------------------------------------------
 func (manager *BuiltinFunctionManager) ListBuiltinFunctionNamesVertically(o *os.File) {
 	for _, builtinFunctionInfo := range *manager.lookupTable {
@@ -1606,6 +1637,18 @@ func (manager *BuiltinFunctionManager) ListBuiltinFunctionNamesAsParagraph(o *os
 	lib.PrintWordsAsParagraph(functionNames, o)
 }
 
+// ----------------------------------------------------------------
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionsAsTable(o *os.File) {
+	fmt.Printf("%-30s %-12s %s\n", "Name", "Class", "Args")
+	for _, builtinFunctionInfo := range *manager.lookupTable {
+		fmt.Printf("%-30s %-12s %s\n",
+			builtinFunctionInfo.name,
+			builtinFunctionInfo.class,
+			describeNargs(&builtinFunctionInfo),
+		)
+	}
+}
+
 // ----------------------------------------------------------------
 func (manager *BuiltinFunctionManager) ListBuiltinFunctionUsages() {
 	for i, builtinFunctionInfo := range *manager.lookupTable {
@@ -1622,6 +1665,30 @@ func (manager *BuiltinFunctionManager) ListBuiltinFunctionUsages() {
 	}
 }
 
+// ----------------------------------------------------------------
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionUsagesByClass() {
+	classesList := manager.getBuiltinFunctionClasses()
+
+	for _, class := range classesList {
+		fmt.Println()
+		fmt.Println(colorizer.MaybeColorizeHelp(strings.ToUpper(class), true))
+		fmt.Println()
+		for _, builtinFunctionInfo := range *manager.lookupTable {
+			if string(builtinFunctionInfo.class) != class {
+				continue
+			}
+			lib.InternalCodingErrorIf(builtinFunctionInfo.help == "")
+			fmt.Print(colorizer.MaybeColorizeHelp(builtinFunctionInfo.name, true))
+			fmt.Printf("  (class=%s #args=%s) %s\n",
+				builtinFunctionInfo.class,
+				describeNargs(&builtinFunctionInfo),
+				builtinFunctionInfo.help,
+			)
+			fmt.Println()
+		}
+	}
+}
+
 func (manager *BuiltinFunctionManager) ListBuiltinFunctionUsage(functionName string, o *os.File) {
 	if !manager.TryListBuiltinFunctionUsage(functionName, o) {
 		fmt.Fprintf(os.Stderr, "Function \"%s\" not found.\n", functionName)
diff --git a/go/todo.txt b/go/todo.txt
index 84a5fab54..a9fee237d 100644
--- a/go/todo.txt
+++ b/go/todo.txt
@@ -113,15 +113,20 @@ w survey
 ----------------------------------------------------------------
 BLOCKERS
 
-* bug:
+* document precedence issue:
+! probably remove this feature !
+
+  μεταμόρφωσις: z = [{"x":3}, {"x":4}]
   μεταμόρφωσις: z[1].x
-  0.3467901443380824
+  3
   μεταμόρφωσις: z[2].x
-  0.7586799647899636
+  4
   μεταμόρφωσις: z[1]["x"] + z[2]["x"]
-  1.105470109128046
+  7
   μεταμόρφωσις: z[1].x + z[2].x
   (absent)
+  μεταμόρφωσις: (z[1].x) + (z[2].x)
+  7
 
 ! repifs
 ! seps "\001" etc

From a916087ca062bea5e90c649863ba6634e2b30315 Mon Sep 17 00:00:00 2001
From: John Kerl 
Date: Sat, 4 Sep 2021 14:02:09 -0400
Subject: [PATCH 3/5] remove extraneous os.Stdout arguments from
 function-manager print methods

---
 docs6/docs/proofreads.txt                    |  5 +++--
 go/src/auxents/help/entry.go                 | 14 ++++++-------
 go/src/auxents/repl/verbs.go                 |  4 ++--
 go/src/cliutil/mlrcli_util.go                | 14 ++++---------
 go/src/dsl/cst/builtin_function_manager.go   | 22 ++++++++++----------
 go/src/dsl/cst/keyword_usage.go              |  3 +--
 go/src/lib/paragraph.go                      |  9 ++++----
 go/src/transformers/aaa_transformer_table.go |  2 +-
 8 files changed, 33 insertions(+), 40 deletions(-)

diff --git a/docs6/docs/proofreads.txt b/docs6/docs/proofreads.txt
index 51384d902..44274ffe6 100644
--- a/docs6/docs/proofreads.txt
+++ b/docs6/docs/proofreads.txt
@@ -3,8 +3,11 @@ TOP:
 c data-types:
   ! don't auto-infer bools; then, data-types page
   ! don't auto-infer Infinity, etc -- ?!? then, data-types page
+  MlrvalPointerFromInferredType -> mk variant
 E flatten/unflatten page
 c! seps \001 etc !
+  mlrc --iusv --oxtab cat regtest/input/example.usv
+  mlr  --iusv --oxtab cat regtest/input/example.usv
 C! repifs !! https://pkg.go.dev/regexp#Regexp.Split 2-for-1 -- get regexp as well ?
 ? twi-dm re all-contribs: all-contributors.org
 
@@ -13,12 +16,10 @@ C flags LUTs
 ----------------------------------------------------------------
 ALL:
 * csv to csv,tsv throughout
-* rid of explicitly passing around os.Stdout in all various help functions, annoying
 * het.dkvp > het.json in more places
 * check each page for adequate h2 coverage
 
 * deduping pass!
-* 1-up/0-up -> one-up/zero-up everywhere
 
 * flags-doc
   C refactor cli-parser w/ LUT & help strings
diff --git a/go/src/auxents/help/entry.go b/go/src/auxents/help/entry.go
index 20d1d3402..f3c024195 100644
--- a/go/src/auxents/help/entry.go
+++ b/go/src/auxents/help/entry.go
@@ -697,26 +697,26 @@ func helpTypeArithmeticInfo() {
 // ----------------------------------------------------------------
 func listFunctions() {
 	if isatty.IsTerminal(os.Stdout.Fd()) {
-		cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesAsParagraph(os.Stdout)
+		cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesAsParagraph()
 	} else {
-		cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesVertically(os.Stdout)
+		cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesVertically()
 	}
 }
 
 func listFunctionClasses() {
-	cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionClasses(os.Stdout)
+	cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionClasses()
 }
 
 func listFunctionsInClass(class string) {
-	cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionsInClass(class, os.Stdout)
+	cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionsInClass(class)
 }
 
 func listFunctionsAsParagraph() {
-	cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesAsParagraph(os.Stdout)
+	cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesAsParagraph()
 }
 
 func listFunctionsAsTable() {
-	cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionsAsTable(os.Stdout)
+	cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionsAsTable()
 }
 
 func usageFunctions() {
@@ -728,7 +728,7 @@ func usageFunctionsByClass() {
 }
 
 func helpForFunction(arg string) {
-	cst.BuiltinFunctionManagerInstance.TryListBuiltinFunctionUsage(arg, os.Stdout)
+	cst.BuiltinFunctionManagerInstance.TryListBuiltinFunctionUsage(arg)
 }
 
 // ----------------------------------------------------------------
diff --git a/go/src/auxents/repl/verbs.go b/go/src/auxents/repl/verbs.go
index e79e0a7c9..74cd483ab 100644
--- a/go/src/auxents/repl/verbs.go
+++ b/go/src/auxents/repl/verbs.go
@@ -855,7 +855,7 @@ func handleHelpSingle(repl *Repl, arg string) {
 	}
 
 	if arg == "function-names" {
-		cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesAsParagraph(os.Stdout)
+		cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionNamesAsParagraph()
 		return
 	}
 
@@ -868,7 +868,7 @@ func handleHelpSingle(repl *Repl, arg string) {
 		return
 	}
 
-	if cst.BuiltinFunctionManagerInstance.TryListBuiltinFunctionUsage(arg, os.Stdout) {
+	if cst.BuiltinFunctionManagerInstance.TryListBuiltinFunctionUsage(arg) {
 		return
 	}
 
diff --git a/go/src/cliutil/mlrcli_util.go b/go/src/cliutil/mlrcli_util.go
index 4e126eb3c..89ad87f17 100644
--- a/go/src/cliutil/mlrcli_util.go
+++ b/go/src/cliutil/mlrcli_util.go
@@ -3,6 +3,8 @@ package cliutil
 import (
 	"fmt"
 	"os"
+
+	"mlr/src/lib"
 )
 
 // For flags with values, e.g. ["-n" "10"], while we're looking at the "-n"
@@ -39,16 +41,8 @@ var SEPARATOR_NAMES_TO_VALUES = map[string]string{
 func SeparatorFromArg(name string) string {
 	sep, ok := SEPARATOR_NAMES_TO_VALUES[name]
 	if !ok {
-		// xxx temp
-		//fmt.Fprintf(os.Stderr, "Miller: could not handle separator \"%s\".\n", name)
-		//os.Exit(1)
-		// It's OK if they do '--ifs ,' -- just pass it back.
-		return name
+		// "\001" -> control-A, etc.
+		return lib.UnbackslashStringLiteral(name)
 	}
 	return sep
-	//	char* chars = lhmss_get(get_desc_to_chars_map(), arg);
-	//	if (chars != nil) // E.g. crlf
-	//		return mlr_strdup_or_die(chars);
-	//	else // E.g. '\r\n'
-	//		return mlr_alloc_unbackslash(arg);
 }
diff --git a/go/src/dsl/cst/builtin_function_manager.go b/go/src/dsl/cst/builtin_function_manager.go
index 95d9c2089..e0adf57ca 100644
--- a/go/src/dsl/cst/builtin_function_manager.go
+++ b/go/src/dsl/cst/builtin_function_manager.go
@@ -1591,7 +1591,7 @@ func hashifyLookupTable(lookupTable *[]BuiltinFunctionInfo) map[string]*BuiltinF
 }
 
 // ----------------------------------------------------------------
-func (manager *BuiltinFunctionManager) ListBuiltinFunctionClasses(o *os.File) {
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionClasses() {
 	classesList := manager.getBuiltinFunctionClasses()
 	for _, class := range classesList {
 		fmt.Println(class)
@@ -1613,32 +1613,32 @@ func (manager *BuiltinFunctionManager) getBuiltinFunctionClasses() []string {
 }
 
 // ----------------------------------------------------------------
-func (manager *BuiltinFunctionManager) ListBuiltinFunctionsInClass(class string, o *os.File) {
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionsInClass(class string) {
 	for _, builtinFunctionInfo := range *manager.lookupTable {
 		if string(builtinFunctionInfo.class) == class {
-			fmt.Fprintln(o, builtinFunctionInfo.name)
+			fmt.Println(builtinFunctionInfo.name)
 		}
 	}
 }
 
 // ----------------------------------------------------------------
-func (manager *BuiltinFunctionManager) ListBuiltinFunctionNamesVertically(o *os.File) {
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionNamesVertically() {
 	for _, builtinFunctionInfo := range *manager.lookupTable {
-		fmt.Fprintln(o, builtinFunctionInfo.name)
+		fmt.Println(builtinFunctionInfo.name)
 	}
 }
 
 // ----------------------------------------------------------------
-func (manager *BuiltinFunctionManager) ListBuiltinFunctionNamesAsParagraph(o *os.File) {
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionNamesAsParagraph() {
 	functionNames := make([]string, len(*manager.lookupTable))
 	for i, builtinFunctionInfo := range *manager.lookupTable {
 		functionNames[i] = builtinFunctionInfo.name
 	}
-	lib.PrintWordsAsParagraph(functionNames, o)
+	lib.PrintWordsAsParagraph(functionNames)
 }
 
 // ----------------------------------------------------------------
-func (manager *BuiltinFunctionManager) ListBuiltinFunctionsAsTable(o *os.File) {
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionsAsTable() {
 	fmt.Printf("%-30s %-12s %s\n", "Name", "Class", "Args")
 	for _, builtinFunctionInfo := range *manager.lookupTable {
 		fmt.Printf("%-30s %-12s %s\n",
@@ -1689,13 +1689,13 @@ func (manager *BuiltinFunctionManager) ListBuiltinFunctionUsagesByClass() {
 	}
 }
 
-func (manager *BuiltinFunctionManager) ListBuiltinFunctionUsage(functionName string, o *os.File) {
-	if !manager.TryListBuiltinFunctionUsage(functionName, o) {
+func (manager *BuiltinFunctionManager) ListBuiltinFunctionUsage(functionName string) {
+	if !manager.TryListBuiltinFunctionUsage(functionName) {
 		fmt.Fprintf(os.Stderr, "Function \"%s\" not found.\n", functionName)
 	}
 }
 
-func (manager *BuiltinFunctionManager) TryListBuiltinFunctionUsage(functionName string, o *os.File) bool {
+func (manager *BuiltinFunctionManager) TryListBuiltinFunctionUsage(functionName string) bool {
 	builtinFunctionInfo := manager.LookUp(functionName)
 	if builtinFunctionInfo == nil {
 		manager.listBuiltinFunctionUsageApproximate(functionName)
diff --git a/go/src/dsl/cst/keyword_usage.go b/go/src/dsl/cst/keyword_usage.go
index e9fc107c5..77ca6a453 100644
--- a/go/src/dsl/cst/keyword_usage.go
+++ b/go/src/dsl/cst/keyword_usage.go
@@ -2,7 +2,6 @@ package cst
 
 import (
 	"fmt"
-	"os"
 
 	"mlr/src/colorizer"
 	"mlr/src/lib"
@@ -114,7 +113,7 @@ func ListKeywordsAsParagraph() {
 	for i, entry := range KEYWORD_USAGE_TABLE {
 		keywords[i] = entry.name
 	}
-	lib.PrintWordsAsParagraph(keywords, os.Stdout)
+	lib.PrintWordsAsParagraph(keywords)
 }
 
 // ----------------------------------------------------------------
diff --git a/go/src/lib/paragraph.go b/go/src/lib/paragraph.go
index 289f59486..4ca9f7409 100644
--- a/go/src/lib/paragraph.go
+++ b/go/src/lib/paragraph.go
@@ -2,12 +2,11 @@ package lib
 
 import (
 	"fmt"
-	"os"
 )
 
 // For online help contexts like printing all the built-in DSL functions, or
 // the list of all verbs.
-func PrintWordsAsParagraph(words []string, o *os.File) {
+func PrintWordsAsParagraph(words []string) {
 	separator := " "
 	maxlen := 80
 
@@ -19,14 +18,14 @@ func PrintWordsAsParagraph(words []string, o *os.File) {
 		wordlen := len(word)
 		linelen += separatorlen + wordlen
 		if linelen >= maxlen {
-			fmt.Fprintf(o, "\n")
+			fmt.Printf("\n")
 			linelen = separatorlen + wordlen
 			j = 0
 		}
 		if j > 0 {
-			fmt.Fprint(o, separator)
+			fmt.Print(separator)
 		}
-		fmt.Fprint(o, word)
+		fmt.Print(word)
 		j++
 	}
 
diff --git a/go/src/transformers/aaa_transformer_table.go b/go/src/transformers/aaa_transformer_table.go
index 53ee4de5b..9b395e969 100644
--- a/go/src/transformers/aaa_transformer_table.go
+++ b/go/src/transformers/aaa_transformer_table.go
@@ -96,7 +96,7 @@ func ListVerbNamesAsParagraph() {
 		verbNames[i] = transformerSetup.Verb
 	}
 
-	lib.PrintWordsAsParagraph(verbNames, os.Stdout)
+	lib.PrintWordsAsParagraph(verbNames)
 }
 
 // ----------------------------------------------------------------

From cfc986b5f4678d8f53e8c549db2aef02c46bb7c6 Mon Sep 17 00:00:00 2001
From: John Kerl 
Date: Sat, 4 Sep 2021 14:19:15 -0400
Subject: [PATCH 4/5] Fix non-auto-infer of true/false/Infinity/NaN/etc in
 non-JSON data files

---
 docs6/docs/10min.md                           | 11 ++----
 docs6/docs/data-cleaning-examples.md          | 14 +++----
 docs6/docs/keystroke-savers.md                |  4 +-
 docs6/docs/proofreads.txt                     |  7 +++-
 docs6/docs/record-heterogeneity.md            |  3 ++
 docs6/docs/reference-main-data-types.md       |  2 +-
 docs6/docs/reference-main-maps.md             |  4 +-
 docs6/docs/repl.md                            |  4 +-
 docs6/docs/special-symbols-and-formatting.md  |  4 +-
 .../cases/dsl-min-max-types/0001/expout       | 12 +++---
 .../cases/dsl-min-max-types/0002/expout       | 16 ++++----
 .../cases/dsl-min-max-types/0003/expout       |  8 ++--
 .../cases/dsl-min-max-types/0004/expout       |  8 ++--
 go/src/input/record_reader_csv.go             |  6 +--
 go/src/input/record_reader_csvlite.go         | 12 +++---
 go/src/input/record_reader_dkvp.go            |  4 +-
 go/src/input/record_reader_nidx.go            |  2 +-
 go/src/input/record_reader_xtab.go            |  2 +-
 go/src/types/mlrval_new.go                    | 38 +++++++++++++++++--
 19 files changed, 97 insertions(+), 64 deletions(-)

diff --git a/docs6/docs/10min.md b/docs6/docs/10min.md
index d75a63584..db6900811 100644
--- a/docs6/docs/10min.md
+++ b/docs6/docs/10min.md
@@ -87,7 +87,7 @@ purple,square,false,10,91,72.3735,8.2430
 {
   "color": "yellow",
   "shape": "circle",
-  "flag": true,
+  "flag": "true",
   "k": 9,
   "index": 87,
   "quantity": 63.5058,
@@ -96,7 +96,7 @@ purple,square,false,10,91,72.3735,8.2430
 {
   "color": "purple",
   "shape": "square",
-  "flag": false,
+  "flag": "false",
   "k": 10,
   "index": 91,
   "quantity": 72.3735,
@@ -212,14 +212,9 @@ red   square false 4 48    77.5542  7.4670
 red   square false 6 64    77.1991  9.5310
-
+
 mlr --icsv --opprint filter '$color == "red" && $flag == true' example.csv
 

-
-color shape  flag k index quantity rate
-red   square true 2 15    79.2778  0.0130
-red   circle true 3 16    13.8103  2.9010
-

 
 You can use `put` to create new fields which are computed from other fields:
 
diff --git a/docs6/docs/data-cleaning-examples.md b/docs6/docs/data-cleaning-examples.md
index b03c96b23..be3b74286 100644
--- a/docs6/docs/data-cleaning-examples.md
+++ b/docs6/docs/data-cleaning-examples.md
@@ -58,9 +58,9 @@ A second option is to flag badly formatted data within the output stream:
 
 name   reachable format_ok
-barney false     false
-betty  true      false
-fred   true      false
+barney false     true
+betty  true      true
+fred   true      true
 wilma  1         false
@@ -72,9 +72,6 @@ Or perhaps to flag badly formatted data outside the output stream: ' data/het-bool.csv 
-Malformed at NR=1
-Malformed at NR=2
-Malformed at NR=3
 Malformed at NR=4
 name   reachable
 barney false
@@ -89,5 +86,8 @@ A third way is to abort the process on first instance of bad data:
 mlr --csv put '$reachable = asserting_string($reachable)' data/het-bool.csv
 
-Miller: is_string type-assertion failed at NR=1 FNR=1 FILENAME=data/het-bool.csv
+name,reachable
+barney,false
+Miller: is_string type-assertion failed at NR=4 FNR=4 FILENAME=data/het-bool.csv
+betty,true
diff --git a/docs6/docs/keystroke-savers.md b/docs6/docs/keystroke-savers.md index fa48095e1..df88f3885 100644 --- a/docs6/docs/keystroke-savers.md +++ b/docs6/docs/keystroke-savers.md @@ -34,7 +34,7 @@ red square true 2 15 79.2778 0.0130 { "color": "yellow", "shape": "triangle", - "flag": true, + "flag": "true", "k": 1, "index": 11, "quantity": 43.6498, @@ -43,7 +43,7 @@ red square true 2 15 79.2778 0.0130 { "color": "red", "shape": "square", - "flag": true, + "flag": "true", "k": 2, "index": 15, "quantity": 79.2778, diff --git a/docs6/docs/proofreads.txt b/docs6/docs/proofreads.txt index 44274ffe6..49505c85c 100644 --- a/docs6/docs/proofreads.txt +++ b/docs6/docs/proofreads.txt @@ -3,14 +3,17 @@ TOP: c data-types: ! don't auto-infer bools; then, data-types page ! don't auto-infer Infinity, etc -- ?!? then, data-types page + RT-check + https://github.com/johnkerl/miller/issues/241 + https://github.com/johnkerl/miller/issues/357 MlrvalPointerFromInferredType -> mk variant -E flatten/unflatten page c! seps \001 etc ! mlrc --iusv --oxtab cat regtest/input/example.usv mlr --iusv --oxtab cat regtest/input/example.usv C! repifs !! https://pkg.go.dev/regexp#Regexp.Split 2-for-1 -- get regexp as well ? -? twi-dm re all-contribs: all-contributors.org +E flatten/unflatten page +? twi-dm re all-contribs: all-contributors.org C flags LUTs ---------------------------------------------------------------- diff --git a/docs6/docs/record-heterogeneity.md b/docs6/docs/record-heterogeneity.md index 6d57396e7..cf2f63136 100644 --- a/docs6/docs/record-heterogeneity.md +++ b/docs6/docs/record-heterogeneity.md @@ -126,6 +126,9 @@ If you `mlr csv cat` this, you'll get an error message: 
 mlr :  Miller: CSV header/data length mismatch 3 != 2 at filename data/het/ragged.csv row 3.
+
+a,b,c
+1,2,3
There are two kinds of raggedness here. Since CSVs form records by zipping the diff --git a/docs6/docs/reference-main-data-types.md b/docs6/docs/reference-main-data-types.md index ed33a0350..afe993d1e 100644 --- a/docs6/docs/reference-main-data-types.md +++ b/docs6/docs/reference-main-data-types.md @@ -92,7 +92,7 @@ ta float b 3 tb int c true -tc bool +tc string d 1.2true td string e 7 diff --git a/docs6/docs/reference-main-maps.md b/docs6/docs/reference-main-maps.md index ad23b1ae4..81debf235 100644 --- a/docs6/docs/reference-main-maps.md +++ b/docs6/docs/reference-main-maps.md @@ -82,7 +82,7 @@ The current record, accessible using `$*`, is a map. { "color": "yellow", "shape": "triangle", - "flag": true, + "flag": "true", "k": 1, "index": 11, "quantity": 43.6498, @@ -92,7 +92,7 @@ Color is yellow { "color": "red", "shape": "square", - "flag": true, + "flag": "true", "k": 2, "index": 15, "quantity": 79.2778, diff --git a/docs6/docs/repl.md b/docs6/docs/repl.md index 3cb2e90a4..61bf3c42e 100644 --- a/docs6/docs/repl.md +++ b/docs6/docs/repl.md @@ -48,7 +48,7 @@ HELLO { "color": "yellow", "shape": "triangle", - "flag": true, + "flag": "true", "k": 1, "index": 11, "quantity": 43.6498, @@ -58,7 +58,7 @@ HELLO { "color": "red", "shape": "square", - "flag": true, + "flag": "true", "k": 2, "index": 15, "quantity": 79.2778, diff --git a/docs6/docs/special-symbols-and-formatting.md b/docs6/docs/special-symbols-and-formatting.md index 44b2bad10..ea50e6ecb 100644 --- a/docs6/docs/special-symbols-and-formatting.md +++ b/docs6/docs/special-symbols-and-formatting.md @@ -83,8 +83,8 @@ characters as delimiters -- here, control-A: mlr --icsv --odkvp --ofs '\001' cat commas.csv | cat -v 
-Name=Xiao, Lin\001Role=administrator
-Name=Khavari, Darius\001Role=tester
+Name=Xiao, Lin^ARole=administrator
+Name=Khavari, Darius^ARole=tester
## How can I handle field names with special symbols in them? diff --git a/go/regtest/cases/dsl-min-max-types/0001/expout b/go/regtest/cases/dsl-min-max-types/0001/expout index 6e03b4091..e021b5638 100644 --- a/go/regtest/cases/dsl-min-max-types/0001/expout +++ b/go/regtest/cases/dsl-min-max-types/0001/expout @@ -1,6 +1,6 @@ { "n": 1, - "b": true, + "b": "true", "v": "", "s": "abc", "min": { @@ -12,19 +12,19 @@ }, "b": { "n": 1, - "b": true, - "v": true, - "s": true + "b": "true", + "v": "", + "s": "abc" }, "v": { "n": 1, - "b": true, + "b": "", "v": "", "s": "" }, "s": { "n": 1, - "b": true, + "b": "abc", "v": "", "s": "abc" } diff --git a/go/regtest/cases/dsl-min-max-types/0002/expout b/go/regtest/cases/dsl-min-max-types/0002/expout index 34f50b766..4c4057af9 100644 --- a/go/regtest/cases/dsl-min-max-types/0002/expout +++ b/go/regtest/cases/dsl-min-max-types/0002/expout @@ -1,30 +1,30 @@ { "n": 1, - "b": true, + "b": "true", "v": "", "s": "abc", "max": { "n": { "n": 1, - "b": true, + "b": "true", "v": "", "s": "abc" }, "b": { - "n": true, - "b": true, - "v": "", - "s": "abc" + "n": "true", + "b": "true", + "v": "true", + "s": "true" }, "v": { "n": "", - "b": "", + "b": "true", "v": "", "s": "abc" }, "s": { "n": "abc", - "b": "abc", + "b": "true", "v": "abc", "s": "abc" } diff --git a/go/regtest/cases/dsl-min-max-types/0003/expout b/go/regtest/cases/dsl-min-max-types/0003/expout index 714c9de22..bd0cd7c8f 100644 --- a/go/regtest/cases/dsl-min-max-types/0003/expout +++ b/go/regtest/cases/dsl-min-max-types/0003/expout @@ -1,12 +1,12 @@ { "n": 1, - "b": true, + "b": "true", "v": "", "s": "abc", "le": { "n": { "n": true, - "b": false, + "b": true, "v": false, "s": true }, @@ -18,13 +18,13 @@ }, "v": { "n": true, - "b": false, + "b": true, "v": true, "s": true }, "s": { "n": false, - "b": false, + "b": true, "v": false, "s": true } diff --git a/go/regtest/cases/dsl-min-max-types/0004/expout b/go/regtest/cases/dsl-min-max-types/0004/expout index 1575d57b5..2e62305c0 100644 --- a/go/regtest/cases/dsl-min-max-types/0004/expout +++ b/go/regtest/cases/dsl-min-max-types/0004/expout @@ -1,6 +1,6 @@ { "n": 1, - "b": true, + "b": "true", "v": "", "s": "abc", "ge": { @@ -11,10 +11,10 @@ "s": false }, "b": { - "n": false, + "n": true, "b": true, - "v": false, - "s": false + "v": true, + "s": true }, "v": { "n": false, diff --git a/go/src/input/record_reader_csv.go b/go/src/input/record_reader_csv.go index 8e3809fdb..0cc3a719b 100644 --- a/go/src/input/record_reader_csv.go +++ b/go/src/input/record_reader_csv.go @@ -140,7 +140,7 @@ func (reader *RecordReaderCSV) processHandle( if nh == nd { for i := 0; i < nh; i++ { key := header[i] - value := types.MlrvalPointerFromInferredType(csvRecord[i]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(csvRecord[i]) record.PutReference(key, value) } @@ -160,13 +160,13 @@ func (reader *RecordReaderCSV) processHandle( n := lib.IntMin2(nh, nd) for i = 0; i < n; i++ { key := header[i] - value := types.MlrvalPointerFromInferredType(csvRecord[i]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(csvRecord[i]) record.PutReference(key, value) } if nh < nd { // if header shorter than data: use 1-up itoa keys key := strconv.Itoa(i + 1) - value := types.MlrvalPointerFromInferredType(csvRecord[i]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(csvRecord[i]) record.PutCopy(key, value) } if nh > nd { diff --git a/go/src/input/record_reader_csvlite.go b/go/src/input/record_reader_csvlite.go index 308d39401..03335c0f3 100644 --- a/go/src/input/record_reader_csvlite.go +++ b/go/src/input/record_reader_csvlite.go @@ -193,7 +193,7 @@ func (reader *RecordReaderCSVLite) processHandleExplicitCSVHeader( record := types.NewMlrmap() if !reader.readerOptions.AllowRaggedCSVInput { for i, field := range fields { - value := types.MlrvalPointerFromInferredType(field) + value := types.MlrvalPointerFromInferredTypeForDataFiles(field) record.PutCopy(headerStrings[i], value) } } else { @@ -202,14 +202,14 @@ func (reader *RecordReaderCSVLite) processHandleExplicitCSVHeader( n := lib.IntMin2(nh, nd) var i int for i = 0; i < n; i++ { - value := types.MlrvalPointerFromInferredType(fields[i]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i]) record.PutCopy(headerStrings[i], value) } if nh < nd { // if header shorter than data: use 1-up itoa keys for i = nh; i < nd; i++ { key := strconv.Itoa(i + 1) - value := types.MlrvalPointerFromInferredType(fields[i]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i]) record.PutCopy(key, value) } } @@ -306,7 +306,7 @@ func (reader *RecordReaderCSVLite) processHandleImplicitCSVHeader( record := types.NewMlrmap() if !reader.readerOptions.AllowRaggedCSVInput { for i, field := range fields { - value := types.MlrvalPointerFromInferredType(field) + value := types.MlrvalPointerFromInferredTypeForDataFiles(field) record.PutCopy(headerStrings[i], value) } } else { @@ -315,13 +315,13 @@ func (reader *RecordReaderCSVLite) processHandleImplicitCSVHeader( n := lib.IntMin2(nh, nd) var i int for i = 0; i < n; i++ { - value := types.MlrvalPointerFromInferredType(fields[i]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i]) record.PutCopy(headerStrings[i], value) } if nh < nd { // if header shorter than data: use 1-up itoa keys key := strconv.Itoa(i + 1) - value := types.MlrvalPointerFromInferredType(fields[i]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i]) record.PutCopy(key, value) } if nh > nd { diff --git a/go/src/input/record_reader_dkvp.go b/go/src/input/record_reader_dkvp.go index 44fdd9786..8a094d8d1 100644 --- a/go/src/input/record_reader_dkvp.go +++ b/go/src/input/record_reader_dkvp.go @@ -122,11 +122,11 @@ func (reader *RecordReaderDKVP) recordFromDKVPLine( // "a=". Here we use the positional index as the key. This way // DKVP is a generalization of NIDX. key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := types.MlrvalPointerFromInferredType(kv[0]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(kv[0]) record.PutReference(key, value) } else { key := kv[0] - value := types.MlrvalPointerFromInferredType(kv[1]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(kv[1]) record.PutReference(key, value) } } diff --git a/go/src/input/record_reader_nidx.go b/go/src/input/record_reader_nidx.go index 0ebdc4d13..6eed7d769 100644 --- a/go/src/input/record_reader_nidx.go +++ b/go/src/input/record_reader_nidx.go @@ -121,7 +121,7 @@ func recordFromNIDXLine( for _, value := range values { i++ key := strconv.Itoa(i) - mval := types.MlrvalPointerFromInferredType(value) + mval := types.MlrvalPointerFromInferredTypeForDataFiles(value) record.PutReference(key, mval) } return record diff --git a/go/src/input/record_reader_xtab.go b/go/src/input/record_reader_xtab.go index f1385d97e..276b7cf64 100644 --- a/go/src/input/record_reader_xtab.go +++ b/go/src/input/record_reader_xtab.go @@ -165,7 +165,7 @@ func (reader *RecordReaderXTAB) recordFromXTABLines( value := types.MLRVAL_VOID record.PutReference(key, value) } else { - value := types.MlrvalPointerFromInferredType(kv[1]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(kv[1]) record.PutReference(key, value) } } diff --git a/go/src/types/mlrval_new.go b/go/src/types/mlrval_new.go index 68c70c3d7..bb8299422 100644 --- a/go/src/types/mlrval_new.go +++ b/go/src/types/mlrval_new.go @@ -101,10 +101,42 @@ func MlrvalPointerFromBoolString(input string) *Mlrval { } } +var floatNamesToNotInfer = map[string]bool{ + "Inf": true, + "+Inf": true, + "-Inf": true, + "Infinity": true, + "+Infinity": true, + "-Infinity": true, + "nan": true, + "NaN": true, +} + +// MlrvalPointerFromInferredTypeForDataFiles is for parsing field values from +// data files (except JSON, which is typed -- "true" and true are distinct). +// Mostly the same as MlrvalPointerFromInferredType, except it doesn't +// auto-infer true/false to bool; don't auto-infer NaN/Inf to float; etc. +func MlrvalPointerFromInferredTypeForDataFiles(input string) *Mlrval { + if input == "" { + return MLRVAL_VOID + } + + _, iok := lib.TryIntFromString(input) + if iok { + return MlrvalPointerFromIntString(input) + } + + if floatNamesToNotInfer[input] == false { + _, fok := lib.TryFloat64FromString(input) + if fok { + return MlrvalPointerFromFloat64String(input) + } + } + + return MlrvalPointerFromString(input) +} + func MlrvalPointerFromInferredType(input string) *Mlrval { - // xxx the parsing has happened so stash it ... - // xxx emphasize the invariant that a non-invalid printrep always - // matches the nval ... if input == "" { return MLRVAL_VOID } From a4d39a1b4b23f380076a052d675d4385e95cd5de Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sat, 4 Sep 2021 22:46:58 -0400 Subject: [PATCH 5/5] Do not type-infer Inf, Infinity, NaN, true, false, etc. on file-read --- docs6/docs/data-cleaning-examples.md | 3 -- docs6/docs/proofreads.txt | 21 +----------- docs6/docs/record-heterogeneity.md | 3 -- docs6/docs/reference-dsl-differences.md | 24 +++++++++++++- docs6/docs/reference-dsl-differences.md.in | 24 +++++++++++++- docs6/docs/reference-main-arrays.md | 6 ++-- docs6/docs/reference-main-arrays.md.in | 6 ++-- docs6/docs/reference-main-data-types.md | 6 ++++ docs6/docs/reference-main-data-types.md.in | 6 ++++ docs6/docs/reference-main-null-data.md | 2 +- docs6/docs/reference-main-null-data.md.in | 2 +- go/regtest/cases/dsl-type-inference/0001/cmd | 2 +- go/regtest/cases/dsl-type-inference/0002/cmd | 2 +- go/regtest/cases/dsl-type-inference/0003/cmd | 2 +- go/regtest/cases/dsl-type-inference/0004/cmd | 2 +- go/regtest/cases/dsl-type-inference/0005/cmd | 2 +- go/regtest/cases/dsl-type-inference/0006/cmd | 2 +- go/regtest/cases/dsl-type-inference/0007/cmd | 2 +- go/regtest/cases/dsl-type-inference/0008/cmd | 2 +- go/regtest/cases/dsl-type-inference/0009/cmd | 2 +- go/regtest/cases/dsl-type-inference/0012/cmd | 2 +- go/regtest/cases/dsl-type-inference/0015/cmd | 2 +- go/regtest/cases/dsl-type-inference/0016/cmd | 2 +- go/regtest/cases/dsl-type-inference/0017/cmd | 2 +- go/regtest/cases/dsl-type-inference/0018/cmd | 2 +- go/regtest/cases/dsl-type-inference/0019/cmd | 2 +- go/regtest/cases/dsl-type-inference/0020/cmd | 2 +- go/regtest/cases/dsl-type-inference/0021/cmd | 2 +- go/regtest/cases/dsl-type-inference/0024/cmd | 2 +- go/regtest/cases/dsl-type-inference/0027/cmd | 2 +- go/regtest/cases/dsl-type-inference/0028/cmd | 2 +- go/regtest/cases/dsl-type-inference/0029/cmd | 2 +- go/regtest/cases/dsl-type-inference/0030/cmd | 2 +- go/regtest/cases/dsl-type-inference/0031/cmd | 2 +- go/regtest/cases/dsl-type-inference/0032/cmd | 2 +- go/regtest/cases/dsl-type-inference/0033/cmd | 2 +- go/regtest/cases/dsl-type-inference/0036/cmd | 2 +- go/regtest/cases/dsl-type-inference/0039/cmd | 2 +- go/regtest/cases/dsl-type-inference/0040/cmd | 2 +- go/regtest/cases/dsl-type-inference/0041/cmd | 2 +- go/regtest/cases/dsl-type-inference/0042/cmd | 2 +- go/regtest/cases/dsl-type-inference/0043/cmd | 2 +- go/regtest/cases/dsl-type-inference/0044/cmd | 2 +- go/regtest/cases/dsl-type-inference/0045/cmd | 2 +- go/regtest/cases/dsl-type-inference/0048/cmd | 2 +- go/regtest/cases/dsl-type-inference/0051/cmd | 2 +- go/regtest/cases/dsl-type-inference/0052/cmd | 2 +- go/regtest/cases/dsl-type-inference/0053/cmd | 2 +- go/regtest/cases/dsl-type-inference/0054/cmd | 2 +- go/regtest/cases/dsl-type-inference/0055/cmd | 2 +- go/regtest/cases/dsl-type-inference/0060/cmd | 2 +- go/regtest/cases/dsl-type-inference/0061/cmd | 2 +- go/regtest/cases/dsl-type-inference/0062/cmd | 2 +- go/regtest/cases/dsl-type-inference/0063/cmd | 2 +- go/regtest/cases/dsl-type-inference/0068/cmd | 2 +- go/regtest/cases/dsl-type-inference/0069/cmd | 2 +- go/regtest/cases/dsl-type-inference/0070/cmd | 2 +- go/regtest/cases/dsl-type-inference/0071/cmd | 2 +- go/regtest/cases/dsl-type-inference/0076/cmd | 2 +- go/regtest/cases/dsl-type-inference/0077/cmd | 2 +- go/regtest/cases/dsl-type-inference/0078/cmd | 2 +- go/regtest/cases/dsl-type-inference/0079/cmd | 2 +- go/regtest/cases/dsl-type-inference/0084/cmd | 2 +- go/regtest/cases/dsl-type-inference/0085/cmd | 2 +- go/regtest/cases/dsl-type-inference/0086/cmd | 2 +- go/regtest/cases/dsl-type-inference/0087/cmd | 2 +- go/regtest/cases/dsl-type-inference/0092/cmd | 2 +- go/regtest/cases/dsl-type-inference/0093/cmd | 2 +- go/regtest/cases/dsl-type-inference/0094/cmd | 2 +- go/regtest/cases/dsl-type-inference/0095/cmd | 2 +- go/regtest/cases/dsl-type-inference/0097/cmd | 1 + .../cases/dsl-type-inference/0097/experr | 0 .../cases/dsl-type-inference/0097/expout | 33 +++++++++++++++++++ .../cases/dsl-type-inference/0097/input | 32 ++++++++++++++++++ go/regtest/cases/dsl-type-inference/0097/mlr | 1 + go/src/types/mlrval_new.go | 18 +++++----- go/todo.txt | 2 ++ 77 files changed, 204 insertions(+), 104 deletions(-) create mode 100644 go/regtest/cases/dsl-type-inference/0097/cmd create mode 100644 go/regtest/cases/dsl-type-inference/0097/experr create mode 100644 go/regtest/cases/dsl-type-inference/0097/expout create mode 100644 go/regtest/cases/dsl-type-inference/0097/input create mode 100644 go/regtest/cases/dsl-type-inference/0097/mlr diff --git a/docs6/docs/data-cleaning-examples.md b/docs6/docs/data-cleaning-examples.md index be3b74286..5c06ecf30 100644 --- a/docs6/docs/data-cleaning-examples.md +++ b/docs6/docs/data-cleaning-examples.md @@ -86,8 +86,5 @@ A third way is to abort the process on first instance of bad data: mlr --csv put '$reachable = asserting_string($reachable)' data/het-bool.csv 
-name,reachable
-barney,false
 Miller: is_string type-assertion failed at NR=4 FNR=4 FILENAME=data/het-bool.csv
-betty,true
diff --git a/docs6/docs/proofreads.txt b/docs6/docs/proofreads.txt index 49505c85c..c6ae1c8e6 100644 --- a/docs6/docs/proofreads.txt +++ b/docs6/docs/proofreads.txt @@ -1,12 +1,5 @@ TOP: -c data-types: - ! don't auto-infer bools; then, data-types page - ! don't auto-infer Infinity, etc -- ?!? then, data-types page - RT-check - https://github.com/johnkerl/miller/issues/241 - https://github.com/johnkerl/miller/issues/357 - MlrvalPointerFromInferredType -> mk variant c! seps \001 etc ! mlrc --iusv --oxtab cat regtest/input/example.usv mlr --iusv --oxtab cat regtest/input/example.usv @@ -112,24 +105,12 @@ reference-verbs: E data/colored-shapes.dkvp (this page & throughout) a CSV file ... * ... I ONLY READ UP TO CUT & PAUSED ... -reference-main-data-types: -! don't auto-infer bool from input data :( -E field values are usually strings -> update -E Field values are treated as numeric for the following: -> update -E true/false -> add info about NaN and Inf et al. -!! very much missing stuff!! where's the listing of mlrval types?!? - - reference-dsl-variables.md.in should link to it - - ditto programming-language.md.in -c UTs for r"a" . r"b" and so on and so on - -reference-dsl-variables.md: -? IFLATSEP & OFLATSEP -- ? +reference-dsl-variables.md: FLATSEP reference-main-null-data: ? more variants of is_... ? ? Records with one or more empty sort-field values sort after records with all sort-field values present -> apparently not true for sort -nr e split out h2's -! spell out JSON-null reference-main-arithmetic: ? test stats1/step -F flag diff --git a/docs6/docs/record-heterogeneity.md b/docs6/docs/record-heterogeneity.md index cf2f63136..6d57396e7 100644 --- a/docs6/docs/record-heterogeneity.md +++ b/docs6/docs/record-heterogeneity.md @@ -126,9 +126,6 @@ If you `mlr csv cat` this, you'll get an error message: 
 mlr :  Miller: CSV header/data length mismatch 3 != 2 at filename data/het/ragged.csv row 3.
-
-a,b,c
-1,2,3
There are two kinds of raggedness here. Since CSVs form records by zipping the diff --git a/docs6/docs/reference-dsl-differences.md b/docs6/docs/reference-dsl-differences.md index a3249219a..d43df8397 100644 --- a/docs6/docs/reference-dsl-differences.md +++ b/docs6/docs/reference-dsl-differences.md @@ -144,6 +144,28 @@ avoid this, use the dot operator for string-concatenation instead. Similarly, a final newline is printed for you; use [`printn`](reference-dsl-output-statements.md#print-statements) to avoid this. +## String literals with double quotes only + +In some languages, like Ruby and Bash, string literals can be in single quotes or double quotes, +where single quotes suppress `\n` converting to a newline character and double quotes allowing it: +`'a\nb'` prints as the four characters `a`, `\`, `n`, and `b` on one line; `"a\nb"` prints as an +`a` on one line and a `b` on another. + +In others, like Python and JavaScript, string literals can be in single quotes or double quotes, +interchangeably -- so you can have `"don't"` or `'the "right" thing'` as you wish. + +In yet others, such as C/C++ and Java, string literals are in double auotes, like `"abc"`, +while single quotes are for character literals like `'a'` or `'\n'`. In these, if `s` is a non-empty string, +then `s[0]` is its first character. + +In the [Miller programming language](programming-language.md): + +* String literals are always in double quotes, like `"abc"`. +* String-indexing/slicing always results in strings (even of length 1): `"abc"[1:1]` is the string `"a"`, and there is no notion in the Miller programming language of a character type. +* The single-quote character plays no role whatsoever in the grammar of the Miller programming language. +* Single quotes are reserved for wrapping expressions at the system command line. For example, in `mlr put '$message = "hello"' ...`, the [`put` verb](reference-dsl.md) gets the string `$message = "hello"`; the shell has consumed the outer single quotes by the time the Miller parser receives it. +* Things are a little different on Windows, where `"""` sequences are sometimes necessary: see the [Miller on Windows page](miller-on-windows.md). + ## Absent-null Miller has a somewhat novel flavor of null data called _absent_: if a record @@ -195,7 +217,7 @@ Miller has a [key-value loop flavor](reference-dsl-control-structures.md#key-val Miller also has a [single-variable loop flavor](reference-dsl-control-structures.md#single-variable-for-loops). If `x` is a map then `for (e in x) { ... }` binds `e` to successive map _keys_ (not values as in PHP). But if `x` is an array then `for e in x) { ... }` binds `e` to successive array _values_ (not indices). -## JSON parse, stringify, decode, encode +## JSON parse, stringify, decode, and encode Miller has the verbs [`json-parse`](reference-verbs.md#json-parse) and diff --git a/docs6/docs/reference-dsl-differences.md.in b/docs6/docs/reference-dsl-differences.md.in index dcb211144..21619c63a 100644 --- a/docs6/docs/reference-dsl-differences.md.in +++ b/docs6/docs/reference-dsl-differences.md.in @@ -116,6 +116,28 @@ GENMD_EOF Similarly, a final newline is printed for you; use [`printn`](reference-dsl-output-statements.md#print-statements) to avoid this. +## String literals with double quotes only + +In some languages, like Ruby and Bash, string literals can be in single quotes or double quotes, +where single quotes suppress `\n` converting to a newline character and double quotes allowing it: +`'a\nb'` prints as the four characters `a`, `\`, `n`, and `b` on one line; `"a\nb"` prints as an +`a` on one line and a `b` on another. + +In others, like Python and JavaScript, string literals can be in single quotes or double quotes, +interchangeably -- so you can have `"don't"` or `'the "right" thing'` as you wish. + +In yet others, such as C/C++ and Java, string literals are in double auotes, like `"abc"`, +while single quotes are for character literals like `'a'` or `'\n'`. In these, if `s` is a non-empty string, +then `s[0]` is its first character. + +In the [Miller programming language](programming-language.md): + +* String literals are always in double quotes, like `"abc"`. +* String-indexing/slicing always results in strings (even of length 1): `"abc"[1:1]` is the string `"a"`, and there is no notion in the Miller programming language of a character type. +* The single-quote character plays no role whatsoever in the grammar of the Miller programming language. +* Single quotes are reserved for wrapping expressions at the system command line. For example, in `mlr put '$message = "hello"' ...`, the [`put` verb](reference-dsl.md) gets the string `$message = "hello"`; the shell has consumed the outer single quotes by the time the Miller parser receives it. +* Things are a little different on Windows, where `"""` sequences are sometimes necessary: see the [Miller on Windows page](miller-on-windows.md). + ## Absent-null Miller has a somewhat novel flavor of null data called _absent_: if a record @@ -156,7 +178,7 @@ Miller has a [key-value loop flavor](reference-dsl-control-structures.md#key-val Miller also has a [single-variable loop flavor](reference-dsl-control-structures.md#single-variable-for-loops). If `x` is a map then `for (e in x) { ... }` binds `e` to successive map _keys_ (not values as in PHP). But if `x` is an array then `for e in x) { ... }` binds `e` to successive array _values_ (not indices). -## JSON parse, stringify, decode, encode +## JSON parse, stringify, decode, and encode Miller has the verbs [`json-parse`](reference-verbs.md#json-parse) and diff --git a/docs6/docs/reference-main-arrays.md b/docs6/docs/reference-main-arrays.md index 2129fb538..22ceae93e 100644 --- a/docs6/docs/reference-main-arrays.md +++ b/docs6/docs/reference-main-arrays.md @@ -162,9 +162,9 @@ If you want to auto-extend an [array](reference-main-arrays.md), initialize it e Once an array is initialized, it can be extended by assigning to indices beyond its length. If each write is one past the end of the array, the array will -grow by one. (Memory management, handled for you, is a little more careful: not -to worry, capacity is doubled so performance doesn't suffer a rellocate on -every single extend.) +grow by one. (Memory management, handled for you, is careful handled here in +Miller: not to worry, capacity is doubled so performance doesn't suffer a +rellocate on every single extend.) This is important in Miller so you can do things like `@records[NR] = $*` with a minimum of keystrokes without worrying about explicitly resizing arrays. In diff --git a/docs6/docs/reference-main-arrays.md.in b/docs6/docs/reference-main-arrays.md.in index 5a94a6fab..be3d44d04 100644 --- a/docs6/docs/reference-main-arrays.md.in +++ b/docs6/docs/reference-main-arrays.md.in @@ -108,9 +108,9 @@ GENMD_EOF Once an array is initialized, it can be extended by assigning to indices beyond its length. If each write is one past the end of the array, the array will -grow by one. (Memory management, handled for you, is a little more careful: not -to worry, capacity is doubled so performance doesn't suffer a rellocate on -every single extend.) +grow by one. (Memory management, handled for you, is careful handled here in +Miller: not to worry, capacity is doubled so performance doesn't suffer a +rellocate on every single extend.) This is important in Miller so you can do things like `@records[NR] = $*` with a minimum of keystrokes without worrying about explicitly resizing arrays. In diff --git a/docs6/docs/reference-main-data-types.md b/docs6/docs/reference-main-data-types.md index afe993d1e..a1d6e4af0 100644 --- a/docs6/docs/reference-main-data-types.md +++ b/docs6/docs/reference-main-data-types.md @@ -125,6 +125,12 @@ of words would result in some strings mixed with some booleans. Use the `boolean` function to coerce: e.g. giving the record `x=1,y=2,w=false` to `mlr filter '$z=($x<$y) || boolean($w)'`. +The same is true for `inf`, `+inf`, `-inf`, `infinity`, `+infinity`, +`-infinity`, `NaN`, and all upper-cased/lower-cased/mixed-case variants of +those. These are valid IEEE floating-point numbers, but Miller treats these as +strings. You can explicit force conversion: if `x=infinity` in a data file, +then `typeof($x)` is `string` but `typeof(float($x))` is `float`. + ## JSON parse and stringify If you have, say, a CSV file whose columns contain strings which are well-formatted JSON, diff --git a/docs6/docs/reference-main-data-types.md.in b/docs6/docs/reference-main-data-types.md.in index 2fdfd5b82..7e7c530c2 100644 --- a/docs6/docs/reference-main-data-types.md.in +++ b/docs6/docs/reference-main-data-types.md.in @@ -75,6 +75,12 @@ of words would result in some strings mixed with some booleans. Use the `boolean` function to coerce: e.g. giving the record `x=1,y=2,w=false` to `mlr filter '$z=($x<$y) || boolean($w)'`. +The same is true for `inf`, `+inf`, `-inf`, `infinity`, `+infinity`, +`-infinity`, `NaN`, and all upper-cased/lower-cased/mixed-case variants of +those. These are valid IEEE floating-point numbers, but Miller treats these as +strings. You can explicit force conversion: if `x=infinity` in a data file, +then `typeof($x)` is `string` but `typeof(float($x))` is `float`. + ## JSON parse and stringify If you have, say, a CSV file whose columns contain strings which are well-formatted JSON, diff --git a/docs6/docs/reference-main-null-data.md b/docs6/docs/reference-main-null-data.md index ae645de8c..4e9a24470 100644 --- a/docs6/docs/reference-main-null-data.md +++ b/docs6/docs/reference-main-null-data.md @@ -22,7 +22,7 @@ Miller has three kinds of null data: * **Absent (key not present)**: a field name is not present, e.g. input record is `x=1,y=2` and a `put` or `filter` expression refers to `$z`. Or, reading an out-of-stream variable which hasn't been assigned a value yet, e.g. `mlr put -q '@sum += $x; end{emit @sum}'` or `mlr put -q '@sum[$a][$b] += $x; end{emit @sum, "a", "b"}'`. -* **JSON null**: (TODO: describe) +* **JSON null**: The main purpose of this is to support reading the `null` type in JSON files. The [Miller programming language](programming-language.md) has a `null` keyword as well, so you can also write the null type using `$x = null`. Addtionally, though, when you write past the end of an array, leaving gaps -- e.g. writing `a[12]` when the array `a` has length 10 -- JSON-null is used to fill the gaps. See also the [arrays page](reference-main-arrays.md#auto-extend-and-null-gaps). You can test these programatically using the functions `is_empty`/`is_not_empty`, `is_absent`/`is_present`, and `is_null`/`is_not_null`. For the last pair, note that null means either empty or absent. diff --git a/docs6/docs/reference-main-null-data.md.in b/docs6/docs/reference-main-null-data.md.in index 3235e6e5d..47ac6c2af 100644 --- a/docs6/docs/reference-main-null-data.md.in +++ b/docs6/docs/reference-main-null-data.md.in @@ -8,7 +8,7 @@ Miller has three kinds of null data: * **Absent (key not present)**: a field name is not present, e.g. input record is `x=1,y=2` and a `put` or `filter` expression refers to `$z`. Or, reading an out-of-stream variable which hasn't been assigned a value yet, e.g. `mlr put -q '@sum += $x; end{emit @sum}'` or `mlr put -q '@sum[$a][$b] += $x; end{emit @sum, "a", "b"}'`. -* **JSON null**: (TODO: describe) +* **JSON null**: The main purpose of this is to support reading the `null` type in JSON files. The [Miller programming language](programming-language.md) has a `null` keyword as well, so you can also write the null type using `$x = null`. Addtionally, though, when you write past the end of an array, leaving gaps -- e.g. writing `a[12]` when the array `a` has length 10 -- JSON-null is used to fill the gaps. See also the [arrays page](reference-main-arrays.md#auto-extend-and-null-gaps). You can test these programatically using the functions `is_empty`/`is_not_empty`, `is_absent`/`is_present`, and `is_null`/`is_not_null`. For the last pair, note that null means either empty or absent. diff --git a/go/regtest/cases/dsl-type-inference/0001/cmd b/go/regtest/cases/dsl-type-inference/0001/cmd index 9ace894e0..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0001/cmd +++ b/go/regtest/cases/dsl-type-inference/0001/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0002/cmd b/go/regtest/cases/dsl-type-inference/0002/cmd index 954d105b9..88c4ca46a 100644 --- a/go/regtest/cases/dsl-type-inference/0002/cmd +++ b/go/regtest/cases/dsl-type-inference/0002/cmd @@ -1 +1 @@ -mlr --xtab filter -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab filter -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0003/cmd b/go/regtest/cases/dsl-type-inference/0003/cmd index 637fd26d9..f1e9713c5 100644 --- a/go/regtest/cases/dsl-type-inference/0003/cmd +++ b/go/regtest/cases/dsl-type-inference/0003/cmd @@ -1 +1 @@ -mlr --oxtab put -f ${CASEDIR}/mlr ./${CASEDIR}/input +mlr --oxtab put -f ${CASEDIR}/mlr ./${CASEDIR}/input diff --git a/go/regtest/cases/dsl-type-inference/0004/cmd b/go/regtest/cases/dsl-type-inference/0004/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0004/cmd +++ b/go/regtest/cases/dsl-type-inference/0004/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0005/cmd b/go/regtest/cases/dsl-type-inference/0005/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0005/cmd +++ b/go/regtest/cases/dsl-type-inference/0005/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0006/cmd b/go/regtest/cases/dsl-type-inference/0006/cmd index 2872eeeb8..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0006/cmd +++ b/go/regtest/cases/dsl-type-inference/0006/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0007/cmd b/go/regtest/cases/dsl-type-inference/0007/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0007/cmd +++ b/go/regtest/cases/dsl-type-inference/0007/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0008/cmd b/go/regtest/cases/dsl-type-inference/0008/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0008/cmd +++ b/go/regtest/cases/dsl-type-inference/0008/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0009/cmd b/go/regtest/cases/dsl-type-inference/0009/cmd index 2872eeeb8..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0009/cmd +++ b/go/regtest/cases/dsl-type-inference/0009/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0012/cmd b/go/regtest/cases/dsl-type-inference/0012/cmd index da804817c..cef492ace 100644 --- a/go/regtest/cases/dsl-type-inference/0012/cmd +++ b/go/regtest/cases/dsl-type-inference/0012/cmd @@ -1 +1 @@ -mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0015/cmd b/go/regtest/cases/dsl-type-inference/0015/cmd index da804817c..cef492ace 100644 --- a/go/regtest/cases/dsl-type-inference/0015/cmd +++ b/go/regtest/cases/dsl-type-inference/0015/cmd @@ -1 +1 @@ -mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0016/cmd b/go/regtest/cases/dsl-type-inference/0016/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0016/cmd +++ b/go/regtest/cases/dsl-type-inference/0016/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0017/cmd b/go/regtest/cases/dsl-type-inference/0017/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0017/cmd +++ b/go/regtest/cases/dsl-type-inference/0017/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0018/cmd b/go/regtest/cases/dsl-type-inference/0018/cmd index 2872eeeb8..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0018/cmd +++ b/go/regtest/cases/dsl-type-inference/0018/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0019/cmd b/go/regtest/cases/dsl-type-inference/0019/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0019/cmd +++ b/go/regtest/cases/dsl-type-inference/0019/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0020/cmd b/go/regtest/cases/dsl-type-inference/0020/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0020/cmd +++ b/go/regtest/cases/dsl-type-inference/0020/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0021/cmd b/go/regtest/cases/dsl-type-inference/0021/cmd index 2872eeeb8..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0021/cmd +++ b/go/regtest/cases/dsl-type-inference/0021/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0024/cmd b/go/regtest/cases/dsl-type-inference/0024/cmd index da804817c..cef492ace 100644 --- a/go/regtest/cases/dsl-type-inference/0024/cmd +++ b/go/regtest/cases/dsl-type-inference/0024/cmd @@ -1 +1 @@ -mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0027/cmd b/go/regtest/cases/dsl-type-inference/0027/cmd index da804817c..cef492ace 100644 --- a/go/regtest/cases/dsl-type-inference/0027/cmd +++ b/go/regtest/cases/dsl-type-inference/0027/cmd @@ -1 +1 @@ -mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0028/cmd b/go/regtest/cases/dsl-type-inference/0028/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0028/cmd +++ b/go/regtest/cases/dsl-type-inference/0028/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0029/cmd b/go/regtest/cases/dsl-type-inference/0029/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0029/cmd +++ b/go/regtest/cases/dsl-type-inference/0029/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0030/cmd b/go/regtest/cases/dsl-type-inference/0030/cmd index 2872eeeb8..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0030/cmd +++ b/go/regtest/cases/dsl-type-inference/0030/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0031/cmd b/go/regtest/cases/dsl-type-inference/0031/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0031/cmd +++ b/go/regtest/cases/dsl-type-inference/0031/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0032/cmd b/go/regtest/cases/dsl-type-inference/0032/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0032/cmd +++ b/go/regtest/cases/dsl-type-inference/0032/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0033/cmd b/go/regtest/cases/dsl-type-inference/0033/cmd index 2872eeeb8..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0033/cmd +++ b/go/regtest/cases/dsl-type-inference/0033/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0036/cmd b/go/regtest/cases/dsl-type-inference/0036/cmd index da804817c..cef492ace 100644 --- a/go/regtest/cases/dsl-type-inference/0036/cmd +++ b/go/regtest/cases/dsl-type-inference/0036/cmd @@ -1 +1 @@ -mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0039/cmd b/go/regtest/cases/dsl-type-inference/0039/cmd index da804817c..cef492ace 100644 --- a/go/regtest/cases/dsl-type-inference/0039/cmd +++ b/go/regtest/cases/dsl-type-inference/0039/cmd @@ -1 +1 @@ -mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0040/cmd b/go/regtest/cases/dsl-type-inference/0040/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0040/cmd +++ b/go/regtest/cases/dsl-type-inference/0040/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0041/cmd b/go/regtest/cases/dsl-type-inference/0041/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0041/cmd +++ b/go/regtest/cases/dsl-type-inference/0041/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0042/cmd b/go/regtest/cases/dsl-type-inference/0042/cmd index 2872eeeb8..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0042/cmd +++ b/go/regtest/cases/dsl-type-inference/0042/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0043/cmd b/go/regtest/cases/dsl-type-inference/0043/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0043/cmd +++ b/go/regtest/cases/dsl-type-inference/0043/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0044/cmd b/go/regtest/cases/dsl-type-inference/0044/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0044/cmd +++ b/go/regtest/cases/dsl-type-inference/0044/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0045/cmd b/go/regtest/cases/dsl-type-inference/0045/cmd index 2872eeeb8..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0045/cmd +++ b/go/regtest/cases/dsl-type-inference/0045/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0048/cmd b/go/regtest/cases/dsl-type-inference/0048/cmd index da804817c..cef492ace 100644 --- a/go/regtest/cases/dsl-type-inference/0048/cmd +++ b/go/regtest/cases/dsl-type-inference/0048/cmd @@ -1 +1 @@ -mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0051/cmd b/go/regtest/cases/dsl-type-inference/0051/cmd index da804817c..cef492ace 100644 --- a/go/regtest/cases/dsl-type-inference/0051/cmd +++ b/go/regtest/cases/dsl-type-inference/0051/cmd @@ -1 +1 @@ -mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -F -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0052/cmd b/go/regtest/cases/dsl-type-inference/0052/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0052/cmd +++ b/go/regtest/cases/dsl-type-inference/0052/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0053/cmd b/go/regtest/cases/dsl-type-inference/0053/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0053/cmd +++ b/go/regtest/cases/dsl-type-inference/0053/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0054/cmd b/go/regtest/cases/dsl-type-inference/0054/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0054/cmd +++ b/go/regtest/cases/dsl-type-inference/0054/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0055/cmd b/go/regtest/cases/dsl-type-inference/0055/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0055/cmd +++ b/go/regtest/cases/dsl-type-inference/0055/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0060/cmd b/go/regtest/cases/dsl-type-inference/0060/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0060/cmd +++ b/go/regtest/cases/dsl-type-inference/0060/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0061/cmd b/go/regtest/cases/dsl-type-inference/0061/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0061/cmd +++ b/go/regtest/cases/dsl-type-inference/0061/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0062/cmd b/go/regtest/cases/dsl-type-inference/0062/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0062/cmd +++ b/go/regtest/cases/dsl-type-inference/0062/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0063/cmd b/go/regtest/cases/dsl-type-inference/0063/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0063/cmd +++ b/go/regtest/cases/dsl-type-inference/0063/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0068/cmd b/go/regtest/cases/dsl-type-inference/0068/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0068/cmd +++ b/go/regtest/cases/dsl-type-inference/0068/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0069/cmd b/go/regtest/cases/dsl-type-inference/0069/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0069/cmd +++ b/go/regtest/cases/dsl-type-inference/0069/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0070/cmd b/go/regtest/cases/dsl-type-inference/0070/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0070/cmd +++ b/go/regtest/cases/dsl-type-inference/0070/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0071/cmd b/go/regtest/cases/dsl-type-inference/0071/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0071/cmd +++ b/go/regtest/cases/dsl-type-inference/0071/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0076/cmd b/go/regtest/cases/dsl-type-inference/0076/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0076/cmd +++ b/go/regtest/cases/dsl-type-inference/0076/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0077/cmd b/go/regtest/cases/dsl-type-inference/0077/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0077/cmd +++ b/go/regtest/cases/dsl-type-inference/0077/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0078/cmd b/go/regtest/cases/dsl-type-inference/0078/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0078/cmd +++ b/go/regtest/cases/dsl-type-inference/0078/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0079/cmd b/go/regtest/cases/dsl-type-inference/0079/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0079/cmd +++ b/go/regtest/cases/dsl-type-inference/0079/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0084/cmd b/go/regtest/cases/dsl-type-inference/0084/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0084/cmd +++ b/go/regtest/cases/dsl-type-inference/0084/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0085/cmd b/go/regtest/cases/dsl-type-inference/0085/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0085/cmd +++ b/go/regtest/cases/dsl-type-inference/0085/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0086/cmd b/go/regtest/cases/dsl-type-inference/0086/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0086/cmd +++ b/go/regtest/cases/dsl-type-inference/0086/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0087/cmd b/go/regtest/cases/dsl-type-inference/0087/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0087/cmd +++ b/go/regtest/cases/dsl-type-inference/0087/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0092/cmd b/go/regtest/cases/dsl-type-inference/0092/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0092/cmd +++ b/go/regtest/cases/dsl-type-inference/0092/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0093/cmd b/go/regtest/cases/dsl-type-inference/0093/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0093/cmd +++ b/go/regtest/cases/dsl-type-inference/0093/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0094/cmd b/go/regtest/cases/dsl-type-inference/0094/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0094/cmd +++ b/go/regtest/cases/dsl-type-inference/0094/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0095/cmd b/go/regtest/cases/dsl-type-inference/0095/cmd index 2f51026ca..6c932f0bf 100644 --- a/go/regtest/cases/dsl-type-inference/0095/cmd +++ b/go/regtest/cases/dsl-type-inference/0095/cmd @@ -1 +1 @@ -mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab +mlr --xtab put -f ${CASEDIR}/mlr regtest/input/mixed-types.xtab diff --git a/go/regtest/cases/dsl-type-inference/0097/cmd b/go/regtest/cases/dsl-type-inference/0097/cmd new file mode 100644 index 000000000..77144583a --- /dev/null +++ b/go/regtest/cases/dsl-type-inference/0097/cmd @@ -0,0 +1 @@ +mlr --idkvp --opprint put -f ${CASEDIR}/mlr ./${CASEDIR}/input diff --git a/go/regtest/cases/dsl-type-inference/0097/experr b/go/regtest/cases/dsl-type-inference/0097/experr new file mode 100644 index 000000000..e69de29bb diff --git a/go/regtest/cases/dsl-type-inference/0097/expout b/go/regtest/cases/dsl-type-inference/0097/expout new file mode 100644 index 000000000..9c0def3cf --- /dev/null +++ b/go/regtest/cases/dsl-type-inference/0097/expout @@ -0,0 +1,33 @@ +a ta +12 int +1.2 float +inf string ++inf string +-inf string +Inf string ++Inf string +-Inf string +INF string ++INF string +-INF string +infinity string ++infinity string +-infinity string +Infinity string ++Infinity string +-Infinity string +INFINITY string ++INFINITY string +-INFINITY string +InFiNiTy string ++InFiNiTy string +-InFiNiTy string +infiniti string ++infiniti string +-infiniti string +true string +false string +True string +False string +TRUE string +FALSE string diff --git a/go/regtest/cases/dsl-type-inference/0097/input b/go/regtest/cases/dsl-type-inference/0097/input new file mode 100644 index 000000000..b4e84a1b5 --- /dev/null +++ b/go/regtest/cases/dsl-type-inference/0097/input @@ -0,0 +1,32 @@ +a=12 +a=1.2 +a=inf +a=+inf +a=-inf +a=Inf +a=+Inf +a=-Inf +a=INF +a=+INF +a=-INF +a=infinity +a=+infinity +a=-infinity +a=Infinity +a=+Infinity +a=-Infinity +a=INFINITY +a=+INFINITY +a=-INFINITY +a=InFiNiTy +a=+InFiNiTy +a=-InFiNiTy +a=infiniti +a=+infiniti +a=-infiniti +a=true +a=false +a=True +a=False +a=TRUE +a=FALSE diff --git a/go/regtest/cases/dsl-type-inference/0097/mlr b/go/regtest/cases/dsl-type-inference/0097/mlr new file mode 100644 index 000000000..7f1766894 --- /dev/null +++ b/go/regtest/cases/dsl-type-inference/0097/mlr @@ -0,0 +1 @@ +$ta = typeof($a) diff --git a/go/src/types/mlrval_new.go b/go/src/types/mlrval_new.go index bb8299422..d7e550650 100644 --- a/go/src/types/mlrval_new.go +++ b/go/src/types/mlrval_new.go @@ -6,6 +6,7 @@ package types import ( "errors" + "strings" "mlr/src/lib" ) @@ -101,15 +102,14 @@ func MlrvalPointerFromBoolString(input string) *Mlrval { } } -var floatNamesToNotInfer = map[string]bool{ - "Inf": true, - "+Inf": true, - "-Inf": true, - "Infinity": true, - "+Infinity": true, - "-Infinity": true, +var downcasedFloatNamesToNotInfer = map[string]bool{ + "inf": true, + "+inf": true, + "-inf": true, + "infinity": true, + "+infinity": true, + "-infinity": true, "nan": true, - "NaN": true, } // MlrvalPointerFromInferredTypeForDataFiles is for parsing field values from @@ -126,7 +126,7 @@ func MlrvalPointerFromInferredTypeForDataFiles(input string) *Mlrval { return MlrvalPointerFromIntString(input) } - if floatNamesToNotInfer[input] == false { + if downcasedFloatNamesToNotInfer[strings.ToLower(input)] == false { _, fok := lib.TryFloat64FromString(input) if fok { return MlrvalPointerFromFloat64String(input) diff --git a/go/todo.txt b/go/todo.txt index a9fee237d..bea9eea01 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -113,6 +113,8 @@ w survey ---------------------------------------------------------------- BLOCKERS +* UTs for r"a" . r"b" and so on and so on + * document precedence issue: ! probably remove this feature !