From d4ffc525f888992b3ea8e9e11eee2077805f37db Mon Sep 17 00:00:00 2001 From: John Kerl Date: Tue, 14 Sep 2021 23:02:29 -0400 Subject: [PATCH] More Miller-6 docs updates (#661) * todo/neatens * bug-fix issue with mlr reorder then something-else Also, gofmt -s -w * flatten/unflatten doc iteration * Include diff output in regtest fail cases, if -vvv * arrayify on unflatten * flatten/unflatten page --- README.md | 2 + build-go-src-test-man-doc.sh | 3 +- docs6/README.md | 10 +- docs6/docs/cpu.md | 53 -- docs6/docs/cpu.md.in | 37 -- docs6/docs/flatten-unflatten.md | 77 --- docs6/docs/flatten-unflatten.md.in | 31 -- docs6/docs/performance.md | 33 -- docs6/docs/performance.md.in | 17 - docs6/docs/reference-main-flag-list.md.in | 3 - docs6/docs/reference-main-io-options.md | 121 ----- docs6/docs/reference-main-io-options.md.in | 80 --- docs6/docs/reference-main-strings.md | 48 -- docs6/docs/reference-main-strings.md.in | 32 -- docs6/mkdocs.yml | 88 ++-- docs6/regen.sh | 2 +- docs6/{docs => src}/.vimrc | 0 docs6/{docs => src}/10min.md | 126 ++++- docs6/{docs => src}/10min.md.in | 38 +- docs6/{docs => src}/Makefile | 0 docs6/{docs => src}/_static/basic.css | 0 docs6/{docs => src}/_static/classic.css | 0 docs6/{docs => src}/_static/mlr.jpg | Bin docs6/{docs => src}/_static/pygments.css | 0 docs6/{docs => src}/_static/scrolls.css | 0 docs6/{docs => src}/_static/top.png | Bin docs6/{docs => src}/absent-example.json | 0 docs6/{docs => src}/build-one | 0 docs6/{docs => src}/build.md | 0 docs6/{docs => src}/build.md.in | 0 docs6/{docs => src}/circle.csv | 0 docs6/{docs => src}/commas.csv | 0 docs6/{docs => src}/community.md | 0 docs6/{docs => src}/community.md.in | 0 docs6/{docs => src}/context-example.mlr | 0 docs6/{docs => src}/contributing.md | 0 docs6/{docs => src}/contributing.md.in | 0 docs6/{docs => src}/coverart/cover-cat.png | Bin .../{docs => src}/coverart/cover-combined.png | Bin .../coverart/cover-filter-to-json.png | Bin .../coverart/cover-sort-to-pprint.png | Bin docs6/{docs => src}/coverart/coverart.key | Bin docs6/src/cpu.md | 65 +++ docs6/src/cpu.md.in | 49 ++ .../csv-with-and-without-headers.md | 0 .../csv-with-and-without-headers.md.in | 0 docs6/{docs => src}/customization.md | 0 docs6/{docs => src}/customization.md.in | 0 docs6/{docs => src}/data-cleaning-examples.md | 0 .../data-cleaning-examples.md.in | 0 docs6/{docs => src}/data-diving-examples.md | 1 + .../{docs => src}/data-diving-examples.md.in | 0 docs6/{docs => src}/data/a.csv | 0 docs6/{docs => src}/data/a.dkvp | 0 docs6/{docs => src}/data/array-example.json | 0 docs6/src/data/array-values-nested.json | 8 + docs6/src/data/array-values-spread-colon.csv | 3 + docs6/src/data/array-values-spread.csv | 3 + docs6/src/data/array-values.json | 8 + docs6/{docs => src}/data/b.csv | 0 docs6/{docs => src}/data/b.dkvp | 0 .../data/begin-end-example-7b.sh | 0 docs6/{docs => src}/data/budget.csv | 0 .../data/bulk-rename-for-loop.mlr | 0 docs6/{docs => src}/data/clean-whitespace.csv | 0 docs6/{docs => src}/data/color-codes.csv | 0 docs6/{docs => src}/data/color-names.csv | 0 docs6/{docs => src}/data/colored-shapes.csv | 0 docs6/{docs => src}/data/colored-shapes.dkvp | 0 docs6/{docs => src}/data/colours.csv | 0 docs6/src/data/consecutive.csv | 2 + docs6/{docs => src}/data/current_counters.csv | 0 docs6/{docs => src}/data/currtemp.csv | 0 docs6/{docs => src}/data/dollar-sign.txt | 0 docs6/{docs => src}/data/english-words.txt | 0 docs6/{docs => src}/data/extra-spaces.txt | 0 docs6/{docs => src}/data/fe-example-3.mlr | 0 docs6/{docs => src}/data/fe-example-4.mlr | 0 docs6/{docs => src}/data/feature-count.mlr | 0 docs6/{docs => src}/data/features.json | 0 docs6/{docs => src}/data/fibo-cached.sh | 0 docs6/{docs => src}/data/fibo-uncached.sh | 0 docs6/{docs => src}/data/fillable.csv | 0 .../data/filter-multiline-example.txt | 0 docs6/{docs => src}/data/flins.csv | 0 docs6/{docs => src}/data/flins.json | 0 .../data/for-oosvar-example-0a.txt | 0 docs6/{docs => src}/data/for-srec-example.tbl | 0 docs6/{docs => src}/data/fraction-example.csv | 0 docs6/{docs => src}/data/headerless.csv | 0 docs6/{docs => src}/data/het-bool.csv | 0 docs6/{docs => src}/data/het.csv | 0 docs6/{docs => src}/data/het.dkvp | 0 docs6/{docs => src}/data/het.json | 0 docs6/{docs => src}/data/het/fillable.csv | 0 docs6/{docs => src}/data/het/hom.csv | 0 docs6/{docs => src}/data/het/irregular.json | 0 docs6/{docs => src}/data/het/ragged.csv | 0 docs6/{docs => src}/data/het/sparse.json | 0 docs6/src/data/hostnames.csv | 3 + docs6/src/data/hostnames.xtab | 9 + docs6/{docs => src}/data/hosts.csv | 0 docs6/{docs => src}/data/inout.csv | 0 .../{docs => src}/data/join-left-example.csv | 0 .../{docs => src}/data/join-right-example.csv | 0 docs6/{docs => src}/data/join-u-left.csv | 0 docs6/{docs => src}/data/join-u-right.csv | 0 docs6/{docs => src}/data/json-example-1.json | 0 docs6/{docs => src}/data/json-example-2.json | 0 docs6/{docs => src}/data/json-example-3.json | 0 docs6/{docs => src}/data/json-in-csv.csv | 0 docs6/{docs => src}/data/label-example.txt | 0 docs6/{docs => src}/data/linreg-example.jpg | Bin docs6/{docs => src}/data/linreg-example.txt | 0 docs6/src/data/map-values-nested.json | 8 + docs6/src/data/map-values-spread-colon.csv | 3 + docs6/src/data/map-values-spread.csv | 3 + docs6/src/data/map-values.json | 8 + docs6/{docs => src}/data/maxrows.mlr | 0 docs6/{docs => src}/data/maxrows.tsv | 0 docs6/{docs => src}/data/medium | 0 docs6/{docs => src}/data/medium-squares | 0 docs6/{docs => src}/data/miss-date.csv | 0 docs6/{docs => src}/data/mkdat2 | 0 docs6/{docs => src}/data/modsep.dkvp | 0 docs6/{docs => src}/data/multicountdown.dat | 0 docs6/{docs => src}/data/multicountdown.rb | 0 docs6/{docs => src}/data/mydata.txt | 0 docs6/{docs => src}/data/nested.tsv | 0 docs6/{docs => src}/data/newline-example.txt | 0 docs6/src/data/non-consecutive.csv | 2 + .../{docs => src}/data/ping-delta-example.txt | 0 .../{docs => src}/data/previous_counters.csv | 0 docs6/{docs => src}/data/prevtemp.csv | 0 .../data/put-gating-example-1.dkvp | 0 .../data/put-gating-example-2.dkvp | 0 docs6/{docs => src}/data/question.dat | 0 docs6/{docs => src}/data/ragged.csv | 0 docs6/{docs => src}/data/rect-inner.txt | 0 docs6/{docs => src}/data/rect-middle.txt | 0 docs6/{docs => src}/data/rect-outer.txt | 0 docs6/{docs => src}/data/rect.txt | 0 docs6/{docs => src}/data/regex-in-data.dat | 0 .../data/remove-empty-columns.csv | 0 docs6/{docs => src}/data/repeat-example.dat | 0 docs6/{docs => src}/data/repeats.dkvp | 0 docs6/{docs => src}/data/sample-example.txt | 0 docs6/{docs => src}/data/sar.csv | 0 docs6/{docs => src}/data/sar.mlr | 0 docs6/{docs => src}/data/scan-example-1.tbl | 0 docs6/{docs => src}/data/scan-example-2.tbl | 0 docs6/{docs => src}/data/scope-example.dat | 0 docs6/{docs => src}/data/scope-example.mlr | 0 docs6/{docs => src}/data/self-join.csv | 0 .../{docs => src}/data/semicolon-example.txt | 0 docs6/src/data/server-log.json | 21 + docs6/{docs => src}/data/short.csv | 0 .../data/single-quote-example.mlr | 0 docs6/{docs => src}/data/small | 0 docs6/{docs => src}/data/small.csv | 0 docs6/{docs => src}/data/small2 | 0 docs6/{docs => src}/data/sort-het.dkvp | 0 docs6/{docs => src}/data/sort-missing.dkvp | 0 docs6/{docs => src}/data/sort-null.dat | 0 .../data/sort-within-records.json | 0 docs6/{docs => src}/data/spaces.csv | 0 docs6/{docs => src}/data/sparse.json | 0 .../{docs => src}/data/subcommand-example.txt | 0 .../data/then-chaining-performance.txt | 0 docs6/{docs => src}/data/then-example.csv | 0 docs6/{docs => src}/data/timing-example.txt | 0 docs6/{docs => src}/data/trivial-records.csv | 0 .../{docs => src}/data/type-decl-example.mlr | 0 docs6/{docs => src}/data/type-infer.csv | 0 docs6/{docs => src}/data/unsparsify.mlr | 0 docs6/{docs => src}/date-time-examples.md | 0 docs6/{docs => src}/date-time-examples.md.in | 0 docs6/{docs => src}/dates.csv | 0 docs6/{docs => src}/dkvp-examples.md | 0 docs6/{docs => src}/dkvp-examples.md.in | 0 docs6/{docs => src}/doc-reorg.txt | 0 docs6/{docs => src}/dsl-example.mlr | 0 docs6/{docs => src}/etymology.md | 0 docs6/{docs => src}/etymology.md.in | 0 docs6/{docs => src}/example.csv | 0 docs6/src/example.json | 18 + docs6/{docs => src}/expo-sample.sh | 0 docs6/{docs => src}/extra.css | 0 docs6/{docs => src}/factorial-example.mlr | 0 docs6/{docs => src}/features.md | 0 docs6/{docs => src}/features.md.in | 0 docs6/{docs => src}/file-formats.md | 9 +- docs6/{docs => src}/file-formats.md.in | 9 +- docs6/{docs => src}/flags.txt | 0 docs6/src/flatten-unflatten.md | 472 ++++++++++++++++++ docs6/src/flatten-unflatten.md.in | 225 +++++++++ docs6/{docs => src}/for-example.mlr | 0 docs6/{docs => src}/genmd-filter | 6 +- docs6/{docs => src}/genmds | 0 docs6/{docs => src}/glossary.md | 107 ++-- docs6/{docs => src}/glossary.md.in | 107 ++-- docs6/{docs => src}/gz-example.csv.gz | Bin docs6/{docs => src}/h1foo | 0 docs6/{docs => src}/h2foo | 0 docs6/{docs => src}/h3foo | 0 docs6/{docs => src}/if-example.mlr | 0 docs6/{docs => src}/images/favicon.ico | Bin docs6/{docs => src}/images/favicon.png | Bin docs6/{docs => src}/index.md | 0 docs6/{docs => src}/index.md.in | 0 docs6/{docs => src}/installation.md | 0 docs6/{docs => src}/installation.md.in | 0 docs6/{docs => src}/installing-miller.md | 0 docs6/{docs => src}/installing-miller.md.in | 0 docs6/{docs => src}/internationalization.md | 0 .../{docs => src}/internationalization.md.in | 0 docs6/{docs => src}/javascripts/tables.js | 0 docs6/{docs => src}/keystroke-savers.md | 0 docs6/{docs => src}/keystroke-savers.md.in | 0 docs6/{docs => src}/link.txt | 0 .../{docs => src}/log-processing-examples.md | 0 .../log-processing-examples.md.in | 0 docs6/{docs => src}/log.txt | 0 docs6/{docs => src}/make.bat | 0 docs6/{docs => src}/manpage.md | 125 +++-- docs6/{docs => src}/manpage.md.in | 0 docs6/{docs => src}/manpage.txt | 125 +++-- docs6/{docs => src}/miller-on-windows.md | 0 docs6/{docs => src}/miller-on-windows.md.in | 0 .../miller-programming-language.md} | 1 + .../miller-programming-language.md.in} | 0 docs6/{docs => src}/misc-examples.md | 0 docs6/{docs => src}/misc-examples.md.in | 0 docs6/{docs => src}/mk-flag-info.rb | 0 docs6/{docs => src}/mk-func-h2s.sh | 0 docs6/{docs => src}/mk-func-info.rb | 2 +- docs6/{docs => src}/mlr.1 | 0 docs6/{docs => src}/morph | 0 docs6/{docs => src}/morphs | 0 docs6/{docs => src}/multi-join/input.csv | 0 .../{docs => src}/multi-join/name-lookup.csv | 0 .../multi-join/status-lookup.csv | 0 docs6/{docs => src}/new-in-miller-6.md | 8 +- docs6/{docs => src}/new-in-miller-6.md.in | 8 +- docs6/{docs => src}/ngrams/gsl-2000.txt | 0 docs6/{docs => src}/ngrams/mkv.mlr | 0 docs6/{docs => src}/ngrams/mkv3.mlr | 0 docs6/{docs => src}/ngrams/mkv4.mlr | 0 docs6/{docs => src}/ngrams/mkv5.mlr | 0 docs6/{docs => src}/ngrams/ng1 | 0 docs6/{docs => src}/ngrams/ng1.mlr | 0 docs6/{docs => src}/ngrams/ng1.mlr.txt | 0 docs6/{docs => src}/ngrams/ng2 | 0 docs6/{docs => src}/ngrams/ng2.mlr | 0 docs6/{docs => src}/ngrams/ng2.mlr.txt | 0 docs6/{docs => src}/ngrams/ng3 | 0 docs6/{docs => src}/ngrams/ng3.mlr | 0 docs6/{docs => src}/ngrams/ng3.mlr.txt | 0 docs6/{docs => src}/ngrams/ng4 | 0 docs6/{docs => src}/ngrams/ng4.mlr | 0 docs6/{docs => src}/ngrams/ng4.mlr.txt | 0 docs6/{docs => src}/ngrams/ng5 | 0 docs6/{docs => src}/ngrams/ng5.mlr | 0 docs6/{docs => src}/ngrams/ng5.mlr.txt | 0 docs6/{docs => src}/ngrams/ngfuncs.mlr | 0 docs6/{docs => src}/ngrams/ngfuncs.mlr.txt | 0 docs6/{docs => src}/online-help.md | 4 +- docs6/{docs => src}/online-help.md.in | 0 .../{docs => src}/operating-on-all-fields.md | 0 .../operating-on-all-fields.md.in | 0 .../{docs => src}/operating-on-all-records.md | 0 .../operating-on-all-records.md.in | 0 docs6/{docs => src}/originality.md | 0 docs6/{docs => src}/originality.md.in | 0 docs6/{docs => src}/output-colorization.md | 0 docs6/{docs => src}/output-colorization.md.in | 0 docs6/src/performance.md | 118 +++++ docs6/src/performance.md.in | 102 ++++ docs6/{docs => src}/pix/colorization.png | Bin docs6/{docs => src}/pix/colorization.png.000 | Bin docs6/{docs => src}/pix/colorization2.png | Bin docs6/{docs => src}/pix/colorization2.png.000 | Bin docs6/{docs => src}/pix/mand.png | Bin .../{docs => src}/pix/miller-msys-complex.png | Bin docs6/{docs => src}/pix/miller-msys.png | Bin .../pix/miller-windows-complex-workaround.png | Bin .../pix/miller-windows-complex.png | Bin .../miller-windows-triple-double-quote.png | Bin docs6/{docs => src}/pix/miller-windows.png | Bin docs6/{docs => src}/pix/mlr-doc-sepia.jpg | Bin docs6/{docs => src}/pix/mlr.jpg | Bin docs6/{docs => src}/pix/omd.png | Bin docs6/{docs => src}/pix/sepia-overlay.jpg | Bin docs6/{docs => src}/pix/vim-syntax-off.png | Bin docs6/{docs => src}/pix/vim-syntax-on.png | Bin .../{docs => src}/polyglot-dkvp-io/dkvp_io.py | 0 .../{docs => src}/polyglot-dkvp-io/dkvp_io.rb | 0 .../{docs => src}/polyglot-dkvp-io/example.py | 0 .../{docs => src}/polyglot-dkvp-io/example.rb | 0 docs6/{docs => src}/programming-examples.md | 0 .../{docs => src}/programming-examples.md.in | 0 docs6/{docs => src}/programs/mand.mlr | 0 docs6/{docs => src}/programs/mand.sh | 0 docs6/{docs => src}/programs/sieve.mlr | 0 docs6/{docs => src}/proofreads.txt | 59 +-- docs6/{docs => src}/purple.csv.gz | Bin docs6/{docs => src}/questions-about-joins.md | 0 .../{docs => src}/questions-about-joins.md.in | 0 .../questions-about-then-chaining.md | 0 .../questions-about-then-chaining.md.in | 0 docs6/{docs => src}/randomizing-examples.md | 0 .../{docs => src}/randomizing-examples.md.in | 0 docs6/{docs => src}/record-heterogeneity.md | 1 + .../{docs => src}/record-heterogeneity.md.in | 0 docs6/{docs => src}/red.csv.gz | Bin .../reference-dsl-builtin-functions.md | 32 +- .../reference-dsl-builtin-functions.md.in | 17 + .../{docs => src}/reference-dsl-complexity.md | 0 .../reference-dsl-complexity.md.in | 0 .../reference-dsl-control-structures.md | 0 .../reference-dsl-control-structures.md.in | 0 .../reference-dsl-differences.md | 41 +- .../reference-dsl-differences.md.in | 40 +- docs6/{docs => src}/reference-dsl-errors.md | 2 +- .../{docs => src}/reference-dsl-errors.md.in | 2 +- .../reference-dsl-filter-statements.md | 0 .../reference-dsl-filter-statements.md.in | 0 .../{docs => src}/reference-dsl-operators.md | 0 .../reference-dsl-operators.md.in | 0 .../reference-dsl-output-statements.md | 0 .../reference-dsl-output-statements.md.in | 0 docs6/{docs => src}/reference-dsl-syntax.md | 0 .../{docs => src}/reference-dsl-syntax.md.in | 0 .../reference-dsl-unset-statements.md | 0 .../reference-dsl-unset-statements.md.in | 0 .../reference-dsl-user-defined-functions.md | 0 ...reference-dsl-user-defined-functions.md.in | 0 .../{docs => src}/reference-dsl-variables.md | 10 +- .../reference-dsl-variables.md.in | 10 +- docs6/{docs => src}/reference-dsl.md | 4 +- docs6/{docs => src}/reference-dsl.md.in | 4 +- .../reference-main-arithmetic.md | 0 .../reference-main-arithmetic.md.in | 0 docs6/{docs => src}/reference-main-arrays.md | 45 +- .../{docs => src}/reference-main-arrays.md.in | 35 +- .../reference-main-auxiliary-commands.md | 0 .../reference-main-auxiliary-commands.md.in | 0 .../reference-main-compressed-data.md | 2 +- .../reference-main-compressed-data.md.in | 2 +- .../reference-main-data-types.md | 4 +- .../reference-main-data-types.md.in | 4 +- .../{docs => src}/reference-main-env-vars.md | 0 .../reference-main-env-vars.md.in | 0 .../{docs => src}/reference-main-flag-list.md | 22 + docs6/src/reference-main-flag-list.md.in | 14 + .../src/reference-main-in-place-processing.md | 28 ++ .../reference-main-in-place-processing.md.in | 12 + docs6/{docs => src}/reference-main-maps.md | 4 + docs6/{docs => src}/reference-main-maps.md.in | 4 + .../{docs => src}/reference-main-null-data.md | 2 +- .../reference-main-null-data.md.in | 2 +- docs6/src/reference-main-number-formatting.md | 82 +++ .../reference-main-number-formatting.md.in | 48 ++ .../{docs => src}/reference-main-overview.md | 6 +- .../reference-main-overview.md.in | 6 +- .../reference-main-regular-expressions.md | 0 .../reference-main-regular-expressions.md.in | 0 .../reference-main-separators.md | 0 .../reference-main-separators.md.in | 0 docs6/src/reference-main-strings.md | 181 +++++++ docs6/src/reference-main-strings.md.in | 129 +++++ .../reference-main-then-chaining.md | 0 .../reference-main-then-chaining.md.in | 0 docs6/{docs => src}/reference-verbs.md | 53 +- docs6/{docs => src}/reference-verbs.md.in | 28 +- docs6/{docs => src}/release-docs.md | 0 docs6/{docs => src}/release-docs.md.in | 0 docs6/{docs => src}/repl.md | 2 +- docs6/{docs => src}/repl.md.in | 2 +- docs6/{docs => src}/sample_mlrrc | 0 docs6/{docs => src}/shapes-of-data.md | 0 docs6/{docs => src}/shapes-of-data.md.in | 0 docs6/{docs => src}/shell-commands.md | 0 docs6/{docs => src}/shell-commands.md.in | 0 .../special-symbols-and-formatting.md | 0 .../special-symbols-and-formatting.md.in | 0 docs6/{docs => src}/sql-examples.md | 0 docs6/{docs => src}/sql-examples.md.in | 0 docs6/{docs => src}/square.csv | 0 docs6/{docs => src}/statistics-examples.md | 0 docs6/{docs => src}/statistics-examples.md.in | 0 docs6/{docs => src}/streaming-and-memory.md | 2 +- .../{docs => src}/streaming-and-memory.md.in | 2 +- .../structure-of-these-documents.md | 2 +- .../structure-of-these-documents.md.in | 2 +- docs6/{docs => src}/triangle.csv | 0 docs6/{docs => src}/two-pass-algorithms.md | 0 docs6/{docs => src}/two-pass-algorithms.md.in | 0 docs6/{docs => src}/unix-toolkit-context.md | 0 .../{docs => src}/unix-toolkit-context.md.in | 0 docs6/{docs => src}/variance.mlr | 0 docs6/{docs => src}/vimrc.md | 2 +- docs6/{docs => src}/vimrc.md.in | 2 +- docs6/{docs => src}/why.md | 0 docs6/{docs => src}/why.md.in | 0 docs6/{docs => src}/yellow.csv.gz | Bin .../cases/dsl-flatten-unflatten/0007/expout | 8 +- .../cases/dsl-flatten-unflatten/0008/cmd | 1 + .../cases/dsl-flatten-unflatten/0008/experr | 0 .../cases/dsl-flatten-unflatten/0008/expout | 5 + .../cases/dsl-flatten-unflatten/0008/input | 2 + .../cases/dsl-flatten-unflatten/0009/cmd | 1 + .../cases/dsl-flatten-unflatten/0009/experr | 0 .../cases/dsl-flatten-unflatten/0009/expout | 8 + .../cases/dsl-flatten-unflatten/0009/input | 2 + go/src/auxents/help/entry.go | 6 +- go/src/auxents/regtest/invoker.go | 43 ++ go/src/auxents/regtest/regtester.go | 14 +- go/src/cli/flag_types.go | 17 +- go/src/cli/option_parse.go | 169 ++++++- go/src/dsl/cst/builtin_functions.go | 2 +- go/src/dsl/cst/collections.go | 24 +- go/src/dsl/cst/emit_emitp.go | 2 +- go/src/dsl/cst/lvalues.go | 4 +- go/src/dsl/cst/udf.go | 4 +- go/src/dsl/cst/uds.go | 2 +- go/src/lib/regex_test.go | 14 +- go/src/lib/util.go | 30 +- go/src/platform/diff_notwindows.go | 12 + go/src/platform/diff_windows.go | 12 + go/src/transformers/most-or-least-frequent.go | 2 +- go/src/transformers/step.go | 4 +- go/src/transformers/top.go | 4 +- go/src/types/mlrmap_accessors.go | 18 +- go/src/types/mlrmap_flatten_unflatten.go | 86 +++- go/src/types/mlrval_collections.go | 2 +- go/src/types/mlrval_functions_arithmetic.go | 4 +- go/src/types/mlrval_functions_collections.go | 25 +- go/todo.txt | 4 + man6/Makefile | 2 +- man6/manpage.txt | 125 +++-- man6/mkman.rb | 4 +- man6/mlr6.1 | 119 +++-- 443 files changed, 3111 insertions(+), 1064 deletions(-) delete mode 100644 docs6/docs/cpu.md delete mode 100644 docs6/docs/cpu.md.in delete mode 100644 docs6/docs/flatten-unflatten.md delete mode 100644 docs6/docs/flatten-unflatten.md.in delete mode 100644 docs6/docs/performance.md delete mode 100644 docs6/docs/performance.md.in delete mode 100644 docs6/docs/reference-main-flag-list.md.in delete mode 100644 docs6/docs/reference-main-io-options.md delete mode 100644 docs6/docs/reference-main-io-options.md.in delete mode 100644 docs6/docs/reference-main-strings.md delete mode 100644 docs6/docs/reference-main-strings.md.in rename docs6/{docs => src}/.vimrc (100%) rename docs6/{docs => src}/10min.md (87%) rename docs6/{docs => src}/10min.md.in (90%) rename docs6/{docs => src}/Makefile (100%) rename docs6/{docs => src}/_static/basic.css (100%) rename docs6/{docs => src}/_static/classic.css (100%) rename docs6/{docs => src}/_static/mlr.jpg (100%) rename docs6/{docs => src}/_static/pygments.css (100%) rename docs6/{docs => src}/_static/scrolls.css (100%) rename docs6/{docs => src}/_static/top.png (100%) rename docs6/{docs => src}/absent-example.json (100%) rename docs6/{docs => src}/build-one (100%) rename docs6/{docs => src}/build.md (100%) rename docs6/{docs => src}/build.md.in (100%) rename docs6/{docs => src}/circle.csv (100%) rename docs6/{docs => src}/commas.csv (100%) rename docs6/{docs => src}/community.md (100%) rename docs6/{docs => src}/community.md.in (100%) rename docs6/{docs => src}/context-example.mlr (100%) rename docs6/{docs => src}/contributing.md (100%) rename docs6/{docs => src}/contributing.md.in (100%) rename docs6/{docs => src}/coverart/cover-cat.png (100%) rename docs6/{docs => src}/coverart/cover-combined.png (100%) rename docs6/{docs => src}/coverart/cover-filter-to-json.png (100%) rename docs6/{docs => src}/coverart/cover-sort-to-pprint.png (100%) rename docs6/{docs => src}/coverart/coverart.key (100%) create mode 100644 docs6/src/cpu.md create mode 100644 docs6/src/cpu.md.in rename docs6/{docs => src}/csv-with-and-without-headers.md (100%) rename docs6/{docs => src}/csv-with-and-without-headers.md.in (100%) rename docs6/{docs => src}/customization.md (100%) rename docs6/{docs => src}/customization.md.in (100%) rename docs6/{docs => src}/data-cleaning-examples.md (100%) rename docs6/{docs => src}/data-cleaning-examples.md.in (100%) rename docs6/{docs => src}/data-diving-examples.md (99%) rename docs6/{docs => src}/data-diving-examples.md.in (100%) rename docs6/{docs => src}/data/a.csv (100%) rename docs6/{docs => src}/data/a.dkvp (100%) rename docs6/{docs => src}/data/array-example.json (100%) create mode 100644 docs6/src/data/array-values-nested.json create mode 100644 docs6/src/data/array-values-spread-colon.csv create mode 100644 docs6/src/data/array-values-spread.csv create mode 100644 docs6/src/data/array-values.json rename docs6/{docs => src}/data/b.csv (100%) rename docs6/{docs => src}/data/b.dkvp (100%) rename docs6/{docs => src}/data/begin-end-example-7b.sh (100%) rename docs6/{docs => src}/data/budget.csv (100%) rename docs6/{docs => src}/data/bulk-rename-for-loop.mlr (100%) rename docs6/{docs => src}/data/clean-whitespace.csv (100%) rename docs6/{docs => src}/data/color-codes.csv (100%) rename docs6/{docs => src}/data/color-names.csv (100%) rename docs6/{docs => src}/data/colored-shapes.csv (100%) rename docs6/{docs => src}/data/colored-shapes.dkvp (100%) rename docs6/{docs => src}/data/colours.csv (100%) create mode 100644 docs6/src/data/consecutive.csv rename docs6/{docs => src}/data/current_counters.csv (100%) rename docs6/{docs => src}/data/currtemp.csv (100%) rename docs6/{docs => src}/data/dollar-sign.txt (100%) rename docs6/{docs => src}/data/english-words.txt (100%) rename docs6/{docs => src}/data/extra-spaces.txt (100%) rename docs6/{docs => src}/data/fe-example-3.mlr (100%) rename docs6/{docs => src}/data/fe-example-4.mlr (100%) rename docs6/{docs => src}/data/feature-count.mlr (100%) rename docs6/{docs => src}/data/features.json (100%) rename docs6/{docs => src}/data/fibo-cached.sh (100%) rename docs6/{docs => src}/data/fibo-uncached.sh (100%) rename docs6/{docs => src}/data/fillable.csv (100%) rename docs6/{docs => src}/data/filter-multiline-example.txt (100%) rename docs6/{docs => src}/data/flins.csv (100%) rename docs6/{docs => src}/data/flins.json (100%) rename docs6/{docs => src}/data/for-oosvar-example-0a.txt (100%) rename docs6/{docs => src}/data/for-srec-example.tbl (100%) rename docs6/{docs => src}/data/fraction-example.csv (100%) rename docs6/{docs => src}/data/headerless.csv (100%) rename docs6/{docs => src}/data/het-bool.csv (100%) rename docs6/{docs => src}/data/het.csv (100%) rename docs6/{docs => src}/data/het.dkvp (100%) rename docs6/{docs => src}/data/het.json (100%) rename docs6/{docs => src}/data/het/fillable.csv (100%) rename docs6/{docs => src}/data/het/hom.csv (100%) rename docs6/{docs => src}/data/het/irregular.json (100%) rename docs6/{docs => src}/data/het/ragged.csv (100%) rename docs6/{docs => src}/data/het/sparse.json (100%) create mode 100644 docs6/src/data/hostnames.csv create mode 100644 docs6/src/data/hostnames.xtab rename docs6/{docs => src}/data/hosts.csv (100%) rename docs6/{docs => src}/data/inout.csv (100%) rename docs6/{docs => src}/data/join-left-example.csv (100%) rename docs6/{docs => src}/data/join-right-example.csv (100%) rename docs6/{docs => src}/data/join-u-left.csv (100%) rename docs6/{docs => src}/data/join-u-right.csv (100%) rename docs6/{docs => src}/data/json-example-1.json (100%) rename docs6/{docs => src}/data/json-example-2.json (100%) rename docs6/{docs => src}/data/json-example-3.json (100%) rename docs6/{docs => src}/data/json-in-csv.csv (100%) rename docs6/{docs => src}/data/label-example.txt (100%) rename docs6/{docs => src}/data/linreg-example.jpg (100%) rename docs6/{docs => src}/data/linreg-example.txt (100%) create mode 100644 docs6/src/data/map-values-nested.json create mode 100644 docs6/src/data/map-values-spread-colon.csv create mode 100644 docs6/src/data/map-values-spread.csv create mode 100644 docs6/src/data/map-values.json rename docs6/{docs => src}/data/maxrows.mlr (100%) rename docs6/{docs => src}/data/maxrows.tsv (100%) rename docs6/{docs => src}/data/medium (100%) rename docs6/{docs => src}/data/medium-squares (100%) rename docs6/{docs => src}/data/miss-date.csv (100%) rename docs6/{docs => src}/data/mkdat2 (100%) rename docs6/{docs => src}/data/modsep.dkvp (100%) rename docs6/{docs => src}/data/multicountdown.dat (100%) rename docs6/{docs => src}/data/multicountdown.rb (100%) rename docs6/{docs => src}/data/mydata.txt (100%) rename docs6/{docs => src}/data/nested.tsv (100%) rename docs6/{docs => src}/data/newline-example.txt (100%) create mode 100644 docs6/src/data/non-consecutive.csv rename docs6/{docs => src}/data/ping-delta-example.txt (100%) rename docs6/{docs => src}/data/previous_counters.csv (100%) rename docs6/{docs => src}/data/prevtemp.csv (100%) rename docs6/{docs => src}/data/put-gating-example-1.dkvp (100%) rename docs6/{docs => src}/data/put-gating-example-2.dkvp (100%) rename docs6/{docs => src}/data/question.dat (100%) rename docs6/{docs => src}/data/ragged.csv (100%) rename docs6/{docs => src}/data/rect-inner.txt (100%) rename docs6/{docs => src}/data/rect-middle.txt (100%) rename docs6/{docs => src}/data/rect-outer.txt (100%) rename docs6/{docs => src}/data/rect.txt (100%) rename docs6/{docs => src}/data/regex-in-data.dat (100%) rename docs6/{docs => src}/data/remove-empty-columns.csv (100%) rename docs6/{docs => src}/data/repeat-example.dat (100%) rename docs6/{docs => src}/data/repeats.dkvp (100%) rename docs6/{docs => src}/data/sample-example.txt (100%) rename docs6/{docs => src}/data/sar.csv (100%) rename docs6/{docs => src}/data/sar.mlr (100%) rename docs6/{docs => src}/data/scan-example-1.tbl (100%) rename docs6/{docs => src}/data/scan-example-2.tbl (100%) rename docs6/{docs => src}/data/scope-example.dat (100%) rename docs6/{docs => src}/data/scope-example.mlr (100%) rename docs6/{docs => src}/data/self-join.csv (100%) rename docs6/{docs => src}/data/semicolon-example.txt (100%) create mode 100644 docs6/src/data/server-log.json rename docs6/{docs => src}/data/short.csv (100%) rename docs6/{docs => src}/data/single-quote-example.mlr (100%) rename docs6/{docs => src}/data/small (100%) rename docs6/{docs => src}/data/small.csv (100%) rename docs6/{docs => src}/data/small2 (100%) rename docs6/{docs => src}/data/sort-het.dkvp (100%) rename docs6/{docs => src}/data/sort-missing.dkvp (100%) rename docs6/{docs => src}/data/sort-null.dat (100%) rename docs6/{docs => src}/data/sort-within-records.json (100%) rename docs6/{docs => src}/data/spaces.csv (100%) rename docs6/{docs => src}/data/sparse.json (100%) rename docs6/{docs => src}/data/subcommand-example.txt (100%) rename docs6/{docs => src}/data/then-chaining-performance.txt (100%) rename docs6/{docs => src}/data/then-example.csv (100%) rename docs6/{docs => src}/data/timing-example.txt (100%) rename docs6/{docs => src}/data/trivial-records.csv (100%) rename docs6/{docs => src}/data/type-decl-example.mlr (100%) rename docs6/{docs => src}/data/type-infer.csv (100%) rename docs6/{docs => src}/data/unsparsify.mlr (100%) rename docs6/{docs => src}/date-time-examples.md (100%) rename docs6/{docs => src}/date-time-examples.md.in (100%) rename docs6/{docs => src}/dates.csv (100%) rename docs6/{docs => src}/dkvp-examples.md (100%) rename docs6/{docs => src}/dkvp-examples.md.in (100%) rename docs6/{docs => src}/doc-reorg.txt (100%) rename docs6/{docs => src}/dsl-example.mlr (100%) rename docs6/{docs => src}/etymology.md (100%) rename docs6/{docs => src}/etymology.md.in (100%) rename docs6/{docs => src}/example.csv (100%) create mode 100644 docs6/src/example.json rename docs6/{docs => src}/expo-sample.sh (100%) rename docs6/{docs => src}/extra.css (100%) rename docs6/{docs => src}/factorial-example.mlr (100%) rename docs6/{docs => src}/features.md (100%) rename docs6/{docs => src}/features.md.in (100%) rename docs6/{docs => src}/file-formats.md (97%) rename docs6/{docs => src}/file-formats.md.in (95%) rename docs6/{docs => src}/flags.txt (100%) create mode 100644 docs6/src/flatten-unflatten.md create mode 100644 docs6/src/flatten-unflatten.md.in rename docs6/{docs => src}/for-example.mlr (100%) rename docs6/{docs => src}/genmd-filter (96%) rename docs6/{docs => src}/genmds (100%) rename docs6/{docs => src}/glossary.md (88%) rename docs6/{docs => src}/glossary.md.in (87%) rename docs6/{docs => src}/gz-example.csv.gz (100%) rename docs6/{docs => src}/h1foo (100%) rename docs6/{docs => src}/h2foo (100%) rename docs6/{docs => src}/h3foo (100%) rename docs6/{docs => src}/if-example.mlr (100%) rename docs6/{docs => src}/images/favicon.ico (100%) rename docs6/{docs => src}/images/favicon.png (100%) rename docs6/{docs => src}/index.md (100%) rename docs6/{docs => src}/index.md.in (100%) rename docs6/{docs => src}/installation.md (100%) rename docs6/{docs => src}/installation.md.in (100%) rename docs6/{docs => src}/installing-miller.md (100%) rename docs6/{docs => src}/installing-miller.md.in (100%) rename docs6/{docs => src}/internationalization.md (100%) rename docs6/{docs => src}/internationalization.md.in (100%) rename docs6/{docs => src}/javascripts/tables.js (100%) rename docs6/{docs => src}/keystroke-savers.md (100%) rename docs6/{docs => src}/keystroke-savers.md.in (100%) rename docs6/{docs => src}/link.txt (100%) rename docs6/{docs => src}/log-processing-examples.md (100%) rename docs6/{docs => src}/log-processing-examples.md.in (100%) rename docs6/{docs => src}/log.txt (100%) rename docs6/{docs => src}/make.bat (100%) rename docs6/{docs => src}/manpage.md (97%) rename docs6/{docs => src}/manpage.md.in (100%) rename docs6/{docs => src}/manpage.txt (97%) rename docs6/{docs => src}/miller-on-windows.md (100%) rename docs6/{docs => src}/miller-on-windows.md.in (100%) rename docs6/{docs/programming-language.md => src/miller-programming-language.md} (99%) rename docs6/{docs/programming-language.md.in => src/miller-programming-language.md.in} (100%) rename docs6/{docs => src}/misc-examples.md (100%) rename docs6/{docs => src}/misc-examples.md.in (100%) rename docs6/{docs => src}/mk-flag-info.rb (100%) rename docs6/{docs => src}/mk-func-h2s.sh (100%) rename docs6/{docs => src}/mk-func-info.rb (99%) rename docs6/{docs => src}/mlr.1 (100%) rename docs6/{docs => src}/morph (100%) rename docs6/{docs => src}/morphs (100%) rename docs6/{docs => src}/multi-join/input.csv (100%) rename docs6/{docs => src}/multi-join/name-lookup.csv (100%) rename docs6/{docs => src}/multi-join/status-lookup.csv (100%) rename docs6/{docs => src}/new-in-miller-6.md (97%) rename docs6/{docs => src}/new-in-miller-6.md.in (96%) rename docs6/{docs => src}/ngrams/gsl-2000.txt (100%) rename docs6/{docs => src}/ngrams/mkv.mlr (100%) rename docs6/{docs => src}/ngrams/mkv3.mlr (100%) rename docs6/{docs => src}/ngrams/mkv4.mlr (100%) rename docs6/{docs => src}/ngrams/mkv5.mlr (100%) rename docs6/{docs => src}/ngrams/ng1 (100%) rename docs6/{docs => src}/ngrams/ng1.mlr (100%) rename docs6/{docs => src}/ngrams/ng1.mlr.txt (100%) rename docs6/{docs => src}/ngrams/ng2 (100%) rename docs6/{docs => src}/ngrams/ng2.mlr (100%) rename docs6/{docs => src}/ngrams/ng2.mlr.txt (100%) rename docs6/{docs => src}/ngrams/ng3 (100%) rename docs6/{docs => src}/ngrams/ng3.mlr (100%) rename docs6/{docs => src}/ngrams/ng3.mlr.txt (100%) rename docs6/{docs => src}/ngrams/ng4 (100%) rename docs6/{docs => src}/ngrams/ng4.mlr (100%) rename docs6/{docs => src}/ngrams/ng4.mlr.txt (100%) rename docs6/{docs => src}/ngrams/ng5 (100%) rename docs6/{docs => src}/ngrams/ng5.mlr (100%) rename docs6/{docs => src}/ngrams/ng5.mlr.txt (100%) rename docs6/{docs => src}/ngrams/ngfuncs.mlr (100%) rename docs6/{docs => src}/ngrams/ngfuncs.mlr.txt (100%) rename docs6/{docs => src}/online-help.md (97%) rename docs6/{docs => src}/online-help.md.in (100%) rename docs6/{docs => src}/operating-on-all-fields.md (100%) rename docs6/{docs => src}/operating-on-all-fields.md.in (100%) rename docs6/{docs => src}/operating-on-all-records.md (100%) rename docs6/{docs => src}/operating-on-all-records.md.in (100%) rename docs6/{docs => src}/originality.md (100%) rename docs6/{docs => src}/originality.md.in (100%) rename docs6/{docs => src}/output-colorization.md (100%) rename docs6/{docs => src}/output-colorization.md.in (100%) create mode 100644 docs6/src/performance.md create mode 100644 docs6/src/performance.md.in rename docs6/{docs => src}/pix/colorization.png (100%) rename docs6/{docs => src}/pix/colorization.png.000 (100%) rename docs6/{docs => src}/pix/colorization2.png (100%) rename docs6/{docs => src}/pix/colorization2.png.000 (100%) rename docs6/{docs => src}/pix/mand.png (100%) rename docs6/{docs => src}/pix/miller-msys-complex.png (100%) rename docs6/{docs => src}/pix/miller-msys.png (100%) rename docs6/{docs => src}/pix/miller-windows-complex-workaround.png (100%) rename docs6/{docs => src}/pix/miller-windows-complex.png (100%) rename docs6/{docs => src}/pix/miller-windows-triple-double-quote.png (100%) rename docs6/{docs => src}/pix/miller-windows.png (100%) rename docs6/{docs => src}/pix/mlr-doc-sepia.jpg (100%) rename docs6/{docs => src}/pix/mlr.jpg (100%) rename docs6/{docs => src}/pix/omd.png (100%) rename docs6/{docs => src}/pix/sepia-overlay.jpg (100%) rename docs6/{docs => src}/pix/vim-syntax-off.png (100%) rename docs6/{docs => src}/pix/vim-syntax-on.png (100%) rename docs6/{docs => src}/polyglot-dkvp-io/dkvp_io.py (100%) rename docs6/{docs => src}/polyglot-dkvp-io/dkvp_io.rb (100%) rename docs6/{docs => src}/polyglot-dkvp-io/example.py (100%) rename docs6/{docs => src}/polyglot-dkvp-io/example.rb (100%) rename docs6/{docs => src}/programming-examples.md (100%) rename docs6/{docs => src}/programming-examples.md.in (100%) rename docs6/{docs => src}/programs/mand.mlr (100%) rename docs6/{docs => src}/programs/mand.sh (100%) rename docs6/{docs => src}/programs/sieve.mlr (100%) rename docs6/{docs => src}/proofreads.txt (84%) rename docs6/{docs => src}/purple.csv.gz (100%) rename docs6/{docs => src}/questions-about-joins.md (100%) rename docs6/{docs => src}/questions-about-joins.md.in (100%) rename docs6/{docs => src}/questions-about-then-chaining.md (100%) rename docs6/{docs => src}/questions-about-then-chaining.md.in (100%) rename docs6/{docs => src}/randomizing-examples.md (100%) rename docs6/{docs => src}/randomizing-examples.md.in (100%) rename docs6/{docs => src}/record-heterogeneity.md (99%) rename docs6/{docs => src}/record-heterogeneity.md.in (100%) rename docs6/{docs => src}/red.csv.gz (100%) rename docs6/{docs => src}/reference-dsl-builtin-functions.md (97%) rename docs6/{docs => src}/reference-dsl-builtin-functions.md.in (69%) rename docs6/{docs => src}/reference-dsl-complexity.md (100%) rename docs6/{docs => src}/reference-dsl-complexity.md.in (100%) rename docs6/{docs => src}/reference-dsl-control-structures.md (100%) rename docs6/{docs => src}/reference-dsl-control-structures.md.in (100%) rename docs6/{docs => src}/reference-dsl-differences.md (88%) rename docs6/{docs => src}/reference-dsl-differences.md.in (87%) rename docs6/{docs => src}/reference-dsl-errors.md (86%) rename docs6/{docs => src}/reference-dsl-errors.md.in (82%) rename docs6/{docs => src}/reference-dsl-filter-statements.md (100%) rename docs6/{docs => src}/reference-dsl-filter-statements.md.in (100%) rename docs6/{docs => src}/reference-dsl-operators.md (100%) rename docs6/{docs => src}/reference-dsl-operators.md.in (100%) rename docs6/{docs => src}/reference-dsl-output-statements.md (100%) rename docs6/{docs => src}/reference-dsl-output-statements.md.in (100%) rename docs6/{docs => src}/reference-dsl-syntax.md (100%) rename docs6/{docs => src}/reference-dsl-syntax.md.in (100%) rename docs6/{docs => src}/reference-dsl-unset-statements.md (100%) rename docs6/{docs => src}/reference-dsl-unset-statements.md.in (100%) rename docs6/{docs => src}/reference-dsl-user-defined-functions.md (100%) rename docs6/{docs => src}/reference-dsl-user-defined-functions.md.in (100%) rename docs6/{docs => src}/reference-dsl-variables.md (99%) rename docs6/{docs => src}/reference-dsl-variables.md.in (98%) rename docs6/{docs => src}/reference-dsl.md (98%) rename docs6/{docs => src}/reference-dsl.md.in (97%) rename docs6/{docs => src}/reference-main-arithmetic.md (100%) rename docs6/{docs => src}/reference-main-arithmetic.md.in (100%) rename docs6/{docs => src}/reference-main-arrays.md (88%) rename docs6/{docs => src}/reference-main-arrays.md.in (87%) rename docs6/{docs => src}/reference-main-auxiliary-commands.md (100%) rename docs6/{docs => src}/reference-main-auxiliary-commands.md.in (100%) rename docs6/{docs => src}/reference-main-compressed-data.md (98%) rename docs6/{docs => src}/reference-main-compressed-data.md.in (98%) rename docs6/{docs => src}/reference-main-data-types.md (98%) rename docs6/{docs => src}/reference-main-data-types.md.in (97%) rename docs6/{docs => src}/reference-main-env-vars.md (100%) rename docs6/{docs => src}/reference-main-env-vars.md.in (100%) rename docs6/{docs => src}/reference-main-flag-list.md (97%) create mode 100644 docs6/src/reference-main-flag-list.md.in create mode 100644 docs6/src/reference-main-in-place-processing.md create mode 100644 docs6/src/reference-main-in-place-processing.md.in rename docs6/{docs => src}/reference-main-maps.md (98%) rename docs6/{docs => src}/reference-main-maps.md.in (97%) rename docs6/{docs => src}/reference-main-null-data.md (95%) rename docs6/{docs => src}/reference-main-null-data.md.in (93%) create mode 100644 docs6/src/reference-main-number-formatting.md create mode 100644 docs6/src/reference-main-number-formatting.md.in rename docs6/{docs => src}/reference-main-overview.md (92%) rename docs6/{docs => src}/reference-main-overview.md.in (88%) rename docs6/{docs => src}/reference-main-regular-expressions.md (100%) rename docs6/{docs => src}/reference-main-regular-expressions.md.in (100%) rename docs6/{docs => src}/reference-main-separators.md (100%) rename docs6/{docs => src}/reference-main-separators.md.in (100%) create mode 100644 docs6/src/reference-main-strings.md create mode 100644 docs6/src/reference-main-strings.md.in rename docs6/{docs => src}/reference-main-then-chaining.md (100%) rename docs6/{docs => src}/reference-main-then-chaining.md.in (100%) rename docs6/{docs => src}/reference-verbs.md (98%) rename docs6/{docs => src}/reference-verbs.md.in (97%) rename docs6/{docs => src}/release-docs.md (100%) rename docs6/{docs => src}/release-docs.md.in (100%) rename docs6/{docs => src}/repl.md (98%) rename docs6/{docs => src}/repl.md.in (97%) rename docs6/{docs => src}/sample_mlrrc (100%) rename docs6/{docs => src}/shapes-of-data.md (100%) rename docs6/{docs => src}/shapes-of-data.md.in (100%) rename docs6/{docs => src}/shell-commands.md (100%) rename docs6/{docs => src}/shell-commands.md.in (100%) rename docs6/{docs => src}/special-symbols-and-formatting.md (100%) rename docs6/{docs => src}/special-symbols-and-formatting.md.in (100%) rename docs6/{docs => src}/sql-examples.md (100%) rename docs6/{docs => src}/sql-examples.md.in (100%) rename docs6/{docs => src}/square.csv (100%) rename docs6/{docs => src}/statistics-examples.md (100%) rename docs6/{docs => src}/statistics-examples.md.in (100%) rename docs6/{docs => src}/streaming-and-memory.md (98%) rename docs6/{docs => src}/streaming-and-memory.md.in (98%) rename docs6/{docs => src}/structure-of-these-documents.md (96%) rename docs6/{docs => src}/structure-of-these-documents.md.in (95%) rename docs6/{docs => src}/triangle.csv (100%) rename docs6/{docs => src}/two-pass-algorithms.md (100%) rename docs6/{docs => src}/two-pass-algorithms.md.in (100%) rename docs6/{docs => src}/unix-toolkit-context.md (100%) rename docs6/{docs => src}/unix-toolkit-context.md.in (100%) rename docs6/{docs => src}/variance.mlr (100%) rename docs6/{docs => src}/vimrc.md (84%) rename docs6/{docs => src}/vimrc.md.in (72%) rename docs6/{docs => src}/why.md (100%) rename docs6/{docs => src}/why.md.in (100%) rename docs6/{docs => src}/yellow.csv.gz (100%) create mode 100644 go/regtest/cases/dsl-flatten-unflatten/0008/cmd create mode 100644 go/regtest/cases/dsl-flatten-unflatten/0008/experr create mode 100644 go/regtest/cases/dsl-flatten-unflatten/0008/expout create mode 100644 go/regtest/cases/dsl-flatten-unflatten/0008/input create mode 100644 go/regtest/cases/dsl-flatten-unflatten/0009/cmd create mode 100644 go/regtest/cases/dsl-flatten-unflatten/0009/experr create mode 100644 go/regtest/cases/dsl-flatten-unflatten/0009/expout create mode 100644 go/regtest/cases/dsl-flatten-unflatten/0009/input create mode 100644 go/src/platform/diff_notwindows.go create mode 100644 go/src/platform/diff_windows.go diff --git a/README.md b/README.md index 8bffc2e12..4902ba6ab 100644 --- a/README.md +++ b/README.md @@ -163,3 +163,5 @@ compile a single binary, `scp` it to a faraway machine, and expect it to work.

Miller: a *format-aware* data munging tool By @__jo_ker__ to overcome limitations with *line-aware* workshorses like awk, sed et al https://t.co/LCyPkhYvt9

The project website is a fantastic example of good software documentation!!

— Donny Daniel (@dnnydnl) September 9, 2018

Holy holly data swiss army knife batman! How did noone suggest Miller https://t.co/JGQpmRAZLv for solving database cleaning / ETL issues to me before

Congrats to @__jo_ker__ for amazingly intuitive tool for critical data management tasks!#DataScienceandLaw #ComputationalLaw

— James Miller (@japanlawprof) June 12, 2018
+ +

🤯@__jo_ker__'s Miller easily reads, transforms, + writes all sorts of tabular data. It's standalone, fast, and built for streaming data (operating on one line at a time, so you can work on files larger than memory).

And the docs are dream. I've been reading them all morning! https://t.co/Be2pGPZK6t

— Benjamin Wolfe (he/him) (@BenjaminWolfe) September 9, 2021
diff --git a/build-go-src-test-man-doc.sh b/build-go-src-test-man-doc.sh index e8efdc27b..85d97aea1 100644 --- a/build-go-src-test-man-doc.sh +++ b/build-go-src-test-man-doc.sh @@ -6,7 +6,7 @@ # go test # man6: # creates manpage mlr.1 and manpage.txt using mlr from the $PATH -# copies the latter to docs6/docs +# copies the latter to docs6/src # docs6: # turn *.md.in into *.md (live code samples), using mlr from the $PATH # turn *.md into docs6/site HTML and CSS files @@ -15,6 +15,7 @@ set -euo pipefail cd go go fmt ./... +gofmt -s -w $(find . -name \*.go | grep -v src/parsing) ./build cd ../man6 diff --git a/docs6/README.md b/docs6/README.md index a4aa28a31..501db7063 100644 --- a/docs6/README.md +++ b/docs6/README.md @@ -14,17 +14,17 @@ * Clone https://github.com/johnkerl/miller and cd into `docs6/` within your clone. * Quick-editing loop: * In one terminal, cd to this directory and leave `mkdocs serve` running. - * In another terminal, cd to the `docs` subdirectory of `docs6` and edit `*.md.in`. + * In another terminal, cd to the `src` subdirectory of `docs6` and edit `*.md.in`. * Run `genmds` to re-create all the `*.md` files, or `genmds foo.md.in` to just re-create the `foo.md.in` file you just edited. * In your browser, visit http://127.0.0.1:8000 * Alternate editing loop: * Leave one terminal open as a place you will run `mkdocs build` - * In one terminal, cd to the `docs` subdirectory of `docs6` and edit `*.md.in`. - * Generate `docs/*.md` from `docs/*.md.in`, and then from that generate the `site/*/*.html`: + * In one terminal, cd to the `src` subdirectory of `docs6` and edit `*.md.in`. + * Generate `src/*.md` from `src/*.md.in`, and then from that generate the `site/*/*.html`: * Run `genmds` to re-create all the `*.md` files, or `genmds foo.md.in` to just re-create the `foo.md.in` file you just edited. * In the first terminal, run `mkdocs build` which will populate the `site` directory. * See also [./regen.sh](./regen.sh) which combines the `genmds` and `mkdocs build` steps. - * In your browser, visit `file:///your/path/to/miller/docs/site/index.html` + * In your browser, visit `file:///your/path/to/miller/docs6/site/index.html` * Link-checking: * `sudo pip3 install git+https://github.com/linkchecker/linkchecker.git` * `cd site` and `linkchecker .` @@ -34,7 +34,7 @@ ## Notes * CSS: - * I used the Mkdocs Readthedocs theme which I like a lot. I customized `docs6/docs/extra.css` for Miller coloring/branding. + * I used the Mkdocs Readthedocs theme which I like a lot. I customized `docs6/src/extra.css` for Miller coloring/branding. * Live code: * I didn't find a way to include non-Python live-code examples within Mkdocs so I adapted the pre-Mkdocs Miller-doc strategy which is to have a generator script read a template file (here, `foo.md.in`), run the marked lines, and generate the output file (`foo.md`). This is `genmds`. * Edit the `*.md.in` files, not `*.md` directly. diff --git a/docs6/docs/cpu.md b/docs6/docs/cpu.md deleted file mode 100644 index cec46180b..000000000 --- a/docs6/docs/cpu.md +++ /dev/null @@ -1,53 +0,0 @@ - -
- -Quick links: -  -Flag list -  -Verb list -  -Function list -  -Glossary -  -Repository ↗ - -
-# CPU/multicore usage - -Miller 6 is written in [Go](https://golang.org/) which supports multicore programming. - -Miller uses Go's _channel_ concept. The following are all separate goroutines: - -* One channel for input-record reader: parsing input file(s) to record objects -* One channel for each [verb](reference-verbs.md) in [then-chains](reference-main-then-chaining.md) -* One channel for output-record writer: formatting output records as text -* One controller channel which coordinates all these, without much work to do. - -For example, `mlr --csv cut -f somefield then sort -f otherfield then put '$z = -$x + $y' a.csv b.csv c.csv` will have 6 goroutines running: input-reader, -`cut`, `sort`, `put`, output-writer, controller. - -If all the verbs in the chain are [streaming](streaming-and-memory.md) -- -operating on each record as it arrives, then passing it on -- then all verbs in -the chain will be active at once. On the other hand, if there is a -non-streaming verb in the chain, which produces output only after receiving all -input -- for example, `sort` -- then we would expect verbs after that in the -chain to sit idle until the end of the input stream is reached, the `sort` does -its computation, then sends its output to downstream verbs. - -In practice, profiling has shown that the input-reader uses the most CPU of all -the above. This means CPUs running verbs may not be 100% utilized, since they -are likely to be spending some of their time waiting for input data. - -Running Miller on a machine with more CPUs than active channels (as listed -above) won't speed up a given invocation of Miller. However, of course, you'll -be able to run more invocations of Miller at the same time if you like. - -You can set the Go-standard environment variable `GOMAXPROCS` if you like. If -you don't, Miller will (as is standard for Go programs in Go 1.16 and above) up -to all available CPUs. - -If you set `$GOMAXPROCS=1` in the environment, that's fine -- the Go runtime -will multiplex different channel-handling goroutines onto the same CPU. diff --git a/docs6/docs/cpu.md.in b/docs6/docs/cpu.md.in deleted file mode 100644 index 6248d5ad3..000000000 --- a/docs6/docs/cpu.md.in +++ /dev/null @@ -1,37 +0,0 @@ -# CPU/multicore usage - -Miller 6 is written in [Go](https://golang.org/) which supports multicore programming. - -Miller uses Go's _channel_ concept. The following are all separate goroutines: - -* One channel for input-record reader: parsing input file(s) to record objects -* One channel for each [verb](reference-verbs.md) in [then-chains](reference-main-then-chaining.md) -* One channel for output-record writer: formatting output records as text -* One controller channel which coordinates all these, without much work to do. - -For example, `mlr --csv cut -f somefield then sort -f otherfield then put '$z = -$x + $y' a.csv b.csv c.csv` will have 6 goroutines running: input-reader, -`cut`, `sort`, `put`, output-writer, controller. - -If all the verbs in the chain are [streaming](streaming-and-memory.md) -- -operating on each record as it arrives, then passing it on -- then all verbs in -the chain will be active at once. On the other hand, if there is a -non-streaming verb in the chain, which produces output only after receiving all -input -- for example, `sort` -- then we would expect verbs after that in the -chain to sit idle until the end of the input stream is reached, the `sort` does -its computation, then sends its output to downstream verbs. - -In practice, profiling has shown that the input-reader uses the most CPU of all -the above. This means CPUs running verbs may not be 100% utilized, since they -are likely to be spending some of their time waiting for input data. - -Running Miller on a machine with more CPUs than active channels (as listed -above) won't speed up a given invocation of Miller. However, of course, you'll -be able to run more invocations of Miller at the same time if you like. - -You can set the Go-standard environment variable `GOMAXPROCS` if you like. If -you don't, Miller will (as is standard for Go programs in Go 1.16 and above) up -to all available CPUs. - -If you set `$GOMAXPROCS=1` in the environment, that's fine -- the Go runtime -will multiplex different channel-handling goroutines onto the same CPU. diff --git a/docs6/docs/flatten-unflatten.md b/docs6/docs/flatten-unflatten.md deleted file mode 100644 index 898480252..000000000 --- a/docs6/docs/flatten-unflatten.md +++ /dev/null @@ -1,77 +0,0 @@ - -
- -Quick links: -  -Flag list -  -Verb list -  -Function list -  -Glossary -  -Repository ↗ - -
-# Flatten/unflatten: JSON vs. tabular formats - -TODO - -* JSON-to-JSON -* JSON-to-not -* not-to-JSON -* not-to-not -* 'concatening keys' -* to-array heuristic -* no-flatten / no-unflatten options - -## TBF - -Suppose we have arrays like this in our input data: - -
-cat data/json-example-3.json
-
-
-{
-  "label": "orange",
-  "values": [12.2, 13.8, 17.2]
-}
-{
-  "label": "purple",
-  "values": [27.0, 32.4]
-}
-
- -comment: - -
-mlr --ijson --oxtab cat data/json-example-3.json
-
-
-label    orange
-values.1 12.2
-values.2 13.8
-values.3 17.2
-
-label    purple
-values.1 27.0
-values.2 32.4
-
- -comment: - -
-mlr --json --jvstack cat data/json-example-3.json
-
-
-{
-  "label": "orange",
-  "values": [12.2, 13.8, 17.2]
-}
-{
-  "label": "purple",
-  "values": [27.0, 32.4]
-}
-
diff --git a/docs6/docs/flatten-unflatten.md.in b/docs6/docs/flatten-unflatten.md.in deleted file mode 100644 index 93aa6aa44..000000000 --- a/docs6/docs/flatten-unflatten.md.in +++ /dev/null @@ -1,31 +0,0 @@ -# Flatten/unflatten: JSON vs. tabular formats - -TODO - -* JSON-to-JSON -* JSON-to-not -* not-to-JSON -* not-to-not -* 'concatening keys' -* to-array heuristic -* no-flatten / no-unflatten options - -## TBF - -Suppose we have arrays like this in our input data: - -GENMD_RUN_COMMAND -cat data/json-example-3.json -GENMD_EOF - -comment: - -GENMD_RUN_COMMAND -mlr --ijson --oxtab cat data/json-example-3.json -GENMD_EOF - -comment: - -GENMD_RUN_COMMAND -mlr --json --jvstack cat data/json-example-3.json -GENMD_EOF diff --git a/docs6/docs/performance.md b/docs6/docs/performance.md deleted file mode 100644 index 9501bad50..000000000 --- a/docs6/docs/performance.md +++ /dev/null @@ -1,33 +0,0 @@ - -
- -Quick links: -  -Flag list -  -Verb list -  -Function list -  -Glossary -  -Repository ↗ - -
-# Performance - -## Disclaimer - -In a previous version of this page, I compared Miller to some items in the Unix toolkit in terms of run time. But such comparisons are very much not apples-to-apples: - -* Miller's principal strength is that it handles **key-value data in various formats** while the system tools **do not**. So if you time `mlr sort` on a CSV file against system `sort`, it's not relevant to say which is faster by how many percent -- Miller will respect the header line, leaving it in place, while the system sort will move it, sorting it along with all the other header lines. This would be comparing the run times of two programs produce different outputs. Likewise, `awk` doesn't respect header lines, although you can code up some CSV-handling using `if (NR==1) { ... } else { ... }`. And that's just CSV: I don't know any simple way to get `sort`, `awk`, etc. to handle DKVP, JSON, etc. -- which is the main reason I wrote Miller. - -* **Implementations differ by platform**: one `awk` may be fundamentally faster than another, and `mawk` has a very efficient bytecode implementation -- which handles positionally indexed data far faster than Miller does. - -* The system `sort` command will, on some systems, handle too-large-for-RAM datasets by spilling to disk; Miller (as of version 5.2.0, mid-2017) does not. Miller sorts are always stable; GNU supports stable and unstable variants. - -* Etc. - -## Summary - -Miller can do many kinds of processing on key-value-pair data using elapsed time roughly of the same order of magnitude as items in the Unix toolkit can handle positionally indexed data. Specific results vary widely by platform, implementation details, and multi-core use (or not). Lastly, specific special-purpose non-record-aware processing will run far faster if implemented in `grep`, `sed`, etc. diff --git a/docs6/docs/performance.md.in b/docs6/docs/performance.md.in deleted file mode 100644 index 9eac506ee..000000000 --- a/docs6/docs/performance.md.in +++ /dev/null @@ -1,17 +0,0 @@ -# Performance - -## Disclaimer - -In a previous version of this page, I compared Miller to some items in the Unix toolkit in terms of run time. But such comparisons are very much not apples-to-apples: - -* Miller's principal strength is that it handles **key-value data in various formats** while the system tools **do not**. So if you time `mlr sort` on a CSV file against system `sort`, it's not relevant to say which is faster by how many percent -- Miller will respect the header line, leaving it in place, while the system sort will move it, sorting it along with all the other header lines. This would be comparing the run times of two programs produce different outputs. Likewise, `awk` doesn't respect header lines, although you can code up some CSV-handling using `if (NR==1) { ... } else { ... }`. And that's just CSV: I don't know any simple way to get `sort`, `awk`, etc. to handle DKVP, JSON, etc. -- which is the main reason I wrote Miller. - -* **Implementations differ by platform**: one `awk` may be fundamentally faster than another, and `mawk` has a very efficient bytecode implementation -- which handles positionally indexed data far faster than Miller does. - -* The system `sort` command will, on some systems, handle too-large-for-RAM datasets by spilling to disk; Miller (as of version 5.2.0, mid-2017) does not. Miller sorts are always stable; GNU supports stable and unstable variants. - -* Etc. - -## Summary - -Miller can do many kinds of processing on key-value-pair data using elapsed time roughly of the same order of magnitude as items in the Unix toolkit can handle positionally indexed data. Specific results vary widely by platform, implementation details, and multi-core use (or not). Lastly, specific special-purpose non-record-aware processing will run far faster if implemented in `grep`, `sed`, etc. diff --git a/docs6/docs/reference-main-flag-list.md.in b/docs6/docs/reference-main-flag-list.md.in deleted file mode 100644 index 8b52cbc91..000000000 --- a/docs6/docs/reference-main-flag-list.md.in +++ /dev/null @@ -1,3 +0,0 @@ -# List of command-line flags - -GENMD_RUN_CONTENT_GENERATOR(./mk-flag-info.rb) diff --git a/docs6/docs/reference-main-io-options.md b/docs6/docs/reference-main-io-options.md deleted file mode 100644 index 15184ad33..000000000 --- a/docs6/docs/reference-main-io-options.md +++ /dev/null @@ -1,121 +0,0 @@ - -
- -Quick links: -  -Flag list -  -Verb list -  -Function list -  -Glossary -  -Repository ↗ - -
-# I/O options - -## Formats - -Options: - -
---dkvp    --idkvp    --odkvp
---nidx    --inidx    --onidx
---csv     --icsv     --ocsv
---csvlite --icsvlite --ocsvlite
---pprint  --ipprint  --opprint  --right
---xtab    --ixtab    --oxtab
---json    --ijson    --ojson
-
- -These are as discussed in [File Formats](file-formats.md), with the exception of `--right` which makes pretty-printed output right-aligned: - -
-mlr --opprint cat data/small
-
-
-a   b   i x        y
-pan pan 1 0.346791 0.726802
-eks pan 2 0.758679 0.522151
-wye wye 3 0.204603 0.338318
-eks wye 4 0.381399 0.134188
-wye pan 5 0.573288 0.863624
-
- -
-mlr --opprint --right cat data/small
-
-
-  a   b i        x        y 
-pan pan 1 0.346791 0.726802 
-eks pan 2 0.758679 0.522151 
-wye wye 3 0.204603 0.338318 
-eks wye 4 0.381399 0.134188 
-wye pan 5 0.573288 0.863624 
-
- -Additional notes: - -* Use `--csv`, `--pprint`, etc. when the input and output formats are the same. - -* Use `--icsv --opprint`, etc. when you want format conversion as part of what Miller does to your data. - -* DKVP (key-value-pair) format is the default for input and output. So, `--oxtab` is the same as `--idkvp --oxtab`. - -**Pro-tip:** Please use either **--format1**, or **--iformat1 --oformat2**. If you use **--format1 --oformat2** then what happens is that flags are set up for input *and* output for format1, some of which are overwritten for output in format2. For technical reasons, having `--oformat2` clobber all the output-related effects of `--format1` also removes some flexibility from the command-line interface. See also Miller issues [180](https://github.com/johnkerl/miller/issues/180) and [199](https://github.com/johnkerl/miller/issues/199). - -## In-place mode - -Use the `mlr -I` flag to process files in-place. For example, `mlr -I --csv cut -x -f unwanted_column_name mydata/*.csv` will remove `unwanted_column_name` from all your `*.csv` files in your `mydata/` subdirectory. - -By default, Miller output goes to the screen (or you can redirect a file using `>` or to another process using `|`). With `-I`, for each file name on the command line, output is written to a temporary file in the same directory. Miller writes its output into that temp file, which is then renamed over the original. Then, processing continues on the next file. Each file is processed in isolation: if the output format is CSV, CSV headers will be present in each output file; statistics are only over each file's own records; and so on. - -Since this replaces your data with modified data, it's often a good idea to back up your original files somewhere -first, to protect against keystroking errors. - -Please see [Choices for printing to files](10min.md#choices-for-printing-to-files) for examples. - -## Compression - -See the separate page on [Compressed data](reference-main-compressed-data.md). - -## Record/field/pair separators - -See the separate page on [separators](reference-main-separators.md). - -## Number formatting - -The command-line option `--ofmt {format string}` is the global number format for commands which generate numeric output, e.g. `stats1`, `stats2`, `histogram`, and `step`, as well as `mlr put`. Examples: - -
---ofmt %.9e  --ofmt %.6f  --ofmt %.0f
-
- -These are just familiar `printf` formats. (TODO: write about type-checking once that's implemented.) Additionally, if you use leading width (e.g. `%18.12f`) then the output will contain embedded whitespace, which may not be what you want if you pipe the output to something else, particularly CSV. I use Miller's pretty-print format (`mlr --opprint`) to column-align numerical data. - -To apply formatting to a single field, overriding the global `ofmt`, use `fmtnum` function within `mlr put`. For example: - -
-echo 'x=3.1,y=4.3' | mlr put '$z=fmtnum($x*$y,"%08f")'
-
-
-x=3.1,y=4.3,z=13.330000
-
- -
-echo 'x=0xffff,y=0xff' | mlr put '$z=fmtnum(int($x*$y),"%08x")'
-
-
-x=0xffff,y=0xff,z=00feff01
-
- -Input conversion from hexadecimal is done automatically on fields handled by `mlr put` and `mlr filter` as long as the field value begins with "0x". To apply output conversion to hexadecimal on a single column, you may use `fmtnum`, or the keystroke-saving `hexfmt` function. Example: - -
-echo 'x=0xffff,y=0xff' | mlr put '$z=hexfmt($x*$y)'
-
-
-x=0xffff,y=0xff,z=0xfeff01
-
diff --git a/docs6/docs/reference-main-io-options.md.in b/docs6/docs/reference-main-io-options.md.in deleted file mode 100644 index 73507fb42..000000000 --- a/docs6/docs/reference-main-io-options.md.in +++ /dev/null @@ -1,80 +0,0 @@ -# I/O options - -## Formats - -Options: - -GENMD_CARDIFY ---dkvp --idkvp --odkvp ---nidx --inidx --onidx ---csv --icsv --ocsv ---csvlite --icsvlite --ocsvlite ---pprint --ipprint --opprint --right ---xtab --ixtab --oxtab ---json --ijson --ojson -GENMD_EOF - -These are as discussed in [File Formats](file-formats.md), with the exception of `--right` which makes pretty-printed output right-aligned: - -GENMD_RUN_COMMAND -mlr --opprint cat data/small -GENMD_EOF - -GENMD_RUN_COMMAND -mlr --opprint --right cat data/small -GENMD_EOF - -Additional notes: - -* Use `--csv`, `--pprint`, etc. when the input and output formats are the same. - -* Use `--icsv --opprint`, etc. when you want format conversion as part of what Miller does to your data. - -* DKVP (key-value-pair) format is the default for input and output. So, `--oxtab` is the same as `--idkvp --oxtab`. - -**Pro-tip:** Please use either **--format1**, or **--iformat1 --oformat2**. If you use **--format1 --oformat2** then what happens is that flags are set up for input *and* output for format1, some of which are overwritten for output in format2. For technical reasons, having `--oformat2` clobber all the output-related effects of `--format1` also removes some flexibility from the command-line interface. See also Miller issues [180](https://github.com/johnkerl/miller/issues/180) and [199](https://github.com/johnkerl/miller/issues/199). - -## In-place mode - -Use the `mlr -I` flag to process files in-place. For example, `mlr -I --csv cut -x -f unwanted_column_name mydata/*.csv` will remove `unwanted_column_name` from all your `*.csv` files in your `mydata/` subdirectory. - -By default, Miller output goes to the screen (or you can redirect a file using `>` or to another process using `|`). With `-I`, for each file name on the command line, output is written to a temporary file in the same directory. Miller writes its output into that temp file, which is then renamed over the original. Then, processing continues on the next file. Each file is processed in isolation: if the output format is CSV, CSV headers will be present in each output file; statistics are only over each file's own records; and so on. - -Since this replaces your data with modified data, it's often a good idea to back up your original files somewhere -first, to protect against keystroking errors. - -Please see [Choices for printing to files](10min.md#choices-for-printing-to-files) for examples. - -## Compression - -See the separate page on [Compressed data](reference-main-compressed-data.md). - -## Record/field/pair separators - -See the separate page on [separators](reference-main-separators.md). - -## Number formatting - -The command-line option `--ofmt {format string}` is the global number format for commands which generate numeric output, e.g. `stats1`, `stats2`, `histogram`, and `step`, as well as `mlr put`. Examples: - -GENMD_CARDIFY ---ofmt %.9e --ofmt %.6f --ofmt %.0f -GENMD_EOF - -These are just familiar `printf` formats. (TODO: write about type-checking once that's implemented.) Additionally, if you use leading width (e.g. `%18.12f`) then the output will contain embedded whitespace, which may not be what you want if you pipe the output to something else, particularly CSV. I use Miller's pretty-print format (`mlr --opprint`) to column-align numerical data. - -To apply formatting to a single field, overriding the global `ofmt`, use `fmtnum` function within `mlr put`. For example: - -GENMD_RUN_COMMAND -echo 'x=3.1,y=4.3' | mlr put '$z=fmtnum($x*$y,"%08f")' -GENMD_EOF - -GENMD_RUN_COMMAND -echo 'x=0xffff,y=0xff' | mlr put '$z=fmtnum(int($x*$y),"%08x")' -GENMD_EOF - -Input conversion from hexadecimal is done automatically on fields handled by `mlr put` and `mlr filter` as long as the field value begins with "0x". To apply output conversion to hexadecimal on a single column, you may use `fmtnum`, or the keystroke-saving `hexfmt` function. Example: - -GENMD_RUN_COMMAND -echo 'x=0xffff,y=0xff' | mlr put '$z=hexfmt($x*$y)' -GENMD_EOF diff --git a/docs6/docs/reference-main-strings.md b/docs6/docs/reference-main-strings.md deleted file mode 100644 index 146093be9..000000000 --- a/docs6/docs/reference-main-strings.md +++ /dev/null @@ -1,48 +0,0 @@ - -
- -Quick links: -  -Flag list -  -Verb list -  -Function list -  -Glossary -  -Repository ↗ - -
-# Strings - -TODO - -* concat w/ dot operator -* index and slice 1-up -* lib functions -* always double-quote -* single-quote for shell; see windows page - -## Escape sequences for string literals - -You can use the following backslash escapes for strings such as between the double quotes in contexts such as `mlr filter '$name =~ "..."'`, `mlr put '$name = $othername . "..."'`, `mlr put '$name = sub($name, "...", "...")`, etc.: - -* `\a`: ASCII code 0x07 (alarm/bell) -* `\b`: ASCII code 0x08 (backspace) -* `\f`: ASCII code 0x0c (formfeed) -* `\n`: ASCII code 0x0a (LF/linefeed/newline) -* `\r`: ASCII code 0x0d (CR/carriage return) -* `\t`: ASCII code 0x09 (tab) -* `\v`: ASCII code 0x0b (vertical tab) -* `\\`: backslash -* `\"`: double quote -* `\123`: Octal 123, etc. for `\000` up to `\377` -* `\x7f`: Hexadecimal 7f, etc. for `\x00` up to `\xff` - -See also [https://en.wikipedia.org/wiki/Escape_sequences_in_C](https://en.wikipedia.org/wiki/Escape_sequences_in_C). - -These replacements apply only to strings you key in for the DSL expressions for `filter` and `put`: that is, if you type `\t` in a string literal for a `filter`/`put` expression, it will be turned into a tab character. If you want a backslash followed by a `t`, then please type `\\t`. - -However, these replacements are done automatically only for string literals within DSL expressions -- they are not done automatically to fields within your data stream. If you wish to make these replacements, you can do (for example) `mlr put '$field = gsub($field, "\\t", "\t")'`. If you need to make such a replacement for all fields in your data, you should probably use the system `sed` command instead. - diff --git a/docs6/docs/reference-main-strings.md.in b/docs6/docs/reference-main-strings.md.in deleted file mode 100644 index b111eab67..000000000 --- a/docs6/docs/reference-main-strings.md.in +++ /dev/null @@ -1,32 +0,0 @@ -# Strings - -TODO - -* concat w/ dot operator -* index and slice 1-up -* lib functions -* always double-quote -* single-quote for shell; see windows page - -## Escape sequences for string literals - -You can use the following backslash escapes for strings such as between the double quotes in contexts such as `mlr filter '$name =~ "..."'`, `mlr put '$name = $othername . "..."'`, `mlr put '$name = sub($name, "...", "...")`, etc.: - -* `\a`: ASCII code 0x07 (alarm/bell) -* `\b`: ASCII code 0x08 (backspace) -* `\f`: ASCII code 0x0c (formfeed) -* `\n`: ASCII code 0x0a (LF/linefeed/newline) -* `\r`: ASCII code 0x0d (CR/carriage return) -* `\t`: ASCII code 0x09 (tab) -* `\v`: ASCII code 0x0b (vertical tab) -* `\\`: backslash -* `\"`: double quote -* `\123`: Octal 123, etc. for `\000` up to `\377` -* `\x7f`: Hexadecimal 7f, etc. for `\x00` up to `\xff` - -See also [https://en.wikipedia.org/wiki/Escape_sequences_in_C](https://en.wikipedia.org/wiki/Escape_sequences_in_C). - -These replacements apply only to strings you key in for the DSL expressions for `filter` and `put`: that is, if you type `\t` in a string literal for a `filter`/`put` expression, it will be turned into a tab character. If you want a backslash followed by a `t`, then please type `\\t`. - -However, these replacements are done automatically only for string literals within DSL expressions -- they are not done automatically to fields within your data stream. If you wish to make these replacements, you can do (for example) `mlr put '$field = gsub($field, "\\t", "\t")'`. If you need to make such a replacement for all fields in your data, you should probably use the system `sed` command instead. - diff --git a/docs6/mkdocs.yml b/docs6/mkdocs.yml index 0f133ec43..e3a444385 100644 --- a/docs6/mkdocs.yml +++ b/docs6/mkdocs.yml @@ -6,6 +6,7 @@ theme: font: text: Lato code: Lato Mono +docs_dir: './src' extra_css: ['extra.css'] extra_javascript: - https://cdnjs.cloudflare.com/ajax/libs/tablesort/5.2.1/tablesort.min.js @@ -16,7 +17,7 @@ nav: - "Installing Miller": "installing-miller.md" - "Miller in 10 minutes": "10min.md" - "Keystroke-savers": "keystroke-savers.md" - - "Intro to Miller's programming language": "programming-language.md" + - "Intro to Miller's programming language": "miller-programming-language.md" - "Miller on Windows": "miller-on-windows.md" - "Community": "community.md" - "Structure of these documents": "structure-of-these-documents.md" @@ -58,46 +59,45 @@ nav: - "Why call it Miller?": "etymology.md" - "How original is Miller?": "originality.md" - "Performance": "performance.md" - - 'Reference': - - 'Main reference': - - "Miller command structure": "reference-main-overview.md" - - "Then-chaining": "reference-main-then-chaining.md" - - "List of command-line flags": "reference-main-flag-list.md" - - "I/O options": "reference-main-io-options.md" - - "Separators": "reference-main-separators.md" - - "List of verbs": "reference-verbs.md" - - "Streaming processing, and memory usage": "streaming-and-memory.md" - - "CPU/multicore usage": "cpu.md" - - "Data types": "reference-main-data-types.md" - - "Strings": "reference-main-strings.md" - - "Regular expressions": "reference-main-regular-expressions.md" - - "Arithmetic": "reference-main-arithmetic.md" - - "Maps": "reference-main-maps.md" - - "Arrays": "reference-main-arrays.md" - - "Null/empty/absent data": "reference-main-null-data.md" - - "Flatten/unflatten: JSON vs. tabular formats": "flatten-unflatten.md" - - "Compressed data": "reference-main-compressed-data.md" - - "Streaming processing, and memory usage": "streaming-and-memory.md" - - "CPU/multicore usage": "cpu.md" - - "Miller environment variables": "reference-main-env-vars.md" - - 'DSL reference': - - "DSL overview": "reference-dsl.md" - - "DSL syntax": "reference-dsl-syntax.md" - - "DSL variables": "reference-dsl-variables.md" - - "DSL operators": "reference-dsl-operators.md" - - "DSL control structures": "reference-dsl-control-structures.md" - - "DSL built-in functions": "reference-dsl-builtin-functions.md" - - "DSL user-defined functions": "reference-dsl-user-defined-functions.md" - - "DSL filter statements": "reference-dsl-filter-statements.md" - - "DSL output statements": "reference-dsl-output-statements.md" - - "DSL unset statements": "reference-dsl-unset-statements.md" - - "DSL errors and transparency": "reference-dsl-errors.md" - - "Differences from other programming languages": "reference-dsl-differences.md" - - "A note on the complexity of Miller's expression language": "reference-dsl-complexity.md" - - 'Misc. reference': - - "Auxiliary commands": "reference-main-auxiliary-commands.md" - - "Manual page": "manpage.md" - - "Installation": "installation.md" - - "Building from source": "build.md" - - "Documents for previous releases": "release-docs.md" - - "Glossary": "glossary.md" + - 'Main reference': + - "Miller command structure": "reference-main-overview.md" + - "Then-chaining": "reference-main-then-chaining.md" + - "List of command-line flags": "reference-main-flag-list.md" + - "List of verbs": "reference-verbs.md" + - "In-place processing": "reference-main-in-place-processing.md" + - "Number formatting": "reference-main-number-formatting.md" + - "Separators": "reference-main-separators.md" + - "Flatten/unflatten: JSON vs. tabular formats": "flatten-unflatten.md" + - "Streaming processing, and memory usage": "streaming-and-memory.md" + - "Compressed data": "reference-main-compressed-data.md" + - "CPU/multicore usage": "cpu.md" + - "Miller environment variables": "reference-main-env-vars.md" + - 'Types reference': + - "Data types": "reference-main-data-types.md" + - "Strings": "reference-main-strings.md" + - "Regular expressions": "reference-main-regular-expressions.md" + - "Arithmetic": "reference-main-arithmetic.md" + - "Maps": "reference-main-maps.md" + - "Arrays": "reference-main-arrays.md" + - "Null/empty/absent data": "reference-main-null-data.md" + - 'DSL reference': + - "DSL overview": "reference-dsl.md" + - "DSL syntax": "reference-dsl-syntax.md" + - "DSL variables": "reference-dsl-variables.md" + - "DSL operators": "reference-dsl-operators.md" + - "DSL control structures": "reference-dsl-control-structures.md" + - "DSL built-in functions": "reference-dsl-builtin-functions.md" + - "DSL user-defined functions": "reference-dsl-user-defined-functions.md" + - "DSL filter statements": "reference-dsl-filter-statements.md" + - "DSL output statements": "reference-dsl-output-statements.md" + - "DSL unset statements": "reference-dsl-unset-statements.md" + - "DSL errors and transparency": "reference-dsl-errors.md" + - "Differences from other programming languages": "reference-dsl-differences.md" + - "A note on the complexity of Miller's expression language": "reference-dsl-complexity.md" + - 'Misc. reference': + - "Auxiliary commands": "reference-main-auxiliary-commands.md" + - "Manual page": "manpage.md" + - "Installation": "installation.md" + - "Building from source": "build.md" + - "Documents for previous releases": "release-docs.md" + - "Glossary": "glossary.md" diff --git a/docs6/regen.sh b/docs6/regen.sh index 47f0f7ff0..12e6fa48f 100755 --- a/docs6/regen.sh +++ b/docs6/regen.sh @@ -2,7 +2,7 @@ set -euo pipefail -pushd docs +pushd src ./genmds popd mkdocs build diff --git a/docs6/docs/.vimrc b/docs6/src/.vimrc similarity index 100% rename from docs6/docs/.vimrc rename to docs6/src/.vimrc diff --git a/docs6/docs/10min.md b/docs6/src/10min.md similarity index 87% rename from docs6/docs/10min.md rename to docs6/src/10min.md index 909805fad..6ecefa827 100644 --- a/docs6/docs/10min.md +++ b/docs6/src/10min.md @@ -305,7 +305,7 @@ purple square false 10 91 72.3735 8.2430 92 8474 For `put` and `filter` we were able to type out expressions using a programming-language syntax. -See the [Miller programming language page](programming-language.md) for more information. +See the [Miller programming language page](miller-programming-language.md) for more information. ## Multiple input files @@ -538,6 +538,130 @@ How to specify these to Miller: You can read more about this at the [File Formats](file-formats.md) page. +If all record values are numbers, strings, etc., then converting back and forth between CSV and JSON is +a matter of specifying input-format and output-format flags: + +
+mlr --json cat example.json
+
+
+{
+  "color": "yellow",
+  "shape": "triangle",
+  "flag": "true",
+  "k": 1,
+  "index": 11,
+  "quantity": 43.6498,
+  "rate": 9.8870
+}
+{
+  "color": "red",
+  "shape": "square",
+  "flag": "true",
+  "k": 2,
+  "index": 15,
+  "quantity": 79.2778,
+  "rate": 0.0130
+}
+
+ +
+mlr --ijson --ocsv cat example.json
+
+
+color,shape,flag,k,index,quantity,rate
+yellow,triangle,true,1,11,43.6498,9.8870
+red,square,true,2,15,79.2778,0.0130
+
+ +However, if JSON data has map-valued or array-valued fields, Miller gives you choices on how to +convert these to CSV columns. For example, here's some JSON data with map-valued fields: + +
+cat data/server-log.json
+
+
+{
+  "hostname": "localhost",
+  "pid": 12345,
+  "req": {
+    "id": 6789,
+    "method": "GET",
+    "path": "api/check",
+    "host": "foo.bar",
+    "headers": {
+      "host": "bar.baz",
+      "user-agent": "browser"
+    }
+  },
+  "res": {
+    "status_code": 200,
+    "header": {
+      "content-type": "text",
+      "content-encoding": "plain"
+    }
+  }
+}
+
+ +We can convert this to CSV, or other tabular formats: + +
+mlr --ijson --ocsv cat data/server-log.json
+
+
+hostname,pid,req.id,req.method,req.path,req.host,req.headers.host,req.headers.user-agent,res.status_code,res.header.content-type,res.header.content-encoding
+localhost,12345,6789,GET,api/check,foo.bar,bar.baz,browser,200,text,plain
+
+ +
+mlr --ijson --oxtab cat data/server-log.json
+
+
+hostname                    localhost
+pid                         12345
+req.id                      6789
+req.method                  GET
+req.path                    api/check
+req.host                    foo.bar
+req.headers.host            bar.baz
+req.headers.user-agent      browser
+res.status_code             200
+res.header.content-type     text
+res.header.content-encoding plain
+
+ +These transformations are reversible: + +
+mlr --ijson --oxtab cat data/server-log.json | mlr --ixtab --ojson cat
+
+
+{
+  "hostname": "localhost",
+  "pid": 12345,
+  "req": {
+    "id": 6789,
+    "method": "GET",
+    "path": "api/check",
+    "host": "foo.bar",
+    "headers": {
+      "host": "bar.baz",
+      "user-agent": "browser"
+    }
+  },
+  "res": {
+    "status_code": 200,
+    "header": {
+      "content-type": "text",
+      "content-encoding": "plain"
+    }
+  }
+}
+
+ +See the [flatten/unflatten page](flatten-unflatten.md) for more information. + ## Choices for printing to files Often we want to print output to the screen. Miller does this by default, as we've seen in the previous examples. diff --git a/docs6/docs/10min.md.in b/docs6/src/10min.md.in similarity index 90% rename from docs6/docs/10min.md.in rename to docs6/src/10min.md.in index b29f278dc..b86a40527 100644 --- a/docs6/docs/10min.md.in +++ b/docs6/src/10min.md.in @@ -105,7 +105,7 @@ mlr --icsv --opprint --from example.csv put ' GENMD_EOF For `put` and `filter` we were able to type out expressions using a programming-language syntax. -See the [Miller programming language page](programming-language.md) for more information. +See the [Miller programming language page](miller-programming-language.md) for more information. ## Multiple input files @@ -255,6 +255,42 @@ How to specify these to Miller: You can read more about this at the [File Formats](file-formats.md) page. +If all record values are numbers, strings, etc., then converting back and forth between CSV and JSON is +a matter of specifying input-format and output-format flags: + +GENMD_RUN_COMMAND +mlr --json cat example.json +GENMD_EOF + +GENMD_RUN_COMMAND +mlr --ijson --ocsv cat example.json +GENMD_EOF + +However, if JSON data has map-valued or array-valued fields, Miller gives you choices on how to +convert these to CSV columns. For example, here's some JSON data with map-valued fields: + +GENMD_RUN_COMMAND +cat data/server-log.json +GENMD_EOF + +We can convert this to CSV, or other tabular formats: + +GENMD_RUN_COMMAND +mlr --ijson --ocsv cat data/server-log.json +GENMD_EOF + +GENMD_RUN_COMMAND +mlr --ijson --oxtab cat data/server-log.json +GENMD_EOF + +These transformations are reversible: + +GENMD_RUN_COMMAND +mlr --ijson --oxtab cat data/server-log.json | mlr --ixtab --ojson cat +GENMD_EOF + +See the [flatten/unflatten page](flatten-unflatten.md) for more information. + ## Choices for printing to files Often we want to print output to the screen. Miller does this by default, as we've seen in the previous examples. diff --git a/docs6/docs/Makefile b/docs6/src/Makefile similarity index 100% rename from docs6/docs/Makefile rename to docs6/src/Makefile diff --git a/docs6/docs/_static/basic.css b/docs6/src/_static/basic.css similarity index 100% rename from docs6/docs/_static/basic.css rename to docs6/src/_static/basic.css diff --git a/docs6/docs/_static/classic.css b/docs6/src/_static/classic.css similarity index 100% rename from docs6/docs/_static/classic.css rename to docs6/src/_static/classic.css diff --git a/docs6/docs/_static/mlr.jpg b/docs6/src/_static/mlr.jpg similarity index 100% rename from docs6/docs/_static/mlr.jpg rename to docs6/src/_static/mlr.jpg diff --git a/docs6/docs/_static/pygments.css b/docs6/src/_static/pygments.css similarity index 100% rename from docs6/docs/_static/pygments.css rename to docs6/src/_static/pygments.css diff --git a/docs6/docs/_static/scrolls.css b/docs6/src/_static/scrolls.css similarity index 100% rename from docs6/docs/_static/scrolls.css rename to docs6/src/_static/scrolls.css diff --git a/docs6/docs/_static/top.png b/docs6/src/_static/top.png similarity index 100% rename from docs6/docs/_static/top.png rename to docs6/src/_static/top.png diff --git a/docs6/docs/absent-example.json b/docs6/src/absent-example.json similarity index 100% rename from docs6/docs/absent-example.json rename to docs6/src/absent-example.json diff --git a/docs6/docs/build-one b/docs6/src/build-one similarity index 100% rename from docs6/docs/build-one rename to docs6/src/build-one diff --git a/docs6/docs/build.md b/docs6/src/build.md similarity index 100% rename from docs6/docs/build.md rename to docs6/src/build.md diff --git a/docs6/docs/build.md.in b/docs6/src/build.md.in similarity index 100% rename from docs6/docs/build.md.in rename to docs6/src/build.md.in diff --git a/docs6/docs/circle.csv b/docs6/src/circle.csv similarity index 100% rename from docs6/docs/circle.csv rename to docs6/src/circle.csv diff --git a/docs6/docs/commas.csv b/docs6/src/commas.csv similarity index 100% rename from docs6/docs/commas.csv rename to docs6/src/commas.csv diff --git a/docs6/docs/community.md b/docs6/src/community.md similarity index 100% rename from docs6/docs/community.md rename to docs6/src/community.md diff --git a/docs6/docs/community.md.in b/docs6/src/community.md.in similarity index 100% rename from docs6/docs/community.md.in rename to docs6/src/community.md.in diff --git a/docs6/docs/context-example.mlr b/docs6/src/context-example.mlr similarity index 100% rename from docs6/docs/context-example.mlr rename to docs6/src/context-example.mlr diff --git a/docs6/docs/contributing.md b/docs6/src/contributing.md similarity index 100% rename from docs6/docs/contributing.md rename to docs6/src/contributing.md diff --git a/docs6/docs/contributing.md.in b/docs6/src/contributing.md.in similarity index 100% rename from docs6/docs/contributing.md.in rename to docs6/src/contributing.md.in diff --git a/docs6/docs/coverart/cover-cat.png b/docs6/src/coverart/cover-cat.png similarity index 100% rename from docs6/docs/coverart/cover-cat.png rename to docs6/src/coverart/cover-cat.png diff --git a/docs6/docs/coverart/cover-combined.png b/docs6/src/coverart/cover-combined.png similarity index 100% rename from docs6/docs/coverart/cover-combined.png rename to docs6/src/coverart/cover-combined.png diff --git a/docs6/docs/coverart/cover-filter-to-json.png b/docs6/src/coverart/cover-filter-to-json.png similarity index 100% rename from docs6/docs/coverart/cover-filter-to-json.png rename to docs6/src/coverart/cover-filter-to-json.png diff --git a/docs6/docs/coverart/cover-sort-to-pprint.png b/docs6/src/coverart/cover-sort-to-pprint.png similarity index 100% rename from docs6/docs/coverart/cover-sort-to-pprint.png rename to docs6/src/coverart/cover-sort-to-pprint.png diff --git a/docs6/docs/coverart/coverart.key b/docs6/src/coverart/coverart.key similarity index 100% rename from docs6/docs/coverart/coverart.key rename to docs6/src/coverart/coverart.key diff --git a/docs6/src/cpu.md b/docs6/src/cpu.md new file mode 100644 index 000000000..f2585c6b3 --- /dev/null +++ b/docs6/src/cpu.md @@ -0,0 +1,65 @@ + +
+ +Quick links: +  +Flag list +  +Verb list +  +Function list +  +Glossary +  +Repository ↗ + +
+# CPU/multicore usage + +Miller 6 is written in [Go](https://golang.org/) which supports multicore programming. + +Miller uses Go's _channel_ concept. The following are all separate "goroutines", +which loosely speaking you can think of as "threads": + +* One for input-record reader: parsing input file(s) to record objects +* One for each [verb](reference-verbs.md) in [then-chains](reference-main-then-chaining.md) +* One for output-record writer: formatting output records as text +* One controller which coordinates all these, without much work to do. + +For example, `mlr --csv cut -f somefield then sort -f otherfield then put '$z = +$x + $y' a.csv b.csv c.csv` will have 6 goroutines running: + +* record-reader, +* `cut`, +* `sort`, +* `rename`, +* record-writer, +* controller. + +Ideally all six of those would utilize 100% of a CPU, for six total. +This is not the general case, for a few reasons: + +* If all the verbs in the chain are [streaming](streaming-and-memory.md) -- +operating on each record as it arrives, then passing it on -- then all verbs +in the chain will be active at once. +* On the other hand, if there is a [non-streaming verb](streaming-and-memory.md) +in the chain, which produces output only after receiving all input -- for +example, `sort` -- then we would expect verbs after that in the chain to sit +idle until the end of the input stream is reached, the `sort` does its +computation, then sends its output to downstream verbs. +* In practice, profiling has shown that the input-reader uses the most CPU of all +the above. This means CPUs running verbs may not be 100% utilized, since they +are likely to be spending some of their time waiting for input data. + +Since the number of CPUs Miller will use is tied to the number of verbs in the +processing chain, running Miller on a machine with more CPUs than active +routines (as listed above) won't speed up a given invocation of Miller. +However, of course, you'll be able to run more invocations of Miller at the +same time if you like. + +You can set the Go-standard environment variable `GOMAXPROCS` if you like. If +you don't, Miller will (as is standard for Go programs in Go 1.16 and above) up +to all available CPUs. + +If you set `$GOMAXPROCS=1` in the environment, that's fine -- the Go runtime +will multiplex different channel-handling goroutines onto the same CPU. diff --git a/docs6/src/cpu.md.in b/docs6/src/cpu.md.in new file mode 100644 index 000000000..88a9d1233 --- /dev/null +++ b/docs6/src/cpu.md.in @@ -0,0 +1,49 @@ +# CPU/multicore usage + +Miller 6 is written in [Go](https://golang.org/) which supports multicore programming. + +Miller uses Go's _channel_ concept. The following are all separate "goroutines", +which loosely speaking you can think of as "threads": + +* One for input-record reader: parsing input file(s) to record objects +* One for each [verb](reference-verbs.md) in [then-chains](reference-main-then-chaining.md) +* One for output-record writer: formatting output records as text +* One controller which coordinates all these, without much work to do. + +For example, `mlr --csv cut -f somefield then sort -f otherfield then put '$z = +$x + $y' a.csv b.csv c.csv` will have 6 goroutines running: + +* record-reader, +* `cut`, +* `sort`, +* `rename`, +* record-writer, +* controller. + +Ideally all six of those would utilize 100% of a CPU, for six total. +This is not the general case, for a few reasons: + +* If all the verbs in the chain are [streaming](streaming-and-memory.md) -- +operating on each record as it arrives, then passing it on -- then all verbs +in the chain will be active at once. +* On the other hand, if there is a [non-streaming verb](streaming-and-memory.md) +in the chain, which produces output only after receiving all input -- for +example, `sort` -- then we would expect verbs after that in the chain to sit +idle until the end of the input stream is reached, the `sort` does its +computation, then sends its output to downstream verbs. +* In practice, profiling has shown that the input-reader uses the most CPU of all +the above. This means CPUs running verbs may not be 100% utilized, since they +are likely to be spending some of their time waiting for input data. + +Since the number of CPUs Miller will use is tied to the number of verbs in the +processing chain, running Miller on a machine with more CPUs than active +routines (as listed above) won't speed up a given invocation of Miller. +However, of course, you'll be able to run more invocations of Miller at the +same time if you like. + +You can set the Go-standard environment variable `GOMAXPROCS` if you like. If +you don't, Miller will (as is standard for Go programs in Go 1.16 and above) up +to all available CPUs. + +If you set `$GOMAXPROCS=1` in the environment, that's fine -- the Go runtime +will multiplex different channel-handling goroutines onto the same CPU. diff --git a/docs6/docs/csv-with-and-without-headers.md b/docs6/src/csv-with-and-without-headers.md similarity index 100% rename from docs6/docs/csv-with-and-without-headers.md rename to docs6/src/csv-with-and-without-headers.md diff --git a/docs6/docs/csv-with-and-without-headers.md.in b/docs6/src/csv-with-and-without-headers.md.in similarity index 100% rename from docs6/docs/csv-with-and-without-headers.md.in rename to docs6/src/csv-with-and-without-headers.md.in diff --git a/docs6/docs/customization.md b/docs6/src/customization.md similarity index 100% rename from docs6/docs/customization.md rename to docs6/src/customization.md diff --git a/docs6/docs/customization.md.in b/docs6/src/customization.md.in similarity index 100% rename from docs6/docs/customization.md.in rename to docs6/src/customization.md.in diff --git a/docs6/docs/data-cleaning-examples.md b/docs6/src/data-cleaning-examples.md similarity index 100% rename from docs6/docs/data-cleaning-examples.md rename to docs6/src/data-cleaning-examples.md diff --git a/docs6/docs/data-cleaning-examples.md.in b/docs6/src/data-cleaning-examples.md.in similarity index 100% rename from docs6/docs/data-cleaning-examples.md.in rename to docs6/src/data-cleaning-examples.md.in diff --git a/docs6/docs/data-diving-examples.md b/docs6/src/data-diving-examples.md similarity index 99% rename from docs6/docs/data-diving-examples.md rename to docs6/src/data-diving-examples.md index 79fa2c21c..7eccc0278 100644 --- a/docs6/docs/data-diving-examples.md +++ b/docs6/src/data-diving-examples.md @@ -262,6 +262,7 @@ u_max 0.999969 v_min -0.092709 v_mean 0.49778696586624427 v_max 1.0725 + The histogram shows the different distribution of 0/1 flags: diff --git a/docs6/docs/data-diving-examples.md.in b/docs6/src/data-diving-examples.md.in similarity index 100% rename from docs6/docs/data-diving-examples.md.in rename to docs6/src/data-diving-examples.md.in diff --git a/docs6/docs/data/a.csv b/docs6/src/data/a.csv similarity index 100% rename from docs6/docs/data/a.csv rename to docs6/src/data/a.csv diff --git a/docs6/docs/data/a.dkvp b/docs6/src/data/a.dkvp similarity index 100% rename from docs6/docs/data/a.dkvp rename to docs6/src/data/a.dkvp diff --git a/docs6/docs/data/array-example.json b/docs6/src/data/array-example.json similarity index 100% rename from docs6/docs/data/array-example.json rename to docs6/src/data/array-example.json diff --git a/docs6/src/data/array-values-nested.json b/docs6/src/data/array-values-nested.json new file mode 100644 index 000000000..e1bfd5c6a --- /dev/null +++ b/docs6/src/data/array-values-nested.json @@ -0,0 +1,8 @@ +{ + "a": 1, + "b": [[2, 3], [4, 5]] +} +{ + "a": 6, + "b": [[7, 8], [9, 10]] +} diff --git a/docs6/src/data/array-values-spread-colon.csv b/docs6/src/data/array-values-spread-colon.csv new file mode 100644 index 000000000..493bc24a5 --- /dev/null +++ b/docs6/src/data/array-values-spread-colon.csv @@ -0,0 +1,3 @@ +a,b:1,b:2 +1,2,3 +4,5,6 diff --git a/docs6/src/data/array-values-spread.csv b/docs6/src/data/array-values-spread.csv new file mode 100644 index 000000000..d17ad1a6d --- /dev/null +++ b/docs6/src/data/array-values-spread.csv @@ -0,0 +1,3 @@ +a,b.1,b.2 +1,2,3 +4,5,6 diff --git a/docs6/src/data/array-values.json b/docs6/src/data/array-values.json new file mode 100644 index 000000000..1d12efa93 --- /dev/null +++ b/docs6/src/data/array-values.json @@ -0,0 +1,8 @@ +{ + "a": 1, + "b": [2, 3] +} +{ + "a": 4, + "b": [5, 6] +} diff --git a/docs6/docs/data/b.csv b/docs6/src/data/b.csv similarity index 100% rename from docs6/docs/data/b.csv rename to docs6/src/data/b.csv diff --git a/docs6/docs/data/b.dkvp b/docs6/src/data/b.dkvp similarity index 100% rename from docs6/docs/data/b.dkvp rename to docs6/src/data/b.dkvp diff --git a/docs6/docs/data/begin-end-example-7b.sh b/docs6/src/data/begin-end-example-7b.sh similarity index 100% rename from docs6/docs/data/begin-end-example-7b.sh rename to docs6/src/data/begin-end-example-7b.sh diff --git a/docs6/docs/data/budget.csv b/docs6/src/data/budget.csv similarity index 100% rename from docs6/docs/data/budget.csv rename to docs6/src/data/budget.csv diff --git a/docs6/docs/data/bulk-rename-for-loop.mlr b/docs6/src/data/bulk-rename-for-loop.mlr similarity index 100% rename from docs6/docs/data/bulk-rename-for-loop.mlr rename to docs6/src/data/bulk-rename-for-loop.mlr diff --git a/docs6/docs/data/clean-whitespace.csv b/docs6/src/data/clean-whitespace.csv similarity index 100% rename from docs6/docs/data/clean-whitespace.csv rename to docs6/src/data/clean-whitespace.csv diff --git a/docs6/docs/data/color-codes.csv b/docs6/src/data/color-codes.csv similarity index 100% rename from docs6/docs/data/color-codes.csv rename to docs6/src/data/color-codes.csv diff --git a/docs6/docs/data/color-names.csv b/docs6/src/data/color-names.csv similarity index 100% rename from docs6/docs/data/color-names.csv rename to docs6/src/data/color-names.csv diff --git a/docs6/docs/data/colored-shapes.csv b/docs6/src/data/colored-shapes.csv similarity index 100% rename from docs6/docs/data/colored-shapes.csv rename to docs6/src/data/colored-shapes.csv diff --git a/docs6/docs/data/colored-shapes.dkvp b/docs6/src/data/colored-shapes.dkvp similarity index 100% rename from docs6/docs/data/colored-shapes.dkvp rename to docs6/src/data/colored-shapes.dkvp diff --git a/docs6/docs/data/colours.csv b/docs6/src/data/colours.csv similarity index 100% rename from docs6/docs/data/colours.csv rename to docs6/src/data/colours.csv diff --git a/docs6/src/data/consecutive.csv b/docs6/src/data/consecutive.csv new file mode 100644 index 000000000..23bc505d0 --- /dev/null +++ b/docs6/src/data/consecutive.csv @@ -0,0 +1,2 @@ +a.1,a.2,a.3 +4,5,6 diff --git a/docs6/docs/data/current_counters.csv b/docs6/src/data/current_counters.csv similarity index 100% rename from docs6/docs/data/current_counters.csv rename to docs6/src/data/current_counters.csv diff --git a/docs6/docs/data/currtemp.csv b/docs6/src/data/currtemp.csv similarity index 100% rename from docs6/docs/data/currtemp.csv rename to docs6/src/data/currtemp.csv diff --git a/docs6/docs/data/dollar-sign.txt b/docs6/src/data/dollar-sign.txt similarity index 100% rename from docs6/docs/data/dollar-sign.txt rename to docs6/src/data/dollar-sign.txt diff --git a/docs6/docs/data/english-words.txt b/docs6/src/data/english-words.txt similarity index 100% rename from docs6/docs/data/english-words.txt rename to docs6/src/data/english-words.txt diff --git a/docs6/docs/data/extra-spaces.txt b/docs6/src/data/extra-spaces.txt similarity index 100% rename from docs6/docs/data/extra-spaces.txt rename to docs6/src/data/extra-spaces.txt diff --git a/docs6/docs/data/fe-example-3.mlr b/docs6/src/data/fe-example-3.mlr similarity index 100% rename from docs6/docs/data/fe-example-3.mlr rename to docs6/src/data/fe-example-3.mlr diff --git a/docs6/docs/data/fe-example-4.mlr b/docs6/src/data/fe-example-4.mlr similarity index 100% rename from docs6/docs/data/fe-example-4.mlr rename to docs6/src/data/fe-example-4.mlr diff --git a/docs6/docs/data/feature-count.mlr b/docs6/src/data/feature-count.mlr similarity index 100% rename from docs6/docs/data/feature-count.mlr rename to docs6/src/data/feature-count.mlr diff --git a/docs6/docs/data/features.json b/docs6/src/data/features.json similarity index 100% rename from docs6/docs/data/features.json rename to docs6/src/data/features.json diff --git a/docs6/docs/data/fibo-cached.sh b/docs6/src/data/fibo-cached.sh similarity index 100% rename from docs6/docs/data/fibo-cached.sh rename to docs6/src/data/fibo-cached.sh diff --git a/docs6/docs/data/fibo-uncached.sh b/docs6/src/data/fibo-uncached.sh similarity index 100% rename from docs6/docs/data/fibo-uncached.sh rename to docs6/src/data/fibo-uncached.sh diff --git a/docs6/docs/data/fillable.csv b/docs6/src/data/fillable.csv similarity index 100% rename from docs6/docs/data/fillable.csv rename to docs6/src/data/fillable.csv diff --git a/docs6/docs/data/filter-multiline-example.txt b/docs6/src/data/filter-multiline-example.txt similarity index 100% rename from docs6/docs/data/filter-multiline-example.txt rename to docs6/src/data/filter-multiline-example.txt diff --git a/docs6/docs/data/flins.csv b/docs6/src/data/flins.csv similarity index 100% rename from docs6/docs/data/flins.csv rename to docs6/src/data/flins.csv diff --git a/docs6/docs/data/flins.json b/docs6/src/data/flins.json similarity index 100% rename from docs6/docs/data/flins.json rename to docs6/src/data/flins.json diff --git a/docs6/docs/data/for-oosvar-example-0a.txt b/docs6/src/data/for-oosvar-example-0a.txt similarity index 100% rename from docs6/docs/data/for-oosvar-example-0a.txt rename to docs6/src/data/for-oosvar-example-0a.txt diff --git a/docs6/docs/data/for-srec-example.tbl b/docs6/src/data/for-srec-example.tbl similarity index 100% rename from docs6/docs/data/for-srec-example.tbl rename to docs6/src/data/for-srec-example.tbl diff --git a/docs6/docs/data/fraction-example.csv b/docs6/src/data/fraction-example.csv similarity index 100% rename from docs6/docs/data/fraction-example.csv rename to docs6/src/data/fraction-example.csv diff --git a/docs6/docs/data/headerless.csv b/docs6/src/data/headerless.csv similarity index 100% rename from docs6/docs/data/headerless.csv rename to docs6/src/data/headerless.csv diff --git a/docs6/docs/data/het-bool.csv b/docs6/src/data/het-bool.csv similarity index 100% rename from docs6/docs/data/het-bool.csv rename to docs6/src/data/het-bool.csv diff --git a/docs6/docs/data/het.csv b/docs6/src/data/het.csv similarity index 100% rename from docs6/docs/data/het.csv rename to docs6/src/data/het.csv diff --git a/docs6/docs/data/het.dkvp b/docs6/src/data/het.dkvp similarity index 100% rename from docs6/docs/data/het.dkvp rename to docs6/src/data/het.dkvp diff --git a/docs6/docs/data/het.json b/docs6/src/data/het.json similarity index 100% rename from docs6/docs/data/het.json rename to docs6/src/data/het.json diff --git a/docs6/docs/data/het/fillable.csv b/docs6/src/data/het/fillable.csv similarity index 100% rename from docs6/docs/data/het/fillable.csv rename to docs6/src/data/het/fillable.csv diff --git a/docs6/docs/data/het/hom.csv b/docs6/src/data/het/hom.csv similarity index 100% rename from docs6/docs/data/het/hom.csv rename to docs6/src/data/het/hom.csv diff --git a/docs6/docs/data/het/irregular.json b/docs6/src/data/het/irregular.json similarity index 100% rename from docs6/docs/data/het/irregular.json rename to docs6/src/data/het/irregular.json diff --git a/docs6/docs/data/het/ragged.csv b/docs6/src/data/het/ragged.csv similarity index 100% rename from docs6/docs/data/het/ragged.csv rename to docs6/src/data/het/ragged.csv diff --git a/docs6/docs/data/het/sparse.json b/docs6/src/data/het/sparse.json similarity index 100% rename from docs6/docs/data/het/sparse.json rename to docs6/src/data/het/sparse.json diff --git a/docs6/src/data/hostnames.csv b/docs6/src/data/hostnames.csv new file mode 100644 index 000000000..05cf9fd60 --- /dev/null +++ b/docs6/src/data/hostnames.csv @@ -0,0 +1,3 @@ +host,status +apoapsis.east.our.org,up +nadir.west.our.org,down diff --git a/docs6/src/data/hostnames.xtab b/docs6/src/data/hostnames.xtab new file mode 100644 index 000000000..cb0bd0da2 --- /dev/null +++ b/docs6/src/data/hostnames.xtab @@ -0,0 +1,9 @@ +host apoapsis.east.our.org +status up +a ["apoapsis", "east", "our", "org"] +b ["apoapsis", "east", "our", "org"] + +host nadir.west.our.org +status down +a ["nadir", "west", "our", "org"] +b ["nadir", "west", "our", "org"] diff --git a/docs6/docs/data/hosts.csv b/docs6/src/data/hosts.csv similarity index 100% rename from docs6/docs/data/hosts.csv rename to docs6/src/data/hosts.csv diff --git a/docs6/docs/data/inout.csv b/docs6/src/data/inout.csv similarity index 100% rename from docs6/docs/data/inout.csv rename to docs6/src/data/inout.csv diff --git a/docs6/docs/data/join-left-example.csv b/docs6/src/data/join-left-example.csv similarity index 100% rename from docs6/docs/data/join-left-example.csv rename to docs6/src/data/join-left-example.csv diff --git a/docs6/docs/data/join-right-example.csv b/docs6/src/data/join-right-example.csv similarity index 100% rename from docs6/docs/data/join-right-example.csv rename to docs6/src/data/join-right-example.csv diff --git a/docs6/docs/data/join-u-left.csv b/docs6/src/data/join-u-left.csv similarity index 100% rename from docs6/docs/data/join-u-left.csv rename to docs6/src/data/join-u-left.csv diff --git a/docs6/docs/data/join-u-right.csv b/docs6/src/data/join-u-right.csv similarity index 100% rename from docs6/docs/data/join-u-right.csv rename to docs6/src/data/join-u-right.csv diff --git a/docs6/docs/data/json-example-1.json b/docs6/src/data/json-example-1.json similarity index 100% rename from docs6/docs/data/json-example-1.json rename to docs6/src/data/json-example-1.json diff --git a/docs6/docs/data/json-example-2.json b/docs6/src/data/json-example-2.json similarity index 100% rename from docs6/docs/data/json-example-2.json rename to docs6/src/data/json-example-2.json diff --git a/docs6/docs/data/json-example-3.json b/docs6/src/data/json-example-3.json similarity index 100% rename from docs6/docs/data/json-example-3.json rename to docs6/src/data/json-example-3.json diff --git a/docs6/docs/data/json-in-csv.csv b/docs6/src/data/json-in-csv.csv similarity index 100% rename from docs6/docs/data/json-in-csv.csv rename to docs6/src/data/json-in-csv.csv diff --git a/docs6/docs/data/label-example.txt b/docs6/src/data/label-example.txt similarity index 100% rename from docs6/docs/data/label-example.txt rename to docs6/src/data/label-example.txt diff --git a/docs6/docs/data/linreg-example.jpg b/docs6/src/data/linreg-example.jpg similarity index 100% rename from docs6/docs/data/linreg-example.jpg rename to docs6/src/data/linreg-example.jpg diff --git a/docs6/docs/data/linreg-example.txt b/docs6/src/data/linreg-example.txt similarity index 100% rename from docs6/docs/data/linreg-example.txt rename to docs6/src/data/linreg-example.txt diff --git a/docs6/src/data/map-values-nested.json b/docs6/src/data/map-values-nested.json new file mode 100644 index 000000000..11721b2a2 --- /dev/null +++ b/docs6/src/data/map-values-nested.json @@ -0,0 +1,8 @@ +{ + "a": 1, + "b": {"s": {"w": 2, "x": 3}, "t": {"y": 4, "z": 5}} +} +{ + "a": 6, + "b": {"s": {"w": 7, "x": 8}, "t": {"y": 9, "z": 10}} +} diff --git a/docs6/src/data/map-values-spread-colon.csv b/docs6/src/data/map-values-spread-colon.csv new file mode 100644 index 000000000..23096a0a8 --- /dev/null +++ b/docs6/src/data/map-values-spread-colon.csv @@ -0,0 +1,3 @@ +a,b:x,b:y +1,2,3 +4,5,6 diff --git a/docs6/src/data/map-values-spread.csv b/docs6/src/data/map-values-spread.csv new file mode 100644 index 000000000..928152d82 --- /dev/null +++ b/docs6/src/data/map-values-spread.csv @@ -0,0 +1,3 @@ +a,b.x,b.y +1,2,3 +4,5,6 diff --git a/docs6/src/data/map-values.json b/docs6/src/data/map-values.json new file mode 100644 index 000000000..9ee91823b --- /dev/null +++ b/docs6/src/data/map-values.json @@ -0,0 +1,8 @@ +{ + "a": 1, + "b": {"x": 2, "y": 3} +} +{ + "a": 4, + "b": {"x": 5, "y": 6} +} diff --git a/docs6/docs/data/maxrows.mlr b/docs6/src/data/maxrows.mlr similarity index 100% rename from docs6/docs/data/maxrows.mlr rename to docs6/src/data/maxrows.mlr diff --git a/docs6/docs/data/maxrows.tsv b/docs6/src/data/maxrows.tsv similarity index 100% rename from docs6/docs/data/maxrows.tsv rename to docs6/src/data/maxrows.tsv diff --git a/docs6/docs/data/medium b/docs6/src/data/medium similarity index 100% rename from docs6/docs/data/medium rename to docs6/src/data/medium diff --git a/docs6/docs/data/medium-squares b/docs6/src/data/medium-squares similarity index 100% rename from docs6/docs/data/medium-squares rename to docs6/src/data/medium-squares diff --git a/docs6/docs/data/miss-date.csv b/docs6/src/data/miss-date.csv similarity index 100% rename from docs6/docs/data/miss-date.csv rename to docs6/src/data/miss-date.csv diff --git a/docs6/docs/data/mkdat2 b/docs6/src/data/mkdat2 similarity index 100% rename from docs6/docs/data/mkdat2 rename to docs6/src/data/mkdat2 diff --git a/docs6/docs/data/modsep.dkvp b/docs6/src/data/modsep.dkvp similarity index 100% rename from docs6/docs/data/modsep.dkvp rename to docs6/src/data/modsep.dkvp diff --git a/docs6/docs/data/multicountdown.dat b/docs6/src/data/multicountdown.dat similarity index 100% rename from docs6/docs/data/multicountdown.dat rename to docs6/src/data/multicountdown.dat diff --git a/docs6/docs/data/multicountdown.rb b/docs6/src/data/multicountdown.rb similarity index 100% rename from docs6/docs/data/multicountdown.rb rename to docs6/src/data/multicountdown.rb diff --git a/docs6/docs/data/mydata.txt b/docs6/src/data/mydata.txt similarity index 100% rename from docs6/docs/data/mydata.txt rename to docs6/src/data/mydata.txt diff --git a/docs6/docs/data/nested.tsv b/docs6/src/data/nested.tsv similarity index 100% rename from docs6/docs/data/nested.tsv rename to docs6/src/data/nested.tsv diff --git a/docs6/docs/data/newline-example.txt b/docs6/src/data/newline-example.txt similarity index 100% rename from docs6/docs/data/newline-example.txt rename to docs6/src/data/newline-example.txt diff --git a/docs6/src/data/non-consecutive.csv b/docs6/src/data/non-consecutive.csv new file mode 100644 index 000000000..4d8176d9e --- /dev/null +++ b/docs6/src/data/non-consecutive.csv @@ -0,0 +1,2 @@ +a.1,a.3,a.5 +4,5,6 diff --git a/docs6/docs/data/ping-delta-example.txt b/docs6/src/data/ping-delta-example.txt similarity index 100% rename from docs6/docs/data/ping-delta-example.txt rename to docs6/src/data/ping-delta-example.txt diff --git a/docs6/docs/data/previous_counters.csv b/docs6/src/data/previous_counters.csv similarity index 100% rename from docs6/docs/data/previous_counters.csv rename to docs6/src/data/previous_counters.csv diff --git a/docs6/docs/data/prevtemp.csv b/docs6/src/data/prevtemp.csv similarity index 100% rename from docs6/docs/data/prevtemp.csv rename to docs6/src/data/prevtemp.csv diff --git a/docs6/docs/data/put-gating-example-1.dkvp b/docs6/src/data/put-gating-example-1.dkvp similarity index 100% rename from docs6/docs/data/put-gating-example-1.dkvp rename to docs6/src/data/put-gating-example-1.dkvp diff --git a/docs6/docs/data/put-gating-example-2.dkvp b/docs6/src/data/put-gating-example-2.dkvp similarity index 100% rename from docs6/docs/data/put-gating-example-2.dkvp rename to docs6/src/data/put-gating-example-2.dkvp diff --git a/docs6/docs/data/question.dat b/docs6/src/data/question.dat similarity index 100% rename from docs6/docs/data/question.dat rename to docs6/src/data/question.dat diff --git a/docs6/docs/data/ragged.csv b/docs6/src/data/ragged.csv similarity index 100% rename from docs6/docs/data/ragged.csv rename to docs6/src/data/ragged.csv diff --git a/docs6/docs/data/rect-inner.txt b/docs6/src/data/rect-inner.txt similarity index 100% rename from docs6/docs/data/rect-inner.txt rename to docs6/src/data/rect-inner.txt diff --git a/docs6/docs/data/rect-middle.txt b/docs6/src/data/rect-middle.txt similarity index 100% rename from docs6/docs/data/rect-middle.txt rename to docs6/src/data/rect-middle.txt diff --git a/docs6/docs/data/rect-outer.txt b/docs6/src/data/rect-outer.txt similarity index 100% rename from docs6/docs/data/rect-outer.txt rename to docs6/src/data/rect-outer.txt diff --git a/docs6/docs/data/rect.txt b/docs6/src/data/rect.txt similarity index 100% rename from docs6/docs/data/rect.txt rename to docs6/src/data/rect.txt diff --git a/docs6/docs/data/regex-in-data.dat b/docs6/src/data/regex-in-data.dat similarity index 100% rename from docs6/docs/data/regex-in-data.dat rename to docs6/src/data/regex-in-data.dat diff --git a/docs6/docs/data/remove-empty-columns.csv b/docs6/src/data/remove-empty-columns.csv similarity index 100% rename from docs6/docs/data/remove-empty-columns.csv rename to docs6/src/data/remove-empty-columns.csv diff --git a/docs6/docs/data/repeat-example.dat b/docs6/src/data/repeat-example.dat similarity index 100% rename from docs6/docs/data/repeat-example.dat rename to docs6/src/data/repeat-example.dat diff --git a/docs6/docs/data/repeats.dkvp b/docs6/src/data/repeats.dkvp similarity index 100% rename from docs6/docs/data/repeats.dkvp rename to docs6/src/data/repeats.dkvp diff --git a/docs6/docs/data/sample-example.txt b/docs6/src/data/sample-example.txt similarity index 100% rename from docs6/docs/data/sample-example.txt rename to docs6/src/data/sample-example.txt diff --git a/docs6/docs/data/sar.csv b/docs6/src/data/sar.csv similarity index 100% rename from docs6/docs/data/sar.csv rename to docs6/src/data/sar.csv diff --git a/docs6/docs/data/sar.mlr b/docs6/src/data/sar.mlr similarity index 100% rename from docs6/docs/data/sar.mlr rename to docs6/src/data/sar.mlr diff --git a/docs6/docs/data/scan-example-1.tbl b/docs6/src/data/scan-example-1.tbl similarity index 100% rename from docs6/docs/data/scan-example-1.tbl rename to docs6/src/data/scan-example-1.tbl diff --git a/docs6/docs/data/scan-example-2.tbl b/docs6/src/data/scan-example-2.tbl similarity index 100% rename from docs6/docs/data/scan-example-2.tbl rename to docs6/src/data/scan-example-2.tbl diff --git a/docs6/docs/data/scope-example.dat b/docs6/src/data/scope-example.dat similarity index 100% rename from docs6/docs/data/scope-example.dat rename to docs6/src/data/scope-example.dat diff --git a/docs6/docs/data/scope-example.mlr b/docs6/src/data/scope-example.mlr similarity index 100% rename from docs6/docs/data/scope-example.mlr rename to docs6/src/data/scope-example.mlr diff --git a/docs6/docs/data/self-join.csv b/docs6/src/data/self-join.csv similarity index 100% rename from docs6/docs/data/self-join.csv rename to docs6/src/data/self-join.csv diff --git a/docs6/docs/data/semicolon-example.txt b/docs6/src/data/semicolon-example.txt similarity index 100% rename from docs6/docs/data/semicolon-example.txt rename to docs6/src/data/semicolon-example.txt diff --git a/docs6/src/data/server-log.json b/docs6/src/data/server-log.json new file mode 100644 index 000000000..98d08daa5 --- /dev/null +++ b/docs6/src/data/server-log.json @@ -0,0 +1,21 @@ +{ + "hostname": "localhost", + "pid": 12345, + "req": { + "id": 6789, + "method": "GET", + "path": "api/check", + "host": "foo.bar", + "headers": { + "host": "bar.baz", + "user-agent": "browser" + } + }, + "res": { + "status_code": 200, + "header": { + "content-type": "text", + "content-encoding": "plain" + } + } +} diff --git a/docs6/docs/data/short.csv b/docs6/src/data/short.csv similarity index 100% rename from docs6/docs/data/short.csv rename to docs6/src/data/short.csv diff --git a/docs6/docs/data/single-quote-example.mlr b/docs6/src/data/single-quote-example.mlr similarity index 100% rename from docs6/docs/data/single-quote-example.mlr rename to docs6/src/data/single-quote-example.mlr diff --git a/docs6/docs/data/small b/docs6/src/data/small similarity index 100% rename from docs6/docs/data/small rename to docs6/src/data/small diff --git a/docs6/docs/data/small.csv b/docs6/src/data/small.csv similarity index 100% rename from docs6/docs/data/small.csv rename to docs6/src/data/small.csv diff --git a/docs6/docs/data/small2 b/docs6/src/data/small2 similarity index 100% rename from docs6/docs/data/small2 rename to docs6/src/data/small2 diff --git a/docs6/docs/data/sort-het.dkvp b/docs6/src/data/sort-het.dkvp similarity index 100% rename from docs6/docs/data/sort-het.dkvp rename to docs6/src/data/sort-het.dkvp diff --git a/docs6/docs/data/sort-missing.dkvp b/docs6/src/data/sort-missing.dkvp similarity index 100% rename from docs6/docs/data/sort-missing.dkvp rename to docs6/src/data/sort-missing.dkvp diff --git a/docs6/docs/data/sort-null.dat b/docs6/src/data/sort-null.dat similarity index 100% rename from docs6/docs/data/sort-null.dat rename to docs6/src/data/sort-null.dat diff --git a/docs6/docs/data/sort-within-records.json b/docs6/src/data/sort-within-records.json similarity index 100% rename from docs6/docs/data/sort-within-records.json rename to docs6/src/data/sort-within-records.json diff --git a/docs6/docs/data/spaces.csv b/docs6/src/data/spaces.csv similarity index 100% rename from docs6/docs/data/spaces.csv rename to docs6/src/data/spaces.csv diff --git a/docs6/docs/data/sparse.json b/docs6/src/data/sparse.json similarity index 100% rename from docs6/docs/data/sparse.json rename to docs6/src/data/sparse.json diff --git a/docs6/docs/data/subcommand-example.txt b/docs6/src/data/subcommand-example.txt similarity index 100% rename from docs6/docs/data/subcommand-example.txt rename to docs6/src/data/subcommand-example.txt diff --git a/docs6/docs/data/then-chaining-performance.txt b/docs6/src/data/then-chaining-performance.txt similarity index 100% rename from docs6/docs/data/then-chaining-performance.txt rename to docs6/src/data/then-chaining-performance.txt diff --git a/docs6/docs/data/then-example.csv b/docs6/src/data/then-example.csv similarity index 100% rename from docs6/docs/data/then-example.csv rename to docs6/src/data/then-example.csv diff --git a/docs6/docs/data/timing-example.txt b/docs6/src/data/timing-example.txt similarity index 100% rename from docs6/docs/data/timing-example.txt rename to docs6/src/data/timing-example.txt diff --git a/docs6/docs/data/trivial-records.csv b/docs6/src/data/trivial-records.csv similarity index 100% rename from docs6/docs/data/trivial-records.csv rename to docs6/src/data/trivial-records.csv diff --git a/docs6/docs/data/type-decl-example.mlr b/docs6/src/data/type-decl-example.mlr similarity index 100% rename from docs6/docs/data/type-decl-example.mlr rename to docs6/src/data/type-decl-example.mlr diff --git a/docs6/docs/data/type-infer.csv b/docs6/src/data/type-infer.csv similarity index 100% rename from docs6/docs/data/type-infer.csv rename to docs6/src/data/type-infer.csv diff --git a/docs6/docs/data/unsparsify.mlr b/docs6/src/data/unsparsify.mlr similarity index 100% rename from docs6/docs/data/unsparsify.mlr rename to docs6/src/data/unsparsify.mlr diff --git a/docs6/docs/date-time-examples.md b/docs6/src/date-time-examples.md similarity index 100% rename from docs6/docs/date-time-examples.md rename to docs6/src/date-time-examples.md diff --git a/docs6/docs/date-time-examples.md.in b/docs6/src/date-time-examples.md.in similarity index 100% rename from docs6/docs/date-time-examples.md.in rename to docs6/src/date-time-examples.md.in diff --git a/docs6/docs/dates.csv b/docs6/src/dates.csv similarity index 100% rename from docs6/docs/dates.csv rename to docs6/src/dates.csv diff --git a/docs6/docs/dkvp-examples.md b/docs6/src/dkvp-examples.md similarity index 100% rename from docs6/docs/dkvp-examples.md rename to docs6/src/dkvp-examples.md diff --git a/docs6/docs/dkvp-examples.md.in b/docs6/src/dkvp-examples.md.in similarity index 100% rename from docs6/docs/dkvp-examples.md.in rename to docs6/src/dkvp-examples.md.in diff --git a/docs6/docs/doc-reorg.txt b/docs6/src/doc-reorg.txt similarity index 100% rename from docs6/docs/doc-reorg.txt rename to docs6/src/doc-reorg.txt diff --git a/docs6/docs/dsl-example.mlr b/docs6/src/dsl-example.mlr similarity index 100% rename from docs6/docs/dsl-example.mlr rename to docs6/src/dsl-example.mlr diff --git a/docs6/docs/etymology.md b/docs6/src/etymology.md similarity index 100% rename from docs6/docs/etymology.md rename to docs6/src/etymology.md diff --git a/docs6/docs/etymology.md.in b/docs6/src/etymology.md.in similarity index 100% rename from docs6/docs/etymology.md.in rename to docs6/src/etymology.md.in diff --git a/docs6/docs/example.csv b/docs6/src/example.csv similarity index 100% rename from docs6/docs/example.csv rename to docs6/src/example.csv diff --git a/docs6/src/example.json b/docs6/src/example.json new file mode 100644 index 000000000..37a5e5cd4 --- /dev/null +++ b/docs6/src/example.json @@ -0,0 +1,18 @@ +{ + "color": "yellow", + "shape": "triangle", + "flag": "true", + "k": 1, + "index": 11, + "quantity": 43.6498, + "rate": 9.8870 +} +{ + "color": "red", + "shape": "square", + "flag": "true", + "k": 2, + "index": 15, + "quantity": 79.2778, + "rate": 0.0130 +} diff --git a/docs6/docs/expo-sample.sh b/docs6/src/expo-sample.sh similarity index 100% rename from docs6/docs/expo-sample.sh rename to docs6/src/expo-sample.sh diff --git a/docs6/docs/extra.css b/docs6/src/extra.css similarity index 100% rename from docs6/docs/extra.css rename to docs6/src/extra.css diff --git a/docs6/docs/factorial-example.mlr b/docs6/src/factorial-example.mlr similarity index 100% rename from docs6/docs/factorial-example.mlr rename to docs6/src/factorial-example.mlr diff --git a/docs6/docs/features.md b/docs6/src/features.md similarity index 100% rename from docs6/docs/features.md rename to docs6/src/features.md diff --git a/docs6/docs/features.md.in b/docs6/src/features.md.in similarity index 100% rename from docs6/docs/features.md.in rename to docs6/src/features.md.in diff --git a/docs6/docs/file-formats.md b/docs6/src/file-formats.md similarity index 97% rename from docs6/docs/file-formats.md rename to docs6/src/file-formats.md index 328db1c16..b9d5c9f0b 100644 --- a/docs6/docs/file-formats.md +++ b/docs6/src/file-formats.md @@ -92,7 +92,7 @@ NIDX: implicitly numerically indexed (Unix-toolkit style) When `mlr` is invoked with the `--csv` or `--csvlite` option, key names are found on the first record and values are taken from subsequent records. This includes the case of CSV-formatted files. See [Record Heterogeneity](record-heterogeneity.md) for how Miller handles changes of field names within a single data stream. -Miller has record separator `RS` and field separator `FS`, just as `awk` does. For TSV, use `--fs tab`; to convert TSV to CSV, use `--ifs tab --ofs comma`, etc. (See also [Record/field/pair separators](reference-main-io-options.md#recordfieldpair-separators).) +Miller has record separator `RS` and field separator `FS`, just as `awk` does. For TSV, use `--fs tab`; to convert TSV to CSV, use `--ifs tab --ofs comma`, etc. (See also the [separators page](reference-main-separators.md).) **TSV (tab-separated values):** the following are synonymous pairs: @@ -276,8 +276,7 @@ TODO: probably remove entirely ### JSON-in-CSV It's quite common to have CSV data which contains stringified JSON as a column. -See the [JSON parse and stringify -section](reference-main-data-types.md#json-parse-and-stringify) for ways to +See the [JSON parse and stringify section](reference-main-data-types.md#json-parse-and-stringify) for ways to decode these in Miller. ## PPRINT: Pretty-printed tabular @@ -467,7 +466,7 @@ resource=/some/other/path,loadsec=0.97,ok=false etc. and I just log them as needed. Then later, I can use `grep`, `mlr --opprint group-like`, etc. to analyze my logs. -See the [I/O options reference](reference-main-io-options.md) regarding how to specify separators other than the default equals-sign and comma. +See the [seprators page](reference-main-separators.md) regarding how to specify separators other than the default equals-sign and comma. ## NIDX: Index-numbered (toolkit style) @@ -611,7 +610,7 @@ If you use `--ors {something else}` with (default or explicitly specified) `--ir If you use `--irs {something else}` with (default or explicitly specified) `--ors auto` then the output line endings used are LF on Unix/Linux/BSD/MacOS X, and CRLF on Windows. -See also [Record/field/pair separators](reference-main-io-options.md#recordfieldpair-separators) for more information about record/field/pair separators. +See also the [separators page](reference-main-separators.md) for more information about record/field/pair separators. ---> ## Comments in data diff --git a/docs6/docs/file-formats.md.in b/docs6/src/file-formats.md.in similarity index 95% rename from docs6/docs/file-formats.md.in rename to docs6/src/file-formats.md.in index 7d1ecea88..82b512224 100644 --- a/docs6/docs/file-formats.md.in +++ b/docs6/src/file-formats.md.in @@ -14,7 +14,7 @@ GENMD_EOF When `mlr` is invoked with the `--csv` or `--csvlite` option, key names are found on the first record and values are taken from subsequent records. This includes the case of CSV-formatted files. See [Record Heterogeneity](record-heterogeneity.md) for how Miller handles changes of field names within a single data stream. -Miller has record separator `RS` and field separator `FS`, just as `awk` does. For TSV, use `--fs tab`; to convert TSV to CSV, use `--ifs tab --ofs comma`, etc. (See also [Record/field/pair separators](reference-main-io-options.md#recordfieldpair-separators).) +Miller has record separator `RS` and field separator `FS`, just as `awk` does. For TSV, use `--fs tab`; to convert TSV to CSV, use `--ifs tab --ofs comma`, etc. (See also the [separators page](reference-main-separators.md).) **TSV (tab-separated values):** the following are synonymous pairs: @@ -121,8 +121,7 @@ TODO: probably remove entirely ### JSON-in-CSV It's quite common to have CSV data which contains stringified JSON as a column. -See the [JSON parse and stringify -section](reference-main-data-types.md#json-parse-and-stringify) for ways to +See the [JSON parse and stringify section](reference-main-data-types.md#json-parse-and-stringify) for ways to decode these in Miller. ## PPRINT: Pretty-printed tabular @@ -263,7 +262,7 @@ GENMD_EOF etc. and I just log them as needed. Then later, I can use `grep`, `mlr --opprint group-like`, etc. to analyze my logs. -See the [I/O options reference](reference-main-io-options.md) regarding how to specify separators other than the default equals-sign and comma. +See the [seprators page](reference-main-separators.md) regarding how to specify separators other than the default equals-sign and comma. ## NIDX: Index-numbered (toolkit style) @@ -324,7 +323,7 @@ If you use `--ors {something else}` with (default or explicitly specified) `--ir If you use `--irs {something else}` with (default or explicitly specified) `--ors auto` then the output line endings used are LF on Unix/Linux/BSD/MacOS X, and CRLF on Windows. -See also [Record/field/pair separators](reference-main-io-options.md#recordfieldpair-separators) for more information about record/field/pair separators. +See also the [separators page](reference-main-separators.md) for more information about record/field/pair separators. ---> ## Comments in data diff --git a/docs6/docs/flags.txt b/docs6/src/flags.txt similarity index 100% rename from docs6/docs/flags.txt rename to docs6/src/flags.txt diff --git a/docs6/src/flatten-unflatten.md b/docs6/src/flatten-unflatten.md new file mode 100644 index 000000000..e2b459112 --- /dev/null +++ b/docs6/src/flatten-unflatten.md @@ -0,0 +1,472 @@ + +
+ +Quick links: +  +Flag list +  +Verb list +  +Function list +  +Glossary +  +Repository ↗ + +
+# Flatten/unflatten: converting between JSON and tabular formats + +Miller has long supported reading and writing multiple [file +formats](file-formats.md) including CSV and JSON, as well as converting back +and forth between them. Two things new in [Miller 6](new-in-miller-6-md), +though, are that [arrays are now fully supported](reference-main-arrays.md), +and that [record values are typed](new-in-miller-6.md#improved-numeric-conversion) +throughout Miller's processing chain from input through [verbs](reference-verbs.md) +to output -- which includes improved handling for [maps](reference-main-maps.md) and +[arrays](reference-main-arrays.md) as record values. + +This raises the question, though, of how to handle maps and arrays as record values. +For [JSON files](file-formats.md#json), this is easy -- JSON is a nested format where values +can be maps or arrays, which can contain other maps or arrays, and so on, with the nesting +happily indicated by curly braces: + +
+cat data/map-values.json
+
+
+{
+  "a": 1,
+  "b": {"x": 2, "y": 3}
+}
+{
+  "a": 4,
+  "b": {"x": 5, "y": 6}
+}
+
+ +
+cat data/map-values-nested.json
+
+
+{
+  "a": 1,
+  "b": {"s": {"w": 2, "x": 3}, "t": {"y": 4, "z": 5}}
+}
+{
+  "a": 6,
+  "b": {"s": {"w": 7, "x": 8}, "t": {"y": 9, "z": 10}}
+}
+
+ +How can we represent these in CSV files? + +Miller's [non-JSON formats](file-formats.md), such as CSV, are all non-nested -- a +cell in a CSV row can't contain another entire row. As we'll see in this +section, there are two main ways to **flatten** nested data structures down to +individual CSV cells -- either by _key-spreading_ (which is the default), or by +_JSON-stringifying_: + +* **Key-spreading** is when the single map-valued field +`b={"x": 2, "y": 3}` spreads into multiple fields `b.x=2,b.y=3`; +* **JSON-stringifying** is when the single map-valued field `"b": {"x": 2, "y": 3}` becomes the single string-valued field `b="{\"x\":2,\"y\":3}"`. + +Miller intends to provide intuitive default behavior for these conversions, while also +providing you with more control when you need it. + +## Converting maps between JSON and non-JSON + +Let's first look at the default behavior with map-valued fields. Miller's +default behavior is to spread the map values into multiple keys -- using +Miller's `flatsep` separator, which defaults to `.` -- to join the original +record key with map keys: + +
+cat data/map-values.json
+
+
+{
+  "a": 1,
+  "b": {"x": 2, "y": 3}
+}
+{
+  "a": 4,
+  "b": {"x": 5, "y": 6}
+}
+
+ +Flattened to CSV format: + +
+mlr --ijson --ocsv cat data/map-values.json
+
+
+a,b.x,b.y
+1,2,3
+4,5,6
+
+ +Flattened to pretty-print format: + +
+mlr --ijson --opprint cat data/map-values.json
+
+
+a b.x b.y
+1 2   3
+4 5   6
+
+ +Using flatten-separator `:` instead of the default `.`: + +
+mlr --ijson --opprint --flatsep : cat data/map-values.json
+
+
+a b:x b:y
+1 2   3
+4 5   6
+
+ +If the maps are more deeply nested, each level of map keys is joined in: + +
+cat data/map-values-nested.json
+
+
+{
+  "a": 1,
+  "b": {"s": {"w": 2, "x": 3}, "t": {"y": 4, "z": 5}}
+}
+{
+  "a": 6,
+  "b": {"s": {"w": 7, "x": 8}, "t": {"y": 9, "z": 10}}
+}
+
+ +
+mlr --ijson --opprint cat data/map-values-nested.json
+
+
+a b.s.w b.s.x b.t.y b.t.z
+1 2     3     4     5
+6 7     8     9     10
+
+ +**Unflattening** is simply the reverse -- from non-JSON back to JSON: + +
+cat data/map-values.json
+
+
+{
+  "a": 1,
+  "b": {"x": 2, "y": 3}
+}
+{
+  "a": 4,
+  "b": {"x": 5, "y": 6}
+}
+
+ +
+mlr --ijson --ocsv cat data/map-values.json
+
+
+a,b.x,b.y
+1,2,3
+4,5,6
+
+ +
+mlr --ijson --ocsv cat data/map-values.json | mlr --icsv --ojson cat
+
+
+{
+  "a": 1,
+  "b": {
+    "x": 2,
+    "y": 3
+  }
+}
+{
+  "a": 4,
+  "b": {
+    "x": 5,
+    "y": 6
+  }
+}
+
+ +## Converting arrays between JSON and non-JSON + +If the input data contains arrays, these are also flattened similarly: the +[1-up array indices](reference-main-arrays.md#1-up-indexing) `1,2,3,...` become string keys +`"1","2","3",...`: + +
+cat data/array-values.json
+
+
+{
+  "a": 1,
+  "b": [2, 3]
+}
+{
+  "a": 4,
+  "b": [5, 6]
+}
+
+ +
+mlr --ijson --opprint cat data/array-values.json
+
+
+a b.1 b.2
+1 2   3
+4 5   6
+
+ +If the arrays are more deeply nested, each level of arrays keys is joined in: + +
+cat data/array-values-nested.json
+
+
+{
+  "a": 1,
+  "b": [[2, 3], [4, 5]]
+}
+{
+  "a": 6,
+  "b": [[7, 8], [9, 10]]
+}
+
+ +
+mlr --ijson --opprint cat data/array-values-nested.json
+
+
+a b.1.1 b.1.2 b.2.1 b.2.2
+1 2     3     4     5
+6 7     8     9     10
+
+ +In the nested-data examples shown here, nested map values are shown containing +maps, and nested array values are shown containing arrays -- of course (even +though not shown here) nested map values can contain arrays, and vice versa. + +**Unflattening** arrays is, again, simply the reverse -- from non-JSON back to JSON: + +
+cat data/array-values.json
+
+
+{
+  "a": 1,
+  "b": [2, 3]
+}
+{
+  "a": 4,
+  "b": [5, 6]
+}
+
+ +
+mlr --ijson --ocsv cat data/array-values.json
+
+
+a,b.1,b.2
+1,2,3
+4,5,6
+
+ +
+mlr --ijson --ocsv cat data/array-values.json | mlr --icsv --ojson cat
+
+
+{
+  "a": 1,
+  "b": [2, 3]
+}
+{
+  "a": 4,
+  "b": [5, 6]
+}
+
+ +## Auto-inferencing of arrays on unflatten + +Note that the CSV field names `b.x` and `b.y` aren't too different from `b.1` +and `b.2`. Miller has the heuristic that if it's unflattening and gets a map +with keys `"1"`, `"2"`, etc. -- starting with `"1"`, consecutively, and with +no gaps -- it turns that back into an array. This is precisely to undo the +flatten conversion. However, it may (or may not) be surprising: + +
+cat data/consecutive.csv
+
+
+a.1,a.2,a.3
+4,5,6
+
+ +
+mlr --c2j cat data/consecutive.csv
+
+
+{
+  "a": [4, 5, 6]
+}
+
+ +
+cat data/non-consecutive.csv
+
+
+a.1,a.3,a.5
+4,5,6
+
+ +
+mlr --c2j cat data/non-consecutive.csv
+
+
+{
+  "a": {
+    "1": 4,
+    "3": 5,
+    "5": 6
+  }
+}
+
+ +## Manual control + +To see what our options are for manually controlling flattening and +unflattening (if the defaults aren't working for us in a particular situation), +let's first look a little into how they're implemented. + +* There are two [verbs](reference-verbs.md) called [flatten](reference-verbs.md#flatten) and [unflatten](reference-verbs.md#unflatten). +* When the output format is not JSON, if you've specified `mlr ... cat then sort ...` (some [chain](reference-main-then-chaining.md) of verbs) then Miller appends, in effect, `then flatten` to the end of the chain. + * This behavior is on by default but it can be suppressed using the `--no-auto-flatten` [flag](reference-main-flag-list.md#flatten-unflatten-flags). +* When the output format is JSON and the input format is not JSON, then (similarly) appends, in effect, `then unflatten` to the end of the chain. + * This behavior is on by default but it can be suppressed using the `--no-auto-unflatten` [flag](reference-main-flag-list.md#flatten-unflatten-flags). + +Note in particular that auto-flatten happens even when the input format and the +output format are both non-JSON, e.g. even for CSV-to-CSV processing. This is +because +[map](reference-main-maps.md)-valued/[array](reference-main-arrays.md)-valued +fields can be produced using [DSL statements](miller-programming-language.md): + +
+cat data/hostnames.csv
+
+
+host,status
+apoapsis.east.our.org,up
+nadir.west.our.org,down
+
+ +Using JSON output, we can see that `splita` has produced an array-valued field named `components`: + +
+mlr --icsv --ojson --from data/hostnames.csv put '$components = splita($host, ".")'
+
+
+{
+  "host": "apoapsis.east.our.org",
+  "status": "up",
+  "components": ["apoapsis", "east", "our", "org"]
+}
+{
+  "host": "nadir.west.our.org",
+  "status": "down",
+  "components": ["nadir", "west", "our", "org"]
+}
+
+ +Using CSV output, with default auto-flatten, we get `components.1` through `components.4`: + +
+mlr --csv --from data/hostnames.csv put '$components = splita($host, ".")'
+
+
+host,status,components.1,components.2,components.3,components.4
+apoapsis.east.our.org,up,apoapsis,east,our,org
+nadir.west.our.org,down,nadir,west,our,org
+
+ +Using CSV output, without default auto-flatten, we get a JSON-stringified encoding of the `components` field: + +
+mlr --csv --from data/hostnames.csv --no-auto-flatten put '$components = splita($host, ".")'
+
+
+host,status,components
+apoapsis.east.our.org,up,"[""apoapsis"", ""east"", ""our"", ""org""]"
+nadir.west.our.org,down,"[""nadir"", ""west"", ""our"", ""org""]"
+
+ +Now suppose we ran this + +
+mlr --icsv --oxtab --from data/hostnames.csv --no-auto-flatten put '
+  $a = splita($host, ".");
+  $b = splita($host, ".");
+'
+
+
+host   apoapsis.east.our.org
+status up
+a      ["apoapsis", "east", "our", "org"]
+b      ["apoapsis", "east", "our", "org"]
+
+host   nadir.west.our.org
+status down
+a      ["nadir", "west", "our", "org"]
+b      ["nadir", "west", "our", "org"]
+
+ +into a file [data/hostnames.xtab](./data/hostnames.xtab): + +
+cat data/hostnames.xtab
+
+
+host   apoapsis.east.our.org
+status up
+a      ["apoapsis", "east", "our", "org"]
+b      ["apoapsis", "east", "our", "org"]
+
+host   nadir.west.our.org
+status down
+a      ["nadir", "west", "our", "org"]
+b      ["nadir", "west", "our", "org"]
+
+ +This was written with `--no-auto-unflatten` so we need to manually revive the +array-valued fields, if we choose -- here, we can JSON-parse the `a` field and +leave `b` JSON-stringified: + +
+mlr --ixtab --ojson json-parse -f a data/hostnames.xtab
+
+
+{
+  "host": "apoapsis.east.our.org",
+  "status": "up",
+  "a": ["apoapsis", "east", "our", "org"],
+  "b": "[\"apoapsis\", \"east\", \"our\", \"org\"]"
+}
+{
+  "host": "nadir.west.our.org",
+  "status": "down",
+  "a": ["nadir", "west", "our", "org"],
+  "b": "[\"nadir\", \"west\", \"our\", \"org\"]"
+}
+
+ +See also the +[JSON parse and stringify section](reference-main-data-types.md#json-parse-and-stringify) section for +more on this -- for example, when Miller is producing SQL-query output from +tables having one or more columns that contain JSON-encoded data. diff --git a/docs6/src/flatten-unflatten.md.in b/docs6/src/flatten-unflatten.md.in new file mode 100644 index 000000000..588a10ec1 --- /dev/null +++ b/docs6/src/flatten-unflatten.md.in @@ -0,0 +1,225 @@ +# Flatten/unflatten: converting between JSON and tabular formats + +Miller has long supported reading and writing multiple [file +formats](file-formats.md) including CSV and JSON, as well as converting back +and forth between them. Two things new in [Miller 6](new-in-miller-6-md), +though, are that [arrays are now fully supported](reference-main-arrays.md), +and that [record values are typed](new-in-miller-6.md#improved-numeric-conversion) +throughout Miller's processing chain from input through [verbs](reference-verbs.md) +to output -- which includes improved handling for [maps](reference-main-maps.md) and +[arrays](reference-main-arrays.md) as record values. + +This raises the question, though, of how to handle maps and arrays as record values. +For [JSON files](file-formats.md#json), this is easy -- JSON is a nested format where values +can be maps or arrays, which can contain other maps or arrays, and so on, with the nesting +happily indicated by curly braces: + +GENMD_RUN_COMMAND +cat data/map-values.json +GENMD_EOF + +GENMD_RUN_COMMAND +cat data/map-values-nested.json +GENMD_EOF + +How can we represent these in CSV files? + +Miller's [non-JSON formats](file-formats.md), such as CSV, are all non-nested -- a +cell in a CSV row can't contain another entire row. As we'll see in this +section, there are two main ways to **flatten** nested data structures down to +individual CSV cells -- either by _key-spreading_ (which is the default), or by +_JSON-stringifying_: + +* **Key-spreading** is when the single map-valued field +`b={"x": 2, "y": 3}` spreads into multiple fields `b.x=2,b.y=3`; +* **JSON-stringifying** is when the single map-valued field `"b": {"x": 2, "y": 3}` becomes the single string-valued field `b="{\"x\":2,\"y\":3}"`. + +Miller intends to provide intuitive default behavior for these conversions, while also +providing you with more control when you need it. + +## Converting maps between JSON and non-JSON + +Let's first look at the default behavior with map-valued fields. Miller's +default behavior is to spread the map values into multiple keys -- using +Miller's `flatsep` separator, which defaults to `.` -- to join the original +record key with map keys: + +GENMD_RUN_COMMAND +cat data/map-values.json +GENMD_EOF + +Flattened to CSV format: + +GENMD_RUN_COMMAND +mlr --ijson --ocsv cat data/map-values.json +GENMD_EOF + +Flattened to pretty-print format: + +GENMD_RUN_COMMAND +mlr --ijson --opprint cat data/map-values.json +GENMD_EOF + +Using flatten-separator `:` instead of the default `.`: + +GENMD_RUN_COMMAND +mlr --ijson --opprint --flatsep : cat data/map-values.json +GENMD_EOF + +If the maps are more deeply nested, each level of map keys is joined in: + +GENMD_RUN_COMMAND +cat data/map-values-nested.json +GENMD_EOF + +GENMD_RUN_COMMAND +mlr --ijson --opprint cat data/map-values-nested.json +GENMD_EOF + +**Unflattening** is simply the reverse -- from non-JSON back to JSON: + +GENMD_RUN_COMMAND +cat data/map-values.json +GENMD_EOF + +GENMD_RUN_COMMAND +mlr --ijson --ocsv cat data/map-values.json +GENMD_EOF + +GENMD_RUN_COMMAND +mlr --ijson --ocsv cat data/map-values.json | mlr --icsv --ojson cat +GENMD_EOF + +## Converting arrays between JSON and non-JSON + +If the input data contains arrays, these are also flattened similarly: the +[1-up array indices](reference-main-arrays.md#1-up-indexing) `1,2,3,...` become string keys +`"1","2","3",...`: + +GENMD_RUN_COMMAND +cat data/array-values.json +GENMD_EOF + +GENMD_RUN_COMMAND +mlr --ijson --opprint cat data/array-values.json +GENMD_EOF + +If the arrays are more deeply nested, each level of arrays keys is joined in: + +GENMD_RUN_COMMAND +cat data/array-values-nested.json +GENMD_EOF + +GENMD_RUN_COMMAND +mlr --ijson --opprint cat data/array-values-nested.json +GENMD_EOF + +In the nested-data examples shown here, nested map values are shown containing +maps, and nested array values are shown containing arrays -- of course (even +though not shown here) nested map values can contain arrays, and vice versa. + +**Unflattening** arrays is, again, simply the reverse -- from non-JSON back to JSON: + +GENMD_RUN_COMMAND +cat data/array-values.json +GENMD_EOF + +GENMD_RUN_COMMAND +mlr --ijson --ocsv cat data/array-values.json +GENMD_EOF + +GENMD_RUN_COMMAND +mlr --ijson --ocsv cat data/array-values.json | mlr --icsv --ojson cat +GENMD_EOF + +## Auto-inferencing of arrays on unflatten + +Note that the CSV field names `b.x` and `b.y` aren't too different from `b.1` +and `b.2`. Miller has the heuristic that if it's unflattening and gets a map +with keys `"1"`, `"2"`, etc. -- starting with `"1"`, consecutively, and with +no gaps -- it turns that back into an array. This is precisely to undo the +flatten conversion. However, it may (or may not) be surprising: + +GENMD_RUN_COMMAND +cat data/consecutive.csv +GENMD_EOF + +GENMD_RUN_COMMAND +mlr --c2j cat data/consecutive.csv +GENMD_EOF + +GENMD_RUN_COMMAND +cat data/non-consecutive.csv +GENMD_EOF + +GENMD_RUN_COMMAND +mlr --c2j cat data/non-consecutive.csv +GENMD_EOF + +## Manual control + +To see what our options are for manually controlling flattening and +unflattening (if the defaults aren't working for us in a particular situation), +let's first look a little into how they're implemented. + +* There are two [verbs](reference-verbs.md) called [flatten](reference-verbs.md#flatten) and [unflatten](reference-verbs.md#unflatten). +* When the output format is not JSON, if you've specified `mlr ... cat then sort ...` (some [chain](reference-main-then-chaining.md) of verbs) then Miller appends, in effect, `then flatten` to the end of the chain. + * This behavior is on by default but it can be suppressed using the `--no-auto-flatten` [flag](reference-main-flag-list.md#flatten-unflatten-flags). +* When the output format is JSON and the input format is not JSON, then (similarly) appends, in effect, `then unflatten` to the end of the chain. + * This behavior is on by default but it can be suppressed using the `--no-auto-unflatten` [flag](reference-main-flag-list.md#flatten-unflatten-flags). + +Note in particular that auto-flatten happens even when the input format and the +output format are both non-JSON, e.g. even for CSV-to-CSV processing. This is +because +[map](reference-main-maps.md)-valued/[array](reference-main-arrays.md)-valued +fields can be produced using [DSL statements](miller-programming-language.md): + +GENMD_RUN_COMMAND +cat data/hostnames.csv +GENMD_EOF + +Using JSON output, we can see that `splita` has produced an array-valued field named `components`: + +GENMD_RUN_COMMAND +mlr --icsv --ojson --from data/hostnames.csv put '$components = splita($host, ".")' +GENMD_EOF + +Using CSV output, with default auto-flatten, we get `components.1` through `components.4`: + +GENMD_RUN_COMMAND +mlr --csv --from data/hostnames.csv put '$components = splita($host, ".")' +GENMD_EOF + +Using CSV output, without default auto-flatten, we get a JSON-stringified encoding of the `components` field: + +GENMD_RUN_COMMAND +mlr --csv --from data/hostnames.csv --no-auto-flatten put '$components = splita($host, ".")' +GENMD_EOF + +Now suppose we ran this + +GENMD_RUN_COMMAND +mlr --icsv --oxtab --from data/hostnames.csv --no-auto-flatten put ' + $a = splita($host, "."); + $b = splita($host, "."); +' +GENMD_EOF + +into a file [data/hostnames.xtab](./data/hostnames.xtab): + +GENMD_RUN_COMMAND +cat data/hostnames.xtab +GENMD_EOF + +This was written with `--no-auto-unflatten` so we need to manually revive the +array-valued fields, if we choose -- here, we can JSON-parse the `a` field and +leave `b` JSON-stringified: + +GENMD_RUN_COMMAND +mlr --ixtab --ojson json-parse -f a data/hostnames.xtab +GENMD_EOF + +See also the +[JSON parse and stringify section](reference-main-data-types.md#json-parse-and-stringify) section for +more on this -- for example, when Miller is producing SQL-query output from +tables having one or more columns that contain JSON-encoded data. diff --git a/docs6/docs/for-example.mlr b/docs6/src/for-example.mlr similarity index 100% rename from docs6/docs/for-example.mlr rename to docs6/src/for-example.mlr diff --git a/docs6/docs/genmd-filter b/docs6/src/genmd-filter similarity index 96% rename from docs6/docs/genmd-filter rename to docs6/src/genmd-filter index 6f1ee63c7..6c7375e6d 100755 --- a/docs6/docs/genmd-filter +++ b/docs6/src/genmd-filter @@ -162,14 +162,16 @@ def run_command(cmd_lines, output_handle) if status != 0 raise "\"#{cmd}\" exited with non-zero code #{status}." end - write_card(cmd_lines, cmd_output.split(/\n/), output_handle) + # chomp and split so "a\nb\nc\n" splits to ["a", "b", "c"] + write_card(cmd_lines, cmd_output.chomp.split(/\n/, -1), output_handle) end # ---------------------------------------------------------------- def run_command_tolerating_error(cmd_lines, output_handle) cmd = cmd_lines.join("\n") cmd_output = `#{cmd} 2>&1` - write_card(cmd_lines, cmd_output.split(/\n/), output_handle) + # chomp and split so "a\nb\nc\n" splits to ["a", "b", "c"] + write_card(cmd_lines, cmd_output.chomp.split(/\n/, -1), output_handle) end # ---------------------------------------------------------------- diff --git a/docs6/docs/genmds b/docs6/src/genmds similarity index 100% rename from docs6/docs/genmds rename to docs6/src/genmds diff --git a/docs6/docs/glossary.md b/docs6/src/glossary.md similarity index 88% rename from docs6/docs/glossary.md rename to docs6/src/glossary.md index 64a0fa49b..b20090bb0 100644 --- a/docs6/docs/glossary.md +++ b/docs6/src/glossary.md @@ -59,14 +59,14 @@ information. ## begin -A [keyword](#keyword) in the [Miller programming language](programming-language.md) +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) indicating the start of a begin [block](#block) within an instance of the [`put`](#put) or [`filter`](#filter) [verb](reference-verbs.md). See [begin/end blocks](reference-dsl-control-structures.md#beginend-blocks). ## block -A group of statements between `{` and `}` in the [Miller programming language](programming-language.md), +A group of statements between `{` and `}` in the [Miller programming language](miller-programming-language.md), including [if-statement bodies](#if), [for-loop bodies](#for), [begin-block bodies](#begin), [end-block bodies](#end), etc. @@ -151,18 +151,18 @@ integer. Stands for _delimited key-value pairs_. A Miller-specific [file format](file-formats.md#dkvp-key-value-pairs), with each line of a file being of the form `x=1,y=2,z=3`. For historical reasons, this is Miller's default -format unless [flags](reference-main-io-options.md) such as `--csv` are +format unless [flags](reference-main-flag-list.md) such as `--csv` are supplied. You can also make CSV your default format using a [.mlrrc file](customization.md). ## do A [keyword](#keyword) which is used to indicate the start of a [do-while loop](reference-dsl-control-structures.md) -in the [Miller programming language](programming-language.md). +in the [Miller programming language](miller-programming-language.md). ## DSL Stands for _domain-specific language_. -The [Miller programming language](programming-language.md) is embedded within +The [Miller programming language](miller-programming-language.md) is embedded within the [put and filter verbs](reference-dsl.md). It's a language with its own syntax and semantics; the Miller executable does not embed, say, Python or Lua as a language for put and filter statements. This makes the Miller programming @@ -172,7 +172,7 @@ or (more briefly) a _DSL_. ## dump A [keyword](#keyword) in the [Miller programming -language](programming-language.md) which is used for printing variables to the +language](miller-programming-language.md) which is used for printing variables to the screen (namely, to [stdout](#stdout)). Largely synonymous with [`print`](#print), except that `print` with no arguments prints nothing, while `dump` with no arguments displays all currently defined [out-of-stream @@ -190,7 +190,7 @@ See also [dump statements](reference-dsl-output-statements.md#dump-statements). A [keyword](#keyword) which is used to indicate the else-if-part of an [if-statement](reference-dsl-control-structures.md) in the [Miller programming -language](programming-language.md). In [some +language](miller-programming-language.md). In [some languages](reference-dsl-differences.md) this is `elsif` or `else if`; in Miller's programming language, `elif`. @@ -198,12 +198,12 @@ Miller's programming language, `elif`. A [keyword](#keyword) which is used to indicate the else-part of an [if-statement](reference-dsl-control-structures.md) in the [Miller programming -language](programming-language.md). See also [`elif`](#elif). +language](miller-programming-language.md). See also [`elif`](#elif). ## emit, emitf, emitp Three [keywords](#keyword) in the -[Miller programming language](programming-language.md) +[Miller programming language](miller-programming-language.md) for injecting new records into the [record](#record) [stream](#stream) using the [put](#put) or [filter](#filter) [verbs](#verb). @@ -216,7 +216,7 @@ Refers to the string with zero characters. For example, in a CSV file with [head ## end -A [keyword](#keyword) in the [Miller programming language](programming-language.md) +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) indicating the start of an end [block](#block) within an instance of the [`put`](#put) or [`filter`](#filter) [verb](reference-verbs.md). See [begin/end blocks](reference-dsl-control-structures.md#beginend-blocks). @@ -225,7 +225,7 @@ indicating the start of an end A [keyword](#keyword) in the -[Miller programming language](programming-language.md) +[Miller programming language](miller-programming-language.md) for accessing a readable/writable [map](#map) of [environment variables](https://en.wikipedia.org/wiki/Environment_variable) @@ -239,7 +239,7 @@ Same as [`printn`](#printn), except it prints to [stderr](#stderr) rather than [ ## false -A [keyword](#keyword) in the [Miller programming language](programming-language.md) for the +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) for the boolean literal; signified by `true` in Python; in some languages (such as C) signified by the zero integer value. @@ -256,7 +256,7 @@ Examples include [CSV](#csv), [TSV](#tsv), and [JSON](#json). See the ## FILENAME A [built-in variable](reference-dsl-variables.md#built-in-variables) in the -[Miller programming language](programming-language.md) referring to the name +[Miller programming language](miller-programming-language.md) referring to the name of the current file being processed as Miller [streams](#stream) through your data. See the section on [built-in variables](reference-dsl-variables.md#built-in-variables). @@ -264,7 +264,7 @@ See the section on [built-in variables](reference-dsl-variables.md#built-in-vari ## FILENUM A [built-in variable](reference-dsl-variables.md#built-in-variables) in the -[Miller programming language](programming-language.md) referring to the [one-up](#one-up) +[Miller programming language](miller-programming-language.md) referring to the [one-up](#one-up) index of the current file being processed as Miller [streams](#stream) through your data. See the section on [built-in variables](reference-dsl-variables.md#built-in-variables). @@ -272,17 +272,25 @@ See the section on [built-in variables](reference-dsl-variables.md#built-in-vari ## filter Along with [put](#put), one of the Miller [verbs](#verb) which use the [Miller -programming language](programming-language.md). +programming language](miller-programming-language.md). Also, a [keyword](#keyword) which you can use within `put` statements: see the page on [DSL filter statements](reference-dsl-filter-statements.md). See the [DSL overview](#reference-dsl). +## flatten + +To convert map-valued and/or array-valued fields to something representable in CSV +and other non-JSON [file formats](file-formats.md) -- either by JSON-stringifying +them or by _key spreading_. See the [flatten/unflatten page](flatten-unflatten.md). + +See also [unflatten](#unflatten). + ## float A floating-point number as a value in Miller records, and in the [Miller -programming language](programming-language.md). Floats interconvert seamlessly +programming language](miller-programming-language.md). Floats interconvert seamlessly with [integers](#int) using Miller's [arithmetic rules](reference-main-arithmetic.md), so usually you only need to think of _numbers_, rather than ints and floats separately. @@ -301,7 +309,7 @@ See also the [section on built-in variables](reference-dsl-variables.md#built-in ## for A [keyword](#keyword) which is used to indicate the start of a [for-loop](reference-dsl-control-structures.md) -in the [Miller programming language](programming-language.md). +in the [Miller programming language](miller-programming-language.md). ## format @@ -311,12 +319,12 @@ See [file format](#file-format). A [keyword](#keyword) used for defining a [user-defined functions](reference-dsl-user-defined-functions.md) in the -[Miller programming language](programming-language.md). +[Miller programming language](miller-programming-language.md). ## function A bit of callable code in the -[Miller programming language](programming-language.md) +[Miller programming language](miller-programming-language.md) which takes zero or more arguments, and optionall returns a value. See the [page on built-in functions](reference-dsl-builtin-functions.md) to see @@ -353,7 +361,7 @@ Referring to data where all records have the same keys, in the same order. See ## if A [keyword](#keyword) which is used to indicate the start of an [if-statement](reference-dsl-control-structures.md) -in the [Miller programming language](programming-language.md). +in the [Miller programming language](miller-programming-language.md). ## IFS @@ -362,7 +370,7 @@ Stands for _input field separator_. See the [separators page](reference-main-sep ## in A [keyword](#keyword) in the -[Miller programming language](programming-language.md) +[Miller programming language](miller-programming-language.md) for [single-variable for-loops](reference-dsl-control-structures.md#single-variable-for-loops) and [key-value for-loops](reference-dsl-control-structures.md#key-value-for-loops). @@ -379,12 +387,12 @@ In all these cases, the original `myfile.csv` is left unmodified. But using Miller's `-I` flag, we can update the original file: e.g. `mlr -I --csv sort myfile.csv` won't print the sorted output to the terminal, but rather will write it back to `myfile.csv`. -See also the [section on in-place mode](reference-main-io-options.md#in-place-mode). +See also the [section on in-place mode](reference-main-in-place-processing.md). ## int A 64-bit signed integer as a value in Miller records, and in the [Miller -programming language](programming-language.md). Ints interconvert seamlessly +programming language](miller-programming-language.md). Ints interconvert seamlessly with [floats](#float) using Miller's [arithmetic rules](reference-main-arithmetic.md), so usually you only need to think of _numbers_, rather than ints and floats separately. @@ -412,7 +420,7 @@ The string index in a [map](#map). Also, the name of a field in a [record](#reco ## keyword A reserved name in the -[Miller programming language](programming-language.md) +[Miller programming language](miller-programming-language.md) which you can't use for any other purpose. For example, `if`, `for`, and `while` are keywords; trying to define a [local variable](#local-variable) `if = 3` will result in a parse error. @@ -425,7 +433,7 @@ as non-line-oriented formats such as [CSV, TSV, JSON, and others](file-formats.m ## local variable -A [variable](#variable) in the [Miller programming language](programming-language.md) +A [variable](#variable) in the [Miller programming language](miller-programming-language.md) whose extent is limited to the expression in which it appears; contrast [out-of-stream variables](#out-of-stream-variable) which endure across the entire [record stream](#streaming). See the @@ -444,7 +452,7 @@ Regardless, you can find the same content [within this documentation site](manpa ## map -A data structure in the [Miller programming language](programming-language.md) containing +A data structure in the [Miller programming language](miller-programming-language.md) containing an ordered sequence of [key](#key)-[value](#value) pairs. See the [maps page](reference-main-maps.md) for more information. @@ -468,13 +476,13 @@ Miller version 6](miller-on-windows.md). ## M_E A [built-in variable](reference-dsl-variables.md#built-in-variables) in the -[Miller programming language](programming-language.md) referring to the mathematic +[Miller programming language](miller-programming-language.md) referring to the mathematic constant [e](https://en.wikipedia.org/wiki/E_(mathematical_constant)). The _M_ is for _math_. ## M_PI A [built-in variable](reference-dsl-variables.md#built-in-variables) in the -[Miller programming language](programming-language.md) referring to the +[Miller programming language](miller-programming-language.md) referring to the mathematic constant [Ï€](https://en.wikipedia.org/wiki/Pi). The _M_ is for _math_. @@ -482,7 +490,7 @@ mathematic constant Stands for _number of fields_. A read-only [built-in variable](reference-dsl-variables.md#built-in-variables) in the [Miller -programming language](programming-language.md) which shows the number of fields +programming language](miller-programming-language.md) which shows the number of fields in the current record. ## NIDX @@ -522,7 +530,7 @@ Miller has more than one kind: see the page on [null/empty/absent data](referenc The `num` [keyword](#keyword) is used for [type declaration]( reference-dsl-variables.md#type-declarations-for-local-variables-function-parameter-and-function-return-values) -in the [Miller programming language](programming-language.md). The `num` type +in the [Miller programming language](miller-programming-language.md). The `num` type encompasses both [`int`](#int) and [`float`](#float). Ints and floats interconvert seamlessly using Miller's [arithmetic rules](reference-main-arithmetic.md), so usually you only need to think of @@ -556,7 +564,7 @@ Stands for _output record separator_. See the [separators page](reference-main-s ## Out-of-stream variable [Variables](#variable), prefixed with the `@` sigil, which persist their values -across multiple records in the [Miller programming language](programming-language.md). +across multiple records in the [Miller programming language](miller-programming-language.md). See [out-of-stream variables](reference-dsl-variables.md#out-of-stream-variables) for more information. @@ -567,7 +575,7 @@ for [key](#key)-[value](#value) pairs, with columns vertically aligned for easy ## print -A [keyword](#keyword) in the [Miller programming language](programming-language.md) for +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) for printing things to the terminal, with final newline printed for you. See also [`printn`](#printn) which does not insert the final newline. @@ -576,7 +584,7 @@ See also [`emit`](#emit) which inserts new [records](#record) into the [record s ## printn -A [keyword](#keyword) in the [Miller programming language](programming-language.md) for +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) for printing things to the terminal, with no final newline printed for you. See also [`print`](#print) which does insert the final newline. @@ -584,7 +592,7 @@ See also [`print`](#print) which does insert the final newline. ## put Along with [filter](#filter), one of the Miller [verbs](#verb) which -use the [Miller programming language](programming-language.md). +use the [Miller programming language](miller-programming-language.md). See the [DSL overview](#reference-dsl). @@ -628,7 +636,7 @@ arguments: a place where you can type `1+2` and get `3`. Miller has a ## return -A [keyword](#keyword) in the [Miller programming language](programming-language.md) which +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) which is used for returning control from a [function](reference-dsl-user-defined-functions.md) to its caller, optionally returning a value from the function. @@ -636,7 +644,7 @@ returning a value from the function. ## semicolon Semicolons are used to [delimit statements](reference-dsl-syntax.md) in the -[Miller programming language](programming-language.md). +[Miller programming language](miller-programming-language.md). ## separator @@ -647,7 +655,7 @@ required after every statement; in others such as Python, they're not required at all; in yet others, they're required _in between_ statements but are optional after the last. Miller is in the third category, so we can say that semicolons are _separators_, not [_terminators_](#terminator), within the -[Miller programming language](programming-language.md). +[Miller programming language](miller-programming-language.md). (2) Refers to character sequences which separate records from one another (like newlines, sometimes), fields from one another (like commas in CSV), and keys @@ -663,14 +671,14 @@ page](record-heterogeneity.md#sparse-data). ## stderr A [keyword](#keyword) in the -[Miller programming language](programming-language.md) +[Miller programming language](miller-programming-language.md) for [print, dump, and tee statements](http://127.0.0.1:8000/reference-dsl-output-statements.md#tee-statements) indicating that data are to be sent to the [_standard output_](https://en.wikipedia.org/wiki/Standard_streams#Standard_output_(stderr)). ## stdout A [keyword](#keyword) in the -[Miller programming language](programming-language.md) +[Miller programming language](miller-programming-language.md) for [print, dump, and tee statements](http://127.0.0.1:8000/reference-dsl-output-statements.md#tee-statements) indicating that data are to be sent to the [_standard output_](https://en.wikipedia.org/wiki/Standard_streams#Standard_output_(stdout)). @@ -699,7 +707,7 @@ A [keyword](#keyword) used for defining a ## subroutine A [user-definable bit of code](reference-dsl-user-defined-functions.md) in the -[Miller programming language](programming-language.md), intended to be +[Miller programming language](miller-programming-language.md), intended to be called for its side effects rather than for returning a value. ## tee @@ -717,7 +725,7 @@ thing4`. Miller has a tee in two places: (1) a [verb](#verb) you can insert into a Miller [then-chain](reference-main-then-chaining.md), and (2) an [output statement](reference-dsl-output-statements.md) in the [Miller programming -language](programming-language.md). Using the latter, you have the additional +language](miller-programming-language.md). Using the latter, you have the additional option of using a tee-to file name which is variable, perhaps depending on the current record. For example, if you have a large file with an `id` column, you can split it into several files, one for each distinct `id`. See the [section @@ -738,7 +746,7 @@ required after every statement; in others such as Python, they're not required at all; in yet others, they're required _in between_ statements but are optional after the last. Miller is in the third category, so we can say that semicolons are [_separators_](#separator), not _terminators_, within the -[Miller programming language](programming-language.md). +[Miller programming language](miller-programming-language.md). ## toolkit @@ -746,7 +754,7 @@ See [Unix toolkit](#unix-toolkit). ## true -A [keyword](#keyword) in the [Miller programming language](programming-language.md) for the boolean +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) for the boolean literal; signified by `True` in Python; in some languages (such as C) signified by non-zero integer values. @@ -756,6 +764,13 @@ Stands for _tab-separated values_. A popular [file format](file-formats.md#csvtsvasvusvetc) for tabular data (tab-separated values) supported by Miller. +## unflatten + +To undo the [flatten](#flatten) operation, restoring map-valued and/or +array-valued fields encoded in CSV and other non-JSON [file +formats](file-formats.md) for JSON output. See the [flatten/unflatten +page](flatten-unflatten.md). + ## Unix toolkit The term [_Unix toolkit_](https://en.wikipedia.org/wiki/List_of_Unix_commands) @@ -774,7 +789,7 @@ to operate on richer [file formats](file-formats.md) such as [CSV](#csv), [TSV]( ## unset -A [keyword](#keyword) in the [Miller programming language](programming-language.md) +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) for removing the definition of a local or out-of-stream variable, or for removing a [key](#key) from the current [record](#record). @@ -800,7 +815,7 @@ letting you declare a new variable, in an inner scope, of the same name as anoth ## variable -A way to access data by name within the [Miller programming language](programming-language.md). +A way to access data by name within the [Miller programming language](miller-programming-language.md). See the [DSL variables page](reference-dsl-variables.md). ## verb @@ -817,7 +832,7 @@ See the [List of verbs](reference-verbs.md) page. A [keyword](#keyword) which is used to indicate the start of a [while-loop](reference-dsl-control-structures.md), and also used in do-while -loops, in the [Miller programming language](programming-language.md). +loops, in the [Miller programming language](miller-programming-language.md). ## XTAB diff --git a/docs6/docs/glossary.md.in b/docs6/src/glossary.md.in similarity index 87% rename from docs6/docs/glossary.md.in rename to docs6/src/glossary.md.in index 9a3007d4b..d209d0d4d 100644 --- a/docs6/docs/glossary.md.in +++ b/docs6/src/glossary.md.in @@ -43,14 +43,14 @@ information. ## begin -A [keyword](#keyword) in the [Miller programming language](programming-language.md) +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) indicating the start of a begin [block](#block) within an instance of the [`put`](#put) or [`filter`](#filter) [verb](reference-verbs.md). See [begin/end blocks](reference-dsl-control-structures.md#beginend-blocks). ## block -A group of statements between `{` and `}` in the [Miller programming language](programming-language.md), +A group of statements between `{` and `}` in the [Miller programming language](miller-programming-language.md), including [if-statement bodies](#if), [for-loop bodies](#for), [begin-block bodies](#begin), [end-block bodies](#end), etc. @@ -135,18 +135,18 @@ integer. Stands for _delimited key-value pairs_. A Miller-specific [file format](file-formats.md#dkvp-key-value-pairs), with each line of a file being of the form `x=1,y=2,z=3`. For historical reasons, this is Miller's default -format unless [flags](reference-main-io-options.md) such as `--csv` are +format unless [flags](reference-main-flag-list.md) such as `--csv` are supplied. You can also make CSV your default format using a [.mlrrc file](customization.md). ## do A [keyword](#keyword) which is used to indicate the start of a [do-while loop](reference-dsl-control-structures.md) -in the [Miller programming language](programming-language.md). +in the [Miller programming language](miller-programming-language.md). ## DSL Stands for _domain-specific language_. -The [Miller programming language](programming-language.md) is embedded within +The [Miller programming language](miller-programming-language.md) is embedded within the [put and filter verbs](reference-dsl.md). It's a language with its own syntax and semantics; the Miller executable does not embed, say, Python or Lua as a language for put and filter statements. This makes the Miller programming @@ -156,7 +156,7 @@ or (more briefly) a _DSL_. ## dump A [keyword](#keyword) in the [Miller programming -language](programming-language.md) which is used for printing variables to the +language](miller-programming-language.md) which is used for printing variables to the screen (namely, to [stdout](#stdout)). Largely synonymous with [`print`](#print), except that `print` with no arguments prints nothing, while `dump` with no arguments displays all currently defined [out-of-stream @@ -174,7 +174,7 @@ See also [dump statements](reference-dsl-output-statements.md#dump-statements). A [keyword](#keyword) which is used to indicate the else-if-part of an [if-statement](reference-dsl-control-structures.md) in the [Miller programming -language](programming-language.md). In [some +language](miller-programming-language.md). In [some languages](reference-dsl-differences.md) this is `elsif` or `else if`; in Miller's programming language, `elif`. @@ -182,12 +182,12 @@ Miller's programming language, `elif`. A [keyword](#keyword) which is used to indicate the else-part of an [if-statement](reference-dsl-control-structures.md) in the [Miller programming -language](programming-language.md). See also [`elif`](#elif). +language](miller-programming-language.md). See also [`elif`](#elif). ## emit, emitf, emitp Three [keywords](#keyword) in the -[Miller programming language](programming-language.md) +[Miller programming language](miller-programming-language.md) for injecting new records into the [record](#record) [stream](#stream) using the [put](#put) or [filter](#filter) [verbs](#verb). @@ -200,7 +200,7 @@ Refers to the string with zero characters. For example, in a CSV file with [head ## end -A [keyword](#keyword) in the [Miller programming language](programming-language.md) +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) indicating the start of an end [block](#block) within an instance of the [`put`](#put) or [`filter`](#filter) [verb](reference-verbs.md). See [begin/end blocks](reference-dsl-control-structures.md#beginend-blocks). @@ -209,7 +209,7 @@ indicating the start of an end A [keyword](#keyword) in the -[Miller programming language](programming-language.md) +[Miller programming language](miller-programming-language.md) for accessing a readable/writable [map](#map) of [environment variables](https://en.wikipedia.org/wiki/Environment_variable) @@ -223,7 +223,7 @@ Same as [`printn`](#printn), except it prints to [stderr](#stderr) rather than [ ## false -A [keyword](#keyword) in the [Miller programming language](programming-language.md) for the +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) for the boolean literal; signified by `true` in Python; in some languages (such as C) signified by the zero integer value. @@ -240,7 +240,7 @@ Examples include [CSV](#csv), [TSV](#tsv), and [JSON](#json). See the ## FILENAME A [built-in variable](reference-dsl-variables.md#built-in-variables) in the -[Miller programming language](programming-language.md) referring to the name +[Miller programming language](miller-programming-language.md) referring to the name of the current file being processed as Miller [streams](#stream) through your data. See the section on [built-in variables](reference-dsl-variables.md#built-in-variables). @@ -248,7 +248,7 @@ See the section on [built-in variables](reference-dsl-variables.md#built-in-vari ## FILENUM A [built-in variable](reference-dsl-variables.md#built-in-variables) in the -[Miller programming language](programming-language.md) referring to the [one-up](#one-up) +[Miller programming language](miller-programming-language.md) referring to the [one-up](#one-up) index of the current file being processed as Miller [streams](#stream) through your data. See the section on [built-in variables](reference-dsl-variables.md#built-in-variables). @@ -256,17 +256,25 @@ See the section on [built-in variables](reference-dsl-variables.md#built-in-vari ## filter Along with [put](#put), one of the Miller [verbs](#verb) which use the [Miller -programming language](programming-language.md). +programming language](miller-programming-language.md). Also, a [keyword](#keyword) which you can use within `put` statements: see the page on [DSL filter statements](reference-dsl-filter-statements.md). See the [DSL overview](#reference-dsl). +## flatten + +To convert map-valued and/or array-valued fields to something representable in CSV +and other non-JSON [file formats](file-formats.md) -- either by JSON-stringifying +them or by _key spreading_. See the [flatten/unflatten page](flatten-unflatten.md). + +See also [unflatten](#unflatten). + ## float A floating-point number as a value in Miller records, and in the [Miller -programming language](programming-language.md). Floats interconvert seamlessly +programming language](miller-programming-language.md). Floats interconvert seamlessly with [integers](#int) using Miller's [arithmetic rules](reference-main-arithmetic.md), so usually you only need to think of _numbers_, rather than ints and floats separately. @@ -285,7 +293,7 @@ See also the [section on built-in variables](reference-dsl-variables.md#built-in ## for A [keyword](#keyword) which is used to indicate the start of a [for-loop](reference-dsl-control-structures.md) -in the [Miller programming language](programming-language.md). +in the [Miller programming language](miller-programming-language.md). ## format @@ -295,12 +303,12 @@ See [file format](#file-format). A [keyword](#keyword) used for defining a [user-defined functions](reference-dsl-user-defined-functions.md) in the -[Miller programming language](programming-language.md). +[Miller programming language](miller-programming-language.md). ## function A bit of callable code in the -[Miller programming language](programming-language.md) +[Miller programming language](miller-programming-language.md) which takes zero or more arguments, and optionall returns a value. See the [page on built-in functions](reference-dsl-builtin-functions.md) to see @@ -337,7 +345,7 @@ Referring to data where all records have the same keys, in the same order. See ## if A [keyword](#keyword) which is used to indicate the start of an [if-statement](reference-dsl-control-structures.md) -in the [Miller programming language](programming-language.md). +in the [Miller programming language](miller-programming-language.md). ## IFS @@ -346,7 +354,7 @@ Stands for _input field separator_. See the [separators page](reference-main-sep ## in A [keyword](#keyword) in the -[Miller programming language](programming-language.md) +[Miller programming language](miller-programming-language.md) for [single-variable for-loops](reference-dsl-control-structures.md#single-variable-for-loops) and [key-value for-loops](reference-dsl-control-structures.md#key-value-for-loops). @@ -363,12 +371,12 @@ In all these cases, the original `myfile.csv` is left unmodified. But using Miller's `-I` flag, we can update the original file: e.g. `mlr -I --csv sort myfile.csv` won't print the sorted output to the terminal, but rather will write it back to `myfile.csv`. -See also the [section on in-place mode](reference-main-io-options.md#in-place-mode). +See also the [section on in-place mode](reference-main-in-place-processing.md). ## int A 64-bit signed integer as a value in Miller records, and in the [Miller -programming language](programming-language.md). Ints interconvert seamlessly +programming language](miller-programming-language.md). Ints interconvert seamlessly with [floats](#float) using Miller's [arithmetic rules](reference-main-arithmetic.md), so usually you only need to think of _numbers_, rather than ints and floats separately. @@ -396,7 +404,7 @@ The string index in a [map](#map). Also, the name of a field in a [record](#reco ## keyword A reserved name in the -[Miller programming language](programming-language.md) +[Miller programming language](miller-programming-language.md) which you can't use for any other purpose. For example, `if`, `for`, and `while` are keywords; trying to define a [local variable](#local-variable) `if = 3` will result in a parse error. @@ -409,7 +417,7 @@ as non-line-oriented formats such as [CSV, TSV, JSON, and others](file-formats.m ## local variable -A [variable](#variable) in the [Miller programming language](programming-language.md) +A [variable](#variable) in the [Miller programming language](miller-programming-language.md) whose extent is limited to the expression in which it appears; contrast [out-of-stream variables](#out-of-stream-variable) which endure across the entire [record stream](#streaming). See the @@ -428,7 +436,7 @@ Regardless, you can find the same content [within this documentation site](manpa ## map -A data structure in the [Miller programming language](programming-language.md) containing +A data structure in the [Miller programming language](miller-programming-language.md) containing an ordered sequence of [key](#key)-[value](#value) pairs. See the [maps page](reference-main-maps.md) for more information. @@ -452,13 +460,13 @@ Miller version 6](miller-on-windows.md). ## M_E A [built-in variable](reference-dsl-variables.md#built-in-variables) in the -[Miller programming language](programming-language.md) referring to the mathematic +[Miller programming language](miller-programming-language.md) referring to the mathematic constant [e](https://en.wikipedia.org/wiki/E_(mathematical_constant)). The _M_ is for _math_. ## M_PI A [built-in variable](reference-dsl-variables.md#built-in-variables) in the -[Miller programming language](programming-language.md) referring to the +[Miller programming language](miller-programming-language.md) referring to the mathematic constant [Ï€](https://en.wikipedia.org/wiki/Pi). The _M_ is for _math_. @@ -466,7 +474,7 @@ mathematic constant Stands for _number of fields_. A read-only [built-in variable](reference-dsl-variables.md#built-in-variables) in the [Miller -programming language](programming-language.md) which shows the number of fields +programming language](miller-programming-language.md) which shows the number of fields in the current record. ## NIDX @@ -506,7 +514,7 @@ Miller has more than one kind: see the page on [null/empty/absent data](referenc The `num` [keyword](#keyword) is used for [type declaration]( reference-dsl-variables.md#type-declarations-for-local-variables-function-parameter-and-function-return-values) -in the [Miller programming language](programming-language.md). The `num` type +in the [Miller programming language](miller-programming-language.md). The `num` type encompasses both [`int`](#int) and [`float`](#float). Ints and floats interconvert seamlessly using Miller's [arithmetic rules](reference-main-arithmetic.md), so usually you only need to think of @@ -540,7 +548,7 @@ Stands for _output record separator_. See the [separators page](reference-main-s ## Out-of-stream variable [Variables](#variable), prefixed with the `@` sigil, which persist their values -across multiple records in the [Miller programming language](programming-language.md). +across multiple records in the [Miller programming language](miller-programming-language.md). See [out-of-stream variables](reference-dsl-variables.md#out-of-stream-variables) for more information. @@ -551,7 +559,7 @@ for [key](#key)-[value](#value) pairs, with columns vertically aligned for easy ## print -A [keyword](#keyword) in the [Miller programming language](programming-language.md) for +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) for printing things to the terminal, with final newline printed for you. See also [`printn`](#printn) which does not insert the final newline. @@ -560,7 +568,7 @@ See also [`emit`](#emit) which inserts new [records](#record) into the [record s ## printn -A [keyword](#keyword) in the [Miller programming language](programming-language.md) for +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) for printing things to the terminal, with no final newline printed for you. See also [`print`](#print) which does insert the final newline. @@ -568,7 +576,7 @@ See also [`print`](#print) which does insert the final newline. ## put Along with [filter](#filter), one of the Miller [verbs](#verb) which -use the [Miller programming language](programming-language.md). +use the [Miller programming language](miller-programming-language.md). See the [DSL overview](#reference-dsl). @@ -612,7 +620,7 @@ arguments: a place where you can type `1+2` and get `3`. Miller has a ## return -A [keyword](#keyword) in the [Miller programming language](programming-language.md) which +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) which is used for returning control from a [function](reference-dsl-user-defined-functions.md) to its caller, optionally returning a value from the function. @@ -620,7 +628,7 @@ returning a value from the function. ## semicolon Semicolons are used to [delimit statements](reference-dsl-syntax.md) in the -[Miller programming language](programming-language.md). +[Miller programming language](miller-programming-language.md). ## separator @@ -631,7 +639,7 @@ required after every statement; in others such as Python, they're not required at all; in yet others, they're required _in between_ statements but are optional after the last. Miller is in the third category, so we can say that semicolons are _separators_, not [_terminators_](#terminator), within the -[Miller programming language](programming-language.md). +[Miller programming language](miller-programming-language.md). (2) Refers to character sequences which separate records from one another (like newlines, sometimes), fields from one another (like commas in CSV), and keys @@ -647,14 +655,14 @@ page](record-heterogeneity.md#sparse-data). ## stderr A [keyword](#keyword) in the -[Miller programming language](programming-language.md) +[Miller programming language](miller-programming-language.md) for [print, dump, and tee statements](http://127.0.0.1:8000/reference-dsl-output-statements.md#tee-statements) indicating that data are to be sent to the [_standard output_](https://en.wikipedia.org/wiki/Standard_streams#Standard_output_(stderr)). ## stdout A [keyword](#keyword) in the -[Miller programming language](programming-language.md) +[Miller programming language](miller-programming-language.md) for [print, dump, and tee statements](http://127.0.0.1:8000/reference-dsl-output-statements.md#tee-statements) indicating that data are to be sent to the [_standard output_](https://en.wikipedia.org/wiki/Standard_streams#Standard_output_(stdout)). @@ -683,7 +691,7 @@ A [keyword](#keyword) used for defining a ## subroutine A [user-definable bit of code](reference-dsl-user-defined-functions.md) in the -[Miller programming language](programming-language.md), intended to be +[Miller programming language](miller-programming-language.md), intended to be called for its side effects rather than for returning a value. ## tee @@ -701,7 +709,7 @@ thing4`. Miller has a tee in two places: (1) a [verb](#verb) you can insert into a Miller [then-chain](reference-main-then-chaining.md), and (2) an [output statement](reference-dsl-output-statements.md) in the [Miller programming -language](programming-language.md). Using the latter, you have the additional +language](miller-programming-language.md). Using the latter, you have the additional option of using a tee-to file name which is variable, perhaps depending on the current record. For example, if you have a large file with an `id` column, you can split it into several files, one for each distinct `id`. See the [section @@ -722,7 +730,7 @@ required after every statement; in others such as Python, they're not required at all; in yet others, they're required _in between_ statements but are optional after the last. Miller is in the third category, so we can say that semicolons are [_separators_](#separator), not _terminators_, within the -[Miller programming language](programming-language.md). +[Miller programming language](miller-programming-language.md). ## toolkit @@ -730,7 +738,7 @@ See [Unix toolkit](#unix-toolkit). ## true -A [keyword](#keyword) in the [Miller programming language](programming-language.md) for the boolean +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) for the boolean literal; signified by `True` in Python; in some languages (such as C) signified by non-zero integer values. @@ -740,6 +748,13 @@ Stands for _tab-separated values_. A popular [file format](file-formats.md#csvtsvasvusvetc) for tabular data (tab-separated values) supported by Miller. +## unflatten + +To undo the [flatten](#flatten) operation, restoring map-valued and/or +array-valued fields encoded in CSV and other non-JSON [file +formats](file-formats.md) for JSON output. See the [flatten/unflatten +page](flatten-unflatten.md). + ## Unix toolkit The term [_Unix toolkit_](https://en.wikipedia.org/wiki/List_of_Unix_commands) @@ -758,7 +773,7 @@ to operate on richer [file formats](file-formats.md) such as [CSV](#csv), [TSV]( ## unset -A [keyword](#keyword) in the [Miller programming language](programming-language.md) +A [keyword](#keyword) in the [Miller programming language](miller-programming-language.md) for removing the definition of a local or out-of-stream variable, or for removing a [key](#key) from the current [record](#record). @@ -784,7 +799,7 @@ letting you declare a new variable, in an inner scope, of the same name as anoth ## variable -A way to access data by name within the [Miller programming language](programming-language.md). +A way to access data by name within the [Miller programming language](miller-programming-language.md). See the [DSL variables page](reference-dsl-variables.md). ## verb @@ -801,7 +816,7 @@ See the [List of verbs](reference-verbs.md) page. A [keyword](#keyword) which is used to indicate the start of a [while-loop](reference-dsl-control-structures.md), and also used in do-while -loops, in the [Miller programming language](programming-language.md). +loops, in the [Miller programming language](miller-programming-language.md). ## XTAB diff --git a/docs6/docs/gz-example.csv.gz b/docs6/src/gz-example.csv.gz similarity index 100% rename from docs6/docs/gz-example.csv.gz rename to docs6/src/gz-example.csv.gz diff --git a/docs6/docs/h1foo b/docs6/src/h1foo similarity index 100% rename from docs6/docs/h1foo rename to docs6/src/h1foo diff --git a/docs6/docs/h2foo b/docs6/src/h2foo similarity index 100% rename from docs6/docs/h2foo rename to docs6/src/h2foo diff --git a/docs6/docs/h3foo b/docs6/src/h3foo similarity index 100% rename from docs6/docs/h3foo rename to docs6/src/h3foo diff --git a/docs6/docs/if-example.mlr b/docs6/src/if-example.mlr similarity index 100% rename from docs6/docs/if-example.mlr rename to docs6/src/if-example.mlr diff --git a/docs6/docs/images/favicon.ico b/docs6/src/images/favicon.ico similarity index 100% rename from docs6/docs/images/favicon.ico rename to docs6/src/images/favicon.ico diff --git a/docs6/docs/images/favicon.png b/docs6/src/images/favicon.png similarity index 100% rename from docs6/docs/images/favicon.png rename to docs6/src/images/favicon.png diff --git a/docs6/docs/index.md b/docs6/src/index.md similarity index 100% rename from docs6/docs/index.md rename to docs6/src/index.md diff --git a/docs6/docs/index.md.in b/docs6/src/index.md.in similarity index 100% rename from docs6/docs/index.md.in rename to docs6/src/index.md.in diff --git a/docs6/docs/installation.md b/docs6/src/installation.md similarity index 100% rename from docs6/docs/installation.md rename to docs6/src/installation.md diff --git a/docs6/docs/installation.md.in b/docs6/src/installation.md.in similarity index 100% rename from docs6/docs/installation.md.in rename to docs6/src/installation.md.in diff --git a/docs6/docs/installing-miller.md b/docs6/src/installing-miller.md similarity index 100% rename from docs6/docs/installing-miller.md rename to docs6/src/installing-miller.md diff --git a/docs6/docs/installing-miller.md.in b/docs6/src/installing-miller.md.in similarity index 100% rename from docs6/docs/installing-miller.md.in rename to docs6/src/installing-miller.md.in diff --git a/docs6/docs/internationalization.md b/docs6/src/internationalization.md similarity index 100% rename from docs6/docs/internationalization.md rename to docs6/src/internationalization.md diff --git a/docs6/docs/internationalization.md.in b/docs6/src/internationalization.md.in similarity index 100% rename from docs6/docs/internationalization.md.in rename to docs6/src/internationalization.md.in diff --git a/docs6/docs/javascripts/tables.js b/docs6/src/javascripts/tables.js similarity index 100% rename from docs6/docs/javascripts/tables.js rename to docs6/src/javascripts/tables.js diff --git a/docs6/docs/keystroke-savers.md b/docs6/src/keystroke-savers.md similarity index 100% rename from docs6/docs/keystroke-savers.md rename to docs6/src/keystroke-savers.md diff --git a/docs6/docs/keystroke-savers.md.in b/docs6/src/keystroke-savers.md.in similarity index 100% rename from docs6/docs/keystroke-savers.md.in rename to docs6/src/keystroke-savers.md.in diff --git a/docs6/docs/link.txt b/docs6/src/link.txt similarity index 100% rename from docs6/docs/link.txt rename to docs6/src/link.txt diff --git a/docs6/docs/log-processing-examples.md b/docs6/src/log-processing-examples.md similarity index 100% rename from docs6/docs/log-processing-examples.md rename to docs6/src/log-processing-examples.md diff --git a/docs6/docs/log-processing-examples.md.in b/docs6/src/log-processing-examples.md.in similarity index 100% rename from docs6/docs/log-processing-examples.md.in rename to docs6/src/log-processing-examples.md.in diff --git a/docs6/docs/log.txt b/docs6/src/log.txt similarity index 100% rename from docs6/docs/log.txt rename to docs6/src/log.txt diff --git a/docs6/docs/make.bat b/docs6/src/make.bat similarity index 100% rename from docs6/docs/make.bat rename to docs6/src/make.bat diff --git a/docs6/docs/manpage.md b/docs6/src/manpage.md similarity index 97% rename from docs6/docs/manpage.md rename to docs6/src/manpage.md index 609e8b955..51e98f27f 100644 --- a/docs6/docs/manpage.md +++ b/docs6/src/manpage.md @@ -29,11 +29,13 @@ NAME SYNOPSIS Usage: mlr [flags] {verb} [verb-dependent options ...] {zero or more - file names} Output of one verb may be chained as input to another using - "then", e.g. - mlr stats1 -a min,mean,max -f flag,u,v -g color then sort -f color - Please see 'mlr help topics' for more information. Please also see - https://johnkerl.org/miller6 + file names} If zero file names are provided, standard input is read. + Example: + mlr --csv sort -f shape example.csv Output of one verb may be chained + as input to another using "then", e.g. + mlr --csv stats1 -a min,mean,max -f quantity then sort -f color + example.csv Please see 'mlr help topics' for more information. Please + also see https://johnkerl.org/miller6 DESCRIPTION @@ -54,54 +56,67 @@ EXAMPLES mlr --icsv --ojson put '$ratio = $quantity / $rate' example.csv mlr --icsv --opprint --from example.csv sort -nr index then cut -f shape,quantity -DATA FORMATS - Type 'mlr help {topic}' for any of the following: - Essentials: - mlr help topics - mlr help basic-examples - mlr help file-formats - Flags: - mlr help flags - mlr help comments-in-data-flags - mlr help compressed-data-flags - mlr help csv-only-flags - mlr help file-format-flags - mlr help flatten-unflatten-flags - mlr help format-conversion-keystroke-saver-flags - mlr help json-only-flags - mlr help legacy-flags - mlr help miscellaneous-flags - mlr help output-colorization-flags - mlr help pprint-only-flags - mlr help separator-flags - Verbs: - mlr help list-verbs - mlr help usage-verbs - mlr help verb - Functions: - mlr help list-functions - mlr help list-function-classes - mlr help list-functions-in-class - mlr help usage-functions - mlr help usage-functions-by-class - mlr help function - Keywords: - mlr help list-keywords - mlr help usage-keywords - mlr help keyword - Other: - mlr help auxents - mlr help mlrrc - mlr help output-colorization - mlr help type-arithmetic-info - Shorthands: - mlr -g = mlr help flags - mlr -l = mlr help list-verbs - mlr -L = mlr help usage-verbs - mlr -f = mlr help list-functions - mlr -F = mlr help usage-functions - mlr -k = mlr help list-keywords - mlr -K = mlr help usage-keywords +FILE FORMATS + CSV/CSV-lite: comma-separated values with separate header line + TSV: same but with tabs in places of commas + +---------------------+ + | apple,bat,cog | + | 1,2,3 | Record 1: "apple":"1", "bat":"2", "cog":"3" + | 4,5,6 | Record 2: "apple":"4", "bat":"5", "cog":"6" + +---------------------+ + + JSON (sequence or array of objects): + +---------------------+ + | { | + | "apple": 1, | Record 1: "apple":"1", "bat":"2", "cog":"3" + | "bat": 2, | + | "cog": 3 | + | } | + | { | + | "dish": { | Record 2: "dish:egg":"7", + | "egg": 7, | "dish:flint":"8", "garlic":"" + | "flint": 8 | + | }, | + | "garlic": "" | + | } | + +---------------------+ + + PPRINT: pretty-printed tabular + +---------------------+ + | apple bat cog | + | 1 2 3 | Record 1: "apple:"1", "bat":"2", "cog":"3" + | 4 5 6 | Record 2: "apple":"4", "bat":"5", "cog":"6" + +---------------------+ + + Markdown tabular (supported for output only): + +-----------------------+ + | | apple | bat | cog | | + | | --- | --- | --- | | + | | 1 | 2 | 3 | | Record 1: "apple:"1", "bat":"2", "cog":"3" + | | 4 | 5 | 6 | | Record 2: "apple":"4", "bat":"5", "cog":"6" + +-----------------------+ + + XTAB: pretty-printed transposed tabular + +---------------------+ + | apple 1 | Record 1: "apple":"1", "bat":"2", "cog":"3" + | bat 2 | + | cog 3 | + | | + | dish 7 | Record 2: "dish":"7", "egg":"8" + | egg 8 | + +---------------------+ + + DKVP: delimited key-value pairs (Miller default format) + +---------------------+ + | apple=1,bat=2,cog=3 | Record 1: "apple":"1", "bat":"2", "cog":"3" + | dish=7,egg=8,flint | Record 2: "dish":"7", "egg":"8", "3":"flint" + +---------------------+ + + NIDX: implicitly numerically indexed (Unix-toolkit style) + +---------------------+ + | the quick brown | Record 1: "1":"the", "2":"quick", "3":"brown" + | fox jumped | Record 2: "1":"fox", "2":"jumped" + +---------------------+ HELP OPTIONS Type 'mlr help {topic}' for any of the following: @@ -380,6 +395,8 @@ FORMAT-CONVERSION KEYSTROKE-SAVER FLAGS | XTAB | --x2c | --x2t | --x2j | --x2d | --x2n | | --x2p | --x2m | | PPRINT | --p2c | --p2t | --p2j | --p2d | --p2n | --p2x | | --p2m | + -p Keystroke-saver for `--nidx --fs space --repifs`. + -T Keystroke-saver for `--nidx --fs tab`. JSON-ONLY FLAGS These are flags which are applicable to JSON format. @@ -2780,5 +2797,5 @@ SEE ALSO - 2021-09-09 MILLER(1) + 2021-09-15 MILLER(1) diff --git a/docs6/docs/manpage.md.in b/docs6/src/manpage.md.in similarity index 100% rename from docs6/docs/manpage.md.in rename to docs6/src/manpage.md.in diff --git a/docs6/docs/manpage.txt b/docs6/src/manpage.txt similarity index 97% rename from docs6/docs/manpage.txt rename to docs6/src/manpage.txt index c70dde832..197e3798a 100644 --- a/docs6/docs/manpage.txt +++ b/docs6/src/manpage.txt @@ -8,11 +8,13 @@ NAME SYNOPSIS Usage: mlr [flags] {verb} [verb-dependent options ...] {zero or more - file names} Output of one verb may be chained as input to another using - "then", e.g. - mlr stats1 -a min,mean,max -f flag,u,v -g color then sort -f color - Please see 'mlr help topics' for more information. Please also see - https://johnkerl.org/miller6 + file names} If zero file names are provided, standard input is read. + Example: + mlr --csv sort -f shape example.csv Output of one verb may be chained + as input to another using "then", e.g. + mlr --csv stats1 -a min,mean,max -f quantity then sort -f color + example.csv Please see 'mlr help topics' for more information. Please + also see https://johnkerl.org/miller6 DESCRIPTION @@ -33,54 +35,67 @@ EXAMPLES mlr --icsv --ojson put '$ratio = $quantity / $rate' example.csv mlr --icsv --opprint --from example.csv sort -nr index then cut -f shape,quantity -DATA FORMATS - Type 'mlr help {topic}' for any of the following: - Essentials: - mlr help topics - mlr help basic-examples - mlr help file-formats - Flags: - mlr help flags - mlr help comments-in-data-flags - mlr help compressed-data-flags - mlr help csv-only-flags - mlr help file-format-flags - mlr help flatten-unflatten-flags - mlr help format-conversion-keystroke-saver-flags - mlr help json-only-flags - mlr help legacy-flags - mlr help miscellaneous-flags - mlr help output-colorization-flags - mlr help pprint-only-flags - mlr help separator-flags - Verbs: - mlr help list-verbs - mlr help usage-verbs - mlr help verb - Functions: - mlr help list-functions - mlr help list-function-classes - mlr help list-functions-in-class - mlr help usage-functions - mlr help usage-functions-by-class - mlr help function - Keywords: - mlr help list-keywords - mlr help usage-keywords - mlr help keyword - Other: - mlr help auxents - mlr help mlrrc - mlr help output-colorization - mlr help type-arithmetic-info - Shorthands: - mlr -g = mlr help flags - mlr -l = mlr help list-verbs - mlr -L = mlr help usage-verbs - mlr -f = mlr help list-functions - mlr -F = mlr help usage-functions - mlr -k = mlr help list-keywords - mlr -K = mlr help usage-keywords +FILE FORMATS + CSV/CSV-lite: comma-separated values with separate header line + TSV: same but with tabs in places of commas + +---------------------+ + | apple,bat,cog | + | 1,2,3 | Record 1: "apple":"1", "bat":"2", "cog":"3" + | 4,5,6 | Record 2: "apple":"4", "bat":"5", "cog":"6" + +---------------------+ + + JSON (sequence or array of objects): + +---------------------+ + | { | + | "apple": 1, | Record 1: "apple":"1", "bat":"2", "cog":"3" + | "bat": 2, | + | "cog": 3 | + | } | + | { | + | "dish": { | Record 2: "dish:egg":"7", + | "egg": 7, | "dish:flint":"8", "garlic":"" + | "flint": 8 | + | }, | + | "garlic": "" | + | } | + +---------------------+ + + PPRINT: pretty-printed tabular + +---------------------+ + | apple bat cog | + | 1 2 3 | Record 1: "apple:"1", "bat":"2", "cog":"3" + | 4 5 6 | Record 2: "apple":"4", "bat":"5", "cog":"6" + +---------------------+ + + Markdown tabular (supported for output only): + +-----------------------+ + | | apple | bat | cog | | + | | --- | --- | --- | | + | | 1 | 2 | 3 | | Record 1: "apple:"1", "bat":"2", "cog":"3" + | | 4 | 5 | 6 | | Record 2: "apple":"4", "bat":"5", "cog":"6" + +-----------------------+ + + XTAB: pretty-printed transposed tabular + +---------------------+ + | apple 1 | Record 1: "apple":"1", "bat":"2", "cog":"3" + | bat 2 | + | cog 3 | + | | + | dish 7 | Record 2: "dish":"7", "egg":"8" + | egg 8 | + +---------------------+ + + DKVP: delimited key-value pairs (Miller default format) + +---------------------+ + | apple=1,bat=2,cog=3 | Record 1: "apple":"1", "bat":"2", "cog":"3" + | dish=7,egg=8,flint | Record 2: "dish":"7", "egg":"8", "3":"flint" + +---------------------+ + + NIDX: implicitly numerically indexed (Unix-toolkit style) + +---------------------+ + | the quick brown | Record 1: "1":"the", "2":"quick", "3":"brown" + | fox jumped | Record 2: "1":"fox", "2":"jumped" + +---------------------+ HELP OPTIONS Type 'mlr help {topic}' for any of the following: @@ -359,6 +374,8 @@ FORMAT-CONVERSION KEYSTROKE-SAVER FLAGS | XTAB | --x2c | --x2t | --x2j | --x2d | --x2n | | --x2p | --x2m | | PPRINT | --p2c | --p2t | --p2j | --p2d | --p2n | --p2x | | --p2m | + -p Keystroke-saver for `--nidx --fs space --repifs`. + -T Keystroke-saver for `--nidx --fs tab`. JSON-ONLY FLAGS These are flags which are applicable to JSON format. @@ -2759,4 +2776,4 @@ SEE ALSO - 2021-09-09 MILLER(1) + 2021-09-15 MILLER(1) diff --git a/docs6/docs/miller-on-windows.md b/docs6/src/miller-on-windows.md similarity index 100% rename from docs6/docs/miller-on-windows.md rename to docs6/src/miller-on-windows.md diff --git a/docs6/docs/miller-on-windows.md.in b/docs6/src/miller-on-windows.md.in similarity index 100% rename from docs6/docs/miller-on-windows.md.in rename to docs6/src/miller-on-windows.md.in diff --git a/docs6/docs/programming-language.md b/docs6/src/miller-programming-language.md similarity index 99% rename from docs6/docs/programming-language.md rename to docs6/src/miller-programming-language.md index 3ecedfb40..41de28b0e 100644 --- a/docs6/docs/programming-language.md +++ b/docs6/src/miller-programming-language.md @@ -357,6 +357,7 @@ KEY IS c VALUE IS 3 KEY IS a VALUE IS 4 KEY IS b VALUE IS 5 KEY IS c VALUE IS 6 + Here we used the local variables `k` and `v`. Now we've seen four kinds of variables: diff --git a/docs6/docs/programming-language.md.in b/docs6/src/miller-programming-language.md.in similarity index 100% rename from docs6/docs/programming-language.md.in rename to docs6/src/miller-programming-language.md.in diff --git a/docs6/docs/misc-examples.md b/docs6/src/misc-examples.md similarity index 100% rename from docs6/docs/misc-examples.md rename to docs6/src/misc-examples.md diff --git a/docs6/docs/misc-examples.md.in b/docs6/src/misc-examples.md.in similarity index 100% rename from docs6/docs/misc-examples.md.in rename to docs6/src/misc-examples.md.in diff --git a/docs6/docs/mk-flag-info.rb b/docs6/src/mk-flag-info.rb similarity index 100% rename from docs6/docs/mk-flag-info.rb rename to docs6/src/mk-flag-info.rb diff --git a/docs6/docs/mk-func-h2s.sh b/docs6/src/mk-func-h2s.sh similarity index 100% rename from docs6/docs/mk-func-h2s.sh rename to docs6/src/mk-func-h2s.sh diff --git a/docs6/docs/mk-func-info.rb b/docs6/src/mk-func-info.rb similarity index 99% rename from docs6/docs/mk-func-info.rb rename to docs6/src/mk-func-info.rb index 2e72121c6..ea80e6520 100755 --- a/docs6/docs/mk-func-info.rb +++ b/docs6/src/mk-func-info.rb @@ -67,7 +67,7 @@ def make_func_details function_classes = `mlr help list-function-classes`.split puts - puts "## Summary" + puts "## Functions by class" puts for function_class in function_classes class_link_name = "#"+"#{function_class}-functions" diff --git a/docs6/docs/mlr.1 b/docs6/src/mlr.1 similarity index 100% rename from docs6/docs/mlr.1 rename to docs6/src/mlr.1 diff --git a/docs6/docs/morph b/docs6/src/morph similarity index 100% rename from docs6/docs/morph rename to docs6/src/morph diff --git a/docs6/docs/morphs b/docs6/src/morphs similarity index 100% rename from docs6/docs/morphs rename to docs6/src/morphs diff --git a/docs6/docs/multi-join/input.csv b/docs6/src/multi-join/input.csv similarity index 100% rename from docs6/docs/multi-join/input.csv rename to docs6/src/multi-join/input.csv diff --git a/docs6/docs/multi-join/name-lookup.csv b/docs6/src/multi-join/name-lookup.csv similarity index 100% rename from docs6/docs/multi-join/name-lookup.csv rename to docs6/src/multi-join/name-lookup.csv diff --git a/docs6/docs/multi-join/status-lookup.csv b/docs6/src/multi-join/status-lookup.csv similarity index 100% rename from docs6/docs/multi-join/status-lookup.csv rename to docs6/src/multi-join/status-lookup.csv diff --git a/docs6/docs/new-in-miller-6.md b/docs6/src/new-in-miller-6.md similarity index 97% rename from docs6/docs/new-in-miller-6.md rename to docs6/src/new-in-miller-6.md index 82a4a9a4d..5e5d3e179 100644 --- a/docs6/docs/new-in-miller-6.md +++ b/docs6/src/new-in-miller-6.md @@ -40,11 +40,11 @@ pages have been split up into separate pages. (See also Since CSV is overwhelmingly the most popular data format for Miller, it is now discussed first, and more examples use CSV. -## JSON support, and arrays +## Improved JSON support, and arrays Arrays are now supported in Miller's `put`/`filter` programming language, as -described in the [Arrays reference](reference-main-arrays.md). Also, `array` is -now a keyword so this is no longer usable as a local-variable or UDF name. +described in the [Arrays reference](reference-main-arrays.md). (Also, `array` is +now a keyword so this is no longer usable as a local-variable or UDF name.) JSON support is improved: @@ -52,6 +52,8 @@ JSON support is improved: * Streamable JSON parsing: Miller's internal record-processing pipeline starts as soon as the first record is read (which was already the case for other file formats). This means that, unless records are wrapped with outermost `[...]`, Miller now handles JSON in `tail -f` contexts like it does for other file formats. * Flatten/unflatten -- conversion of JSON nested data structures (arrays and/or maps in record values) to/from non-JSON formats is a powerful new feature, discussed in the page [Flatten/unflatten: JSON vs. tabular formats](flatten-unflatten.md). +See the [Arrays reference](reference-main-arrays.md) for more information. + ## Improved Windows experience Stronger support for Windows (with or without MSYS2), with a couple of diff --git a/docs6/docs/new-in-miller-6.md.in b/docs6/src/new-in-miller-6.md.in similarity index 96% rename from docs6/docs/new-in-miller-6.md.in rename to docs6/src/new-in-miller-6.md.in index a17c80902..f5c6a45cb 100644 --- a/docs6/docs/new-in-miller-6.md.in +++ b/docs6/src/new-in-miller-6.md.in @@ -24,11 +24,11 @@ pages have been split up into separate pages. (See also Since CSV is overwhelmingly the most popular data format for Miller, it is now discussed first, and more examples use CSV. -## JSON support, and arrays +## Improved JSON support, and arrays Arrays are now supported in Miller's `put`/`filter` programming language, as -described in the [Arrays reference](reference-main-arrays.md). Also, `array` is -now a keyword so this is no longer usable as a local-variable or UDF name. +described in the [Arrays reference](reference-main-arrays.md). (Also, `array` is +now a keyword so this is no longer usable as a local-variable or UDF name.) JSON support is improved: @@ -36,6 +36,8 @@ JSON support is improved: * Streamable JSON parsing: Miller's internal record-processing pipeline starts as soon as the first record is read (which was already the case for other file formats). This means that, unless records are wrapped with outermost `[...]`, Miller now handles JSON in `tail -f` contexts like it does for other file formats. * Flatten/unflatten -- conversion of JSON nested data structures (arrays and/or maps in record values) to/from non-JSON formats is a powerful new feature, discussed in the page [Flatten/unflatten: JSON vs. tabular formats](flatten-unflatten.md). +See the [Arrays reference](reference-main-arrays.md) for more information. + ## Improved Windows experience Stronger support for Windows (with or without MSYS2), with a couple of diff --git a/docs6/docs/ngrams/gsl-2000.txt b/docs6/src/ngrams/gsl-2000.txt similarity index 100% rename from docs6/docs/ngrams/gsl-2000.txt rename to docs6/src/ngrams/gsl-2000.txt diff --git a/docs6/docs/ngrams/mkv.mlr b/docs6/src/ngrams/mkv.mlr similarity index 100% rename from docs6/docs/ngrams/mkv.mlr rename to docs6/src/ngrams/mkv.mlr diff --git a/docs6/docs/ngrams/mkv3.mlr b/docs6/src/ngrams/mkv3.mlr similarity index 100% rename from docs6/docs/ngrams/mkv3.mlr rename to docs6/src/ngrams/mkv3.mlr diff --git a/docs6/docs/ngrams/mkv4.mlr b/docs6/src/ngrams/mkv4.mlr similarity index 100% rename from docs6/docs/ngrams/mkv4.mlr rename to docs6/src/ngrams/mkv4.mlr diff --git a/docs6/docs/ngrams/mkv5.mlr b/docs6/src/ngrams/mkv5.mlr similarity index 100% rename from docs6/docs/ngrams/mkv5.mlr rename to docs6/src/ngrams/mkv5.mlr diff --git a/docs6/docs/ngrams/ng1 b/docs6/src/ngrams/ng1 similarity index 100% rename from docs6/docs/ngrams/ng1 rename to docs6/src/ngrams/ng1 diff --git a/docs6/docs/ngrams/ng1.mlr b/docs6/src/ngrams/ng1.mlr similarity index 100% rename from docs6/docs/ngrams/ng1.mlr rename to docs6/src/ngrams/ng1.mlr diff --git a/docs6/docs/ngrams/ng1.mlr.txt b/docs6/src/ngrams/ng1.mlr.txt similarity index 100% rename from docs6/docs/ngrams/ng1.mlr.txt rename to docs6/src/ngrams/ng1.mlr.txt diff --git a/docs6/docs/ngrams/ng2 b/docs6/src/ngrams/ng2 similarity index 100% rename from docs6/docs/ngrams/ng2 rename to docs6/src/ngrams/ng2 diff --git a/docs6/docs/ngrams/ng2.mlr b/docs6/src/ngrams/ng2.mlr similarity index 100% rename from docs6/docs/ngrams/ng2.mlr rename to docs6/src/ngrams/ng2.mlr diff --git a/docs6/docs/ngrams/ng2.mlr.txt b/docs6/src/ngrams/ng2.mlr.txt similarity index 100% rename from docs6/docs/ngrams/ng2.mlr.txt rename to docs6/src/ngrams/ng2.mlr.txt diff --git a/docs6/docs/ngrams/ng3 b/docs6/src/ngrams/ng3 similarity index 100% rename from docs6/docs/ngrams/ng3 rename to docs6/src/ngrams/ng3 diff --git a/docs6/docs/ngrams/ng3.mlr b/docs6/src/ngrams/ng3.mlr similarity index 100% rename from docs6/docs/ngrams/ng3.mlr rename to docs6/src/ngrams/ng3.mlr diff --git a/docs6/docs/ngrams/ng3.mlr.txt b/docs6/src/ngrams/ng3.mlr.txt similarity index 100% rename from docs6/docs/ngrams/ng3.mlr.txt rename to docs6/src/ngrams/ng3.mlr.txt diff --git a/docs6/docs/ngrams/ng4 b/docs6/src/ngrams/ng4 similarity index 100% rename from docs6/docs/ngrams/ng4 rename to docs6/src/ngrams/ng4 diff --git a/docs6/docs/ngrams/ng4.mlr b/docs6/src/ngrams/ng4.mlr similarity index 100% rename from docs6/docs/ngrams/ng4.mlr rename to docs6/src/ngrams/ng4.mlr diff --git a/docs6/docs/ngrams/ng4.mlr.txt b/docs6/src/ngrams/ng4.mlr.txt similarity index 100% rename from docs6/docs/ngrams/ng4.mlr.txt rename to docs6/src/ngrams/ng4.mlr.txt diff --git a/docs6/docs/ngrams/ng5 b/docs6/src/ngrams/ng5 similarity index 100% rename from docs6/docs/ngrams/ng5 rename to docs6/src/ngrams/ng5 diff --git a/docs6/docs/ngrams/ng5.mlr b/docs6/src/ngrams/ng5.mlr similarity index 100% rename from docs6/docs/ngrams/ng5.mlr rename to docs6/src/ngrams/ng5.mlr diff --git a/docs6/docs/ngrams/ng5.mlr.txt b/docs6/src/ngrams/ng5.mlr.txt similarity index 100% rename from docs6/docs/ngrams/ng5.mlr.txt rename to docs6/src/ngrams/ng5.mlr.txt diff --git a/docs6/docs/ngrams/ngfuncs.mlr b/docs6/src/ngrams/ngfuncs.mlr similarity index 100% rename from docs6/docs/ngrams/ngfuncs.mlr rename to docs6/src/ngrams/ngfuncs.mlr diff --git a/docs6/docs/ngrams/ngfuncs.mlr.txt b/docs6/src/ngrams/ngfuncs.mlr.txt similarity index 100% rename from docs6/docs/ngrams/ngfuncs.mlr.txt rename to docs6/src/ngrams/ngfuncs.mlr.txt diff --git a/docs6/docs/online-help.md b/docs6/src/online-help.md similarity index 97% rename from docs6/docs/online-help.md rename to docs6/src/online-help.md index 0fa1b7a6b..778bf0cb2 100644 --- a/docs6/docs/online-help.md +++ b/docs6/src/online-help.md @@ -27,8 +27,10 @@ The front door is `mlr --help` or its synonym `mlr -h`. This leads you to `mlr h 
 Usage: mlr [flags] {verb} [verb-dependent options ...] {zero or more file names}
+If zero file names are provided, standard input is read. Example:
+  mlr --csv sort -f shape example.csv
 Output of one verb may be chained as input to another using "then", e.g.
-  mlr stats1 -a min,mean,max -f flag,u,v -g color then sort -f color
+  mlr --csv stats1 -a min,mean,max -f quantity then sort -f color example.csv
 Please see 'mlr help topics' for more information.
 Please also see https://johnkerl.org/miller6
diff --git a/docs6/docs/online-help.md.in b/docs6/src/online-help.md.in similarity index 100% rename from docs6/docs/online-help.md.in rename to docs6/src/online-help.md.in diff --git a/docs6/docs/operating-on-all-fields.md b/docs6/src/operating-on-all-fields.md similarity index 100% rename from docs6/docs/operating-on-all-fields.md rename to docs6/src/operating-on-all-fields.md diff --git a/docs6/docs/operating-on-all-fields.md.in b/docs6/src/operating-on-all-fields.md.in similarity index 100% rename from docs6/docs/operating-on-all-fields.md.in rename to docs6/src/operating-on-all-fields.md.in diff --git a/docs6/docs/operating-on-all-records.md b/docs6/src/operating-on-all-records.md similarity index 100% rename from docs6/docs/operating-on-all-records.md rename to docs6/src/operating-on-all-records.md diff --git a/docs6/docs/operating-on-all-records.md.in b/docs6/src/operating-on-all-records.md.in similarity index 100% rename from docs6/docs/operating-on-all-records.md.in rename to docs6/src/operating-on-all-records.md.in diff --git a/docs6/docs/originality.md b/docs6/src/originality.md similarity index 100% rename from docs6/docs/originality.md rename to docs6/src/originality.md diff --git a/docs6/docs/originality.md.in b/docs6/src/originality.md.in similarity index 100% rename from docs6/docs/originality.md.in rename to docs6/src/originality.md.in diff --git a/docs6/docs/output-colorization.md b/docs6/src/output-colorization.md similarity index 100% rename from docs6/docs/output-colorization.md rename to docs6/src/output-colorization.md diff --git a/docs6/docs/output-colorization.md.in b/docs6/src/output-colorization.md.in similarity index 100% rename from docs6/docs/output-colorization.md.in rename to docs6/src/output-colorization.md.in diff --git a/docs6/src/performance.md b/docs6/src/performance.md new file mode 100644 index 000000000..7070061bf --- /dev/null +++ b/docs6/src/performance.md @@ -0,0 +1,118 @@ + +
+ +Quick links: +  +Flag list +  +Verb list +  +Function list +  +Glossary +  +Repository ↗ + +
+# Performance + +## Disclaimer + +In a previous version of this page, I compared Miller to some items in the Unix toolkit in terms of run time. But such comparisons are very much not apples-to-apples: + +* Miller's principal strength is that it handles **key-value data in various formats** while the system tools **do not**. So if you time `mlr sort` on a CSV file against system `sort`, it's not relevant to say which is faster by how many percent -- Miller will respect the header line, leaving it in place, while the system sort will move it, sorting it along with all the other header lines. This would be comparing the run times of two programs produce different outputs. Likewise, `awk` doesn't respect header lines, although you can code up some CSV-handling using `if (NR==1) { ... } else { ... }`. And that's just CSV: I don't know any simple way to get `sort`, `awk`, etc. to handle DKVP, JSON, etc. -- which is the main reason I wrote Miller. + +* **Implementations differ by platform**: one `awk` may be fundamentally faster than another, and `mawk` has a very efficient bytecode implementation -- which handles positionally indexed data far faster than Miller does. + +* The system `sort` command will, on some systems, handle too-large-for-RAM datasets by spilling to disk; Miller (as of version 5.2.0, mid-2017) does not. Miller sorts are always stable; GNU supports stable and unstable variants. + +* Etc. + +## Summary + +Miller can do many kinds of processing on key-value-pair data using elapsed time roughly of the same order of magnitude as items in the Unix toolkit can handle positionally indexed data. Specific results vary widely by platform, implementation details, and multi-core use (or not). Lastly, specific special-purpose non-record-aware processing will run far faster if implemented in `grep`, `sed`, etc. + +## Some examples + +This is some data from [https://community.opencellid.org](https://community.opencellid.org): approximately 40 +million records, 0.9GB compressed, 3.2GB uncommpressed. + +First we see that decompression is much cheaper than compression: 10 seconds vs. 2.5 minutes: + +``` +$ time gunzip cell_towers.csv.gz + +real 0m10.431s +user 0m9.235s +sys 0m1.030s + +$ ls -lh cell_towers.csv +-rw-r--r-- 1 johnkerl staff 3.2G Sep 8 17:13 cell_towers.csv + +$ time gzip cell_towers.csv + +real 2m30.171s +user 2m28.508s +sys 0m1.257s + +$ ls -lh cell_towers.csv.gz +-rw-r--r-- 1 johnkerl staff 917M Sep 7 12:34 cell_towers.csv.gz + +``` + +Next we look at system `cut` which needs to split on lines and fields. Since `cut` is in the +[Unix toolkit](unix-toolkit-context.md) it handles integer column names, starting with 1. + +``` +$ gunzip < cell_towers.csv.gz | head -n 5 +radio,mcc,net,area,cell,unit,lon,lat,range,samples,changeable,created,updated,averageSignal +UMTS,262,2,801,86355,0,13.285512,52.522202,1000,7,1,1282569574,1300155341,0 +GSM,262,2,801,1795,0,13.276907,52.525714,5716,9,1,1282569574,1300155341,0 +GSM,262,2,801,1794,0,13.285064,52.524,6280,13,1,1282569574,1300796207,0 +UMTS,262,2,801,211250,0,13.285446,52.521744,1000,3,1,1282569574,1299466955,0 +UMTS,262,2,801,86353,0,13.293457,52.521515,1000,2,1,1282569574,1291380444,0 +``` + +This takes about a minute and a half: + +``` +$ time cut -d, -f 1,2,12,13 cell_towers.csv > /dev/null + +real 1m29.228s +user 1m26.347s +sys 0m2.426s +``` + +Columns `1,2,12,13` are the same as `radio,mcc,created,updated`. Since +decompression is quick, it's perhaps unsurprising that whether we decompress +and have Miller read uncompressed data, or have it [decompress +in-process](reference-main-compressed-data.md#automatic-detection-on-input), or +use an [external decompressor with +`--prepipe`](reference-main-compressed-data.md#external-decompressors-on-input), +the results are about the same. This is not as fast as `cut`, but it's in the ballpark. + +``` + +$ gunzip cell_towers.csv.gz +$ time mlr --csv cut -f radio,mcc,created,updated cell_towers.csv > /dev/null + +real 4m10.097s +user 8m56.975s +sys 4m40.046s + +$ gzip cell_towers.csv +$ time mlr --csv cut -f radio,mcc,created,updated cell_towers.csv.gz > /dev/null + +real 4m14.185s +user 9m5.044s +sys 4m23.886s + +$ time mlr --csv --prepipe gunzip cut -f radio,mcc,created,updated cell_towers.csv.gz > /dev/null + +real 4m13.614s +user 9m5.623s +sys 4m57.827s + +``` + + diff --git a/docs6/src/performance.md.in b/docs6/src/performance.md.in new file mode 100644 index 000000000..7c642e136 --- /dev/null +++ b/docs6/src/performance.md.in @@ -0,0 +1,102 @@ +# Performance + +## Disclaimer + +In a previous version of this page, I compared Miller to some items in the Unix toolkit in terms of run time. But such comparisons are very much not apples-to-apples: + +* Miller's principal strength is that it handles **key-value data in various formats** while the system tools **do not**. So if you time `mlr sort` on a CSV file against system `sort`, it's not relevant to say which is faster by how many percent -- Miller will respect the header line, leaving it in place, while the system sort will move it, sorting it along with all the other header lines. This would be comparing the run times of two programs produce different outputs. Likewise, `awk` doesn't respect header lines, although you can code up some CSV-handling using `if (NR==1) { ... } else { ... }`. And that's just CSV: I don't know any simple way to get `sort`, `awk`, etc. to handle DKVP, JSON, etc. -- which is the main reason I wrote Miller. + +* **Implementations differ by platform**: one `awk` may be fundamentally faster than another, and `mawk` has a very efficient bytecode implementation -- which handles positionally indexed data far faster than Miller does. + +* The system `sort` command will, on some systems, handle too-large-for-RAM datasets by spilling to disk; Miller (as of version 5.2.0, mid-2017) does not. Miller sorts are always stable; GNU supports stable and unstable variants. + +* Etc. + +## Summary + +Miller can do many kinds of processing on key-value-pair data using elapsed time roughly of the same order of magnitude as items in the Unix toolkit can handle positionally indexed data. Specific results vary widely by platform, implementation details, and multi-core use (or not). Lastly, specific special-purpose non-record-aware processing will run far faster if implemented in `grep`, `sed`, etc. + +## Some examples + +This is some data from [https://community.opencellid.org](https://community.opencellid.org): approximately 40 +million records, 0.9GB compressed, 3.2GB uncommpressed. + +First we see that decompression is much cheaper than compression: 10 seconds vs. 2.5 minutes: + +``` +$ time gunzip cell_towers.csv.gz + +real 0m10.431s +user 0m9.235s +sys 0m1.030s + +$ ls -lh cell_towers.csv +-rw-r--r-- 1 johnkerl staff 3.2G Sep 8 17:13 cell_towers.csv + +$ time gzip cell_towers.csv + +real 2m30.171s +user 2m28.508s +sys 0m1.257s + +$ ls -lh cell_towers.csv.gz +-rw-r--r-- 1 johnkerl staff 917M Sep 7 12:34 cell_towers.csv.gz + +``` + +Next we look at system `cut` which needs to split on lines and fields. Since `cut` is in the +[Unix toolkit](unix-toolkit-context.md) it handles integer column names, starting with 1. + +``` +$ gunzip < cell_towers.csv.gz | head -n 5 +radio,mcc,net,area,cell,unit,lon,lat,range,samples,changeable,created,updated,averageSignal +UMTS,262,2,801,86355,0,13.285512,52.522202,1000,7,1,1282569574,1300155341,0 +GSM,262,2,801,1795,0,13.276907,52.525714,5716,9,1,1282569574,1300155341,0 +GSM,262,2,801,1794,0,13.285064,52.524,6280,13,1,1282569574,1300796207,0 +UMTS,262,2,801,211250,0,13.285446,52.521744,1000,3,1,1282569574,1299466955,0 +UMTS,262,2,801,86353,0,13.293457,52.521515,1000,2,1,1282569574,1291380444,0 +``` + +This takes about a minute and a half: + +``` +$ time cut -d, -f 1,2,12,13 cell_towers.csv > /dev/null + +real 1m29.228s +user 1m26.347s +sys 0m2.426s +``` + +Columns `1,2,12,13` are the same as `radio,mcc,created,updated`. Since +decompression is quick, it's perhaps unsurprising that whether we decompress +and have Miller read uncompressed data, or have it [decompress +in-process](reference-main-compressed-data.md#automatic-detection-on-input), or +use an [external decompressor with +`--prepipe`](reference-main-compressed-data.md#external-decompressors-on-input), +the results are about the same. This is not as fast as `cut`, but it's in the ballpark. + +``` + +$ gunzip cell_towers.csv.gz +$ time mlr --csv cut -f radio,mcc,created,updated cell_towers.csv > /dev/null + +real 4m10.097s +user 8m56.975s +sys 4m40.046s + +$ gzip cell_towers.csv +$ time mlr --csv cut -f radio,mcc,created,updated cell_towers.csv.gz > /dev/null + +real 4m14.185s +user 9m5.044s +sys 4m23.886s + +$ time mlr --csv --prepipe gunzip cut -f radio,mcc,created,updated cell_towers.csv.gz > /dev/null + +real 4m13.614s +user 9m5.623s +sys 4m57.827s + +``` + + diff --git a/docs6/docs/pix/colorization.png b/docs6/src/pix/colorization.png similarity index 100% rename from docs6/docs/pix/colorization.png rename to docs6/src/pix/colorization.png diff --git a/docs6/docs/pix/colorization.png.000 b/docs6/src/pix/colorization.png.000 similarity index 100% rename from docs6/docs/pix/colorization.png.000 rename to docs6/src/pix/colorization.png.000 diff --git a/docs6/docs/pix/colorization2.png b/docs6/src/pix/colorization2.png similarity index 100% rename from docs6/docs/pix/colorization2.png rename to docs6/src/pix/colorization2.png diff --git a/docs6/docs/pix/colorization2.png.000 b/docs6/src/pix/colorization2.png.000 similarity index 100% rename from docs6/docs/pix/colorization2.png.000 rename to docs6/src/pix/colorization2.png.000 diff --git a/docs6/docs/pix/mand.png b/docs6/src/pix/mand.png similarity index 100% rename from docs6/docs/pix/mand.png rename to docs6/src/pix/mand.png diff --git a/docs6/docs/pix/miller-msys-complex.png b/docs6/src/pix/miller-msys-complex.png similarity index 100% rename from docs6/docs/pix/miller-msys-complex.png rename to docs6/src/pix/miller-msys-complex.png diff --git a/docs6/docs/pix/miller-msys.png b/docs6/src/pix/miller-msys.png similarity index 100% rename from docs6/docs/pix/miller-msys.png rename to docs6/src/pix/miller-msys.png diff --git a/docs6/docs/pix/miller-windows-complex-workaround.png b/docs6/src/pix/miller-windows-complex-workaround.png similarity index 100% rename from docs6/docs/pix/miller-windows-complex-workaround.png rename to docs6/src/pix/miller-windows-complex-workaround.png diff --git a/docs6/docs/pix/miller-windows-complex.png b/docs6/src/pix/miller-windows-complex.png similarity index 100% rename from docs6/docs/pix/miller-windows-complex.png rename to docs6/src/pix/miller-windows-complex.png diff --git a/docs6/docs/pix/miller-windows-triple-double-quote.png b/docs6/src/pix/miller-windows-triple-double-quote.png similarity index 100% rename from docs6/docs/pix/miller-windows-triple-double-quote.png rename to docs6/src/pix/miller-windows-triple-double-quote.png diff --git a/docs6/docs/pix/miller-windows.png b/docs6/src/pix/miller-windows.png similarity index 100% rename from docs6/docs/pix/miller-windows.png rename to docs6/src/pix/miller-windows.png diff --git a/docs6/docs/pix/mlr-doc-sepia.jpg b/docs6/src/pix/mlr-doc-sepia.jpg similarity index 100% rename from docs6/docs/pix/mlr-doc-sepia.jpg rename to docs6/src/pix/mlr-doc-sepia.jpg diff --git a/docs6/docs/pix/mlr.jpg b/docs6/src/pix/mlr.jpg similarity index 100% rename from docs6/docs/pix/mlr.jpg rename to docs6/src/pix/mlr.jpg diff --git a/docs6/docs/pix/omd.png b/docs6/src/pix/omd.png similarity index 100% rename from docs6/docs/pix/omd.png rename to docs6/src/pix/omd.png diff --git a/docs6/docs/pix/sepia-overlay.jpg b/docs6/src/pix/sepia-overlay.jpg similarity index 100% rename from docs6/docs/pix/sepia-overlay.jpg rename to docs6/src/pix/sepia-overlay.jpg diff --git a/docs6/docs/pix/vim-syntax-off.png b/docs6/src/pix/vim-syntax-off.png similarity index 100% rename from docs6/docs/pix/vim-syntax-off.png rename to docs6/src/pix/vim-syntax-off.png diff --git a/docs6/docs/pix/vim-syntax-on.png b/docs6/src/pix/vim-syntax-on.png similarity index 100% rename from docs6/docs/pix/vim-syntax-on.png rename to docs6/src/pix/vim-syntax-on.png diff --git a/docs6/docs/polyglot-dkvp-io/dkvp_io.py b/docs6/src/polyglot-dkvp-io/dkvp_io.py similarity index 100% rename from docs6/docs/polyglot-dkvp-io/dkvp_io.py rename to docs6/src/polyglot-dkvp-io/dkvp_io.py diff --git a/docs6/docs/polyglot-dkvp-io/dkvp_io.rb b/docs6/src/polyglot-dkvp-io/dkvp_io.rb similarity index 100% rename from docs6/docs/polyglot-dkvp-io/dkvp_io.rb rename to docs6/src/polyglot-dkvp-io/dkvp_io.rb diff --git a/docs6/docs/polyglot-dkvp-io/example.py b/docs6/src/polyglot-dkvp-io/example.py similarity index 100% rename from docs6/docs/polyglot-dkvp-io/example.py rename to docs6/src/polyglot-dkvp-io/example.py diff --git a/docs6/docs/polyglot-dkvp-io/example.rb b/docs6/src/polyglot-dkvp-io/example.rb similarity index 100% rename from docs6/docs/polyglot-dkvp-io/example.rb rename to docs6/src/polyglot-dkvp-io/example.rb diff --git a/docs6/docs/programming-examples.md b/docs6/src/programming-examples.md similarity index 100% rename from docs6/docs/programming-examples.md rename to docs6/src/programming-examples.md diff --git a/docs6/docs/programming-examples.md.in b/docs6/src/programming-examples.md.in similarity index 100% rename from docs6/docs/programming-examples.md.in rename to docs6/src/programming-examples.md.in diff --git a/docs6/docs/programs/mand.mlr b/docs6/src/programs/mand.mlr similarity index 100% rename from docs6/docs/programs/mand.mlr rename to docs6/src/programs/mand.mlr diff --git a/docs6/docs/programs/mand.sh b/docs6/src/programs/mand.sh similarity index 100% rename from docs6/docs/programs/mand.sh rename to docs6/src/programs/mand.sh diff --git a/docs6/docs/programs/sieve.mlr b/docs6/src/programs/sieve.mlr similarity index 100% rename from docs6/docs/programs/sieve.mlr rename to docs6/src/programs/sieve.mlr diff --git a/docs6/docs/proofreads.txt b/docs6/src/proofreads.txt similarity index 84% rename from docs6/docs/proofreads.txt rename to docs6/src/proofreads.txt index 0b1f45c05..69c697b6b 100644 --- a/docs6/docs/proofreads.txt +++ b/docs6/src/proofreads.txt @@ -1,16 +1,23 @@ TOP: +c array+string slice UTs +! mapselect int-key etc -- fix + UT +! https://github.com/johnkerl/miller/issues/653 -- stats1 w/ empties? check stats2 + - needs UTs as well + +? nidx no output-coloring? + +e fix -I + auto-decompress + +? twi-dm re all-contribs: all-contributors.org + +* nikos materials -> fold in + c! seps \001 etc ! mlrc --iusv --oxtab cat regtest/input/example.usv mlr --iusv --oxtab cat regtest/input/example.usv C! repifs !! https://pkg.go.dev/regexp#Regexp.Split 2-for-1 -- get regexp as well ? -E flatten/unflatten page - -? twi-dm re all-contribs: all-contributors.org - -* nikos materials -> fold in - ---------------------------------------------------------------- ALL: * csv to csv,tsv throughout @@ -37,17 +44,6 @@ E fill out flatten/unflatten page echo a:b:c=3 | mlr --ojson --no-jvstack --flatsep : cat -> call it --flatsep -E fill out strings page - e substr0/substr1/substr. slices & string-indexing new in m6. - t RT for string/array indexing/slicing: like python: idx oob -> error; slice out -> empty -E fill out arrays page - - xref to fla/unfla page - - slices including semantics for out-of-range slices - t oob like python: idx oob -> error; slice out -> empty -E fill out maps page - - xref to fla/unfla page - - move from reference-dsl-variables - * regex: more about what is / is not https://github.com/johnkerl/miller/issues/77#issuecomment-538553828 @@ -57,12 +53,14 @@ c? mlr -S / -F outside of put/filter -- ? e docs6/rmd: big picture .md.in -> .md -> build -> isp/rtd; or live -c -i, -o, --io need to set more than just file format -t test --ifs pipe --icsv as well as --icsv --ifs pipe - ? entrypoint? or cli? map -i foo to --ifoo, -o foo to --ofoo, and --io foo to --foo +csv-with-and-without-headers: +? Headerless CSV with duplicate field values -> typo-fix -- duplicate keys actually -- ?!? +? cat out nas.csv first +? ag -> just grep ? +l Miller handles compliant CSV: include rfc4180 link +? Regularizing ragged CSV: use regularize verb?!? ---------------------------------------------------------------- - record-heterogeneity: ? current csv-reader code handles this? i think not. check csvlite-reader too. ? --allow-ragged-csv-input: link to new flags page once it's written @@ -76,13 +74,6 @@ l link-to's: two-pass-algorithms.md.in:Suppose you have some heterogeneous data like this: ? back out of ref section -- ? -csv-with-and-without-headers: -? Headerless CSV with duplicate field values -> typo-fix -- duplicate keys actually -- ?!? -? cat out nas.csv first -? ag -> just grep ? -l Miller handles compliant CSV: include rfc4180 link -? Regularizing ragged CSV: use regularize verb?!? - special-symbols-and-formatting: ? mlr --icsv --odkvp --ofs '\001' cat commas.csv | cat -v o test this @@ -144,19 +135,9 @@ E Keep in mind that out-of-stream variables are a nested, multi-level hashmap (d values. - new: mlr: lashed-emit node types must be local variables, field names, or oosvars; got map literal. c ?make FunctionCallsite an Emittable? does this work in the BNF? -? Use --flatsep -> check -- jflatsep, iflatsep, flatsep as --csv/--icsv/--ocsv -- ? what was the thinking at the time? ? mlr --oxtab put -q --flatsep / ' o 2 examples not 3? o why not '--flatsep /' respected? -diffs other langs: -e slice incl/excl; split-in-half: x[:2] x[2:] vs x[:2] x[3:] -e localvars in if/else -e "arrays, including ..." -> 1-up array/string indexing/slicing -e no boolish truthy falsy - separators: -E write up which file formats support which flags - -manpage: -? [NEEDS READ-THROUGH] +E write up which file formats support which flags (TODO comments) diff --git a/docs6/docs/purple.csv.gz b/docs6/src/purple.csv.gz similarity index 100% rename from docs6/docs/purple.csv.gz rename to docs6/src/purple.csv.gz diff --git a/docs6/docs/questions-about-joins.md b/docs6/src/questions-about-joins.md similarity index 100% rename from docs6/docs/questions-about-joins.md rename to docs6/src/questions-about-joins.md diff --git a/docs6/docs/questions-about-joins.md.in b/docs6/src/questions-about-joins.md.in similarity index 100% rename from docs6/docs/questions-about-joins.md.in rename to docs6/src/questions-about-joins.md.in diff --git a/docs6/docs/questions-about-then-chaining.md b/docs6/src/questions-about-then-chaining.md similarity index 100% rename from docs6/docs/questions-about-then-chaining.md rename to docs6/src/questions-about-then-chaining.md diff --git a/docs6/docs/questions-about-then-chaining.md.in b/docs6/src/questions-about-then-chaining.md.in similarity index 100% rename from docs6/docs/questions-about-then-chaining.md.in rename to docs6/src/questions-about-then-chaining.md.in diff --git a/docs6/docs/randomizing-examples.md b/docs6/src/randomizing-examples.md similarity index 100% rename from docs6/docs/randomizing-examples.md rename to docs6/src/randomizing-examples.md diff --git a/docs6/docs/randomizing-examples.md.in b/docs6/src/randomizing-examples.md.in similarity index 100% rename from docs6/docs/randomizing-examples.md.in rename to docs6/src/randomizing-examples.md.in diff --git a/docs6/docs/record-heterogeneity.md b/docs6/src/record-heterogeneity.md similarity index 99% rename from docs6/docs/record-heterogeneity.md rename to docs6/src/record-heterogeneity.md index 811eb1e77..91951e76d 100644 --- a/docs6/docs/record-heterogeneity.md +++ b/docs6/src/record-heterogeneity.md @@ -130,6 +130,7 @@ If you `mlr csv cat` this, you'll get an error message: a,b,c 1,2,3 mlr : Miller: CSV header/data length mismatch 3 != 2 at filename data/het/ragged.csv row 3. + There are two kinds of raggedness here. Since CSVs form records by zipping the diff --git a/docs6/docs/record-heterogeneity.md.in b/docs6/src/record-heterogeneity.md.in similarity index 100% rename from docs6/docs/record-heterogeneity.md.in rename to docs6/src/record-heterogeneity.md.in diff --git a/docs6/docs/red.csv.gz b/docs6/src/red.csv.gz similarity index 100% rename from docs6/docs/red.csv.gz rename to docs6/src/red.csv.gz diff --git a/docs6/docs/reference-dsl-builtin-functions.md b/docs6/src/reference-dsl-builtin-functions.md similarity index 97% rename from docs6/docs/reference-dsl-builtin-functions.md rename to docs6/src/reference-dsl-builtin-functions.md index e96f4c93b..f2c30c7fd 100644 --- a/docs6/docs/reference-dsl-builtin-functions.md +++ b/docs6/src/reference-dsl-builtin-functions.md @@ -16,6 +16,36 @@ Quick links: # DSL built-in functions +These are functions in the [Miller programming language](miller-programming-language.md) +that you can call when you use `mlr put` and `mlr filter`. For example, when you type + +
+mlr --icsv --opprint --from example.csv put '
+  $color = toupper($color);
+  $shape = gsub($shape, "[aeiou]", "*");
+'
+
+
+color  shape    flag  k  index quantity rate
+YELLOW tr**ngl* true  1  11    43.6498  9.8870
+RED    sq**r*   true  2  15    79.2778  0.0130
+RED    c*rcl*   true  3  16    13.8103  2.9010
+RED    sq**r*   false 4  48    77.5542  7.4670
+PURPLE tr**ngl* false 5  51    81.2290  8.5910
+RED    sq**r*   false 6  64    77.1991  9.5310
+PURPLE tr**ngl* false 7  65    80.1405  5.8240
+YELLOW c*rcl*   true  8  73    63.9785  4.2370
+YELLOW c*rcl*   true  9  87    63.5058  8.3350
+PURPLE sq**r*   false 10 91    72.3735  8.2430
+
+ +the `toupper` and `gsub` bits are _functions_. + +## Overview + +At the command line, you can use `mlr -f` and `mlr -F` for information much +like what's on this page. + Each function takes a specific number of arguments, as shown below, except for functions marked as variadic such as `min` and `max`. (The latter compute min and max of any number of arguments.) There is no notion of optional or @@ -35,7 +65,7 @@ is 2. Unary operators such as `!` and `~` show argument-count of 1; the ternary `? :` operator shows an argument-count of 3. -## Summary +## Functions by class * [**Arithmetic functions**](#arithmetic-functions): [bitcount](#bitcount), [madd](#madd), [mexp](#mexp), [mmul](#mmul), [msub](#msub), [pow](#pow), [%](#percent), [&](#bitwise-and), [\*](#times), [\**](#exponentiation), [\+](#plus), [\-](#minus), [\.\*](#dot-times), [\.\+](#dot-plus), [\.\-](#dot-minus), [\./](#dot-slash), [/](#slash), [//](#slash-slash), [<<](#lsh), [>>](#srsh), [>>>](#ursh), [^](#bitwise-xor), [\|](#bitwise-or), [~](#bitwise-not). * [**Boolean functions**](#boolean-functions): [\!](#exclamation-point), [\!=](#exclamation-point-equals), [!=~](#regnotmatch), [&&](#logical-and), [<](#less-than), [<=](#less-than-or-equals), [==](#double-equals), [=~](#regmatch), [>](#greater-than), [>=](#greater-than-or-equals), [?:](#question-mark-colon), [??](#absent-coalesce), [???](#absent-empty-coalesce), [^^](#logical-xor), [\|\|](#logical-or). diff --git a/docs6/docs/reference-dsl-builtin-functions.md.in b/docs6/src/reference-dsl-builtin-functions.md.in similarity index 69% rename from docs6/docs/reference-dsl-builtin-functions.md.in rename to docs6/src/reference-dsl-builtin-functions.md.in index 0cf7e516c..a0797a291 100644 --- a/docs6/docs/reference-dsl-builtin-functions.md.in +++ b/docs6/src/reference-dsl-builtin-functions.md.in @@ -1,5 +1,22 @@ # DSL built-in functions +These are functions in the [Miller programming language](miller-programming-language.md) +that you can call when you use `mlr put` and `mlr filter`. For example, when you type + +GENMD_RUN_COMMAND +mlr --icsv --opprint --from example.csv put ' + $color = toupper($color); + $shape = gsub($shape, "[aeiou]", "*"); +' +GENMD_EOF + +the `toupper` and `gsub` bits are _functions_. + +## Overview + +At the command line, you can use `mlr -f` and `mlr -F` for information much +like what's on this page. + Each function takes a specific number of arguments, as shown below, except for functions marked as variadic such as `min` and `max`. (The latter compute min and max of any number of arguments.) There is no notion of optional or diff --git a/docs6/docs/reference-dsl-complexity.md b/docs6/src/reference-dsl-complexity.md similarity index 100% rename from docs6/docs/reference-dsl-complexity.md rename to docs6/src/reference-dsl-complexity.md diff --git a/docs6/docs/reference-dsl-complexity.md.in b/docs6/src/reference-dsl-complexity.md.in similarity index 100% rename from docs6/docs/reference-dsl-complexity.md.in rename to docs6/src/reference-dsl-complexity.md.in diff --git a/docs6/docs/reference-dsl-control-structures.md b/docs6/src/reference-dsl-control-structures.md similarity index 100% rename from docs6/docs/reference-dsl-control-structures.md rename to docs6/src/reference-dsl-control-structures.md diff --git a/docs6/docs/reference-dsl-control-structures.md.in b/docs6/src/reference-dsl-control-structures.md.in similarity index 100% rename from docs6/docs/reference-dsl-control-structures.md.in rename to docs6/src/reference-dsl-control-structures.md.in diff --git a/docs6/docs/reference-dsl-differences.md b/docs6/src/reference-dsl-differences.md similarity index 88% rename from docs6/docs/reference-dsl-differences.md rename to docs6/src/reference-dsl-differences.md index 469c10fe1..aaf25ebbb 100644 --- a/docs6/docs/reference-dsl-differences.md +++ b/docs6/src/reference-dsl-differences.md @@ -69,12 +69,40 @@ Please check for missing semicolon. Expected one of: $ ; > >> | ? || ^^ && ?? ??? =~ !=~ == != >= < <= ^ & << >>> + - .+ .- . * / // % .* ./ .// ** [ [[ [[[ + ## elif Miller has [`elif`](reference-dsl-control-structures.md#if-statements), not `else if` or `elsif`. +## If-statement variable scoping + +Miller is simple-minded about scoping [local variables](reference-dsl-variables.md#local-variables) to blocks. +If you have + +
+  if (something) {
+    x = 1
+  } else {
+    x = 2
+  }
+
+ +then there are two `x` variable, each confined only to their enclosing curly +braces; there is no hoisting out of the `if` and `else` blocks. + +A suggestion is + +
+  var x
+  if (something) {
+    x = 1
+  } else {
+    x = 2
+  }
+
+ ## Required curly braces Bodies for all compound statements must be enclosed in curly braces, even if the body is a single statement: @@ -89,6 +117,7 @@ Bodies for all compound statements must be enclosed in curly braces, even if the ## No autoconvert to boolean +I.e. ints/strings/etc are neither "truthy" nor "falsy". Boolean tests in `if`/`while`/`for`/etc must always take a boolean expression: `if (1) {...}` results in the parse error `Miller: conditional expression did not evaluate to boolean.`, @@ -160,7 +189,7 @@ In yet others, such as C/C++ and Java, string literals are in double auotes, lik while single quotes are for character literals like `'a'` or `'\n'`. In these, if `s` is a non-empty string, then `s[0]` is its first character. -In the [Miller programming language](programming-language.md): +In the [Miller programming language](miller-programming-language.md): * String literals are always in double quotes, like `"abc"`. * String-indexing/slicing always results in strings (even of length 1): `"abc"[1:1]` is the string `"a"`, and there is no notion in the Miller programming language of a character type. @@ -181,7 +210,9 @@ See the [maps page](reference-main-maps.md). ## Arrays, including 1-up array indices -Arrays are indexed starting with 1, not 0. This is discussed in detail on the [arrays page](reference-main-arrays.md). +Arrays and strings are indexed starting with 1, not 0. This is discussed in +detail on the [arrays page](reference-main-arrays.md) and the [strings +page](reference-main-strings.md). 
 mlr --csv --from data/short.csv cat
@@ -209,7 +240,11 @@ Record 2 has word ball
 Record 3 has word cat
-See the [arrays page](reference-main-arrays.md) for more about arrays. +Also, slices for arrays and strings are _doubly inclusive_: `x[3:5]` gets you +elements 3, 4, and 5 of the array or string named `x`. + +See the [arrays page](reference-main-arrays.md) for more about arrays; see the +[strings page](reference-main-strings.md) for more about strings. ## Two-variable for-loops diff --git a/docs6/docs/reference-dsl-differences.md.in b/docs6/src/reference-dsl-differences.md.in similarity index 87% rename from docs6/docs/reference-dsl-differences.md.in rename to docs6/src/reference-dsl-differences.md.in index 21619c63a..097ed3cca 100644 --- a/docs6/docs/reference-dsl-differences.md.in +++ b/docs6/src/reference-dsl-differences.md.in @@ -51,6 +51,33 @@ GENMD_EOF Miller has [`elif`](reference-dsl-control-structures.md#if-statements), not `else if` or `elsif`. +## If-statement variable scoping + +Miller is simple-minded about scoping [local variables](reference-dsl-variables.md#local-variables) to blocks. +If you have + +GENMD_CARDIFY + if (something) { + x = 1 + } else { + x = 2 + } +GENMD_EOF + +then there are two `x` variable, each confined only to their enclosing curly +braces; there is no hoisting out of the `if` and `else` blocks. + +A suggestion is + +GENMD_CARDIFY + var x + if (something) { + x = 1 + } else { + x = 2 + } +GENMD_EOF + ## Required curly braces Bodies for all compound statements must be enclosed in curly braces, even if the body is a single statement: @@ -65,6 +92,7 @@ GENMD_EOF ## No autoconvert to boolean +I.e. ints/strings/etc are neither "truthy" nor "falsy". Boolean tests in `if`/`while`/`for`/etc must always take a boolean expression: `if (1) {...}` results in the parse error `Miller: conditional expression did not evaluate to boolean.`, @@ -130,7 +158,7 @@ In yet others, such as C/C++ and Java, string literals are in double auotes, lik while single quotes are for character literals like `'a'` or `'\n'`. In these, if `s` is a non-empty string, then `s[0]` is its first character. -In the [Miller programming language](programming-language.md): +In the [Miller programming language](miller-programming-language.md): * String literals are always in double quotes, like `"abc"`. * String-indexing/slicing always results in strings (even of length 1): `"abc"[1:1]` is the string `"a"`, and there is no notion in the Miller programming language of a character type. @@ -151,7 +179,9 @@ See the [maps page](reference-main-maps.md). ## Arrays, including 1-up array indices -Arrays are indexed starting with 1, not 0. This is discussed in detail on the [arrays page](reference-main-arrays.md). +Arrays and strings are indexed starting with 1, not 0. This is discussed in +detail on the [arrays page](reference-main-arrays.md) and the [strings +page](reference-main-strings.md). GENMD_RUN_COMMAND mlr --csv --from data/short.csv cat @@ -168,7 +198,11 @@ mlr --csv --from data/short.csv put -q ' ' GENMD_EOF -See the [arrays page](reference-main-arrays.md) for more about arrays. +Also, slices for arrays and strings are _doubly inclusive_: `x[3:5]` gets you +elements 3, 4, and 5 of the array or string named `x`. + +See the [arrays page](reference-main-arrays.md) for more about arrays; see the +[strings page](reference-main-strings.md) for more about strings. ## Two-variable for-loops diff --git a/docs6/docs/reference-dsl-errors.md b/docs6/src/reference-dsl-errors.md similarity index 86% rename from docs6/docs/reference-dsl-errors.md rename to docs6/src/reference-dsl-errors.md index 8083ead5b..37e4af7a3 100644 --- a/docs6/docs/reference-dsl-errors.md +++ b/docs6/src/reference-dsl-errors.md @@ -16,7 +16,7 @@ Quick links: # DSL errors and transparency -As soon as you have a [programming language](programming-language.md), you start having the problem *What is my code doing, and why?* This includes getting syntax errors -- which are always annoying -- as well as the even more annoying problem of a program which parses without syntax error but doesn't do what you expect. +As soon as you have a [programming language](miller-programming-language.md), you start having the problem *What is my code doing, and why?* This includes getting syntax errors -- which are always annoying -- as well as the even more annoying problem of a program which parses without syntax error but doesn't do what you expect. The syntax-error message gives you line/column position for the syntax that couldn't be parsed. The cause may be clear from that information, or perhaps not. Here are some common causes of syntax errors: diff --git a/docs6/docs/reference-dsl-errors.md.in b/docs6/src/reference-dsl-errors.md.in similarity index 82% rename from docs6/docs/reference-dsl-errors.md.in rename to docs6/src/reference-dsl-errors.md.in index 5e58bd180..d50b4ed1a 100644 --- a/docs6/docs/reference-dsl-errors.md.in +++ b/docs6/src/reference-dsl-errors.md.in @@ -1,6 +1,6 @@ # DSL errors and transparency -As soon as you have a [programming language](programming-language.md), you start having the problem *What is my code doing, and why?* This includes getting syntax errors -- which are always annoying -- as well as the even more annoying problem of a program which parses without syntax error but doesn't do what you expect. +As soon as you have a [programming language](miller-programming-language.md), you start having the problem *What is my code doing, and why?* This includes getting syntax errors -- which are always annoying -- as well as the even more annoying problem of a program which parses without syntax error but doesn't do what you expect. The syntax-error message gives you line/column position for the syntax that couldn't be parsed. The cause may be clear from that information, or perhaps not. Here are some common causes of syntax errors: diff --git a/docs6/docs/reference-dsl-filter-statements.md b/docs6/src/reference-dsl-filter-statements.md similarity index 100% rename from docs6/docs/reference-dsl-filter-statements.md rename to docs6/src/reference-dsl-filter-statements.md diff --git a/docs6/docs/reference-dsl-filter-statements.md.in b/docs6/src/reference-dsl-filter-statements.md.in similarity index 100% rename from docs6/docs/reference-dsl-filter-statements.md.in rename to docs6/src/reference-dsl-filter-statements.md.in diff --git a/docs6/docs/reference-dsl-operators.md b/docs6/src/reference-dsl-operators.md similarity index 100% rename from docs6/docs/reference-dsl-operators.md rename to docs6/src/reference-dsl-operators.md diff --git a/docs6/docs/reference-dsl-operators.md.in b/docs6/src/reference-dsl-operators.md.in similarity index 100% rename from docs6/docs/reference-dsl-operators.md.in rename to docs6/src/reference-dsl-operators.md.in diff --git a/docs6/docs/reference-dsl-output-statements.md b/docs6/src/reference-dsl-output-statements.md similarity index 100% rename from docs6/docs/reference-dsl-output-statements.md rename to docs6/src/reference-dsl-output-statements.md diff --git a/docs6/docs/reference-dsl-output-statements.md.in b/docs6/src/reference-dsl-output-statements.md.in similarity index 100% rename from docs6/docs/reference-dsl-output-statements.md.in rename to docs6/src/reference-dsl-output-statements.md.in diff --git a/docs6/docs/reference-dsl-syntax.md b/docs6/src/reference-dsl-syntax.md similarity index 100% rename from docs6/docs/reference-dsl-syntax.md rename to docs6/src/reference-dsl-syntax.md diff --git a/docs6/docs/reference-dsl-syntax.md.in b/docs6/src/reference-dsl-syntax.md.in similarity index 100% rename from docs6/docs/reference-dsl-syntax.md.in rename to docs6/src/reference-dsl-syntax.md.in diff --git a/docs6/docs/reference-dsl-unset-statements.md b/docs6/src/reference-dsl-unset-statements.md similarity index 100% rename from docs6/docs/reference-dsl-unset-statements.md rename to docs6/src/reference-dsl-unset-statements.md diff --git a/docs6/docs/reference-dsl-unset-statements.md.in b/docs6/src/reference-dsl-unset-statements.md.in similarity index 100% rename from docs6/docs/reference-dsl-unset-statements.md.in rename to docs6/src/reference-dsl-unset-statements.md.in diff --git a/docs6/docs/reference-dsl-user-defined-functions.md b/docs6/src/reference-dsl-user-defined-functions.md similarity index 100% rename from docs6/docs/reference-dsl-user-defined-functions.md rename to docs6/src/reference-dsl-user-defined-functions.md diff --git a/docs6/docs/reference-dsl-user-defined-functions.md.in b/docs6/src/reference-dsl-user-defined-functions.md.in similarity index 100% rename from docs6/docs/reference-dsl-user-defined-functions.md.in rename to docs6/src/reference-dsl-user-defined-functions.md.in diff --git a/docs6/docs/reference-dsl-variables.md b/docs6/src/reference-dsl-variables.md similarity index 99% rename from docs6/docs/reference-dsl-variables.md rename to docs6/src/reference-dsl-variables.md index 1863ed9de..e65296722 100644 --- a/docs6/docs/reference-dsl-variables.md +++ b/docs6/src/reference-dsl-variables.md @@ -548,6 +548,8 @@ Like out-of-stream and local variables, map literals can be multi-level: } +See also the [Maps page](reference-main-maps.md). + ## Built-in variables These are written all in capital letters, and only a small, specific set of them is defined by Miller. @@ -556,9 +558,11 @@ Namely, Miller supports the following five built-in variables for [filter and put](reference-dsl.md), all `awk`-inspired: `NF`, `NR`, `FNR`, `FILENUM`, and `FILENAME`, as well as the mathematical constants `M_PI` and `M_E`. As well, there are the read-only separator variables `IRS`, `ORS`, `IFS`, `OFS`, `IPS`, -and `OPS` as discussed on the [separators page](reference-main-separators.md). -Lastly, the `ENV` map allows read/write access to environment variables, e.g. -`ENV["HOME"]` or `ENV["foo_".$hostname]` or `ENV["VERSION"]="1.2.3"`. +and `OPS` as discussed on the [separators page](reference-main-separators.md), +and the flatten/unflatten separator `FLATSEP` discussed on the +[flatten/unflatten page](flatten-unflatten.md). Lastly, the `ENV` map allows +read/write access to environment variables, e.g. `ENV["HOME"]` or +`ENV["foo_".$hostname]` or `ENV["VERSION"]="1.2.3"`. diff --git a/docs6/docs/reference-dsl-variables.md.in b/docs6/src/reference-dsl-variables.md.in similarity index 98% rename from docs6/docs/reference-dsl-variables.md.in rename to docs6/src/reference-dsl-variables.md.in index 01e63aada..fc33164d5 100644 --- a/docs6/docs/reference-dsl-variables.md.in +++ b/docs6/src/reference-dsl-variables.md.in @@ -283,6 +283,8 @@ mlr --from data/small put -q ' ' GENMD_EOF +See also the [Maps page](reference-main-maps.md). + ## Built-in variables These are written all in capital letters, and only a small, specific set of them is defined by Miller. @@ -291,9 +293,11 @@ Namely, Miller supports the following five built-in variables for [filter and put](reference-dsl.md), all `awk`-inspired: `NF`, `NR`, `FNR`, `FILENUM`, and `FILENAME`, as well as the mathematical constants `M_PI` and `M_E`. As well, there are the read-only separator variables `IRS`, `ORS`, `IFS`, `OFS`, `IPS`, -and `OPS` as discussed on the [separators page](reference-main-separators.md). -Lastly, the `ENV` map allows read/write access to environment variables, e.g. -`ENV["HOME"]` or `ENV["foo_".$hostname]` or `ENV["VERSION"]="1.2.3"`. +and `OPS` as discussed on the [separators page](reference-main-separators.md), +and the flatten/unflatten separator `FLATSEP` discussed on the +[flatten/unflatten page](flatten-unflatten.md). Lastly, the `ENV` map allows +read/write access to environment variables, e.g. `ENV["HOME"]` or +`ENV["foo_".$hostname]` or `ENV["VERSION"]="1.2.3"`. diff --git a/docs6/docs/reference-dsl.md b/docs6/src/reference-dsl.md similarity index 98% rename from docs6/docs/reference-dsl.md rename to docs6/src/reference-dsl.md index 07ef7d077..1b5729fc2 100644 --- a/docs6/docs/reference-dsl.md +++ b/docs6/src/reference-dsl.md @@ -18,7 +18,7 @@ Quick links: _DSL_ stands for _domain-specific language_: it's language particular to Miller which you can use to write expressions to specify customer transformations to -your data. (See [Miller programming language](programming-language.md) for an +your data. (See [Miller programming language](miller-programming-language.md) for an introduction.) ## Verbs compared to DSL @@ -84,7 +84,7 @@ the body of the loop, not the loop itself. (But you can, if you like, use those per-record statements to grow a list of records, then loop over them all in an `end` block. This is described in the -page on [operating on all records](operating-on-all-records.md)). +page on [operating on all records](operating-on-all-records.md).) To see this in action, let's take a look at the [data/short.csv](./data/short.csv) file: diff --git a/docs6/docs/reference-dsl.md.in b/docs6/src/reference-dsl.md.in similarity index 97% rename from docs6/docs/reference-dsl.md.in rename to docs6/src/reference-dsl.md.in index 755fef4f7..42f53835d 100644 --- a/docs6/docs/reference-dsl.md.in +++ b/docs6/src/reference-dsl.md.in @@ -2,7 +2,7 @@ _DSL_ stands for _domain-specific language_: it's language particular to Miller which you can use to write expressions to specify customer transformations to -your data. (See [Miller programming language](programming-language.md) for an +your data. (See [Miller programming language](miller-programming-language.md) for an introduction.) ## Verbs compared to DSL @@ -58,7 +58,7 @@ the body of the loop, not the loop itself. (But you can, if you like, use those per-record statements to grow a list of records, then loop over them all in an `end` block. This is described in the -page on [operating on all records](operating-on-all-records.md)). +page on [operating on all records](operating-on-all-records.md).) To see this in action, let's take a look at the [data/short.csv](./data/short.csv) file: diff --git a/docs6/docs/reference-main-arithmetic.md b/docs6/src/reference-main-arithmetic.md similarity index 100% rename from docs6/docs/reference-main-arithmetic.md rename to docs6/src/reference-main-arithmetic.md diff --git a/docs6/docs/reference-main-arithmetic.md.in b/docs6/src/reference-main-arithmetic.md.in similarity index 100% rename from docs6/docs/reference-main-arithmetic.md.in rename to docs6/src/reference-main-arithmetic.md.in diff --git a/docs6/docs/reference-main-arrays.md b/docs6/src/reference-main-arrays.md similarity index 88% rename from docs6/docs/reference-main-arrays.md rename to docs6/src/reference-main-arrays.md index 6629820f9..553d6639c 100644 --- a/docs6/docs/reference-main-arrays.md +++ b/docs6/src/reference-main-arrays.md @@ -71,7 +71,8 @@ Also note that several [built-in functions](reference-dsl-builtin-functions.md) ## 1-up indexing The most important difference between Miller's arrays and arrays in other -languages is that indices start with 1, not 0. This is intentional. +languages is that indices start with 1, not 0. (The same is true for [Miller +strings](reference-main-strings.md).) This is intentional. 1-up array indices may feel like a thing of the past, belonging to Fortran and Matlab, say; or R and Julia as well, which are more modern. But the overall @@ -136,6 +137,44 @@ x[4], x[5]]`. [20, 30, 40] +## Out-of-bounds indexing + +Somewhat imitating Python, out-of-bounds index accesses are +[absent](reference-main-null-data.md), but out-of-bounds slice accesses result +in trimming the indices, resulting in a short array or even the empty array: + +
+mlr -n put '
+  end {
+    x = [10, 20, 30, 40, 50];
+    print x[1];
+    print x[5];
+    print x[6]; # absent
+  }
+'
+
+
+10
+50
+
+
+ +
+mlr -n put '
+  end {
+    x = [10, 20, 30, 40, 50];
+    print x[1:2];
+    print x[1:6];
+    print x[10:20];
+  }
+'
+
+
+[10, 20]
+[10, 20, 30, 40, 50]
+[]
+
+ ## Auto-create results in maps As noted on the [maps page](reference-main-maps.md), indexing any @@ -271,3 +310,7 @@ More generally, you can get shift and pop operations by unsetting indices 1 and ## Looping See [single-variable for-loops](reference-dsl-control-structures.md#single-variable-for-loops) and [key-value for-loops](reference-dsl-control-structures.md#key-value-for-loops). + +## Array-valued fields in CSV files + +See the [flatten/unflatten page](flatten-unflatten.md). diff --git a/docs6/docs/reference-main-arrays.md.in b/docs6/src/reference-main-arrays.md.in similarity index 87% rename from docs6/docs/reference-main-arrays.md.in rename to docs6/src/reference-main-arrays.md.in index c30205128..031e84688 100644 --- a/docs6/docs/reference-main-arrays.md.in +++ b/docs6/src/reference-main-arrays.md.in @@ -39,7 +39,8 @@ Also note that several [built-in functions](reference-dsl-builtin-functions.md) ## 1-up indexing The most important difference between Miller's arrays and arrays in other -languages is that indices start with 1, not 0. This is intentional. +languages is that indices start with 1, not 0. (The same is true for [Miller +strings](reference-main-strings.md).) This is intentional. 1-up array indices may feel like a thing of the past, belonging to Fortran and Matlab, say; or R and Julia as well, which are more modern. But the overall @@ -91,6 +92,34 @@ mlr -n put ' ' GENMD_EOF +## Out-of-bounds indexing + +Somewhat imitating Python, out-of-bounds index accesses are +[absent](reference-main-null-data.md), but out-of-bounds slice accesses result +in trimming the indices, resulting in a short array or even the empty array: + +GENMD_RUN_COMMAND +mlr -n put ' + end { + x = [10, 20, 30, 40, 50]; + print x[1]; + print x[5]; + print x[6]; # absent + } +' +GENMD_EOF + +GENMD_RUN_COMMAND +mlr -n put ' + end { + x = [10, 20, 30, 40, 50]; + print x[1:2]; + print x[1:6]; + print x[10:20]; + } +' +GENMD_EOF + ## Auto-create results in maps As noted on the [maps page](reference-main-maps.md), indexing any @@ -198,3 +227,7 @@ GENMD_EOF ## Looping See [single-variable for-loops](reference-dsl-control-structures.md#single-variable-for-loops) and [key-value for-loops](reference-dsl-control-structures.md#key-value-for-loops). + +## Array-valued fields in CSV files + +See the [flatten/unflatten page](flatten-unflatten.md). diff --git a/docs6/docs/reference-main-auxiliary-commands.md b/docs6/src/reference-main-auxiliary-commands.md similarity index 100% rename from docs6/docs/reference-main-auxiliary-commands.md rename to docs6/src/reference-main-auxiliary-commands.md diff --git a/docs6/docs/reference-main-auxiliary-commands.md.in b/docs6/src/reference-main-auxiliary-commands.md.in similarity index 100% rename from docs6/docs/reference-main-auxiliary-commands.md.in rename to docs6/src/reference-main-auxiliary-commands.md.in diff --git a/docs6/docs/reference-main-compressed-data.md b/docs6/src/reference-main-compressed-data.md similarity index 98% rename from docs6/docs/reference-main-compressed-data.md rename to docs6/src/reference-main-compressed-data.md index 284fef741..36871db4a 100644 --- a/docs6/docs/reference-main-compressed-data.md +++ b/docs6/src/reference-main-compressed-data.md @@ -151,7 +151,7 @@ yellow,circle,true,8,73,63.9785,4.2370 yellow,circle,true,9,87,63.5058,8.3350 -* Using the [in-place flag](reference-main-io-options.md#in-place-mode) `-I`, +* Using the [in-place flag](reference-main-in-place-processing.md) `-I`, as of August 2021 the overwritten file will _not_ be compressed as it was when it was read: e.g. `mlr -I --csv cat gz-example.csv.gz` will write `gz-example.csv.gz` which contains a plain, uncompressed CSV contents. This is a bug and will be fixed. diff --git a/docs6/docs/reference-main-compressed-data.md.in b/docs6/src/reference-main-compressed-data.md.in similarity index 98% rename from docs6/docs/reference-main-compressed-data.md.in rename to docs6/src/reference-main-compressed-data.md.in index 100598b24..e04a4ba07 100644 --- a/docs6/docs/reference-main-compressed-data.md.in +++ b/docs6/src/reference-main-compressed-data.md.in @@ -103,7 +103,7 @@ yellow,circle,true,8,73,63.9785,4.2370 yellow,circle,true,9,87,63.5058,8.3350 GENMD_EOF -* Using the [in-place flag](reference-main-io-options.md#in-place-mode) `-I`, +* Using the [in-place flag](reference-main-in-place-processing.md) `-I`, as of August 2021 the overwritten file will _not_ be compressed as it was when it was read: e.g. `mlr -I --csv cat gz-example.csv.gz` will write `gz-example.csv.gz` which contains a plain, uncompressed CSV contents. This is a bug and will be fixed. diff --git a/docs6/docs/reference-main-data-types.md b/docs6/src/reference-main-data-types.md similarity index 98% rename from docs6/docs/reference-main-data-types.md rename to docs6/src/reference-main-data-types.md index ddd28990a..80d77e82d 100644 --- a/docs6/docs/reference-main-data-types.md +++ b/docs6/src/reference-main-data-types.md @@ -34,7 +34,7 @@ Miller's types are: See also the list of [type-checking functions](reference-dsl-builtin-functions.md#type-checkin -functions) for the -[Miller programming language](programming-language.md). +[Miller programming language](miller-programming-language.md). See also [Differences from other programming languages](reference-dsl-differences.md). @@ -47,7 +47,7 @@ such as CSV, TSV, and JSON; binary formats such as not supported (as of mid-2021). In this sense, everything is a string in and out of Miller -- be it in data files, or in DSL expressions you key in. -In the [DSL](programming-language.md), `7` is an `int` and `8.9` is a float, as +In the [DSL](miller-programming-language.md), `7` is an `int` and `8.9` is a float, as one would expect. Likewise, on input from [data files](file-formats.md), string values representable as numbers, e.g. `1.2` or `3`, are treated as int or float, respectively. If a record has `x=1,y=2` then `mlr put '$z=$x+$y'` diff --git a/docs6/docs/reference-main-data-types.md.in b/docs6/src/reference-main-data-types.md.in similarity index 97% rename from docs6/docs/reference-main-data-types.md.in rename to docs6/src/reference-main-data-types.md.in index 7e7c530c2..2dcef0f87 100644 --- a/docs6/docs/reference-main-data-types.md.in +++ b/docs6/src/reference-main-data-types.md.in @@ -18,7 +18,7 @@ Miller's types are: See also the list of [type-checking functions](reference-dsl-builtin-functions.md#type-checkin -functions) for the -[Miller programming language](programming-language.md). +[Miller programming language](miller-programming-language.md). See also [Differences from other programming languages](reference-dsl-differences.md). @@ -31,7 +31,7 @@ such as CSV, TSV, and JSON; binary formats such as not supported (as of mid-2021). In this sense, everything is a string in and out of Miller -- be it in data files, or in DSL expressions you key in. -In the [DSL](programming-language.md), `7` is an `int` and `8.9` is a float, as +In the [DSL](miller-programming-language.md), `7` is an `int` and `8.9` is a float, as one would expect. Likewise, on input from [data files](file-formats.md), string values representable as numbers, e.g. `1.2` or `3`, are treated as int or float, respectively. If a record has `x=1,y=2` then `mlr put '$z=$x+$y'` diff --git a/docs6/docs/reference-main-env-vars.md b/docs6/src/reference-main-env-vars.md similarity index 100% rename from docs6/docs/reference-main-env-vars.md rename to docs6/src/reference-main-env-vars.md diff --git a/docs6/docs/reference-main-env-vars.md.in b/docs6/src/reference-main-env-vars.md.in similarity index 100% rename from docs6/docs/reference-main-env-vars.md.in rename to docs6/src/reference-main-env-vars.md.in diff --git a/docs6/docs/reference-main-flag-list.md b/docs6/src/reference-main-flag-list.md similarity index 97% rename from docs6/docs/reference-main-flag-list.md rename to docs6/src/reference-main-flag-list.md index a0ff25517..cbe109965 100644 --- a/docs6/docs/reference-main-flag-list.md +++ b/docs6/src/reference-main-flag-list.md @@ -16,6 +16,28 @@ Quick links: # List of command-line flags +Here are flags you can use when invoking Miller. For example, when you type + +
+mlr --icsv --ojson head -n 1 example.csv
+
+
+{
+  "color": "yellow",
+  "shape": "triangle",
+  "flag": "true",
+  "k": 1,
+  "index": 11,
+  "quantity": 43.6498,
+  "rate": 9.8870
+}
+
+ +the `--icsv` and `--ojson` bits are _flags_. See the [Miller command +structure](reference-main-overview.md) page for context. + +Also, at the command line, you can use `mlr -g` for a list much like this one. + ## Comments-in-data flags Miller lets you put comments in your data, such as diff --git a/docs6/src/reference-main-flag-list.md.in b/docs6/src/reference-main-flag-list.md.in new file mode 100644 index 000000000..66d4f7d37 --- /dev/null +++ b/docs6/src/reference-main-flag-list.md.in @@ -0,0 +1,14 @@ +# List of command-line flags + +Here are flags you can use when invoking Miller. For example, when you type + +GENMD_RUN_COMMAND +mlr --icsv --ojson head -n 1 example.csv +GENMD_EOF + +the `--icsv` and `--ojson` bits are _flags_. See the [Miller command +structure](reference-main-overview.md) page for context. + +Also, at the command line, you can use `mlr -g` for a list much like this one. + +GENMD_RUN_CONTENT_GENERATOR(./mk-flag-info.rb) diff --git a/docs6/src/reference-main-in-place-processing.md b/docs6/src/reference-main-in-place-processing.md new file mode 100644 index 000000000..5da98bc63 --- /dev/null +++ b/docs6/src/reference-main-in-place-processing.md @@ -0,0 +1,28 @@ + +
+ +Quick links: +  +Flag list +  +Verb list +  +Function list +  +Glossary +  +Repository ↗ + +
+# In-place mode + +Use the `mlr -I` flag to process files in-place. For example, `mlr -I --csv cut -x -f unwanted_column_name mydata/*.csv` will remove `unwanted_column_name` from all your `*.csv` files in your `mydata/` subdirectory. + +By default, Miller output goes to the screen (or you can redirect a file using `>` or to another process using `|`). With `-I`, for each file name on the command line, output is written to a temporary file in the same directory. Miller writes its output into that temp file, which is then renamed over the original. Then, processing continues on the next file. Each file is processed in isolation: if the output format is CSV, CSV headers will be present in each output file; statistics are only over each file's own records; and so on. + +Since this replaces your data with modified data, it's often a good idea to back up your original files somewhere +first, to protect against keystroking errors. + +TODO: fix the combination of `-I` and compressed input. + +Please see [Choices for printing to files](10min.md#choices-for-printing-to-files) for examples. diff --git a/docs6/src/reference-main-in-place-processing.md.in b/docs6/src/reference-main-in-place-processing.md.in new file mode 100644 index 000000000..19d142198 --- /dev/null +++ b/docs6/src/reference-main-in-place-processing.md.in @@ -0,0 +1,12 @@ +# In-place mode + +Use the `mlr -I` flag to process files in-place. For example, `mlr -I --csv cut -x -f unwanted_column_name mydata/*.csv` will remove `unwanted_column_name` from all your `*.csv` files in your `mydata/` subdirectory. + +By default, Miller output goes to the screen (or you can redirect a file using `>` or to another process using `|`). With `-I`, for each file name on the command line, output is written to a temporary file in the same directory. Miller writes its output into that temp file, which is then renamed over the original. Then, processing continues on the next file. Each file is processed in isolation: if the output format is CSV, CSV headers will be present in each output file; statistics are only over each file's own records; and so on. + +Since this replaces your data with modified data, it's often a good idea to back up your original files somewhere +first, to protect against keystroking errors. + +TODO: fix the combination of `-I` and compressed input. + +Please see [Choices for printing to files](10min.md#choices-for-printing-to-files) for examples. diff --git a/docs6/docs/reference-main-maps.md b/docs6/src/reference-main-maps.md similarity index 98% rename from docs6/docs/reference-main-maps.md rename to docs6/src/reference-main-maps.md index e906ffa03..a377af097 100644 --- a/docs6/docs/reference-main-maps.md +++ b/docs6/src/reference-main-maps.md @@ -222,3 +222,7 @@ purple square 8.2430 ## Looping See [single-variable for-loops](reference-dsl-control-structures.md#single-variable-for-loops) and [key-value for-loops](reference-dsl-control-structures.md#key-value-for-loops). + +## Map-valued fields in CSV files + +See the [flatten/unflatten page](flatten-unflatten.md). diff --git a/docs6/docs/reference-main-maps.md.in b/docs6/src/reference-main-maps.md.in similarity index 97% rename from docs6/docs/reference-main-maps.md.in rename to docs6/src/reference-main-maps.md.in index ba80bab70..b875dd407 100644 --- a/docs6/docs/reference-main-maps.md.in +++ b/docs6/src/reference-main-maps.md.in @@ -128,3 +128,7 @@ GENMD_EOF ## Looping See [single-variable for-loops](reference-dsl-control-structures.md#single-variable-for-loops) and [key-value for-loops](reference-dsl-control-structures.md#key-value-for-loops). + +## Map-valued fields in CSV files + +See the [flatten/unflatten page](flatten-unflatten.md). diff --git a/docs6/docs/reference-main-null-data.md b/docs6/src/reference-main-null-data.md similarity index 95% rename from docs6/docs/reference-main-null-data.md rename to docs6/src/reference-main-null-data.md index 16bc2caf7..907a1e9b5 100644 --- a/docs6/docs/reference-main-null-data.md +++ b/docs6/src/reference-main-null-data.md @@ -24,7 +24,7 @@ Miller has three kinds of null data: * **Absent (key not present)**: a field name is not present, e.g. input record is `x=1,y=2` and a `put` or `filter` expression refers to `$z`. Or, reading an out-of-stream variable which hasn't been assigned a value yet, e.g. `mlr put -q '@sum += $x; end{emit @sum}'` or `mlr put -q '@sum[$a][$b] += $x; end{emit @sum, "a", "b"}'`. -* **JSON null**: The main purpose of this is to support reading the `null` type in JSON files. The [Miller programming language](programming-language.md) has a `null` keyword as well, so you can also write the null type using `$x = null`. Addtionally, though, when you write past the end of an array, leaving gaps -- e.g. writing `a[12]` when the array `a` has length 10 -- JSON-null is used to fill the gaps. See also the [arrays page](reference-main-arrays.md#auto-extend-and-null-gaps). +* **JSON null**: The main purpose of this is to support reading the `null` type in JSON files. The [Miller programming language](miller-programming-language.md) has a `null` keyword as well, so you can also write the null type using `$x = null`. Addtionally, though, when you write past the end of an array, leaving gaps -- e.g. writing `a[12]` when the array `a` has length 10 -- JSON-null is used to fill the gaps. See also the [arrays page](reference-main-arrays.md#auto-extend-and-null-gaps). You can test these programatically using the functions `is_empty`/`is_not_empty`, `is_absent`/`is_present`, and `is_null`/`is_not_null`. For the last pair, note that null means either empty or absent. diff --git a/docs6/docs/reference-main-null-data.md.in b/docs6/src/reference-main-null-data.md.in similarity index 93% rename from docs6/docs/reference-main-null-data.md.in rename to docs6/src/reference-main-null-data.md.in index 47ac6c2af..700dbfcbd 100644 --- a/docs6/docs/reference-main-null-data.md.in +++ b/docs6/src/reference-main-null-data.md.in @@ -8,7 +8,7 @@ Miller has three kinds of null data: * **Absent (key not present)**: a field name is not present, e.g. input record is `x=1,y=2` and a `put` or `filter` expression refers to `$z`. Or, reading an out-of-stream variable which hasn't been assigned a value yet, e.g. `mlr put -q '@sum += $x; end{emit @sum}'` or `mlr put -q '@sum[$a][$b] += $x; end{emit @sum, "a", "b"}'`. -* **JSON null**: The main purpose of this is to support reading the `null` type in JSON files. The [Miller programming language](programming-language.md) has a `null` keyword as well, so you can also write the null type using `$x = null`. Addtionally, though, when you write past the end of an array, leaving gaps -- e.g. writing `a[12]` when the array `a` has length 10 -- JSON-null is used to fill the gaps. See also the [arrays page](reference-main-arrays.md#auto-extend-and-null-gaps). +* **JSON null**: The main purpose of this is to support reading the `null` type in JSON files. The [Miller programming language](miller-programming-language.md) has a `null` keyword as well, so you can also write the null type using `$x = null`. Addtionally, though, when you write past the end of an array, leaving gaps -- e.g. writing `a[12]` when the array `a` has length 10 -- JSON-null is used to fill the gaps. See also the [arrays page](reference-main-arrays.md#auto-extend-and-null-gaps). You can test these programatically using the functions `is_empty`/`is_not_empty`, `is_absent`/`is_present`, and `is_null`/`is_not_null`. For the last pair, note that null means either empty or absent. diff --git a/docs6/src/reference-main-number-formatting.md b/docs6/src/reference-main-number-formatting.md new file mode 100644 index 000000000..c7ba6e389 --- /dev/null +++ b/docs6/src/reference-main-number-formatting.md @@ -0,0 +1,82 @@ + +
+ +Quick links: +  +Flag list +  +Verb list +  +Function list +  +Glossary +  +Repository ↗ + +
+# Number formatting + +## The --ofmt flag + +The command-line option `--ofmt {format string}` is the global number format for all numeric fields. Examples: + +
+--ofmt %.9e --ofmt %.6f --ofmt %.0f
+
+ +These are just familiar `printf` formats. (TODO: write about type-checking once that's implemented.) Additionally, if you use leading width (e.g. `%18.12f`) then the output will contain embedded whitespace, which may not be what you want if you pipe the output to something else, particularly CSV. I use Miller's pretty-print format (`mlr --opprint`) to column-align numerical data. + +
+echo 'x=3.1,y=4.3' | mlr --ofmt '%8.3f' cat
+
+
+x=   3.100,y=   4.300
+
+ +
+echo 'x=3.1,y=4.3' | mlr --ofmt '%11.8e' cat
+
+
+x=3.10000000e+00,y=4.30000000e+00
+
+ +## The format-values verb + +To separately specify formatting for string, int, and float fields, you can use +the [`format-values`](reference-verbs.md#format-values) verb -- see that section for examples. + +## The fmtnum and hexfmt functions + +To apply formatting to a single field, you can also use +[`fmtnum`](reference-dsl-builtin-functions.md#fmtnum) function within `mlr +put`. For example: + +
+echo 'x=3.1,y=4.3' | mlr put '$z=fmtnum($x*$y,"%08f")'
+
+
+x=3.1,y=4.3,z=13.330000
+
+ +
+echo 'x=0xffff,y=0xff' | mlr put '$z=fmtnum(int($x*$y),"%08x")'
+
+
+x=0xffff,y=0xff,z=00feff01
+
+ +Input conversion from hexadecimal is done automatically on fields handled by `mlr put` and `mlr filter` as long as the field value begins with `0x`. To apply output conversion to hexadecimal on a single column, you may use `fmtnum`, or the keystroke-saving [`hexfmt`](reference-dsl-builtin-functions.md#hexfmt) function. Example: + +
+echo 'x=0xffff,y=0xff' | mlr put '$z=$x*$y'
+
+
+x=0xffff,y=0xff,z=16711425
+
+ +
+echo 'x=0xffff,y=0xff' | mlr put '$z=hexfmt($x*$y)'
+
+
+x=0xffff,y=0xff,z=0xfeff01
+
diff --git a/docs6/src/reference-main-number-formatting.md.in b/docs6/src/reference-main-number-formatting.md.in new file mode 100644 index 000000000..89b4ac78f --- /dev/null +++ b/docs6/src/reference-main-number-formatting.md.in @@ -0,0 +1,48 @@ +# Number formatting + +## The --ofmt flag + +The command-line option `--ofmt {format string}` is the global number format for all numeric fields. Examples: + +GENMD_CARDIFY +--ofmt %.9e --ofmt %.6f --ofmt %.0f +GENMD_EOF + +These are just familiar `printf` formats. (TODO: write about type-checking once that's implemented.) Additionally, if you use leading width (e.g. `%18.12f`) then the output will contain embedded whitespace, which may not be what you want if you pipe the output to something else, particularly CSV. I use Miller's pretty-print format (`mlr --opprint`) to column-align numerical data. + +GENMD_RUN_COMMAND +echo 'x=3.1,y=4.3' | mlr --ofmt '%8.3f' cat +GENMD_EOF + +GENMD_RUN_COMMAND +echo 'x=3.1,y=4.3' | mlr --ofmt '%11.8e' cat +GENMD_EOF + +## The format-values verb + +To separately specify formatting for string, int, and float fields, you can use +the [`format-values`](reference-verbs.md#format-values) verb -- see that section for examples. + +## The fmtnum and hexfmt functions + +To apply formatting to a single field, you can also use +[`fmtnum`](reference-dsl-builtin-functions.md#fmtnum) function within `mlr +put`. For example: + +GENMD_RUN_COMMAND +echo 'x=3.1,y=4.3' | mlr put '$z=fmtnum($x*$y,"%08f")' +GENMD_EOF + +GENMD_RUN_COMMAND +echo 'x=0xffff,y=0xff' | mlr put '$z=fmtnum(int($x*$y),"%08x")' +GENMD_EOF + +Input conversion from hexadecimal is done automatically on fields handled by `mlr put` and `mlr filter` as long as the field value begins with `0x`. To apply output conversion to hexadecimal on a single column, you may use `fmtnum`, or the keystroke-saving [`hexfmt`](reference-dsl-builtin-functions.md#hexfmt) function. Example: + +GENMD_RUN_COMMAND +echo 'x=0xffff,y=0xff' | mlr put '$z=$x*$y' +GENMD_EOF + +GENMD_RUN_COMMAND +echo 'x=0xffff,y=0xff' | mlr put '$z=hexfmt($x*$y)' +GENMD_EOF diff --git a/docs6/docs/reference-main-overview.md b/docs6/src/reference-main-overview.md similarity index 92% rename from docs6/docs/reference-main-overview.md rename to docs6/src/reference-main-overview.md index 366275f3a..bb2db11ca 100644 --- a/docs6/docs/reference-main-overview.md +++ b/docs6/src/reference-main-overview.md @@ -21,8 +21,8 @@ Quick links: The outline of an invocation of Miller is: * The program name `mlr`. -* Options controlling input/output formatting, etc. (See [I/O options](reference-main-io-options.md)). -* One or more verbs -- such as `cut`, `sort`, etc. (see [Verbs Reference](reference-verbs.md)) -- chained together using [then](reference-main-then-chaining.md). You use these to transform your data. +* Flags controlling input/output formatting, etc. (see the [flags reference](reference-main-flag-list.md)). +* One or more verbs -- such as `cut`, `sort`, etc. (see [verbs reference](reference-verbs.md)) -- chained together using [then](reference-main-then-chaining.md). You use these to transform your data. * Zero or more filenames, with input taken from standard input if there are no filenames present. (You can place the filenames up front using `--from` or `--mfrom` as described on the [keystroke-savers page](keystroke-savers.md#file-names-up-front-including-from).) For example, reading from a file: @@ -66,7 +66,7 @@ See also the [Glossary](glossary.md) for more about terms such as When you type `mlr {something} myfile.dat`, the `{something}` part is called a **verb**. It specifies how you want to transform your data. Most of the verbs are counterparts of built-in system tools like `cut` and `sort` -- but with file-format awareness, and giving you the ability to refer to fields by name. -The verbs `put` and `filter` are special in that they have a rich expression language (domain-specific language, or "DSL"). More information about them can be found at on the [Intro to Miller's programming language page](programming-language.md); see also [DSL reference](reference-dsl.md) for more details. +The verbs `put` and `filter` are special in that they have a rich expression language (domain-specific language, or "DSL"). More information about them can be found at on the [Intro to Miller's programming language page](miller-programming-language.md); see also [DSL reference](reference-dsl.md) for more details. Here's a comparison of verbs and `put`/`filter` DSL expressions: diff --git a/docs6/docs/reference-main-overview.md.in b/docs6/src/reference-main-overview.md.in similarity index 88% rename from docs6/docs/reference-main-overview.md.in rename to docs6/src/reference-main-overview.md.in index 9faa796c1..b3311cd37 100644 --- a/docs6/docs/reference-main-overview.md.in +++ b/docs6/src/reference-main-overview.md.in @@ -5,8 +5,8 @@ The outline of an invocation of Miller is: * The program name `mlr`. -* Options controlling input/output formatting, etc. (See [I/O options](reference-main-io-options.md)). -* One or more verbs -- such as `cut`, `sort`, etc. (see [Verbs Reference](reference-verbs.md)) -- chained together using [then](reference-main-then-chaining.md). You use these to transform your data. +* Flags controlling input/output formatting, etc. (see the [flags reference](reference-main-flag-list.md)). +* One or more verbs -- such as `cut`, `sort`, etc. (see [verbs reference](reference-verbs.md)) -- chained together using [then](reference-main-then-chaining.md). You use these to transform your data. * Zero or more filenames, with input taken from standard input if there are no filenames present. (You can place the filenames up front using `--from` or `--mfrom` as described on the [keystroke-savers page](keystroke-savers.md#file-names-up-front-including-from).) For example, reading from a file: @@ -35,7 +35,7 @@ See also the [Glossary](glossary.md) for more about terms such as When you type `mlr {something} myfile.dat`, the `{something}` part is called a **verb**. It specifies how you want to transform your data. Most of the verbs are counterparts of built-in system tools like `cut` and `sort` -- but with file-format awareness, and giving you the ability to refer to fields by name. -The verbs `put` and `filter` are special in that they have a rich expression language (domain-specific language, or "DSL"). More information about them can be found at on the [Intro to Miller's programming language page](programming-language.md); see also [DSL reference](reference-dsl.md) for more details. +The verbs `put` and `filter` are special in that they have a rich expression language (domain-specific language, or "DSL"). More information about them can be found at on the [Intro to Miller's programming language page](miller-programming-language.md); see also [DSL reference](reference-dsl.md) for more details. Here's a comparison of verbs and `put`/`filter` DSL expressions: diff --git a/docs6/docs/reference-main-regular-expressions.md b/docs6/src/reference-main-regular-expressions.md similarity index 100% rename from docs6/docs/reference-main-regular-expressions.md rename to docs6/src/reference-main-regular-expressions.md diff --git a/docs6/docs/reference-main-regular-expressions.md.in b/docs6/src/reference-main-regular-expressions.md.in similarity index 100% rename from docs6/docs/reference-main-regular-expressions.md.in rename to docs6/src/reference-main-regular-expressions.md.in diff --git a/docs6/docs/reference-main-separators.md b/docs6/src/reference-main-separators.md similarity index 100% rename from docs6/docs/reference-main-separators.md rename to docs6/src/reference-main-separators.md diff --git a/docs6/docs/reference-main-separators.md.in b/docs6/src/reference-main-separators.md.in similarity index 100% rename from docs6/docs/reference-main-separators.md.in rename to docs6/src/reference-main-separators.md.in diff --git a/docs6/src/reference-main-strings.md b/docs6/src/reference-main-strings.md new file mode 100644 index 000000000..17b0dc7ad --- /dev/null +++ b/docs6/src/reference-main-strings.md @@ -0,0 +1,181 @@ + +
+ +Quick links: +  +Flag list +  +Verb list +  +Function list +  +Glossary +  +Repository ↗ + +
+# Strings + +## Essentials + +Miller string literals are always written with double quotes, like `"abcde"`; single quotes +are not part of the grammar of [Miller's programming language](miller-programming-language.md). +Single quotes are used for wrapping `put`/`filter` statements, as in `mlr put '$b=$a.".suffix"' myfile.csv'`: +the single-quotes are consumed by the shell and Miller gets `$b=$a.".suffix"`. (See however the +[Miller on Windows page](miller-on-windows.md).) + +A basic string operation is the `.` (concatenation) operator: + +
+mlr --c2p --from example.csv put '$output = $color . ":" . $shape'
+
+
+color  shape    flag  k  index quantity rate   output
+yellow triangle true  1  11    43.6498  9.8870 yellow:triangle
+red    square   true  2  15    79.2778  0.0130 red:square
+red    circle   true  3  16    13.8103  2.9010 red:circle
+red    square   false 4  48    77.5542  7.4670 red:square
+purple triangle false 5  51    81.2290  8.5910 purple:triangle
+red    square   false 6  64    77.1991  9.5310 red:square
+purple triangle false 7  65    80.1405  5.8240 purple:triangle
+yellow circle   true  8  73    63.9785  4.2370 yellow:circle
+yellow circle   true  9  87    63.5058  8.3350 yellow:circle
+purple square   false 10 91    72.3735  8.2430 purple:square
+
+ +Also see the [list of string-related built-in functions](reference-dsl-builtin-functions.md#string-functions). + +## 1-up indexing + +The most important difference between Miller's strings and strings in other +languages is that indices start with 1, not 0. (The same is true for [Miller +arrays](reference-main-arrays.md).) This is intentional. + +1-up indices may feel like a thing of the past, belonging to Fortran and Matlab, +say; or R and Julia as well, which are more modern. But the overall trend is +decidedly toward 0-up. This means that if Miller does 1-up indices, it should +do so for good reasons. + +Miller strings are indexed 1-up simply because Miller arrays are indexed 1-up. +See [this section](reference-main-arrays.md#1-up-indexing) for the reasoning. + +Strings have been in Miller since the beginning, but they weren't accessible +using indices or slices until [Miller 6](new-in-miller-6.md). Also, the +[`substr`](reference-dsl-builtin-functions.md#substr) function predates Miller +6. This function was implemented to take 0-up indices. When Miller 6 was +implemented, this became inconsistent. As a result, there are +[`substr0`](reference-dsl-builtin-functions.md#substr0) and +[`substr1`](reference-dsl-builtin-functions.md#substr1) functions. For backward +compatibility with existing Miller scripts, `substr` is the same as `substr0`. +But users starting out with Miller 6 will probably want `substr1`. + +## Negative-index aliasing + +Imitating Python and other languages, you can use negative indices to read +backward from the end of the string, while positive indices read forward from +the start. If a string has length `n` then `-n..-1` are aliases for `1..n`, +respectively; 0 is never a valid string index in Miller. + +
+mlr -n put '
+  end {
+    x = "abcde";
+    print x[1];
+    print x[-1];
+    print x[1:2];
+    print x[-2:-1];
+  }
+'
+
+
+a
+e
+ab
+de
+
+ +## Slicing + +Miller supports slicing using `[lo:hi]` syntax. Either or both of the indices +in a slice can be negatively aliased as described above. Unlike in Python, +Miller string-slice indices are inclusive on both sides: `x[3:5]` means `x[3] . x[4] . x[5]`. + +
+mlr -n put '
+  end {
+    x = "abcde";
+    print x[3:4];
+    print x[:2];
+    print x[3:];
+    print x[1:-1];
+    print x[2:-2];
+  }
+'
+
+
+cd
+ab
+cde
+abcde
+bcd
+
+ +## Out-of-bounds indexing + +Somewhat imitating Python, out-of-bounds index accesses are +[errors](reference-main-data-types.md), but out-of-bounds slice accesses result +in trimming the indices, resulting in a short string or even the empty string: + +
+mlr -n put '
+  end {
+    x = "abcde";
+    print x[1];
+    print x[5];
+    print x[6]; # absent
+  }
+'
+
+
+a
+e
+(error)
+
+ +
+mlr -n put '
+  end {
+    x = "abcde";
+    print x[1:2];
+    print x[1:6];
+    print x[10:20];
+  }
+'
+
+
+ab
+
+
+
+ +## Escape sequences for string literals + +You can use the following backslash escapes for strings such as between the double quotes in contexts such as `mlr filter '$name =~ "..."'`, `mlr put '$name = $othername . "..."'`, `mlr put '$name = sub($name, "...", "...")`, etc.: + +* `\a`: ASCII code 0x07 (alarm/bell) +* `\b`: ASCII code 0x08 (backspace) +* `\f`: ASCII code 0x0c (formfeed) +* `\n`: ASCII code 0x0a (LF/linefeed/newline) +* `\r`: ASCII code 0x0d (CR/carriage return) +* `\t`: ASCII code 0x09 (tab) +* `\v`: ASCII code 0x0b (vertical tab) +* `\\`: backslash +* `\"`: double quote +* `\123`: Octal 123, etc. for `\000` up to `\377` +* `\x7f`: Hexadecimal 7f, etc. for `\x00` up to `\xff` + +See also [https://en.wikipedia.org/wiki/Escape_sequences_in_C](https://en.wikipedia.org/wiki/Escape_sequences_in_C). + +These replacements apply only to strings you key in for the DSL expressions for `filter` and `put`: that is, if you type `\t` in a string literal for a `filter`/`put` expression, it will be turned into a tab character. If you want a backslash followed by a `t`, then please type `\\t`. + +However, these replacements are done automatically only for string literals within DSL expressions -- they are not done automatically to fields within your data stream. If you wish to make these replacements, you can do (for example) `mlr put '$field = gsub($field, "\\t", "\t")'`. If you need to make such a replacement for all fields in your data, you should probably use the system `sed` command instead. diff --git a/docs6/src/reference-main-strings.md.in b/docs6/src/reference-main-strings.md.in new file mode 100644 index 000000000..ef4f95701 --- /dev/null +++ b/docs6/src/reference-main-strings.md.in @@ -0,0 +1,129 @@ +# Strings + +## Essentials + +Miller string literals are always written with double quotes, like `"abcde"`; single quotes +are not part of the grammar of [Miller's programming language](miller-programming-language.md). +Single quotes are used for wrapping `put`/`filter` statements, as in `mlr put '$b=$a.".suffix"' myfile.csv'`: +the single-quotes are consumed by the shell and Miller gets `$b=$a.".suffix"`. (See however the +[Miller on Windows page](miller-on-windows.md).) + +A basic string operation is the `.` (concatenation) operator: + +GENMD_RUN_COMMAND +mlr --c2p --from example.csv put '$output = $color . ":" . $shape' +GENMD_EOF + +Also see the [list of string-related built-in functions](reference-dsl-builtin-functions.md#string-functions). + +## 1-up indexing + +The most important difference between Miller's strings and strings in other +languages is that indices start with 1, not 0. (The same is true for [Miller +arrays](reference-main-arrays.md).) This is intentional. + +1-up indices may feel like a thing of the past, belonging to Fortran and Matlab, +say; or R and Julia as well, which are more modern. But the overall trend is +decidedly toward 0-up. This means that if Miller does 1-up indices, it should +do so for good reasons. + +Miller strings are indexed 1-up simply because Miller arrays are indexed 1-up. +See [this section](reference-main-arrays.md#1-up-indexing) for the reasoning. + +Strings have been in Miller since the beginning, but they weren't accessible +using indices or slices until [Miller 6](new-in-miller-6.md). Also, the +[`substr`](reference-dsl-builtin-functions.md#substr) function predates Miller +6. This function was implemented to take 0-up indices. When Miller 6 was +implemented, this became inconsistent. As a result, there are +[`substr0`](reference-dsl-builtin-functions.md#substr0) and +[`substr1`](reference-dsl-builtin-functions.md#substr1) functions. For backward +compatibility with existing Miller scripts, `substr` is the same as `substr0`. +But users starting out with Miller 6 will probably want `substr1`. + +## Negative-index aliasing + +Imitating Python and other languages, you can use negative indices to read +backward from the end of the string, while positive indices read forward from +the start. If a string has length `n` then `-n..-1` are aliases for `1..n`, +respectively; 0 is never a valid string index in Miller. + +GENMD_RUN_COMMAND +mlr -n put ' + end { + x = "abcde"; + print x[1]; + print x[-1]; + print x[1:2]; + print x[-2:-1]; + } +' +GENMD_EOF + +## Slicing + +Miller supports slicing using `[lo:hi]` syntax. Either or both of the indices +in a slice can be negatively aliased as described above. Unlike in Python, +Miller string-slice indices are inclusive on both sides: `x[3:5]` means `x[3] . x[4] . x[5]`. + +GENMD_RUN_COMMAND +mlr -n put ' + end { + x = "abcde"; + print x[3:4]; + print x[:2]; + print x[3:]; + print x[1:-1]; + print x[2:-2]; + } +' +GENMD_EOF + +## Out-of-bounds indexing + +Somewhat imitating Python, out-of-bounds index accesses are +[errors](reference-main-data-types.md), but out-of-bounds slice accesses result +in trimming the indices, resulting in a short string or even the empty string: + +GENMD_RUN_COMMAND +mlr -n put ' + end { + x = "abcde"; + print x[1]; + print x[5]; + print x[6]; # absent + } +' +GENMD_EOF + +GENMD_RUN_COMMAND +mlr -n put ' + end { + x = "abcde"; + print x[1:2]; + print x[1:6]; + print x[10:20]; + } +' +GENMD_EOF + +## Escape sequences for string literals + +You can use the following backslash escapes for strings such as between the double quotes in contexts such as `mlr filter '$name =~ "..."'`, `mlr put '$name = $othername . "..."'`, `mlr put '$name = sub($name, "...", "...")`, etc.: + +* `\a`: ASCII code 0x07 (alarm/bell) +* `\b`: ASCII code 0x08 (backspace) +* `\f`: ASCII code 0x0c (formfeed) +* `\n`: ASCII code 0x0a (LF/linefeed/newline) +* `\r`: ASCII code 0x0d (CR/carriage return) +* `\t`: ASCII code 0x09 (tab) +* `\v`: ASCII code 0x0b (vertical tab) +* `\\`: backslash +* `\"`: double quote +* `\123`: Octal 123, etc. for `\000` up to `\377` +* `\x7f`: Hexadecimal 7f, etc. for `\x00` up to `\xff` + +See also [https://en.wikipedia.org/wiki/Escape_sequences_in_C](https://en.wikipedia.org/wiki/Escape_sequences_in_C). + +These replacements apply only to strings you key in for the DSL expressions for `filter` and `put`: that is, if you type `\t` in a string literal for a `filter`/`put` expression, it will be turned into a tab character. If you want a backslash followed by a `t`, then please type `\\t`. + +However, these replacements are done automatically only for string literals within DSL expressions -- they are not done automatically to fields within your data stream. If you wish to make these replacements, you can do (for example) `mlr put '$field = gsub($field, "\\t", "\t")'`. If you need to make such a replacement for all fields in your data, you should probably use the system `sed` command instead. diff --git a/docs6/docs/reference-main-then-chaining.md b/docs6/src/reference-main-then-chaining.md similarity index 100% rename from docs6/docs/reference-main-then-chaining.md rename to docs6/src/reference-main-then-chaining.md diff --git a/docs6/docs/reference-main-then-chaining.md.in b/docs6/src/reference-main-then-chaining.md.in similarity index 100% rename from docs6/docs/reference-main-then-chaining.md.in rename to docs6/src/reference-main-then-chaining.md.in diff --git a/docs6/docs/reference-verbs.md b/docs6/src/reference-verbs.md similarity index 98% rename from docs6/docs/reference-verbs.md rename to docs6/src/reference-verbs.md index 4b03dc932..b555c382d 100644 --- a/docs6/docs/reference-verbs.md +++ b/docs6/src/reference-verbs.md @@ -16,10 +16,31 @@ Quick links: # List of verbs +Verbs are the building blocks of how you can use Miller to process your data. +When you type + +
+mlr --icsv --opprint sort -n quanity then head -n 4 example.csv
+
+
+color  shape    flag  k index quantity rate
+yellow triangle true  1 11    43.6498  9.8870
+red    square   true  2 15    79.2778  0.0130
+red    circle   true  3 16    13.8103  2.9010
+red    square   false 4 48    77.5542  7.4670
+
+ +the `sort` and `head` bits are _verbs_. See the [Miller command +structure](reference-main-overview.md) page for context. + +At the command line, you can use `mlr -l` and `mlr -L` for information much +like what's on this page. + ## Overview Whereas the Unix toolkit is made of the separate executables `cat`, `tail`, `cut`, -`sort`, etc., Miller has subcommands, or **verbs**, invoked as follows: +`sort`, etc., Miller has subcommands, or **verbs**, such as `mlr cat`, `mlr tail`, `mlr cut`, and +`mlr sort`, invoked as follows: 
 mlr tac *.dat
@@ -952,6 +973,21 @@ Parser-info options:
 
 Please see [DSL reference](reference-dsl.md) for more information about the expression language for `mlr filter`.
 
+## flatten
+
+
+mlr flatten --help
+

+
+Usage: mlr flatten [options]
+Flattens multi-level maps to single-level ones. Example: field with name 'a'
+and value '{"b": { "c": 4 }}' becomes name 'a.b.c' and value 4.
+Options:
+-f Comma-separated list of field names to flatten (default all).
+-s Separator, defaulting to mlr --flatsep value.
+-h|--help Show this message.
+

+
 ## format-values
 
 
@@ -3482,6 +3518,21 @@ zee 1       0.9994904324789629
 zee 2       0.9994378171787394
 

 
+## unflatten
+
+
+mlr unflatten --help
+

+
+Usage: mlr unflatten [options]
+Reverses flatten. Example: field with name 'a.b.c' and value 4
+becomes name 'a' and value '{"b": { "c": 4 }}'.
+Options:
+-f {a,b,c} Comma-separated list of field names to unflatten (default all).
+-s {string} Separator, defaulting to mlr --flatsep value.
+-h|--help Show this message.
+

+
 ## uniq
 
 
diff --git a/docs6/docs/reference-verbs.md.in b/docs6/src/reference-verbs.md.in
similarity index 97%
rename from docs6/docs/reference-verbs.md.in
rename to docs6/src/reference-verbs.md.in
index cadb0fa8b..68c5335a8 100644
--- a/docs6/docs/reference-verbs.md.in
+++ b/docs6/src/reference-verbs.md.in
@@ -1,9 +1,23 @@
 # List of verbs
 
+Verbs are the building blocks of how you can use Miller to process your data.
+When you type
+
+GENMD_RUN_COMMAND
+mlr --icsv --opprint sort -n quanity then head -n 4 example.csv
+GENMD_EOF
+
+the `sort` and `head` bits are _verbs_.  See the [Miller command
+structure](reference-main-overview.md) page for context.
+
+At the command line, you can use `mlr -l` and `mlr -L` for information much
+like what's on this page.
+
 ## Overview
 
 Whereas the Unix toolkit is made of the separate executables `cat`, `tail`, `cut`,
-`sort`, etc., Miller has subcommands, or **verbs**, invoked as follows:
+`sort`, etc., Miller has subcommands, or **verbs**, such as `mlr cat`, `mlr tail`, `mlr cut`, and
+`mlr sort`, invoked as follows:
 
 GENMD_INCLUDE_ESCAPED(data/subcommand-example.txt)
 
@@ -330,6 +344,12 @@ GENMD_EOF
 
 Please see [DSL reference](reference-dsl.md) for more information about the expression language for `mlr filter`.
 
+## flatten
+
+GENMD_RUN_COMMAND
+mlr flatten --help
+GENMD_EOF
+
 ## format-values
 
 GENMD_RUN_COMMAND
@@ -1091,6 +1111,12 @@ GENMD_RUN_COMMAND
 mlr --opprint top -n 2 -f x -g a then sort -f a data/medium
 GENMD_EOF
 
+## unflatten
+
+GENMD_RUN_COMMAND
+mlr unflatten --help
+GENMD_EOF
+
 ## uniq
 
 GENMD_RUN_COMMAND
diff --git a/docs6/docs/release-docs.md b/docs6/src/release-docs.md
similarity index 100%
rename from docs6/docs/release-docs.md
rename to docs6/src/release-docs.md
diff --git a/docs6/docs/release-docs.md.in b/docs6/src/release-docs.md.in
similarity index 100%
rename from docs6/docs/release-docs.md.in
rename to docs6/src/release-docs.md.in
diff --git a/docs6/docs/repl.md b/docs6/src/repl.md
similarity index 98%
rename from docs6/docs/repl.md
rename to docs6/src/repl.md
index cdc211c2a..13399fdfe 100644
--- a/docs6/docs/repl.md
+++ b/docs6/src/repl.md
@@ -18,7 +18,7 @@ Quick links:
 
 The Miller REPL (read-evaluate-print loop) is an interactive counterpart to record-processing using the `put`/`filter` language. (A REPL is anything that evaluates what you type into it -- like `python` with no arguments, or Ruby's `irb`, or `node` with no arguments, etc.)
 
-Miller's REPL isn't a source-level debugger which lets you execute one source-code *statement* at a time -- however, it does let you operate on one *record* at a time. Further, it lets you use "immediate expressions", namely, you can interact with the [Miller programming language](programming-language.md) without having to provide data from an input file.
+Miller's REPL isn't a source-level debugger which lets you execute one source-code *statement* at a time -- however, it does let you operate on one *record* at a time. Further, it lets you use "immediate expressions", namely, you can interact with the [Miller programming language](miller-programming-language.md) without having to provide data from an input file.
 
 
 mlr repl
diff --git a/docs6/docs/repl.md.in b/docs6/src/repl.md.in
similarity index 97%
rename from docs6/docs/repl.md.in
rename to docs6/src/repl.md.in
index e2312104b..41171510f 100644
--- a/docs6/docs/repl.md.in
+++ b/docs6/src/repl.md.in
@@ -2,7 +2,7 @@
 
 The Miller REPL (read-evaluate-print loop) is an interactive counterpart to record-processing using the `put`/`filter` language. (A REPL is anything that evaluates what you type into it -- like `python` with no arguments, or Ruby's `irb`, or `node` with no arguments, etc.)
 
-Miller's REPL isn't a source-level debugger which lets you execute one source-code *statement* at a time -- however, it does let you operate on one *record* at a time. Further, it lets you use "immediate expressions", namely, you can interact with the [Miller programming language](programming-language.md) without having to provide data from an input file.
+Miller's REPL isn't a source-level debugger which lets you execute one source-code *statement* at a time -- however, it does let you operate on one *record* at a time. Further, it lets you use "immediate expressions", namely, you can interact with the [Miller programming language](miller-programming-language.md) without having to provide data from an input file.
 
 GENMD_CARDIFY_HIGHLIGHT_ONE
 mlr repl
diff --git a/docs6/docs/sample_mlrrc b/docs6/src/sample_mlrrc
similarity index 100%
rename from docs6/docs/sample_mlrrc
rename to docs6/src/sample_mlrrc
diff --git a/docs6/docs/shapes-of-data.md b/docs6/src/shapes-of-data.md
similarity index 100%
rename from docs6/docs/shapes-of-data.md
rename to docs6/src/shapes-of-data.md
diff --git a/docs6/docs/shapes-of-data.md.in b/docs6/src/shapes-of-data.md.in
similarity index 100%
rename from docs6/docs/shapes-of-data.md.in
rename to docs6/src/shapes-of-data.md.in
diff --git a/docs6/docs/shell-commands.md b/docs6/src/shell-commands.md
similarity index 100%
rename from docs6/docs/shell-commands.md
rename to docs6/src/shell-commands.md
diff --git a/docs6/docs/shell-commands.md.in b/docs6/src/shell-commands.md.in
similarity index 100%
rename from docs6/docs/shell-commands.md.in
rename to docs6/src/shell-commands.md.in
diff --git a/docs6/docs/special-symbols-and-formatting.md b/docs6/src/special-symbols-and-formatting.md
similarity index 100%
rename from docs6/docs/special-symbols-and-formatting.md
rename to docs6/src/special-symbols-and-formatting.md
diff --git a/docs6/docs/special-symbols-and-formatting.md.in b/docs6/src/special-symbols-and-formatting.md.in
similarity index 100%
rename from docs6/docs/special-symbols-and-formatting.md.in
rename to docs6/src/special-symbols-and-formatting.md.in
diff --git a/docs6/docs/sql-examples.md b/docs6/src/sql-examples.md
similarity index 100%
rename from docs6/docs/sql-examples.md
rename to docs6/src/sql-examples.md
diff --git a/docs6/docs/sql-examples.md.in b/docs6/src/sql-examples.md.in
similarity index 100%
rename from docs6/docs/sql-examples.md.in
rename to docs6/src/sql-examples.md.in
diff --git a/docs6/docs/square.csv b/docs6/src/square.csv
similarity index 100%
rename from docs6/docs/square.csv
rename to docs6/src/square.csv
diff --git a/docs6/docs/statistics-examples.md b/docs6/src/statistics-examples.md
similarity index 100%
rename from docs6/docs/statistics-examples.md
rename to docs6/src/statistics-examples.md
diff --git a/docs6/docs/statistics-examples.md.in b/docs6/src/statistics-examples.md.in
similarity index 100%
rename from docs6/docs/statistics-examples.md.in
rename to docs6/src/statistics-examples.md.in
diff --git a/docs6/docs/streaming-and-memory.md b/docs6/src/streaming-and-memory.md
similarity index 98%
rename from docs6/docs/streaming-and-memory.md
rename to docs6/src/streaming-and-memory.md
index fce6649f1..64326422e 100644
--- a/docs6/docs/streaming-and-memory.md
+++ b/docs6/src/streaming-and-memory.md
@@ -40,7 +40,7 @@ seeing partial results. This also means if you pipe Miller's output to other
 streaming tools (like `cat`, `grep`, `sed`, and so on), they will also output
 partial results as data arrives.
 
-The statements in the [Miller programming language](programming-language.md)
+The statements in the [Miller programming language](miller-programming-language.md)
 (outside of optional `begin`/`end` blocks which execute before and after all
 records have been read, respectively) are implicit callbacks which are executed
 once per record. For example, using `mlr --csv put '$z = $x + $y' myfile.csv`,
diff --git a/docs6/docs/streaming-and-memory.md.in b/docs6/src/streaming-and-memory.md.in
similarity index 98%
rename from docs6/docs/streaming-and-memory.md.in
rename to docs6/src/streaming-and-memory.md.in
index 06a2a4d76..93e71436c 100644
--- a/docs6/docs/streaming-and-memory.md.in
+++ b/docs6/src/streaming-and-memory.md.in
@@ -24,7 +24,7 @@ seeing partial results. This also means if you pipe Miller's output to other
 streaming tools (like `cat`, `grep`, `sed`, and so on), they will also output
 partial results as data arrives.
 
-The statements in the [Miller programming language](programming-language.md)
+The statements in the [Miller programming language](miller-programming-language.md)
 (outside of optional `begin`/`end` blocks which execute before and after all
 records have been read, respectively) are implicit callbacks which are executed
 once per record. For example, using `mlr --csv put '$z = $x + $y' myfile.csv`,
diff --git a/docs6/docs/structure-of-these-documents.md b/docs6/src/structure-of-these-documents.md
similarity index 96%
rename from docs6/docs/structure-of-these-documents.md
rename to docs6/src/structure-of-these-documents.md
index 993d323f6..92ebc8db8 100644
--- a/docs6/docs/structure-of-these-documents.md
+++ b/docs6/src/structure-of-these-documents.md
@@ -25,7 +25,7 @@ The goal is _multiple levels of detail_.
 * The _Background_ section is some **non-essential historical** and meta material on why Miller was created.
 * The _Reference_ section aims to answer all questions the previous sections didn't. The discussion is **concept-driven**, although there are still plenty of examples throughout for concreteness.
     * _Main reference_ goes carefully through various aspects of Miller, concept by concept.
-    * _DSL reference_ focuses on the [Miller programming language](programming-language.md), again following a concept-at-a-time approach.
+    * _DSL reference_ focuses on the [Miller programming language](miller-programming-language.md), again following a concept-at-a-time approach.
     * _Misc. reference_ is aptly named, with things like build-from-source notes.
     * _Documents for previous releases_ is not only for historical curiosity -- experience has shown that various Linux/BSD distros update their Miller versions on their own cadences, so the version on your system (as shown by `mlr --version`) might be best-served by its respective documentation version.
 * Lastly, new with the Miller 6 documents is a very easy-to-access **Search field** at the top of each page.
diff --git a/docs6/docs/structure-of-these-documents.md.in b/docs6/src/structure-of-these-documents.md.in
similarity index 95%
rename from docs6/docs/structure-of-these-documents.md.in
rename to docs6/src/structure-of-these-documents.md.in
index 50c710125..9cb40d3a1 100644
--- a/docs6/docs/structure-of-these-documents.md.in
+++ b/docs6/src/structure-of-these-documents.md.in
@@ -9,7 +9,7 @@ The goal is _multiple levels of detail_.
 * The _Background_ section is some **non-essential historical** and meta material on why Miller was created.
 * The _Reference_ section aims to answer all questions the previous sections didn't. The discussion is **concept-driven**, although there are still plenty of examples throughout for concreteness.
     * _Main reference_ goes carefully through various aspects of Miller, concept by concept.
-    * _DSL reference_ focuses on the [Miller programming language](programming-language.md), again following a concept-at-a-time approach.
+    * _DSL reference_ focuses on the [Miller programming language](miller-programming-language.md), again following a concept-at-a-time approach.
     * _Misc. reference_ is aptly named, with things like build-from-source notes.
     * _Documents for previous releases_ is not only for historical curiosity -- experience has shown that various Linux/BSD distros update their Miller versions on their own cadences, so the version on your system (as shown by `mlr --version`) might be best-served by its respective documentation version.
 * Lastly, new with the Miller 6 documents is a very easy-to-access **Search field** at the top of each page.
diff --git a/docs6/docs/triangle.csv b/docs6/src/triangle.csv
similarity index 100%
rename from docs6/docs/triangle.csv
rename to docs6/src/triangle.csv
diff --git a/docs6/docs/two-pass-algorithms.md b/docs6/src/two-pass-algorithms.md
similarity index 100%
rename from docs6/docs/two-pass-algorithms.md
rename to docs6/src/two-pass-algorithms.md
diff --git a/docs6/docs/two-pass-algorithms.md.in b/docs6/src/two-pass-algorithms.md.in
similarity index 100%
rename from docs6/docs/two-pass-algorithms.md.in
rename to docs6/src/two-pass-algorithms.md.in
diff --git a/docs6/docs/unix-toolkit-context.md b/docs6/src/unix-toolkit-context.md
similarity index 100%
rename from docs6/docs/unix-toolkit-context.md
rename to docs6/src/unix-toolkit-context.md
diff --git a/docs6/docs/unix-toolkit-context.md.in b/docs6/src/unix-toolkit-context.md.in
similarity index 100%
rename from docs6/docs/unix-toolkit-context.md.in
rename to docs6/src/unix-toolkit-context.md.in
diff --git a/docs6/docs/variance.mlr b/docs6/src/variance.mlr
similarity index 100%
rename from docs6/docs/variance.mlr
rename to docs6/src/variance.mlr
diff --git a/docs6/docs/vimrc.md b/docs6/src/vimrc.md
similarity index 84%
rename from docs6/docs/vimrc.md
rename to docs6/src/vimrc.md
index 4fd9621a7..22e73f89d 100644
--- a/docs6/docs/vimrc.md
+++ b/docs6/src/vimrc.md
@@ -19,7 +19,7 @@ Quick links:
 At [https://github.com/johnkerl/miller/tree/main/vim](https://github.com/johnkerl/miller/tree/main/vim) you will
 find [ftdetect](https://github.com/johnkerl/miller/tree/main/vim/ftdetect) for telling the `vim` text editor
 about the `.mlr` file extension, [syntax](https://github.com/johnkerl/miller/tree/main/vim/syntax) for syntax-highlighting source files in the
-[Miller programming language](programming-language.md), and a [README file](https://github.com/johnkerl/miller/blob/main/vim/README.md) shwoing you where to put these where `vim` can find them.
+[Miller programming language](miller-programming-language.md), and a [README file](https://github.com/johnkerl/miller/blob/main/vim/README.md) shwoing you where to put these where `vim` can find them.
 
 Before-and-after images:
 
diff --git a/docs6/docs/vimrc.md.in b/docs6/src/vimrc.md.in
similarity index 72%
rename from docs6/docs/vimrc.md.in
rename to docs6/src/vimrc.md.in
index 6d2355d16..c1a4d481f 100644
--- a/docs6/docs/vimrc.md.in
+++ b/docs6/src/vimrc.md.in
@@ -3,7 +3,7 @@
 At [https://github.com/johnkerl/miller/tree/main/vim](https://github.com/johnkerl/miller/tree/main/vim) you will
 find [ftdetect](https://github.com/johnkerl/miller/tree/main/vim/ftdetect) for telling the `vim` text editor
 about the `.mlr` file extension, [syntax](https://github.com/johnkerl/miller/tree/main/vim/syntax) for syntax-highlighting source files in the
-[Miller programming language](programming-language.md), and a [README file](https://github.com/johnkerl/miller/blob/main/vim/README.md) shwoing you where to put these where `vim` can find them.
+[Miller programming language](miller-programming-language.md), and a [README file](https://github.com/johnkerl/miller/blob/main/vim/README.md) shwoing you where to put these where `vim` can find them.
 
 Before-and-after images:
 
diff --git a/docs6/docs/why.md b/docs6/src/why.md
similarity index 100%
rename from docs6/docs/why.md
rename to docs6/src/why.md
diff --git a/docs6/docs/why.md.in b/docs6/src/why.md.in
similarity index 100%
rename from docs6/docs/why.md.in
rename to docs6/src/why.md.in
diff --git a/docs6/docs/yellow.csv.gz b/docs6/src/yellow.csv.gz
similarity index 100%
rename from docs6/docs/yellow.csv.gz
rename to docs6/src/yellow.csv.gz
diff --git a/go/regtest/cases/dsl-flatten-unflatten/0007/expout b/go/regtest/cases/dsl-flatten-unflatten/0007/expout
index 2c424af3a..0a3d2cfeb 100644
--- a/go/regtest/cases/dsl-flatten-unflatten/0007/expout
+++ b/go/regtest/cases/dsl-flatten-unflatten/0007/expout
@@ -18,10 +18,10 @@
       "content-encoding": "plain"
     }
   },
-  "empty1": "{}",
-  "empty2": "[]",
+  "empty1": {},
+  "empty2": [],
   "wrapper": {
-    "empty3": "{}",
-    "emtpy4": "[]"
+    "empty3": {},
+    "emtpy4": []
   }
 }
diff --git a/go/regtest/cases/dsl-flatten-unflatten/0008/cmd b/go/regtest/cases/dsl-flatten-unflatten/0008/cmd
new file mode 100644
index 000000000..e3df6131f
--- /dev/null
+++ b/go/regtest/cases/dsl-flatten-unflatten/0008/cmd
@@ -0,0 +1 @@
+mlr --icsv --ojson --from ${CASEDIR}/input cat
diff --git a/go/regtest/cases/dsl-flatten-unflatten/0008/experr b/go/regtest/cases/dsl-flatten-unflatten/0008/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/go/regtest/cases/dsl-flatten-unflatten/0008/expout b/go/regtest/cases/dsl-flatten-unflatten/0008/expout
new file mode 100644
index 000000000..5da45409d
--- /dev/null
+++ b/go/regtest/cases/dsl-flatten-unflatten/0008/expout
@@ -0,0 +1,5 @@
+{
+  "a": 5,
+  "b": [6, 7, 8],
+  "c": 9
+}
diff --git a/go/regtest/cases/dsl-flatten-unflatten/0008/input b/go/regtest/cases/dsl-flatten-unflatten/0008/input
new file mode 100644
index 000000000..6cdc72451
--- /dev/null
+++ b/go/regtest/cases/dsl-flatten-unflatten/0008/input
@@ -0,0 +1,2 @@
+a,b.1,b.2,b.3,c
+5,6,7,8,9
diff --git a/go/regtest/cases/dsl-flatten-unflatten/0009/cmd b/go/regtest/cases/dsl-flatten-unflatten/0009/cmd
new file mode 100644
index 000000000..e3df6131f
--- /dev/null
+++ b/go/regtest/cases/dsl-flatten-unflatten/0009/cmd
@@ -0,0 +1 @@
+mlr --icsv --ojson --from ${CASEDIR}/input cat
diff --git a/go/regtest/cases/dsl-flatten-unflatten/0009/experr b/go/regtest/cases/dsl-flatten-unflatten/0009/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/go/regtest/cases/dsl-flatten-unflatten/0009/expout b/go/regtest/cases/dsl-flatten-unflatten/0009/expout
new file mode 100644
index 000000000..1539e9e12
--- /dev/null
+++ b/go/regtest/cases/dsl-flatten-unflatten/0009/expout
@@ -0,0 +1,8 @@
+{
+  "a": 5,
+  "b": {
+    "1": 6,
+    "3": 8
+  },
+  "c": 9
+}
diff --git a/go/regtest/cases/dsl-flatten-unflatten/0009/input b/go/regtest/cases/dsl-flatten-unflatten/0009/input
new file mode 100644
index 000000000..70c763786
--- /dev/null
+++ b/go/regtest/cases/dsl-flatten-unflatten/0009/input
@@ -0,0 +1,2 @@
+a,b.1,b.3,c
+5,6,8,9
diff --git a/go/src/auxents/help/entry.go b/go/src/auxents/help/entry.go
index 57bd3c592..790c65c29 100644
--- a/go/src/auxents/help/entry.go
+++ b/go/src/auxents/help/entry.go
@@ -150,7 +150,7 @@ func init() {
 		// Note: `j, _` rather than `_, downdashSectionName` since the latter
 		// is a data copy while the former allows us to do a reference. The
 		// former won't produce correct lookup-table data.
-		for j, _ := range downdashSectionNames {
+		for j := range downdashSectionNames {
 			downdashSectionName := downdashSectionNames[j]
 			// Patch a new entry into the "Flags" section of our lookup table.
 			entry := tHandlerInfo{
@@ -229,8 +229,10 @@ func HelpMain(args []string) int {
 func MainUsage(o *os.File) {
 	fmt.Fprintf(o,
 		`Usage: mlr [flags] {verb} [verb-dependent options ...] {zero or more file names}
+If zero file names are provided, standard input is read. Example:
+  mlr --csv sort -f shape example.csv
 Output of one verb may be chained as input to another using "then", e.g.
-  mlr stats1 -a min,mean,max -f flag,u,v -g color then sort -f color
+  mlr --csv stats1 -a min,mean,max -f quantity then sort -f color example.csv
 Please see 'mlr help topics' for more information.
 `)
 	fmt.Fprintf(o, "Please also see %s\n", lib.DOC_URL)
diff --git a/go/src/auxents/regtest/invoker.go b/go/src/auxents/regtest/invoker.go
index 7cb6aa515..2f8493d6b 100644
--- a/go/src/auxents/regtest/invoker.go
+++ b/go/src/auxents/regtest/invoker.go
@@ -2,9 +2,11 @@ package regtest
 
 import (
 	"bytes"
+	"os"
 	"os/exec"
 	"strings"
 
+	"mlr/src/lib"
 	"mlr/src/platform"
 )
 
@@ -59,3 +61,44 @@ func RunMillerCommand(
 
 	return stdout, stderr, exitCode, nil
 }
+
+// RunCompareCommand runs either diff or fc (not-Windows / Windows respectively)
+// to show differences between actual and expected regression-test output.
+func RunDiffCommand(
+	actualOutput string,
+	expectedOutput string,
+) (
+	diffOutput string,
+) {
+	actualOutputFileName := lib.WriteTempFileOrDie(actualOutput)
+	expectedOutputFileName := lib.WriteTempFileOrDie(expectedOutput)
+	defer os.Remove(actualOutputFileName)
+	defer os.Remove(expectedOutputFileName)
+
+	// This is diff or fc
+	diffRunArray := platform.GetDiffRunArray(actualOutputFileName, expectedOutputFileName)
+
+	cmd := exec.Command(diffRunArray[0], diffRunArray[1:]...)
+
+	var stdoutBuffer bytes.Buffer
+	var stderrBuffer bytes.Buffer
+	cmd.Stdout = &stdoutBuffer
+	cmd.Stderr = &stderrBuffer
+
+	// Ignore the error-return since it's likely the fact that diff exits
+	// non-zero when files differ at all. Otherwise it's a failure to invoke
+	// diff itself, about which we can do little within the regtest.  A diff
+	// output is simply something (in addition to printing the actual &
+	// expected outputs) to help people debug, and hey, we tried.
+
+	// err := cmd.Run()
+	_ = cmd.Run()
+	//if err != nil {
+	//	fmt.Printf("Error executing %s:\n", strings.Join(diffRunArray, " "))
+	//	fmt.Println(err)
+	//	fmt.Println(stderrBuffer.String())
+	//	os.Exit(1)
+	//}
+
+	return stdoutBuffer.String()
+}
diff --git a/go/src/auxents/regtest/regtester.go b/go/src/auxents/regtest/regtester.go
index a91e7cd72..a51458cd1 100644
--- a/go/src/auxents/regtest/regtester.go
+++ b/go/src/auxents/regtest/regtester.go
@@ -445,8 +445,9 @@ func (regtester *RegTester) executeSingleCmdFile(
 	}
 
 	if regtester.doPopulate {
-		// Populate mode: write out the actual stdout/stderr/exit-code to disk
-		// as expected values for subsequent runs.
+		// Populate mode, not verify mode: write out the actual
+		// stdout/stderr/exit-code to disk as expected values for subsequent
+		// runs.
 
 		// TODO: temp replace-all for CR/LF to LF. Will need re-work once auto-detect is ported.
 		actualStdout = strings.ReplaceAll(actualStdout, "\r\n", "\n")
@@ -504,7 +505,7 @@ func (regtester *RegTester) executeSingleCmdFile(
 		}
 
 	} else {
-		// Verify mode: check actuals against expecteds
+		// Verify mode, not populate mode: check actuals against expecteds
 
 		// Load the .expout file
 		expectedStdout, err := regtester.loadFile(expectedStdoutFileName, caseDir)
@@ -573,6 +574,10 @@ func (regtester *RegTester) executeSingleCmdFile(
 					expectedStdoutFileName,
 				)
 			}
+
+			if verbosityLevel >= 3 {
+				fmt.Println(RunDiffCommand(actualStdout, expectedStdout))
+			}
 			passed = false
 		}
 
@@ -585,6 +590,9 @@ func (regtester *RegTester) executeSingleCmdFile(
 					expectedStderrFileName,
 				)
 			}
+			if verbosityLevel >= 3 {
+				fmt.Println(RunDiffCommand(actualStderr, expectedStderr))
+			}
 			passed = false
 		}
 
diff --git a/go/src/cli/flag_types.go b/go/src/cli/flag_types.go
index 4d1c69eb7..d02a7b025 100644
--- a/go/src/cli/flag_types.go
+++ b/go/src/cli/flag_types.go
@@ -85,9 +85,6 @@ type FlagSection struct {
 	name        string
 	infoPrinter func()
 	flags       []Flag
-	// For format-conversion keystroke-savers, a matrix is plenty -- we don't
-	// need to print a tedious 60-line list.
-	suppressFlagEnumeration bool
 }
 
 // Flag is a container for all runtime as well as documentation information for
@@ -120,6 +117,10 @@ type Flag struct {
 
 	// A function for parsing the command line, as described above.
 	parser FlagParser
+
+	// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+	// need to print a tedious 60-line list.
+	suppressFlagEnumeration bool
 }
 
 // ================================================================
@@ -315,12 +316,12 @@ func (fs *FlagSection) Sort() {
 // ShowHelpForFlags prints all-in-one on-line help, nominally for `mlr help
 // flags`.
 func (fs *FlagSection) ShowHelpForFlags() {
-	// For format-conversion keystroke-savers, a matrix is plenty -- we don't
-	// need to print a tedious 60-line list.
-	if fs.suppressFlagEnumeration {
-		return
-	}
 	for _, flag := range fs.flags {
+		// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+		// need to print a tedious 60-line list.
+		if flag.suppressFlagEnumeration {
+			continue
+		}
 		flag.ShowHelp()
 	}
 }
diff --git a/go/src/cli/option_parse.go b/go/src/cli/option_parse.go
index 91fac1d6c..2f80c021f 100644
--- a/go/src/cli/option_parse.go
+++ b/go/src/cli/option_parse.go
@@ -1054,10 +1054,6 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 	name:        "Format-conversion keystroke-saver flags",
 	infoPrinter: FormatConversionKeystrokeSaverPrintInfo,
 
-	// For format-conversion keystroke-savers, a matrix is plenty -- we don't
-	// need to print a tedious 60-line list.
-	suppressFlagEnumeration: true,
-
 	flags: []Flag{
 
 		{
@@ -1093,6 +1089,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--c2t",
 			help: "Use CSV for input, TSV for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IRS = "auto"
@@ -1108,6 +1107,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--c2d",
 			help: "Use CSV for input, DKVP for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IRS = "auto"
@@ -1119,6 +1121,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--c2n",
 			help: "Use CSV for input, NIDX for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IRS = "auto"
@@ -1132,6 +1137,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--c2j",
 			help: "Use CSV for input, JSON for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IRS = "auto"
@@ -1143,6 +1151,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--c2p",
 			help: "Use CSV for input, PPRINT for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IRS = "auto"
@@ -1154,6 +1165,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--c2b",
 			help: "Use CSV for input, PPRINT with `--barred` for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IRS = "auto"
@@ -1166,6 +1180,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--c2x",
 			help: "Use CSV for input, XTAB for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IRS = "auto"
@@ -1177,6 +1194,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--c2m",
 			help: "Use CSV for input, markdown-tabular for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IRS = "auto"
@@ -1189,6 +1209,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--t2c",
 			help: "Use TSV for input, CSV for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IFS = "\t"
@@ -1204,6 +1227,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--t2d",
 			help: "Use TSV for input, DKVP for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IFS = "\t"
@@ -1217,6 +1243,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--t2n",
 			help: "Use TSV for input, NIDX for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IFS = "\t"
@@ -1232,6 +1261,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--t2j",
 			help: "Use TSV for input, JSON for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IFS = "\t"
@@ -1245,6 +1277,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--t2p",
 			help: "Use TSV for input, PPRINT for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IFS = "\t"
@@ -1258,6 +1293,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--t2b",
 			help: "Use TSV for input, PPRINT with `--barred` for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IFS = "\t"
@@ -1272,6 +1310,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--t2x",
 			help: "Use TSV for input, XTAB for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IFS = "\t"
@@ -1285,6 +1326,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--t2m",
 			help: "Use TSV for input, markdown-tabular for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "csv"
 				options.ReaderOptions.IFS = "\t"
@@ -1299,6 +1343,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--d2c",
 			help: "Use DKVP for input, CSV for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "dkvp"
 				options.WriterOptions.OutputFileFormat = "csv"
@@ -1309,6 +1356,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--d2t",
 			help: "Use DKVP for input, TSV for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "dkvp"
 				options.WriterOptions.OutputFileFormat = "csv"
@@ -1322,6 +1372,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--d2n",
 			help: "Use DKVP for input, NIDX for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "dkvp"
 				options.WriterOptions.OutputFileFormat = "nidx"
@@ -1333,6 +1386,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--d2j",
 			help: "Use DKVP for input, JSON for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "dkvp"
 				options.WriterOptions.OutputFileFormat = "json"
@@ -1342,6 +1398,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--d2p",
 			help: "Use DKVP for input, PPRINT for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "dkvp"
 				options.WriterOptions.OutputFileFormat = "pprint"
@@ -1351,6 +1410,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--d2b",
 			help: "Use DKVP for input, PPRINT with `--barred` for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "dkvp"
 				options.WriterOptions.OutputFileFormat = "pprint"
@@ -1361,6 +1423,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--d2x",
 			help: "Use DKVP for input, XTAB for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "dkvp"
 				options.WriterOptions.OutputFileFormat = "xtab"
@@ -1370,6 +1435,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--d2m",
 			help: "Use DKVP for input, markdown-tabular for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "dkvp"
 				options.WriterOptions.OutputFileFormat = "markdown"
@@ -1380,6 +1448,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--n2c",
 			help: "Use NIDX for input, CSV for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "nidx"
 				options.WriterOptions.OutputFileFormat = "csv"
@@ -1391,6 +1462,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--n2t",
 			help: "Use NIDX for input, TSV for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "nidx"
 				options.WriterOptions.OutputFileFormat = "csv"
@@ -1404,6 +1478,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--n2d",
 			help: "Use NIDX for input, DKVP for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "nidx"
 				options.WriterOptions.OutputFileFormat = "dkvp"
@@ -1413,6 +1490,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--n2j",
 			help: "Use NIDX for input, JSON for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "nidx"
 				options.WriterOptions.OutputFileFormat = "json"
@@ -1422,6 +1502,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--n2p",
 			help: "Use NIDX for input, PPRINT for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "nidx"
 				options.WriterOptions.OutputFileFormat = "pprint"
@@ -1431,6 +1514,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--n2b",
 			help: "Use NIDX for input, PPRINT with `--barred` for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "nidx"
 				options.WriterOptions.OutputFileFormat = "pprint"
@@ -1441,6 +1527,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--n2x",
 			help: "Use NIDX for input, XTAB for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "nidx"
 				options.WriterOptions.OutputFileFormat = "xtab"
@@ -1450,6 +1539,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--n2m",
 			help: "Use NIDX for input, markdown-tabular for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "nidx"
 				options.WriterOptions.OutputFileFormat = "markdown"
@@ -1460,6 +1552,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--j2c",
 			help: "Use JSON for input, CSV for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "json"
 				options.WriterOptions.OutputFileFormat = "csv"
@@ -1471,6 +1566,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--j2t",
 			help: "Use JSON for input, TSV for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "json"
 				options.WriterOptions.OutputFileFormat = "csv"
@@ -1484,6 +1582,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--j2d",
 			help: "Use JSON for input, DKVP for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "json"
 				options.WriterOptions.OutputFileFormat = "dkvp"
@@ -1493,6 +1594,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--j2n",
 			help: "Use JSON for input, NIDX for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "json"
 				options.WriterOptions.OutputFileFormat = "nidx"
@@ -1502,6 +1606,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--j2p",
 			help: "Use JSON for input, PPRINT for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "json"
 				options.WriterOptions.OutputFileFormat = "pprint"
@@ -1511,6 +1618,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--j2b",
 			help: "Use JSON for input, PPRINT with --barred for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "json"
 				options.WriterOptions.OutputFileFormat = "pprint"
@@ -1521,6 +1631,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--j2x",
 			help: "Use JSON for input, XTAB for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "json"
 				options.WriterOptions.OutputFileFormat = "xtab"
@@ -1530,6 +1643,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--j2m",
 			help: "Use JSON for input, markdown-tabular for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "json"
 				options.WriterOptions.OutputFileFormat = "markdown"
@@ -1540,6 +1656,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--p2c",
 			help: "Use PPRINT for input, CSV for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "pprint"
 				options.ReaderOptions.IFS = " "
@@ -1553,6 +1672,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--p2t",
 			help: "Use PPRINT for input, TSV for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "pprint"
 				options.ReaderOptions.IFS = " "
@@ -1568,6 +1690,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--p2d",
 			help: "Use PPRINT for input, DKVP for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "pprint"
 				options.ReaderOptions.IFS = " "
@@ -1579,6 +1704,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--p2n",
 			help: "Use PPRINT for input, NIDX for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "pprint"
 				options.ReaderOptions.IFS = " "
@@ -1590,6 +1718,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--p2j",
 			help: "Use PPRINT for input, JSON for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "pprint"
 				options.ReaderOptions.IFS = " "
@@ -1601,6 +1732,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--p2x",
 			help: "Use PPRINT for input, XTAB for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "pprint"
 				options.ReaderOptions.IFS = " "
@@ -1612,6 +1746,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--p2m",
 			help: "Use PPRINT for input, markdown-tabular for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "pprint"
 				options.ReaderOptions.IFS = " "
@@ -1624,6 +1761,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--x2c",
 			help: "Use XTAB for input, CSV for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "xtab"
 				options.WriterOptions.OutputFileFormat = "csv"
@@ -1635,6 +1775,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--x2t",
 			help: "Use XTAB for input, TSV for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "xtab"
 				options.WriterOptions.OutputFileFormat = "csv"
@@ -1648,6 +1791,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--x2d",
 			help: "Use XTAB for input, DKVP for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "xtab"
 				options.WriterOptions.OutputFileFormat = "dkvp"
@@ -1657,6 +1803,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--x2n",
 			help: "Use XTAB for input, NIDX for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "xtab"
 				options.WriterOptions.OutputFileFormat = "nidx"
@@ -1666,6 +1815,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--x2j",
 			help: "Use XTAB for input, JSON for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "xtab"
 				options.WriterOptions.OutputFileFormat = "json"
@@ -1675,6 +1827,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--x2p",
 			help: "Use XTAB for input, PPRINT for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "xtab"
 				options.WriterOptions.OutputFileFormat = "pprint"
@@ -1684,6 +1839,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--x2b",
 			help: "Use XTAB for input, PPRINT with `--barred` for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "xtab"
 				options.WriterOptions.OutputFileFormat = "pprint"
@@ -1694,6 +1852,9 @@ var FormatConversionKeystrokeSaverFlagSection = FlagSection{
 		{
 			name: "--x2m",
 			help: "Use XTAB for input, markdown-tabular for output.",
+			// For format-conversion keystroke-savers, a matrix is plenty -- we don't
+			// need to print a tedious 60-line list.
+			suppressFlagEnumeration: true,
 			parser: func(args []string, argc int, pargi *int, options *TOptions) {
 				options.ReaderOptions.InputFileFormat = "xtab"
 				options.WriterOptions.OutputFileFormat = "markdown"
diff --git a/go/src/dsl/cst/builtin_functions.go b/go/src/dsl/cst/builtin_functions.go
index 31516391e..7b09e4298 100644
--- a/go/src/dsl/cst/builtin_functions.go
+++ b/go/src/dsl/cst/builtin_functions.go
@@ -531,7 +531,7 @@ func (node *VariadicFunctionCallsiteNode) Evaluate(
 	state *runtime.State,
 ) *types.Mlrval {
 	args := make([]*types.Mlrval, len(node.evaluables))
-	for i, _ := range node.evaluables {
+	for i := range node.evaluables {
 		args[i] = node.evaluables[i].Evaluate(state)
 	}
 	return node.variadicFunc(args)
diff --git a/go/src/dsl/cst/collections.go b/go/src/dsl/cst/collections.go
index 54bce5113..7e46a1d95 100644
--- a/go/src/dsl/cst/collections.go
+++ b/go/src/dsl/cst/collections.go
@@ -44,7 +44,7 @@ func (node *ArrayLiteralNode) Evaluate(
 	state *runtime.State,
 ) *types.Mlrval {
 	mlrvals := make([]types.Mlrval, len(node.evaluables))
-	for i, _ := range node.evaluables {
+	for i := range node.evaluables {
 		mlrvals[i] = *node.evaluables[i].Evaluate(state)
 	}
 	return types.MlrvalPointerFromArrayLiteralReference(mlrvals)
@@ -206,18 +206,36 @@ func (node *ArraySliceAccessNode) Evaluate(
 		return types.MlrvalPointerFromArrayLiteralReference(make([]types.Mlrval, 0))
 	}
 
-	// Say x=[1,2,3,4,5]. Then x[3:10] is [3,4,5].
+	// Semantics: say x=[1,2,3,4,5]. Then x[3:10] is [3,4,5].
+	//
+	// Cases:
+	//      [* * * * *]              actual data
+	//  [o o]                        1. attempted indexing: lo, hi both out of bounds
+	//  [o o o o o o ]               2. attempted indexing: hi in bounds, lo out
+	//  [o o o o o o o o o o o o]    3. attempted indexing: lo, hi both out of bounds
+	//        [o o o]                4. attempted indexing: lo, hi in bounds
+	//        [o o o o o o ]         5. attempted indexing: lo in bounds, hi out
+	//                  [o o o o]    6. attempted indexing: lo, hi both out of bounds
+
 	if lowerZindex < 0 {
 		lowerZindex = 0
+		if lowerZindex > upperZindex {
+			return types.MlrvalPointerFromArrayLiteralReference(make([]types.Mlrval, 0))
+		}
 	}
 	if upperZindex > n-1 {
 		upperZindex = n - 1
+		if lowerZindex > upperZindex {
+			return types.MlrvalPointerFromArrayLiteralReference(make([]types.Mlrval, 0))
+		}
 	}
 
 	// Go     slices have inclusive lower bound, exclusive upper bound.
 	// Miller slices have inclusive lower bound, inclusive upper bound.
 	var m = upperZindex - lowerZindex + 1
+
 	retval := make([]types.Mlrval, m)
+
 	di := 0
 	for si := lowerZindex; si <= upperZindex; si++ {
 		retval[di] = *array[si].Copy()
@@ -521,7 +539,7 @@ func (node *MapLiteralNode) Evaluate(
 ) *types.Mlrval {
 	output := types.MlrvalPointerFromEmptyMap()
 
-	for i, _ := range node.evaluablePairs {
+	for i := range node.evaluablePairs {
 		key := node.evaluablePairs[i].Key.Evaluate(state)
 		value := node.evaluablePairs[i].Value.Evaluate(state)
 
diff --git a/go/src/dsl/cst/emit_emitp.go b/go/src/dsl/cst/emit_emitp.go
index dd3f5daef..41cb97f45 100644
--- a/go/src/dsl/cst/emit_emitp.go
+++ b/go/src/dsl/cst/emit_emitp.go
@@ -527,7 +527,7 @@ func (node *EmitXStatementNode) executeIndexed(
 
 	// TODO: libify this
 	indices := make([]*types.Mlrval, len(node.indexEvaluables))
-	for i, _ := range node.indexEvaluables {
+	for i := range node.indexEvaluables {
 		indices[i] = node.indexEvaluables[i].Evaluate(state)
 		if indices[i].IsAbsent() {
 			return nil
diff --git a/go/src/dsl/cst/lvalues.go b/go/src/dsl/cst/lvalues.go
index 97d711753..5c4e448b0 100644
--- a/go/src/dsl/cst/lvalues.go
+++ b/go/src/dsl/cst/lvalues.go
@@ -1012,7 +1012,7 @@ func (node *IndexedLvalueNode) Assign(
 ) error {
 	indices := make([]*types.Mlrval, len(node.indexEvaluables))
 
-	for i, _ := range node.indexEvaluables {
+	for i := range node.indexEvaluables {
 		indices[i] = node.indexEvaluables[i].Evaluate(state)
 		if indices[i].IsAbsent() {
 			return nil
@@ -1042,7 +1042,7 @@ func (node *IndexedLvalueNode) Unassign(
 	state *runtime.State,
 ) {
 	indices := make([]*types.Mlrval, len(node.indexEvaluables))
-	for i, _ := range node.indexEvaluables {
+	for i := range node.indexEvaluables {
 		indices[i] = node.indexEvaluables[i].Evaluate(state)
 	}
 
diff --git a/go/src/dsl/cst/udf.go b/go/src/dsl/cst/udf.go
index 3161b03c2..4d3e8faba 100644
--- a/go/src/dsl/cst/udf.go
+++ b/go/src/dsl/cst/udf.go
@@ -112,7 +112,7 @@ func (site *UDFCallsite) Evaluate(
 	numArguments := len(site.udf.signature.typeGatedParameterNames)
 	arguments := make([]*types.Mlrval, numArguments)
 
-	for i, _ := range site.udf.signature.typeGatedParameterNames {
+	for i := range site.udf.signature.typeGatedParameterNames {
 		arguments[i] = site.argumentNodes[i].Evaluate(state)
 
 		err := site.udf.signature.typeGatedParameterNames[i].Check(arguments[i])
@@ -127,7 +127,7 @@ func (site *UDFCallsite) Evaluate(
 	state.Stack.PushStackFrameSet()
 	defer state.Stack.PopStackFrameSet()
 
-	for i, _ := range arguments {
+	for i := range arguments {
 		err := state.Stack.DefineTypedAtScope(
 			runtime.NewStackVariable(site.udf.signature.typeGatedParameterNames[i].Name),
 			site.udf.signature.typeGatedParameterNames[i].TypeName,
diff --git a/go/src/dsl/cst/uds.go b/go/src/dsl/cst/uds.go
index e954096a5..c179ed983 100644
--- a/go/src/dsl/cst/uds.go
+++ b/go/src/dsl/cst/uds.go
@@ -121,7 +121,7 @@ func (site *UDSCallsite) Execute(state *runtime.State) (*BlockExitPayload, error
 	state.Stack.PushStackFrameSet()
 	defer state.Stack.PopStackFrameSet()
 
-	for i, _ := range arguments {
+	for i := range arguments {
 		err := state.Stack.DefineTypedAtScope(
 			runtime.NewStackVariable(site.uds.signature.typeGatedParameterNames[i].Name),
 			site.uds.signature.typeGatedParameterNames[i].TypeName,
diff --git a/go/src/lib/regex_test.go b/go/src/lib/regex_test.go
index a4579ebdb..961d73f8d 100644
--- a/go/src/lib/regex_test.go
+++ b/go/src/lib/regex_test.go
@@ -33,10 +33,10 @@ type tDataForMatches struct {
 // ----------------------------------------------------------------
 var dataForHasCaptures = []tDataForHasCaptures{
 	{"foo", false, nil},
-	{"\\0", true, [][]int{[]int{0, 2, 0, 2}}},
-	{"\\3", true, [][]int{[]int{0, 2, 0, 2}}},
-	{"\\34", true, [][]int{[]int{0, 2, 0, 2}}},
-	{"abc\\1def\\2ghi", true, [][]int{[]int{3, 5, 3, 5}, []int{8, 10, 8, 10}}},
+	{"\\0", true, [][]int{{0, 2, 0, 2}}},
+	{"\\3", true, [][]int{{0, 2, 0, 2}}},
+	{"\\34", true, [][]int{{0, 2, 0, 2}}},
+	{"abc\\1def\\2ghi", true, [][]int{{3, 5, 3, 5}, {8, 10, 8, 10}}},
 }
 
 var dataForSub = []tDataForSubGsub{
@@ -153,13 +153,13 @@ func compareMatrices(
 	if len(actualMatrix) != len(expectedMatrix) {
 		return false
 	}
-	for i, _ := range expectedMatrix {
+	for i := range expectedMatrix {
 		actualRow := actualMatrix[i]
 		expectedRow := expectedMatrix[i]
 		if len(actualRow) != len(expectedRow) {
 			return false
 		}
-		for j, _ := range expectedRow {
+		for j := range expectedRow {
 			if actualRow[j] != expectedRow[j] {
 				return false
 			}
@@ -181,7 +181,7 @@ func compareCaptures(
 	if len(actualCaptures) != len(expectedCaptures) {
 		return false
 	}
-	for i, _ := range expectedCaptures {
+	for i := range expectedCaptures {
 		if actualCaptures[i] != expectedCaptures[i] {
 			return false
 		}
diff --git a/go/src/lib/util.go b/go/src/lib/util.go
index 8b3c09e06..1c5e88413 100644
--- a/go/src/lib/util.go
+++ b/go/src/lib/util.go
@@ -1,6 +1,9 @@
 package lib
 
 import (
+	"fmt"
+	"io/ioutil"
+	"os"
 	"sort"
 	"strconv"
 	"strings"
@@ -124,10 +127,35 @@ func TryBoolFromBoolString(input string) (bool, bool) {
 func GetArrayKeysSorted(input map[string]string) []string {
 	keys := make([]string, len(input))
 	i := 0
-	for key, _ := range input {
+	for key := range input {
 		keys[i] = key
 		i++
 	}
 	sort.Strings(keys)
 	return keys
 }
+
+// WriteTempFile places the contents string into a temp file, which the caller
+// must remove.
+func WriteTempFileOrDie(contents string) string {
+	// Use "" as first argument to ioutil.TempFile to use default directory.
+	// Nominally "/tmp" or somesuch on all unix-like systems, but not for Windows.
+	handle, err := ioutil.TempFile("", "mlr-temp")
+	if err != nil {
+		fmt.Printf("mlr: could not create temp file.\n")
+		os.Exit(1)
+	}
+
+	_, err = handle.WriteString(contents)
+	if err != nil {
+		fmt.Printf("mlr: could not populate temp file.\n")
+		os.Exit(1)
+	}
+
+	err = handle.Close()
+	if err != nil {
+		fmt.Printf("mlr: could not finish write of  temp file.\n")
+		os.Exit(1)
+	}
+	return handle.Name()
+}
diff --git a/go/src/platform/diff_notwindows.go b/go/src/platform/diff_notwindows.go
new file mode 100644
index 000000000..d9d89fd75
--- /dev/null
+++ b/go/src/platform/diff_notwindows.go
@@ -0,0 +1,12 @@
+// ================================================================
+// Handling diff or fc for regression-test.
+// ================================================================
+
+//go:build !windows
+// +build !windows
+
+package platform
+
+func GetDiffRunArray(filename1, filename2 string) []string {
+	return []string{"diff", "-u", filename1, filename2}
+}
diff --git a/go/src/platform/diff_windows.go b/go/src/platform/diff_windows.go
new file mode 100644
index 000000000..8ab77f1ef
--- /dev/null
+++ b/go/src/platform/diff_windows.go
@@ -0,0 +1,12 @@
+// ================================================================
+// Handling diff or fc for regression-test.
+// ================================================================
+
+//go:build windows
+// +build windows
+
+package platform
+
+func GetDiffRunArray(filename1, filename2 string) []string {
+	return []string{"fc", filename1, filename2}
+}
diff --git a/go/src/transformers/most-or-least-frequent.go b/go/src/transformers/most-or-least-frequent.go
index c24116977..9bcbfc18c 100644
--- a/go/src/transformers/most-or-least-frequent.go
+++ b/go/src/transformers/most-or-least-frequent.go
@@ -261,7 +261,7 @@ func (tr *TransformerMostOrLeastFrequent) Transform(
 		for i := 0; i < outputLength; i++ {
 			outrec := types.NewMlrmapAsRecord()
 			groupByFieldValues := tr.valuesForGroup[sortPairs[i].groupingKey]
-			for j, _ := range tr.groupByFieldNames {
+			for j := range tr.groupByFieldNames {
 				outrec.PutCopy(
 					tr.groupByFieldNames[j],
 					groupByFieldValues[j],
diff --git a/go/src/transformers/step.go b/go/src/transformers/step.go
index 1fc70c45e..8f39e9a9e 100644
--- a/go/src/transformers/step.go
+++ b/go/src/transformers/step.go
@@ -584,13 +584,13 @@ func (stepper *tStepperEWMA) process(
 	inrec *types.Mlrmap,
 ) {
 	if !stepper.havePrevs {
-		for i, _ := range stepper.alphas {
+		for i := range stepper.alphas {
 			inrec.PutCopy(stepper.outputFieldNames[i], valueFieldValue)
 			stepper.prevs[i] = valueFieldValue.Copy()
 		}
 		stepper.havePrevs = true
 	} else {
-		for i, _ := range stepper.alphas {
+		for i := range stepper.alphas {
 			curr := valueFieldValue.Copy()
 			// xxx pending pointer-output refactor
 			product1 := types.MlrvalTimes(curr, &stepper.alphas[i])
diff --git a/go/src/transformers/top.go b/go/src/transformers/top.go
index 177fa4dfb..e030538a9 100644
--- a/go/src/transformers/top.go
+++ b/go/src/transformers/top.go
@@ -197,7 +197,7 @@ func (tr *TransformerTop) ingest(
 	}
 
 	// for "x", "y" and "1", "2"
-	for i, _ := range tr.valueFieldNames {
+	for i := range tr.valueFieldNames {
 		valueFieldName := tr.valueFieldNames[i]
 		valueFieldValue := valueFieldValues[i]
 		if valueFieldValue.IsVoid() {
@@ -253,7 +253,7 @@ func (tr *TransformerTop) emit(
 				newrec := types.NewMlrmapAsRecord()
 
 				// Add in a=s,b=t fields:
-				for j, _ := range tr.groupByFieldNames {
+				for j := range tr.groupByFieldNames {
 					newrec.PutCopy(tr.groupByFieldNames[j], groupByFieldValues[j])
 				}
 
diff --git a/go/src/types/mlrmap_accessors.go b/go/src/types/mlrmap_accessors.go
index f107c6134..fe7a7cd36 100644
--- a/go/src/types/mlrmap_accessors.go
+++ b/go/src/types/mlrmap_accessors.go
@@ -7,6 +7,11 @@ import (
 	"mlr/src/lib"
 )
 
+// ----------------------------------------------------------------
+func (mlrmap *Mlrmap) IsEmpty() bool {
+	return mlrmap.Head == nil
+}
+
 // ----------------------------------------------------------------
 func (mlrmap *Mlrmap) Has(key string) bool {
 	return mlrmap.findEntry(key) != nil
@@ -377,6 +382,9 @@ func (mlrmap *Mlrmap) Clear() {
 	// Assuming everything unreferenced is getting GC'ed by the Go runtime
 	mlrmap.Head = nil
 	mlrmap.Tail = nil
+	if mlrmap.keysToEntries != nil {
+		mlrmap.keysToEntries = make(map[string]*MlrmapEntry)
+	}
 }
 
 // ----------------------------------------------------------------
@@ -635,7 +643,9 @@ func (mlrmap *Mlrmap) Rename(oldKey string, newKey string) bool {
 		// Rename field from 'a' to 'b' where there are both 'a' and 'b':
 		// remove old 'a' and put its value into the slot of 'b'.
 		existing.Value = entry.Value
-		delete(mlrmap.keysToEntries, oldKey)
+		if mlrmap.keysToEntries != nil {
+			delete(mlrmap.keysToEntries, oldKey)
+		}
 		mlrmap.Unlink(entry)
 	}
 
@@ -812,6 +822,9 @@ func (mlrmap *Mlrmap) linkAtHead(pe *MlrmapEntry) {
 		mlrmap.Head.Prev = pe
 		mlrmap.Head = pe
 	}
+	if mlrmap.keysToEntries != nil {
+		mlrmap.keysToEntries[pe.Key] = pe
+	}
 	mlrmap.FieldCount++
 }
 
@@ -828,6 +841,9 @@ func (mlrmap *Mlrmap) linkAtTail(pe *MlrmapEntry) {
 		mlrmap.Tail.Next = pe
 		mlrmap.Tail = pe
 	}
+	if mlrmap.keysToEntries != nil {
+		mlrmap.keysToEntries[pe.Key] = pe
+	}
 	mlrmap.FieldCount++
 }
 
diff --git a/go/src/types/mlrmap_flatten_unflatten.go b/go/src/types/mlrmap_flatten_unflatten.go
index 8e3a85f27..3dba50e78 100644
--- a/go/src/types/mlrmap_flatten_unflatten.go
+++ b/go/src/types/mlrmap_flatten_unflatten.go
@@ -34,6 +34,12 @@ import (
 // nullity of the fieldNameSet) since the flatten/unflatten check is done by
 // default on ALL Miller records whenever we convert to/from JSON. So, the
 // default path should be fast.
+//
+// Examples:
+// * The single field x = {"a": 7, "b": 8, "c": 9} becomes the three fields
+//   x.a = 7, x.b = 8, x.c = 9.
+// * The single field x = [7,8,9] becomes the three fields
+//   x.1 = 7, x.2 = 8, x.3 = 9.
 
 func (mlrmap *Mlrmap) Flatten(separator string) {
 	if !mlrmap.isFlattenable() { // fast path: don't modify the record at all
@@ -97,12 +103,45 @@ func (mlrmap *Mlrmap) isFlattenable() bool {
 }
 
 // ----------------------------------------------------------------
-func (mlrmap *Mlrmap) Unflatten(separator string) {
-	other := NewMlrmapAsRecord()
+// For mlr unflatten without -f. This undoes Unflatten.  This is for conversion
+// from non-JSON to JSON.  If there are fields x.a, x.b, x.c, etc. they're put
+// into a single field x with map-valued value keyed by "a", "b", "c".
 
+// Special case: if the resulting string keys are string representations of 1,
+// 2, 3, etc -- without gaps -- then the map is converted to an array.
+//
+// Examples:
+//
+// * The three fields x.a = 7, x.b = 8, x.c = 9  become
+//   the single field x = {"a": 7, "b": 8, "c": 9}.
+//
+// * The three fields x.1 = 7, x.2 = 8, x.3 = 9 become
+//   the single field x = [7,8,9].
+//
+// * The two fields x.1 = 7, x.3 = 9 become
+//   the single field x = {"1": 7, "3": 9}
+func (mlrmap *Mlrmap) Unflatten(
+	separator string,
+) {
+	*mlrmap = *(mlrmap.CopyUnflattened(separator))
+}
+
+func (mlrmap *Mlrmap) CopyUnflattened(
+	separator string,
+) *Mlrmap {
+	other := NewMlrmapAsRecord()
+	affectedBaseIndices := make(map[string]bool)
+
+	// We'll come through this loop once for x.a, another for x.b, etc.
 	for pe := mlrmap.Head; pe != nil; pe = pe.Next {
+		// Is the field name something dot something?
 		if strings.Contains(pe.Key, separator) {
 			arrayOfIndices := mlrvalSplitAXHelper(pe.Key, separator)
+			lib.InternalCodingErrorIf(len(arrayOfIndices.arrayval) < 1)
+			// If the input field name was "x.a" then remember the "x".
+			baseIndex := arrayOfIndices.arrayval[0].String()
+			affectedBaseIndices[baseIndex] = true
+			// Use PutIndexed to assign $x["a"] = 7, or $x["b"] = 8, etc.
 			other.PutIndexed(
 				MakePointerArray(arrayOfIndices.arrayval),
 				unflattenTerminal(pe.Value).Copy(),
@@ -112,27 +151,52 @@ func (mlrmap *Mlrmap) Unflatten(separator string) {
 		}
 	}
 
-	*mlrmap = *other
+	// Go through all the field names which were turned into maps -- e.g.  "x"
+	// in the example above -- and see if the keys were like "1", "2", etc and
+	// if so then convert to array. This undoes how Flatten flattens arrays.
+	for baseIndex := range affectedBaseIndices {
+		oldValue := other.Get(baseIndex)
+		lib.InternalCodingErrorIf(oldValue == nil)
+		newValue := MlrvalArrayify(oldValue)
+		other.PutReference(baseIndex, newValue)
+	}
+
+	return other
 }
 
 // ----------------------------------------------------------------
-// For mlr unflatten -f.
+// For mlr unflatten -f. See comments on Unflatten. Largely copypasta of
+// Unflatten, but split out separately since Flatten needn't check a
+// fieldNameSet.
 func (mlrmap *Mlrmap) UnflattenFields(
 	fieldNameSet map[string]bool,
 	separator string,
 ) {
-	other := NewMlrmapAsRecord()
+	*mlrmap = *(mlrmap.CopyUnflattenFields(fieldNameSet, separator))
+}
 
+func (mlrmap *Mlrmap) CopyUnflattenFields(
+	fieldNameSet map[string]bool,
+	separator string,
+) *Mlrmap {
+	other := NewMlrmapAsRecord()
+	affectedBaseIndices := make(map[string]bool)
+
+	// We'll come through this loop once for x.a, another for x.b, etc.
 	for pe := mlrmap.Head; pe != nil; pe = pe.Next {
+		// Is the field name something dot something?
 		if strings.Contains(pe.Key, separator) {
 			arrayOfIndices := mlrvalSplitAXHelper(pe.Key, separator)
 			lib.InternalCodingErrorIf(len(arrayOfIndices.arrayval) < 1)
+			// If the input field name was "x.a" then remember the "x".
 			baseIndex := arrayOfIndices.arrayval[0].String()
 			if fieldNameSet[baseIndex] {
+				// Use PutIndexed to assign $x["a"] = 7, or $x["b"] = 8, etc.
 				other.PutIndexed(
 					MakePointerArray(arrayOfIndices.arrayval),
 					unflattenTerminal(pe.Value).Copy(),
 				)
+				affectedBaseIndices[baseIndex] = true
 			} else {
 				other.PutReference(pe.Key, unflattenTerminal(pe.Value))
 			}
@@ -141,7 +205,17 @@ func (mlrmap *Mlrmap) UnflattenFields(
 		}
 	}
 
-	*mlrmap = *other
+	// Go through all the field names which were turned into maps -- e.g.  "x"
+	// in the example above -- and see if the keys were like "1", "2", etc and
+	// if so then convert to array. This undoes how Flatten flattens arrays.
+	for baseIndex := range affectedBaseIndices {
+		oldValue := other.Get(baseIndex)
+		lib.InternalCodingErrorIf(oldValue == nil)
+		newValue := MlrvalArrayify(oldValue)
+		other.PutReference(baseIndex, newValue)
+	}
+
+	return other
 }
 
 // ----------------------------------------------------------------
diff --git a/go/src/types/mlrval_collections.go b/go/src/types/mlrval_collections.go
index 961b77a15..47b0bfc52 100644
--- a/go/src/types/mlrval_collections.go
+++ b/go/src/types/mlrval_collections.go
@@ -546,7 +546,7 @@ func MakePointerArray(
 	valueArray []Mlrval,
 ) (pointerArray []*Mlrval) {
 	pointerArray = make([]*Mlrval, len(valueArray))
-	for i, _ := range valueArray {
+	for i := range valueArray {
 		pointerArray[i] = &valueArray[i]
 	}
 	return pointerArray
diff --git a/go/src/types/mlrval_functions_arithmetic.go b/go/src/types/mlrval_functions_arithmetic.go
index 3dae7fb3f..2a6c77870 100644
--- a/go/src/types/mlrval_functions_arithmetic.go
+++ b/go/src/types/mlrval_functions_arithmetic.go
@@ -819,7 +819,7 @@ func MlrvalVariadicMin(mlrvals []*Mlrval) *Mlrval {
 		return MLRVAL_VOID
 	} else {
 		retval := mlrvals[0]
-		for i, _ := range mlrvals {
+		for i := range mlrvals {
 			if i > 0 {
 				retval = MlrvalBinaryMin(retval, mlrvals[i])
 			}
@@ -902,7 +902,7 @@ func MlrvalVariadicMax(mlrvals []*Mlrval) *Mlrval {
 		return MLRVAL_VOID
 	} else {
 		retval := mlrvals[0]
-		for i, _ := range mlrvals {
+		for i := range mlrvals {
 			if i > 0 {
 				retval = MlrvalBinaryMax(retval, mlrvals[i])
 			}
diff --git a/go/src/types/mlrval_functions_collections.go b/go/src/types/mlrval_functions_collections.go
index 939ee1284..a5d7568d4 100644
--- a/go/src/types/mlrval_functions_collections.go
+++ b/go/src/types/mlrval_functions_collections.go
@@ -319,7 +319,7 @@ func MlrvalJoinK(input1, input2 *Mlrval) *Mlrval {
 	} else if input1.mvtype == MT_ARRAY {
 		var buffer bytes.Buffer
 
-		for i, _ := range input1.arrayval {
+		for i := range input1.arrayval {
 			if i > 0 {
 				buffer.WriteString(fieldSeparator)
 			}
@@ -553,8 +553,8 @@ func MlrvalSplitA(input1, input2 *Mlrval) *Mlrval {
 }
 
 // ----------------------------------------------------------------
-// splitax("3,4,5", ",") -> ["3","4","5"]
-
+// MlrvalSplitAX splits a string to an array, without type-inference:
+// e.g. splitax("3,4,5", ",") -> ["3","4","5"]
 func MlrvalSplitAX(input1, input2 *Mlrval) *Mlrval {
 	if input1.mvtype != MT_STRING {
 		return MLRVAL_ERROR
@@ -568,7 +568,8 @@ func MlrvalSplitAX(input1, input2 *Mlrval) *Mlrval {
 	return mlrvalSplitAXHelper(input, fieldSeparator)
 }
 
-// Split out for MlrvalSplitAX and MlrvalUnflatten
+// mlrvalSplitAXHelper is Split out for the benefit of MlrvalSplitAX and
+// MlrvalUnflatten.
 func mlrvalSplitAXHelper(input string, separator string) *Mlrval {
 	fields := lib.SplitString(input, separator)
 
@@ -596,7 +597,7 @@ func MlrvalGetKeys(input1 *Mlrval) *Mlrval {
 
 	} else if input1.mvtype == MT_ARRAY {
 		output := NewSizedMlrvalArray(int(len(input1.arrayval)))
-		for i, _ := range input1.arrayval {
+		for i := range input1.arrayval {
 			output.arrayval[i] = MlrvalFromInt(int(i + 1)) // Miller user-space indices are 1-up
 		}
 		return output
@@ -683,13 +684,7 @@ func MlrvalUnflatten(input1, input2 *Mlrval) *Mlrval {
 	}
 	oldmap := input1.mapval
 	separator := input2.printrep
-	newmap := NewMlrmap()
-
-	for pe := oldmap.Head; pe != nil; pe = pe.Next {
-		// TODO: factor out a shared helper function bewteen here and MlrvalSplitAX.
-		arrayOfIndices := mlrvalSplitAXHelper(pe.Key, separator)
-		newmap.PutIndexed(MakePointerArray(arrayOfIndices.arrayval), pe.Value.Copy())
-	}
+	newmap := oldmap.CopyUnflattened(separator)
 	return MlrvalPointerFromMapReferenced(newmap)
 }
 
@@ -697,6 +692,10 @@ func MlrvalUnflatten(input1, input2 *Mlrval) *Mlrval {
 // Converts maps with "1", "2", ... keys into arrays. Recurses nested data structures.
 func MlrvalArrayify(input1 *Mlrval) *Mlrval {
 	if input1.mvtype == MT_MAP {
+		if input1.mapval.IsEmpty() {
+			return input1
+		}
+
 		convertible := true
 		i := 0
 		for pe := input1.mapval.Head; pe != nil; pe = pe.Next {
@@ -724,7 +723,7 @@ func MlrvalArrayify(input1 *Mlrval) *Mlrval {
 	} else if input1.mvtype == MT_ARRAY {
 		// TODO: comment (or rethink) that this modifies its inputs!!
 		output := input1.Copy()
-		for i, _ := range input1.arrayval {
+		for i := range input1.arrayval {
 			output.arrayval[i] = *MlrvalArrayify(&output.arrayval[i])
 		}
 		return output
diff --git a/go/todo.txt b/go/todo.txt
index 184701c1a..7dd300008 100644
--- a/go/todo.txt
+++ b/go/todo.txt
@@ -197,6 +197,8 @@ BLOCKERS
 ================================================================
 NON-BLOCKERS
 
+* unarrayify function
+
 * XYZWasSpecified -> XYZ = "default" w/ check-after -- ?
 
 * parquet -- ?
@@ -277,6 +279,8 @@ d type up a note somewhere in the regtest tree that all the .mlrs and the -f 00n
   o multiple-valued return/assign -- ?
     - array destructure at LHS for multi-retval assign (maps too?)
 
+* UT per se for lrec ops
+
 * "Miller: " prefixes on all errors.New
 * libify errors.New callsites for DSL/CST
 * record-readers are fully in-channel/loop; record-writers are multi with in-channel/loop being
diff --git a/man6/Makefile b/man6/Makefile
index d4b55e1c9..02076cb22 100644
--- a/man6/Makefile
+++ b/man6/Makefile
@@ -9,7 +9,7 @@ top: .always
 	echo mkman start
 	./mkman.rb > mlr6.1
 	./mkman.rb | groff -man -Tascii | col -b | expand -8 > manpage.txt
-	cp manpage.txt ../docs6/docs
+	cp manpage.txt ../docs6/src/
 	echo mkman end
 
 # These targets are only for local dev work.
diff --git a/man6/manpage.txt b/man6/manpage.txt
index c70dde832..197e3798a 100644
--- a/man6/manpage.txt
+++ b/man6/manpage.txt
@@ -8,11 +8,13 @@ NAME
 
 SYNOPSIS
        Usage: mlr [flags] {verb} [verb-dependent options ...] {zero or more
-       file names} Output of one verb may be chained as input to another using
-       "then", e.g.
-         mlr stats1 -a min,mean,max -f flag,u,v -g color then sort -f color
-       Please see 'mlr help topics' for more information.  Please also see
-       https://johnkerl.org/miller6
+       file names} If zero file names are provided, standard input is read.
+       Example:
+         mlr --csv sort -f shape example.csv Output of one verb may be chained
+       as input to another using "then", e.g.
+         mlr --csv stats1 -a min,mean,max -f quantity then sort -f color
+       example.csv Please see 'mlr help topics' for more information.  Please
+       also see https://johnkerl.org/miller6
 
 
 DESCRIPTION
@@ -33,54 +35,67 @@ EXAMPLES
        mlr --icsv --ojson put '$ratio = $quantity / $rate' example.csv
        mlr --icsv --opprint --from example.csv sort -nr index then cut -f shape,quantity
 
-DATA FORMATS
-       Type 'mlr help {topic}' for any of the following:
-       Essentials:
-         mlr help topics
-         mlr help basic-examples
-         mlr help file-formats
-       Flags:
-         mlr help flags
-         mlr help comments-in-data-flags
-         mlr help compressed-data-flags
-         mlr help csv-only-flags
-         mlr help file-format-flags
-         mlr help flatten-unflatten-flags
-         mlr help format-conversion-keystroke-saver-flags
-         mlr help json-only-flags
-         mlr help legacy-flags
-         mlr help miscellaneous-flags
-         mlr help output-colorization-flags
-         mlr help pprint-only-flags
-         mlr help separator-flags
-       Verbs:
-         mlr help list-verbs
-         mlr help usage-verbs
-         mlr help verb
-       Functions:
-         mlr help list-functions
-         mlr help list-function-classes
-         mlr help list-functions-in-class
-         mlr help usage-functions
-         mlr help usage-functions-by-class
-         mlr help function
-       Keywords:
-         mlr help list-keywords
-         mlr help usage-keywords
-         mlr help keyword
-       Other:
-         mlr help auxents
-         mlr help mlrrc
-         mlr help output-colorization
-         mlr help type-arithmetic-info
-       Shorthands:
-         mlr -g = mlr help flags
-         mlr -l = mlr help list-verbs
-         mlr -L = mlr help usage-verbs
-         mlr -f = mlr help list-functions
-         mlr -F = mlr help usage-functions
-         mlr -k = mlr help list-keywords
-         mlr -K = mlr help usage-keywords
+FILE FORMATS
+       CSV/CSV-lite: comma-separated values with separate header line
+       TSV: same but with tabs in places of commas
+       +---------------------+
+       | apple,bat,cog       |
+       | 1,2,3               | Record 1: "apple":"1", "bat":"2", "cog":"3"
+       | 4,5,6               | Record 2: "apple":"4", "bat":"5", "cog":"6"
+       +---------------------+
+
+       JSON (sequence or array of objects):
+       +---------------------+
+       | {                   |
+       |  "apple": 1,        | Record 1: "apple":"1", "bat":"2", "cog":"3"
+       |  "bat": 2,          |
+       |  "cog": 3           |
+       | }                   |
+       | {                   |
+       |   "dish": {         | Record 2: "dish:egg":"7",
+       |     "egg": 7,       | "dish:flint":"8", "garlic":""
+       |     "flint": 8      |
+       |   },                |
+       |   "garlic": ""      |
+       | }                   |
+       +---------------------+
+
+       PPRINT: pretty-printed tabular
+       +---------------------+
+       | apple bat cog       |
+       | 1     2   3         | Record 1: "apple:"1", "bat":"2", "cog":"3"
+       | 4     5   6         | Record 2: "apple":"4", "bat":"5", "cog":"6"
+       +---------------------+
+
+       Markdown tabular (supported for output only):
+       +-----------------------+
+       | | apple | bat | cog | |
+       | | ---   | --- | --- | |
+       | | 1     | 2   | 3   | | Record 1: "apple:"1", "bat":"2", "cog":"3"
+       | | 4     | 5   | 6   | | Record 2: "apple":"4", "bat":"5", "cog":"6"
+       +-----------------------+
+
+       XTAB: pretty-printed transposed tabular
+       +---------------------+
+       | apple 1             | Record 1: "apple":"1", "bat":"2", "cog":"3"
+       | bat   2             |
+       | cog   3             |
+       |                     |
+       | dish 7              | Record 2: "dish":"7", "egg":"8"
+       | egg  8              |
+       +---------------------+
+
+       DKVP: delimited key-value pairs (Miller default format)
+       +---------------------+
+       | apple=1,bat=2,cog=3 | Record 1: "apple":"1", "bat":"2", "cog":"3"
+       | dish=7,egg=8,flint  | Record 2: "dish":"7", "egg":"8", "3":"flint"
+       +---------------------+
+
+       NIDX: implicitly numerically indexed (Unix-toolkit style)
+       +---------------------+
+       | the quick brown     | Record 1: "1":"the", "2":"quick", "3":"brown"
+       | fox jumped          | Record 2: "1":"fox", "2":"jumped"
+       +---------------------+
 
 HELP OPTIONS
        Type 'mlr help {topic}' for any of the following:
@@ -359,6 +374,8 @@ FORMAT-CONVERSION KEYSTROKE-SAVER FLAGS
        | XTAB   | --x2c | --x2t | --x2j  | --x2d  | --x2n  |        | --x2p  | --x2m    |
        | PPRINT | --p2c | --p2t | --p2j  | --p2d  | --p2n  | --p2x  |        | --p2m    |
 
+       -p                       Keystroke-saver for `--nidx --fs space --repifs`.
+       -T                       Keystroke-saver for `--nidx --fs tab`.
 
 JSON-ONLY FLAGS
        These are flags which are applicable to JSON format.
@@ -2759,4 +2776,4 @@ SEE ALSO
 
 
 
-                                  2021-09-09                         MILLER(1)
+                                  2021-09-15                         MILLER(1)
diff --git a/man6/mkman.rb b/man6/mkman.rb
index e40047afe..dfb78ae60 100755
--- a/man6/mkman.rb
+++ b/man6/mkman.rb
@@ -40,8 +40,8 @@ a special case.) This manpage documents #{`mlr --version`.chomp}."""
   ])
 	print make_code_block(`mlr help basic-examples`)
 
-	print make_section('DATA FORMATS', [])
-	print make_code_block(`mlr help data-formats`)
+	print make_section('FILE FORMATS', [])
+	print make_code_block(`mlr help file-formats`)
 
 	print make_section('HELP OPTIONS', [])
 	print make_code_block(`mlr help topics`)
diff --git a/man6/mlr6.1 b/man6/mlr6.1
index 0c37e3b74..8b15c7352 100644
--- a/man6/mlr6.1
+++ b/man6/mlr6.1
@@ -2,12 +2,12 @@
 .\"     Title: mlr
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: ./mkman.rb
-.\"      Date: 2021-09-09
+.\"      Date: 2021-09-15
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "MILLER" "1" "2021-09-09" "\ \&" "\ \&"
+.TH "MILLER" "1" "2021-09-15" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Portability definitions
 .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -30,8 +30,10 @@ miller \- like awk, sed, cut, join, and sort for name-indexed data such as CSV a
 .SH "SYNOPSIS"
 .sp
 Usage: mlr [flags] {verb} [verb-dependent options ...] {zero or more file names}
+If zero file names are provided, standard input is read. Example:
+  mlr --csv sort -f shape example.csv
 Output of one verb may be chained as input to another using "then", e.g.
-  mlr stats1 -a min,mean,max -f flag,u,v -g color then sort -f color
+  mlr --csv stats1 -a min,mean,max -f quantity then sort -f color example.csv
 Please see 'mlr help topics' for more information.
 Please also see https://johnkerl.org/miller6
 
@@ -60,58 +62,71 @@ mlr --icsv --opprint --from example.csv sort -nr index then cut -f shape,quantit
 .fi
 .if n \{\
 .RE
-.SH "DATA FORMATS"
+.SH "FILE FORMATS"
 .if n \{\
 .RS 0
 .\}
 .nf
-Type 'mlr help {topic}' for any of the following:
-Essentials:
-  mlr help topics
-  mlr help basic-examples
-  mlr help file-formats
-Flags:
-  mlr help flags
-  mlr help comments-in-data-flags
-  mlr help compressed-data-flags
-  mlr help csv-only-flags
-  mlr help file-format-flags
-  mlr help flatten-unflatten-flags
-  mlr help format-conversion-keystroke-saver-flags
-  mlr help json-only-flags
-  mlr help legacy-flags
-  mlr help miscellaneous-flags
-  mlr help output-colorization-flags
-  mlr help pprint-only-flags
-  mlr help separator-flags
-Verbs:
-  mlr help list-verbs
-  mlr help usage-verbs
-  mlr help verb
-Functions:
-  mlr help list-functions
-  mlr help list-function-classes
-  mlr help list-functions-in-class
-  mlr help usage-functions
-  mlr help usage-functions-by-class
-  mlr help function
-Keywords:
-  mlr help list-keywords
-  mlr help usage-keywords
-  mlr help keyword
-Other:
-  mlr help auxents
-  mlr help mlrrc
-  mlr help output-colorization
-  mlr help type-arithmetic-info
-Shorthands:
-  mlr -g = mlr help flags
-  mlr -l = mlr help list-verbs
-  mlr -L = mlr help usage-verbs
-  mlr -f = mlr help list-functions
-  mlr -F = mlr help usage-functions
-  mlr -k = mlr help list-keywords
-  mlr -K = mlr help usage-keywords
+CSV/CSV-lite: comma-separated values with separate header line
+TSV: same but with tabs in places of commas
++---------------------+
+| apple,bat,cog       |
+| 1,2,3               | Record 1: "apple":"1", "bat":"2", "cog":"3"
+| 4,5,6               | Record 2: "apple":"4", "bat":"5", "cog":"6"
++---------------------+
+
+JSON (sequence or array of objects):
++---------------------+
+| {                   |
+|  "apple": 1,        | Record 1: "apple":"1", "bat":"2", "cog":"3"
+|  "bat": 2,          |
+|  "cog": 3           |
+| }                   |
+| {                   |
+|   "dish": {         | Record 2: "dish:egg":"7",
+|     "egg": 7,       | "dish:flint":"8", "garlic":""
+|     "flint": 8      |
+|   },                |
+|   "garlic": ""      |
+| }                   |
++---------------------+
+
+PPRINT: pretty-printed tabular
++---------------------+
+| apple bat cog       |
+| 1     2   3         | Record 1: "apple:"1", "bat":"2", "cog":"3"
+| 4     5   6         | Record 2: "apple":"4", "bat":"5", "cog":"6"
++---------------------+
+
+Markdown tabular (supported for output only):
++-----------------------+
+| | apple | bat | cog | |
+| | ---   | --- | --- | |
+| | 1     | 2   | 3   | | Record 1: "apple:"1", "bat":"2", "cog":"3"
+| | 4     | 5   | 6   | | Record 2: "apple":"4", "bat":"5", "cog":"6"
++-----------------------+
+
+XTAB: pretty-printed transposed tabular
++---------------------+
+| apple 1             | Record 1: "apple":"1", "bat":"2", "cog":"3"
+| bat   2             |
+| cog   3             |
+|                     |
+| dish 7              | Record 2: "dish":"7", "egg":"8"
+| egg  8              |
++---------------------+
+
+DKVP: delimited key-value pairs (Miller default format)
++---------------------+
+| apple=1,bat=2,cog=3 | Record 1: "apple":"1", "bat":"2", "cog":"3"
+| dish=7,egg=8,flint  | Record 2: "dish":"7", "egg":"8", "3":"flint"
++---------------------+
+
+NIDX: implicitly numerically indexed (Unix-toolkit style)
++---------------------+
+| the quick brown     | Record 1: "1":"the", "2":"quick", "3":"brown"
+| fox jumped          | Record 2: "1":"fox", "2":"jumped"
++---------------------+
 .fi
 .if n \{\
 .RE
@@ -456,6 +471,8 @@ available for output only.
 | XTAB   | --x2c | --x2t | --x2j  | --x2d  | --x2n  |        | --x2p  | --x2m    |
 | PPRINT | --p2c | --p2t | --p2j  | --p2d  | --p2n  | --p2x  |        | --p2m    |
 
+-p                       Keystroke-saver for `--nidx --fs space --repifs`.
+-T                       Keystroke-saver for `--nidx --fs tab`.
 .fi
 .if n \{\
 .RE