From fa3ee05822de7d4239d291b56262405c5496a421 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Wed, 4 Aug 2021 21:50:41 -0400 Subject: [PATCH] More miller-6 mkdocs porting from sphinx (#617) * More mkdocs porting * mkdocs-ify http/external links * move README.md up one level * Further sharpen code-sample CSS * fix last internal-ref links --- docs6b/README.md | 42 + docs6b/docs/.vimrc | 1 - docs6b/docs/10min.md | 342 ++- docs6b/docs/10min.md.in | 174 +- docs6b/docs/README.md | 40 - docs6b/docs/build.md | 67 +- docs6b/docs/build.md.in | 65 +- docs6b/docs/community.md | 6 +- docs6b/docs/community.md.in | 6 +- docs6b/docs/contributing.md | 8 +- docs6b/docs/contributing.md.in | 8 +- docs6b/docs/csv-with-and-without-headers.md | 54 +- .../docs/csv-with-and-without-headers.md.in | 12 +- docs6b/docs/customization.md | 46 +- docs6b/docs/customization.md.in | 34 +- docs6b/docs/data-cleaning-examples.md | 26 +- docs6b/docs/data-cleaning-examples.md.in | 2 +- docs6b/docs/data-diving-examples.md | 78 +- docs6b/docs/data-diving-examples.md.in | 16 +- docs6b/docs/data/sar.mlr | 6 +- docs6b/docs/dates-and-times.md | 38 +- docs6b/docs/dates-and-times.md.in | 6 +- docs6b/docs/dkvp-examples.md | 28 +- docs6b/docs/etymology.md | 2 +- docs6b/docs/etymology.md.in | 2 +- docs6b/docs/extra.css | 55 +- docs6b/docs/feature-comparison.md | 40 +- docs6b/docs/feature-comparison.md.in | 24 +- docs6b/docs/features.md | 10 +- docs6b/docs/features.md.in | 10 +- docs6b/docs/file-formats.md | 200 +- docs6b/docs/file-formats.md.in | 80 +- docs6b/docs/genmd-filter | 18 +- docs6b/docs/index.md | 4 +- docs6b/docs/index.md.in | 4 +- docs6b/docs/installation.md | 14 +- docs6b/docs/installation.md.in | 2 +- docs6b/docs/internationalization.md | 6 +- docs6b/docs/internationalization.md.in | 6 +- docs6b/docs/joins.md | 58 +- docs6b/docs/joins.md.in | 10 +- docs6b/docs/keystroke-savers.md | 26 +- docs6b/docs/keystroke-savers.md.in | 8 +- docs6b/docs/log-processing-examples.md | 30 +- docs6b/docs/log-processing-examples.md.in | 6 +- docs6b/docs/manpage.md | 2 +- docs6b/docs/miller-on-windows.md | 14 +- docs6b/docs/miller-on-windows.md.in | 14 +- docs6b/docs/misc-examples.md | 74 +- docs6b/docs/misc-examples.md.in | 8 +- docs6b/docs/mk-func-h2s.sh | 40 +- docs6b/docs/new-in-miller-6.md | 54 +- docs6b/docs/new-in-miller-6.md.in | 44 +- docs6b/docs/operating-on-all-fields.md | 52 +- docs6b/docs/operating-on-all-fields.md.in | 6 +- docs6b/docs/originality.md | 26 +- docs6b/docs/originality.md.in | 26 +- docs6b/docs/output-colorization.md | 29 +- docs6b/docs/output-colorization.md.in | 29 +- docs6b/docs/performance.md | 8 +- docs6b/docs/performance.md.in | 8 +- docs6b/docs/programming-examples.md | 20 +- docs6b/docs/programming-examples.md.in | 2 +- docs6b/docs/programming-language.md | 152 +- docs6b/docs/programming-language.md.in | 72 +- docs6b/docs/randomizing-examples.md | 24 +- docs6b/docs/randomizing-examples.md.in | 4 +- docs6b/docs/record-heterogeneity.md | 58 +- docs6b/docs/record-heterogeneity.md.in | 6 +- docs6b/docs/reference-dsl-arrays.md | 4 +- .../docs/reference-dsl-builtin-functions.md | 2437 ++++++----------- .../reference-dsl-builtin-functions.md.in | 2 +- docs6b/docs/reference-dsl-complexity.md | 6 +- docs6b/docs/reference-dsl-complexity.md.in | 6 +- .../docs/reference-dsl-control-structures.md | 158 +- .../reference-dsl-control-structures.md.in | 55 +- docs6b/docs/reference-dsl-errors.md | 20 +- docs6b/docs/reference-dsl-errors.md.in | 20 +- .../docs/reference-dsl-filter-statements.md | 18 +- .../reference-dsl-filter-statements.md.in | 2 +- docs6b/docs/reference-dsl-operators.md | 10 +- docs6b/docs/reference-dsl-operators.md.in | 8 +- .../docs/reference-dsl-output-statements.md | 184 +- .../reference-dsl-output-statements.md.in | 68 +- docs6b/docs/reference-dsl-syntax.md | 66 +- docs6b/docs/reference-dsl-syntax.md.in | 8 +- docs6b/docs/reference-dsl-unset-statements.md | 22 +- .../docs/reference-dsl-unset-statements.md.in | 6 +- .../reference-dsl-user-defined-functions.md | 34 +- ...reference-dsl-user-defined-functions.md.in | 26 +- docs6b/docs/reference-dsl-variables.md | 254 +- docs6b/docs/reference-dsl-variables.md.in | 102 +- docs6b/docs/reference-dsl.md | 36 +- docs6b/docs/reference-dsl.md.in | 16 +- docs6b/docs/reference-main-arithmetic.md | 14 +- docs6b/docs/reference-main-arithmetic.md.in | 12 +- .../docs/reference-main-auxiliary-commands.md | 40 +- docs6b/docs/reference-main-data-types.md | 46 +- docs6b/docs/reference-main-data-types.md.in | 46 +- docs6b/docs/reference-main-env-vars.md | 6 +- docs6b/docs/reference-main-env-vars.md.in | 6 +- docs6b/docs/reference-main-io-options.md | 70 +- docs6b/docs/reference-main-io-options.md.in | 36 +- docs6b/docs/reference-main-null-data.md | 74 +- docs6b/docs/reference-main-null-data.md.in | 22 +- docs6b/docs/reference-main-online-help.md | 8 +- docs6b/docs/reference-main-overview.md | 28 +- docs6b/docs/reference-main-overview.md.in | 12 +- .../reference-main-regular-expressions.md | 48 +- .../reference-main-regular-expressions.md.in | 32 +- docs6b/docs/reference-main-then-chaining.md | 12 +- .../docs/reference-main-then-chaining.md.in | 6 +- docs6b/docs/reference-verbs.md | 1024 ++++--- docs6b/docs/reference-verbs.md.in | 208 +- docs6b/docs/release-docs.md | 2 +- docs6b/docs/release-docs.md.in | 2 +- docs6b/docs/repl.md | 92 +- docs6b/docs/repl.md.in | 72 +- docs6b/docs/shapes-of-data.md | 158 +- docs6b/docs/shapes-of-data.md.in | 40 +- docs6b/docs/shell-commands.md | 24 +- docs6b/docs/shell-commands.md.in | 4 +- docs6b/docs/special-symbols-and-formatting.md | 76 +- .../docs/special-symbols-and-formatting.md.in | 26 +- docs6b/docs/sql-examples.md | 40 +- docs6b/docs/sql-examples.md.in | 10 +- docs6b/docs/statistics-examples.md | 14 +- docs6b/docs/statistics-examples.md.in | 4 +- docs6b/docs/then-chaining.md | 50 +- docs6b/docs/then-chaining.md.in | 14 +- docs6b/docs/two-pass-algorithms.md | 154 +- docs6b/docs/two-pass-algorithms.md.in | 14 +- docs6b/docs/why.md | 16 +- docs6b/docs/why.md.in | 16 +- docs6b/docs/x | 19 +- docs6b/mkdocs.yml | 6 - docs6b/mkdocs.yml.000 | 9 - docs6b/mkdocs.yml.001 | 12 - docs6b/mkdocs.yml.002 | 84 - docs6b/setup.txt | 7 - docs6b/wishlist.txt | 26 +- 141 files changed, 4449 insertions(+), 4352 deletions(-) create mode 100644 docs6b/README.md delete mode 100644 docs6b/docs/README.md delete mode 100644 docs6b/mkdocs.yml.000 delete mode 100644 docs6b/mkdocs.yml.001 delete mode 100644 docs6b/mkdocs.yml.002 delete mode 100644 docs6b/setup.txt diff --git a/docs6b/README.md b/docs6b/README.md new file mode 100644 index 000000000..e7c748407 --- /dev/null +++ b/docs6b/README.md @@ -0,0 +1,42 @@ +# Miller docs + +## Why use Mkdocs + +* Connects to https://miller.readthedocs.io so people can get their docmods onto the web instead of the self-hosted https://johnkerl.org/miller/doc. Thanks to @pabloab for the great advice! +* More standard look and feel -- lots of people use readthedocs for other things so this should feel familiar. +* We get a Search feature for free. +* Mkdocs vs Sphinx: these are similar tools, but I find that I more easily get better desktop+mobile formatting using Mkdocs. + +## Contributing + +* You need `pip install mkdocs` (or `pip3 install mkdocs`). +* The docs include lots of live code examples which will be invoked using `mlr` which must be somewhere in your `$PATH`. +* Clone https://github.com/johnkerl/miller and cd into `docs/` within your clone. +* Quick-editing loop: + * In one terminal, cd to this directory and leave `mkdocs serve` running. + * In another terminal, cd to the `docs` subdirectory and edit `*.md.in`. + * In your browser, visit http://127.0.0.1:8000 +* Alternate editing loop: + * Leave one terminal open as a place you will run `mkdocs build` + * In one terminal, cd to the `docs` subdirectory and edit `*.md.in`. + * In the first terminal, run `mkdocs build` which will populate the `site` directory. + * In your browser, visit `file:///your/path/to/miller/docs/site/index.html` + * Link-checking: + * `sudo pip3 install git+https://github.com/linkchecker/linkchecker.git` + * `cd site` and `linkchecker .` +* Submitting: + * `git add` your modified files, `git commit`, `git push`, and submit a PR at https://github.com/johnkerl/miller. + +## Notes + +* CSS: + * I used the Mkdocs Readthedocs theme which I like a lot. I customized `docs/extra.css` for Miller coloring/branding. +* Live code: + * I didn't find a way to include non-Python live-code examples within Mkdocs so I adapted the pre-Mkdocs Miller-doc strategy which is to have a generator script read a template file (here, `foo.md.in`), run the marked lines, and generate the output file (`foo.md`). This is `genmds`. + * Edit the `*.md.in` files, not `*.md` directly. + * Within the `*.md.in` files are lines like `GENMD_RUN_COMMAND`. These will be run, and their output included, by `genmds` which calls the `genmds` script for you. +* readthedocs: + * https://readthedocs.org/ + * https://readthedocs.org/projects/miller/ + * https://readthedocs.org/projects/miller/builds/ + * https://miller.readthedocs.io/en/latest/ diff --git a/docs6b/docs/.vimrc b/docs6b/docs/.vimrc index 3c9b75a7a..5f2e150af 100644 --- a/docs6b/docs/.vimrc +++ b/docs6b/docs/.vimrc @@ -1,2 +1 @@ map \d :w:!clear;build-one % -map \f :w:!clear;make html diff --git a/docs6b/docs/10min.md b/docs6b/docs/10min.md index 448bd521d..e4f952018 100644 --- a/docs6b/docs/10min.md +++ b/docs6b/docs/10min.md @@ -5,23 +5,28 @@ You can install Miller for various platforms as follows: -* Linux: ``yum install miller`` or ``apt-get install miller`` depending on your flavor of Linux -* MacOS: ``brew install miller`` or ``port install miller`` depending on your preference of [Homebrew](https://brew.sh>`_ or `MacPorts +
 mlr --version
+
+
 Miller v6.0.0-dev
 
As a second check, given [example.csv](./example.csv) you should be able to do -
+
 mlr --csv cat example.csv
+
+
 color,shape,flag,index,quantity,rate
 yellow,triangle,true,11,43.6498,9.8870
 red,square,true,15,79.2778,0.0130
@@ -35,8 +40,10 @@ yellow,circle,true,87,63.5058,8.3350
 purple,square,false,91,72.3735,8.2430
 
-
+
 mlr --icsv --opprint cat example.csv
+
+
 color  shape    flag  index quantity rate
 yellow triangle true  11    43.6498  9.8870
 red    square   true  15    79.2778  0.0130
@@ -56,10 +63,12 @@ If you run into issues on these checks, please check out the resources on the [c
 
 Let's take a quick look at some of the most useful Miller verbs -- file-format-aware, name-index-empowered equivalents of standard system commands.
 
-``mlr cat`` is like system ``cat`` (or ``type`` on Windows) -- it passes the data through unmodified:
+`mlr cat` is like system `cat` (or `type` on Windows) -- it passes the data through unmodified:
 
-
+
 mlr --csv cat example.csv
+
+
 color,shape,flag,index,quantity,rate
 yellow,triangle,true,11,43.6498,9.8870
 red,square,true,15,79.2778,0.0130
@@ -73,10 +82,12 @@ yellow,circle,true,87,63.5058,8.3350
 purple,square,false,91,72.3735,8.2430
 
-But ``mlr cat`` can also do format conversion -- for example, you can pretty-print in tabular format: +But `mlr cat` can also do format conversion -- for example, you can pretty-print in tabular format: -
+
 mlr --icsv --opprint cat example.csv
+
+
 color  shape    flag  index quantity rate
 yellow triangle true  11    43.6498  9.8870
 red    square   true  15    79.2778  0.0130
@@ -90,10 +101,12 @@ yellow circle   true  87    63.5058  8.3350
 purple square   false 91    72.3735  8.2430
 
-``mlr head`` and ``mlr tail`` count records rather than lines. Whether you're getting the first few records or the last few, the CSV header is included either way: +`mlr head` and `mlr tail` count records rather than lines. Whether you're getting the first few records or the last few, the CSV header is included either way: -
+
 mlr --csv head -n 4 example.csv
+
+
 color,shape,flag,index,quantity,rate
 yellow,triangle,true,11,43.6498,9.8870
 red,square,true,15,79.2778,0.0130
@@ -101,8 +114,10 @@ red,circle,true,16,13.8103,2.9010
 red,square,false,48,77.5542,7.4670
 
-
+
 mlr --csv tail -n 4 example.csv
+
+
 color,shape,flag,index,quantity,rate
 purple,triangle,false,65,80.1405,5.8240
 yellow,circle,true,73,63.9785,4.2370
@@ -110,8 +125,10 @@ yellow,circle,true,87,63.5058,8.3350
 purple,square,false,91,72.3735,8.2430
 
-
+
 mlr --icsv --ojson tail -n 2 example.csv
+
+
 {
   "color": "yellow",
   "shape": "circle",
@@ -132,8 +149,10 @@ purple,square,false,91,72.3735,8.2430
 
 You can sort on a single field:
 
-
+
 mlr --icsv --opprint sort -f shape example.csv
+
+
 color  shape    flag  index quantity rate
 red    circle   true  16    13.8103  2.9010
 yellow circle   true  73    63.9785  4.2370
@@ -149,8 +168,10 @@ purple triangle false 65    80.1405  5.8240
 
 Or, you can sort primarily alphabetically on one field, then secondarily numerically descending on another field, and so on:
 
-
+
 mlr --icsv --opprint sort -f shape -nr index example.csv
+
+
 color  shape    flag  index quantity rate
 yellow circle   true  87    63.5058  8.3350
 yellow circle   true  73    63.9785  4.2370
@@ -164,10 +185,12 @@ purple triangle false 51    81.2290  8.5910
 yellow triangle true  11    43.6498  9.8870
 
-If there are fields you don't want to see in your data, you can use ``cut`` to keep only the ones you want, in the same order they appeared in the input data: +If there are fields you don't want to see in your data, you can use `cut` to keep only the ones you want, in the same order they appeared in the input data: -
+
 mlr --icsv --opprint cut -f flag,shape example.csv
+
+
 shape    flag
 triangle true
 square   true
@@ -181,10 +204,12 @@ circle   true
 square   false
 
-You can also use ``cut -o`` to keep specified fields, but in your preferred order: +You can also use `cut -o` to keep specified fields, but in your preferred order: -
+
 mlr --icsv --opprint cut -o -f flag,shape example.csv
+
+
 flag  shape
 true  triangle
 true  square
@@ -198,10 +223,12 @@ true  circle
 false square
 
-You can use ``cut -x`` to omit fields you don't care about: +You can use `cut -x` to omit fields you don't care about: -
+
 mlr --icsv --opprint cut -x -f flag,shape example.csv
+
+
 color  index quantity rate
 yellow 11    43.6498  9.8870
 red    15    79.2778  0.0130
@@ -215,10 +242,12 @@ yellow 87    63.5058  8.3350
 purple 91    72.3735  8.2430
 
-You can use ``filter`` to keep only records you care about: +You can use `filter` to keep only records you care about: -
+
 mlr --icsv --opprint filter '$color == "red"' example.csv
+
+
 color shape  flag  index quantity rate
 red   square true  15    79.2778  0.0130
 red   circle true  16    13.8103  2.9010
@@ -226,20 +255,24 @@ red   square false 48    77.5542  7.4670
 red   square false 64    77.1991  9.5310
 
-
+
 mlr --icsv --opprint filter '$color == "red" && $flag == true' example.csv
+
+
 color shape  flag index quantity rate
 red   square true 15    79.2778  0.0130
 red   circle true 16    13.8103  2.9010
 
-You can use ``put`` to create new fields which are computed from other fields: +You can use `put` to create new fields which are computed from other fields: -
+
 mlr --icsv --opprint put '
   $ratio = $quantity / $rate;
   $color_shape = $color . "_" . $shape
 ' example.csv
+
+
 color  shape    flag  index quantity rate   ratio              color_shape
 yellow triangle true  11    43.6498  9.8870 4.414868008496004  yellow_triangle
 red    square   true  15    79.2778  0.0130 6098.292307692308  red_square
@@ -253,10 +286,12 @@ yellow circle   true  87    63.5058  8.3350 7.619172165566886  yellow_circle
 purple square   false 91    72.3735  8.2430 8.779995147397793  purple_square
 
-Even though Miller's main selling point is name-indexing, sometimes you really want to refer to a field name by its positional index. Use ``$[[3]]`` to access the name of field 3 or ``$[[[3]]]`` to access the value of field 3: +Even though Miller's main selling point is name-indexing, sometimes you really want to refer to a field name by its positional index. Use `$[[3]]` to access the name of field 3 or `$[[[3]]]` to access the value of field 3: -
+
 mlr --icsv --opprint put '$[[3]] = "NEW"' example.csv
+
+
 color  shape    NEW   index quantity rate
 yellow triangle true  11    43.6498  9.8870
 red    square   true  15    79.2778  0.0130
@@ -270,8 +305,10 @@ yellow circle   true  87    63.5058  8.3350
 purple square   false 91    72.3735  8.2430
 
-
+
 mlr --icsv --opprint put '$[[[3]]] = "NEW"' example.csv
+
+
 color  shape    flag index quantity rate
 yellow triangle NEW  11    43.6498  9.8870
 red    square   NEW  15    79.2778  0.0130
@@ -289,23 +326,29 @@ You can find the full list of verbs at the [Verbs Reference](reference-verbs.md)
 
 ## Multiple input files
 
-Miller takes all the files from the command line as an input stream. But it's format-aware, so it doesn't repeat CSV header lines. For example, with input files [data/a.csv](data/a.csv and [data/b.csv](data/b.csv), the system ``cat`` command will repeat header lines:
+Miller takes all the files from the command line as an input stream. But it's format-aware, so it doesn't repeat CSV header lines. For example, with input files [data/a.csv](data/a.csv and [data/b.csv](data/b.csv), the system `cat` command will repeat header lines:
 
-
+
 cat data/a.csv
+
+
 a,b,c
 1,2,3
 4,5,6
 
-
+
 cat data/b.csv
+
+
 a,b,c
 7,8,9
 
-
+
 cat data/a.csv data/b.csv
+
+
 a,b,c
 1,2,3
 4,5,6
@@ -313,10 +356,12 @@ a,b,c
 7,8,9
 
-However, ``mlr cat`` will not: +However, `mlr cat` will not: -
+
 mlr --csv cat data/a.csv data/b.csv
+
+
 a,b,c
 1,2,3
 4,5,6
@@ -327,39 +372,47 @@ a,b,c
 
 Often we want to chain queries together -- for example, sorting by a field and taking the top few values. We can do this using pipes:
 
-
+
 mlr --csv sort -nr index example.csv | mlr --icsv --opprint head -n 3
+
+
 color  shape  flag  index quantity rate
 purple square false 91    72.3735  8.2430
 yellow circle true  87    63.5058  8.3350
 yellow circle true  73    63.9785  4.2370
 
-This works fine -- but Miller also lets you chain verbs together using the word ``then``. Think of this as a Miller-internal pipe that lets you use fewer keystrokes: +This works fine -- but Miller also lets you chain verbs together using the word `then`. Think of this as a Miller-internal pipe that lets you use fewer keystrokes: -
+
 mlr --icsv --opprint sort -nr index then head -n 3 example.csv
+
+
 color  shape  flag  index quantity rate
 purple square false 91    72.3735  8.2430
 yellow circle true  87    63.5058  8.3350
 yellow circle true  73    63.9785  4.2370
 
-As another convenience, you can put the filename first using ``--from``. When you're interacting with your data at the command line, this makes it easier to up-arrow and append to the previous command: +As another convenience, you can put the filename first using `--from`. When you're interacting with your data at the command line, this makes it easier to up-arrow and append to the previous command: -
+
 mlr --icsv --opprint --from example.csv sort -nr index then head -n 3
+
+
 color  shape  flag  index quantity rate
 purple square false 91    72.3735  8.2430
 yellow circle true  87    63.5058  8.3350
 yellow circle true  73    63.9785  4.2370
 
-
+
 mlr --icsv --opprint --from example.csv \
   sort -nr index \
   then head -n 3 \
   then cut -f shape,quantity
+
+
 shape  quantity
 square 72.3735
 circle 63.5058
@@ -368,22 +421,26 @@ circle 63.9785
 
 ## Sorts and stats
 
-Now suppose you want to sort the data on a given column, *and then* take the top few in that ordering. You can use Miller's ``then`` feature to pipe commands together.
+Now suppose you want to sort the data on a given column, *and then* take the top few in that ordering. You can use Miller's `then` feature to pipe commands together.
 
-Here are the records with the top three ``index`` values:
+Here are the records with the top three `index` values:
 
-
+
 mlr --icsv --opprint sort -nr index then head -n 3 example.csv
+
+
 color  shape  flag  index quantity rate
 purple square false 91    72.3735  8.2430
 yellow circle true  87    63.5058  8.3350
 yellow circle true  73    63.9785  4.2370
 
-Lots of Miller commands take a ``-g`` option for group-by: here, ``head -n 1 -g shape`` outputs the first record for each distinct value of the ``shape`` field. This means we're finding the record with highest ``index`` field for each distinct ``shape`` field: +Lots of Miller commands take a `-g` option for group-by: here, `head -n 1 -g shape` outputs the first record for each distinct value of the `shape` field. This means we're finding the record with highest `index` field for each distinct `shape` field: -
+
 mlr --icsv --opprint sort -f shape -nr index then head -n 1 -g shape example.csv
+
+
 color  shape    flag  index quantity rate
 yellow circle   true  87    63.5058  8.3350
 purple square   false 91    72.3735  8.2430
@@ -392,18 +449,22 @@ purple triangle false 65    80.1405  5.8240
 
 Statistics can be computed with or without group-by field(s):
 
-
+
 mlr --icsv --opprint --from example.csv \
   stats1 -a count,min,mean,max -f quantity -g shape
+
+
 shape    quantity_count quantity_min quantity_mean     quantity_max
 triangle 3              43.6498      68.33976666666666 81.229
 square   4              72.3735      76.60114999999999 79.2778
 circle   3              13.8103      47.0982           63.9785
 
-
+
 mlr --icsv --opprint --from example.csv \
   stats1 -a count,min,mean,max -f quantity -g shape,color
+
+
 shape    color  quantity_count quantity_min quantity_mean      quantity_max
 triangle yellow 1              43.6498      43.6498            43.6498
 square   red    3              77.1991      78.01036666666666  79.2778
@@ -415,9 +476,11 @@ square   purple 1              72.3735      72.3735            72.3735
 
 If your output has a lot of columns, you can use XTAB format to line things up vertically for you instead:
 
-
+
 mlr --icsv --oxtab --from example.csv \
   stats1 -a p0,p10,p25,p50,p75,p90,p99,p100 -f rate
+
+
 rate_p0   0.0130
 rate_p10  2.9010
 rate_p25  4.2370
@@ -438,14 +501,14 @@ Miller supports the following formats:
 * JSON (JavaScript Object Notation)
 * PPRINT (pretty-printed tabular)
 * XTAB (vertical-tabular or sideways-tabular)
-* NIDX (numerically indexed, label-free, with implicit labels ``"1"``, ``"2"``, etc.)
+* NIDX (numerically indexed, label-free, with implicit labels `"1"`, `"2"`, etc.)
 * DKVP (delimited key-value pairs).
 
 What's a CSV file, really? It's an array of rows, or *records*, each being a list of key-value pairs, or *fields*: for CSV it so happens that all the keys are shared in the header line and the values vary from one data line to another.
 
 For example, if you have:
 
-
+
 shape,flag,index
 circle,1,24
 square,0,36
@@ -453,14 +516,14 @@ square,0,36
 
 then that's a way of saying:
 
-
+
 shape=circle,flag=1,index=24
 shape=square,flag=0,index=36
 
Other ways to write the same data: -
+
 CSV                   PPRINT
 shape,flag,index      shape  flag index
 circle,1,24           circle 1    24
@@ -487,111 +550,106 @@ Anything we can do with CSV input data, we can do with any other format input da
 
 How to specify these to Miller:
 
-* If you use ``--csv`` or ``--json`` or ``--pprint``, etc., then Miller will use that format for input and output.
-* If you use ``--icsv`` and ``--ojson`` (note the extra ``i`` and ``o``) then Miller will use CSV for input and JSON for output, etc.  See also [Keystroke Savers](keystroke-savers.md) for even shorter options like ``--c2j``.
+* If you use `--csv` or `--json` or `--pprint`, etc., then Miller will use that format for input and output.
+* If you use `--icsv` and `--ojson` (note the extra `i` and `o`) then Miller will use CSV for input and JSON for output, etc.  See also [Keystroke Savers](keystroke-savers.md) for even shorter options like `--c2j`.
 
 You can read more about this at the [File Formats](file-formats.md) page.
 
-.. _10min-choices-for-printing-to-files:
-
 ## Choices for printing to files
 
 Often we want to print output to the screen. Miller does this by default, as we've seen in the previous examples.
 
 Sometimes, though, we want to print output to another file. Just use **> outputfilenamegoeshere** at the end of your command:
 
-.. code-block:: none
-   :emphasize-lines: 1,1
-
-    mlr --icsv --opprint cat example.csv > newfile.csv
-    # Output goes to the new file;
-    # nothing is printed to the screen.
-
-.. code-block:: none
-   :emphasize-lines: 1,1
-
-    cat newfile.csv
-    color  shape    flag     index quantity rate
-    yellow triangle true     11    43.6498  9.8870
-    red    square   true     15    79.2778  0.0130
-    red    circle   true     16    13.8103  2.9010
-    red    square   false    48    77.5542  7.4670
-    purple triangle false    51    81.2290  8.5910
-    red    square   false    64    77.1991  9.5310
-    purple triangle false    65    80.1405  5.8240
-    yellow circle   true     73    63.9785  4.2370
-    yellow circle   true     87    63.5058  8.3350
-    purple square   false    91    72.3735  8.2430
-
-Other times we just want our files to be **changed in-place**: just use **mlr -I**:
-
-.. code-block:: none
-   :emphasize-lines: 1,1
-
-    cp example.csv newfile.txt
-
-.. code-block:: none
-   :emphasize-lines: 1,1
-
-    cat newfile.txt
-    color,shape,flag,index,quantity,rate
-    yellow,triangle,true,11,43.6498,9.8870
-    red,square,true,15,79.2778,0.0130
-    red,circle,true,16,13.8103,2.9010
-    red,square,false,48,77.5542,7.4670
-    purple,triangle,false,51,81.2290,8.5910
-    red,square,false,64,77.1991,9.5310
-    purple,triangle,false,65,80.1405,5.8240
-    yellow,circle,true,73,63.9785,4.2370
-    yellow,circle,true,87,63.5058,8.3350
-    purple,square,false,91,72.3735,8.2430
-
-.. code-block:: none
-   :emphasize-lines: 1,1
-
-    mlr -I --csv sort -f shape newfile.txt
-
-.. code-block:: none
-   :emphasize-lines: 1,1
-
-    cat newfile.txt
-    color,shape,flag,index,quantity,rate
-    red,circle,true,16,13.8103,2.9010
-    yellow,circle,true,73,63.9785,4.2370
-    yellow,circle,true,87,63.5058,8.3350
-    red,square,true,15,79.2778,0.0130
-    red,square,false,48,77.5542,7.4670
-    red,square,false,64,77.1991,9.5310
-    purple,square,false,91,72.3735,8.2430
-    yellow,triangle,true,11,43.6498,9.8870
-    purple,triangle,false,51,81.2290,8.5910
-    purple,triangle,false,65,80.1405,5.8240
-
-Also using ``mlr -I`` you can bulk-operate on lots of files: e.g.:
-
-.. code-block:: none
-   :emphasize-lines: 1,1
-
-    mlr -I --csv cut -x -f unwanted_column_name *.csv
-
-If you like, you can first copy off your original data somewhere else, before doing in-place operations.
-
-Lastly, using ``tee`` within ``put``, you can split your input data into separate files per one or more field names:
-
 
-mlr --csv --from example.csv put -q 'tee > $shape.".csv", $*'
+mlr --icsv --opprint cat example.csv > newfile.csv
+# Output goes to the new file;
+# nothing is printed to the screen.
 
+cat newfile.csv
+color  shape    flag     index quantity rate
+yellow triangle true     11    43.6498  9.8870
+red    square   true     15    79.2778  0.0130
+red    circle   true     16    13.8103  2.9010
+red    square   false    48    77.5542  7.4670
+purple triangle false    51    81.2290  8.5910
+red    square   false    64    77.1991  9.5310
+purple triangle false    65    80.1405  5.8240
+yellow circle   true     73    63.9785  4.2370
+yellow circle   true     87    63.5058  8.3350
+purple square   false    91    72.3735  8.2430
+
+ +Other times we just want our files to be **changed in-place**: just use **mlr -I**: + +
+cp example.csv newfile.txt
+
+ +
+cat newfile.txt
+color,shape,flag,index,quantity,rate
+yellow,triangle,true,11,43.6498,9.8870
+red,square,true,15,79.2778,0.0130
+red,circle,true,16,13.8103,2.9010
+red,square,false,48,77.5542,7.4670
+purple,triangle,false,51,81.2290,8.5910
+red,square,false,64,77.1991,9.5310
+purple,triangle,false,65,80.1405,5.8240
+yellow,circle,true,73,63.9785,4.2370
+yellow,circle,true,87,63.5058,8.3350
+purple,square,false,91,72.3735,8.2430
+
+ +
+mlr -I --csv sort -f shape newfile.txt
+
+ +
+cat newfile.txt
+color,shape,flag,index,quantity,rate
+red,circle,true,16,13.8103,2.9010
+yellow,circle,true,73,63.9785,4.2370
+yellow,circle,true,87,63.5058,8.3350
+red,square,true,15,79.2778,0.0130
+red,square,false,48,77.5542,7.4670
+red,square,false,64,77.1991,9.5310
+purple,square,false,91,72.3735,8.2430
+yellow,triangle,true,11,43.6498,9.8870
+purple,triangle,false,51,81.2290,8.5910
+purple,triangle,false,65,80.1405,5.8240
+
+ +Also using `mlr -I` you can bulk-operate on lots of files: e.g.: + +
+mlr -I --csv cut -x -f unwanted_column_name *.csv
+
+ +If you like, you can first copy off your original data somewhere else, before doing in-place operations. + +Lastly, using `tee` within `put`, you can split your input data into separate files per one or more field names: + +
+mlr --csv --from example.csv put -q 'tee > $shape.".csv", $*'
+
+ +
 cat circle.csv
+
+
 color,shape,flag,index,quantity,rate
 red,circle,true,16,13.8103,2.9010
 yellow,circle,true,73,63.9785,4.2370
 yellow,circle,true,87,63.5058,8.3350
 
-
+
 cat square.csv
+
+
 color,shape,flag,index,quantity,rate
 red,square,true,15,79.2778,0.0130
 red,square,false,48,77.5542,7.4670
@@ -599,8 +657,10 @@ red,square,false,64,77.1991,9.5310
 purple,square,false,91,72.3735,8.2430
 
-
+
 cat triangle.csv
+
+
 color,shape,flag,index,quantity,rate
 yellow,triangle,true,11,43.6498,9.8870
 purple,triangle,false,51,81.2290,8.5910
diff --git a/docs6b/docs/10min.md.in b/docs6b/docs/10min.md.in
index d44e3b18f..be1e5f5c0 100644
--- a/docs6b/docs/10min.md.in
+++ b/docs6b/docs/10min.md.in
@@ -4,13 +4,14 @@
 
 You can install Miller for various platforms as follows:
 
-* Linux: ``yum install miller`` or ``apt-get install miller`` depending on your flavor of Linux
-* MacOS: ``brew install miller`` or ``port install miller`` depending on your preference of [Homebrew](https://brew.sh>`_ or `MacPorts  outputfilenamegoeshere** at the end of your command:
 
-.. code-block:: none
-   :emphasize-lines: 1,1
+
+mlr --icsv --opprint cat example.csv > newfile.csv
+# Output goes to the new file;
+# nothing is printed to the screen.
+
- mlr --icsv --opprint cat example.csv > newfile.csv - # Output goes to the new file; - # nothing is printed to the screen. - -.. code-block:: none - :emphasize-lines: 1,1 - - cat newfile.csv - color shape flag index quantity rate - yellow triangle true 11 43.6498 9.8870 - red square true 15 79.2778 0.0130 - red circle true 16 13.8103 2.9010 - red square false 48 77.5542 7.4670 - purple triangle false 51 81.2290 8.5910 - red square false 64 77.1991 9.5310 - purple triangle false 65 80.1405 5.8240 - yellow circle true 73 63.9785 4.2370 - yellow circle true 87 63.5058 8.3350 - purple square false 91 72.3735 8.2430 +
+cat newfile.csv
+color  shape    flag     index quantity rate
+yellow triangle true     11    43.6498  9.8870
+red    square   true     15    79.2778  0.0130
+red    circle   true     16    13.8103  2.9010
+red    square   false    48    77.5542  7.4670
+purple triangle false    51    81.2290  8.5910
+red    square   false    64    77.1991  9.5310
+purple triangle false    65    80.1405  5.8240
+yellow circle   true     73    63.9785  4.2370
+yellow circle   true     87    63.5058  8.3350
+purple square   false    91    72.3735  8.2430
+
Other times we just want our files to be **changed in-place**: just use **mlr -I**: -.. code-block:: none - :emphasize-lines: 1,1 +
+cp example.csv newfile.txt
+
- cp example.csv newfile.txt +
+cat newfile.txt
+color,shape,flag,index,quantity,rate
+yellow,triangle,true,11,43.6498,9.8870
+red,square,true,15,79.2778,0.0130
+red,circle,true,16,13.8103,2.9010
+red,square,false,48,77.5542,7.4670
+purple,triangle,false,51,81.2290,8.5910
+red,square,false,64,77.1991,9.5310
+purple,triangle,false,65,80.1405,5.8240
+yellow,circle,true,73,63.9785,4.2370
+yellow,circle,true,87,63.5058,8.3350
+purple,square,false,91,72.3735,8.2430
+
-.. code-block:: none - :emphasize-lines: 1,1 +
+mlr -I --csv sort -f shape newfile.txt
+
- cat newfile.txt - color,shape,flag,index,quantity,rate - yellow,triangle,true,11,43.6498,9.8870 - red,square,true,15,79.2778,0.0130 - red,circle,true,16,13.8103,2.9010 - red,square,false,48,77.5542,7.4670 - purple,triangle,false,51,81.2290,8.5910 - red,square,false,64,77.1991,9.5310 - purple,triangle,false,65,80.1405,5.8240 - yellow,circle,true,73,63.9785,4.2370 - yellow,circle,true,87,63.5058,8.3350 - purple,square,false,91,72.3735,8.2430 +
+cat newfile.txt
+color,shape,flag,index,quantity,rate
+red,circle,true,16,13.8103,2.9010
+yellow,circle,true,73,63.9785,4.2370
+yellow,circle,true,87,63.5058,8.3350
+red,square,true,15,79.2778,0.0130
+red,square,false,48,77.5542,7.4670
+red,square,false,64,77.1991,9.5310
+purple,square,false,91,72.3735,8.2430
+yellow,triangle,true,11,43.6498,9.8870
+purple,triangle,false,51,81.2290,8.5910
+purple,triangle,false,65,80.1405,5.8240
+
-.. code-block:: none - :emphasize-lines: 1,1 +Also using `mlr -I` you can bulk-operate on lots of files: e.g.: - mlr -I --csv sort -f shape newfile.txt - -.. code-block:: none - :emphasize-lines: 1,1 - - cat newfile.txt - color,shape,flag,index,quantity,rate - red,circle,true,16,13.8103,2.9010 - yellow,circle,true,73,63.9785,4.2370 - yellow,circle,true,87,63.5058,8.3350 - red,square,true,15,79.2778,0.0130 - red,square,false,48,77.5542,7.4670 - red,square,false,64,77.1991,9.5310 - purple,square,false,91,72.3735,8.2430 - yellow,triangle,true,11,43.6498,9.8870 - purple,triangle,false,51,81.2290,8.5910 - purple,triangle,false,65,80.1405,5.8240 - -Also using ``mlr -I`` you can bulk-operate on lots of files: e.g.: - -.. code-block:: none - :emphasize-lines: 1,1 - - mlr -I --csv cut -x -f unwanted_column_name *.csv +
+mlr -I --csv cut -x -f unwanted_column_name *.csv
+
If you like, you can first copy off your original data somewhere else, before doing in-place operations. -Lastly, using ``tee`` within ``put``, you can split your input data into separate files per one or more field names: +Lastly, using `tee` within `put`, you can split your input data into separate files per one or more field names: GENMD_RUN_COMMAND mlr --csv --from example.csv put -q 'tee > $shape.".csv", $*' diff --git a/docs6b/docs/README.md b/docs6b/docs/README.md deleted file mode 100644 index d434ab8f5..000000000 --- a/docs6b/docs/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Miller Sphinx docs - -## Why use Sphinx - -* Connects to https://miller.readthedocs.io so people can get their docmods onto the web instead of the self-hosted https://johnkerl.org/miller/doc. Thanks to @pabloab for the great advice! -* More standard look and feel -- lots of people use readthedocs for other things so this should feel familiar -* We get a Search feature for free - -## Contributing - -* You need `pip install sphinx` (or `pip3 install sphinx`) -* The docs include lots of live code examples which will be invoked using `mlr` which must be somewhere in your `$PATH` -* Clone https://github.com/johnkerl/miller and cd into `docs/` within your clone -* Editing loop: - * Edit `*.md.in` - * Run `make html` - * Either `open _build/html/index.html` (MacOS) or point your browser to `file:///path/to/your/clone/of/miller/docs/_build/html/index.html` -* Submitting: - * `git add` your modified files, `git commit`, `git push`, and submit a PR at https://github.com/johnkerl/miller -* A nice markup reference: https://www.sphinx-doc.org/en/1.8/usage/restructuredtext/basics.html - -## Notes - -* CSS: - * I used the Sphinx Classic theme which I like a lot except the colors -- it's a blue scheme and Miller has never been blue. - * Files are in `docs/_static/*.css` where I marked my mods with `/* CHANGE ME */`. - * If you modify the CSS you must run `make clean html` (not just `make html`) then reload in your browser. -* Live code: - * I didn't find a way to include non-Python live-code examples within Sphinx so I adapted the pre-Sphinx Miller-doc strategy which is to have a generator script read a template file (here, `foo.md.in`), run the marked lines, and generate the output file (`foo.md`). - * Edit the `*.md.in` files, not `*.md` directly. - * Within the `*.md.in` files are lines like `GENMD_RUN_COMMAND`. These will be run, and their output included, by `make html` which calls the `genmds` script for you. -* readthedocs: - * https://readthedocs.org/ - * https://readthedocs.org/projects/miller/ - * https://readthedocs.org/projects/miller/builds/ - * https://miller.readthedocs.io/en/latest/ - -## To do - -* Let's all discuss if/how we want the v2 docs to be structured better than the v1 docs. diff --git a/docs6b/docs/build.md b/docs6b/docs/build.md index 898748b0f..a76ea9102 100644 --- a/docs6b/docs/build.md +++ b/docs6b/docs/build.md @@ -5,40 +5,40 @@ Please also see [Installation](installation.md) for information about pre-built ## Miller license -Two-clause BSD license https://github.com/johnkerl/miller/blob/master/LICENSE.txt. +Two-clause BSD license [https://github.com/johnkerl/miller/blob/master/LICENSE.txt](https://github.com/johnkerl/miller/blob/master/LICENSE.txt). ## From release tarball -* Obtain ``mlr-i.j.k.tar.gz`` from https://github.com/johnkerl/miller/tags, replacing ``i.j.k`` with the desired release, e.g. ``6.1.0``. -* ``tar zxvf mlr-i.j.k.tar.gz`` -* ``cd mlr-i.j.k`` -* ``cd go`` -* ``./build`` creates the ``go/mlr`` executable and runs regression tests -* ``go build mlr.go`` creates the ``go/mlr`` executable without running regression tests +* Obtain `mlr-i.j.k.tar.gz` from [https://github.com/johnkerl/miller/tags](https://github.com/johnkerl/miller/tags), replacing `i.j.k` with the desired release, e.g. `6.1.0`. +* `tar zxvf mlr-i.j.k.tar.gz` +* `cd mlr-i.j.k` +* `cd go` +* `./build` creates the `go/mlr` executable and runs regression tests +* `go build mlr.go` creates the `go/mlr` executable without running regression tests ## From git clone -* ``git clone https://github.com/johnkerl/miller`` -* ``cd miller/go`` -* ``./build`` creates the ``go/mlr`` executable and runs regression tests -* ``go build mlr.go`` creates the ``go/mlr`` executable without running regression tests +* `git clone https://github.com/johnkerl/miller` +* `cd miller/go` +* `./build` creates the `go/mlr` executable and runs regression tests +* `go build mlr.go` creates the `go/mlr` executable without running regression tests ## In case of problems -If you have any build errors, feel free to open an issue with "New Issue" at https://github.com/johnkerl/miller/issues. +If you have any build errors, feel free to open an issue with "New Issue" at [https://github.com/johnkerl/miller/issues](https://github.com/johnkerl/miller/issues). ## Dependencies ### Required external dependencies -These are necessary to produce the ``mlr`` executable. +These are necessary to produce the `mlr` executable. * Go version 1.16 or higher -* Others packaged within ``go.mod`` and ``go.sum`` which you don't need to deal with manually -- the Go build process handles them for us +* Others packaged within `go.mod` and `go.sum` which you don't need to deal with manually -- the Go build process handles them for us ### Optional external dependencies -This documentation pageset is built using Sphinx. Please see https://github.com/johnkerl/miller/blob/main/docs6/README.md for details. +This documentation pageset is built using Sphinx. Please see [https://github.com/johnkerl/miller/blob/main/docs6/README.md](https://github.com/johnkerl/miller/blob/main/docs6/README.md) for details. ## Creating a new release: for developers @@ -46,40 +46,40 @@ At present I'm the primary developer so this is just my checklist for making new In this example I am using version 6.1.0 to 6.2.0; of course that will change for subsequent revisions. -* Update version found in ``mlr --version`` and ``man mlr``: +* Update version found in `mlr --version` and `man mlr`: - * Edit ``go/src/version/version.go`` from ``6.1.0-dev`` to ``6.2.0``. - * Likewise ``docs6/conf.py`` - * ``cd ../docs6`` - * ``export PATH=../go:$PATH`` - * ``make html`` - * The ordering is important: the first build creates ``mlr``; the second runs ``mlr`` to create ``manpage.txt``; the third includes ``manpage.txt`` into one of its outputs. + * Edit `go/src/version/version.go` from `6.1.0-dev` to `6.2.0`. + * Likewise `docs6/conf.py` + * `cd ../docs6` + * `export PATH=../go:$PATH` + * `make html` + * The ordering is important: the first build creates `mlr`; the second runs `mlr` to create `manpage.txt`; the third includes `manpage.txt` into one of its outputs. * Commit and push. * Create the release tarball and SRPM: * TBD for the Go port ... * Linux/MacOS/Windows binaries from GitHub Actions ... - * Pull back release tarball ``mlr-6.2.0.tar.gz`` from buildbox, and ``mlr.{arch}`` binaries from whatever buildboxes. + * Pull back release tarball `mlr-6.2.0.tar.gz` from buildbox, and `mlr.{arch}` binaries from whatever buildboxes. * Create the Github release tag: - * Don't forget the ``v`` in ``v6.2.0`` + * Don't forget the `v` in `v6.2.0` * Write the release notes * Attach the release tarball and binaries. Double-check assets were successfully uploaded. * Publish the release * Check the release-specific docs: - * Look at https://miller.readthedocs.io for new-version docs, after a few minutes' propagation time. + * Look at [https://miller.readthedocs.io](https://miller.readthedocs.io) for new-version docs, after a few minutes' propagation time. * Notify: - * Submit ``brew`` pull request; notify any other distros which don't appear to have autoupdated since the previous release (notes below) - * Similarly for ``macports``: https://github.com/macports/macports-ports/blob/master/textproc/miller/Portfile. + * Submit `brew` pull request; notify any other distros which don't appear to have autoupdated since the previous release (notes below) + * Similarly for `macports`: [https://github.com/macports/macports-ports/blob/master/textproc/miller/Portfile](https://github.com/macports/macports-ports/blob/master/textproc/miller/Portfile) * Social-media updates. -
+
 git remote add upstream https://github.com/Homebrew/homebrew-core # one-time setup only
 git fetch upstream
 git rebase upstream/master
@@ -88,8 +88,9 @@ shasum -a 256 /path/to/mlr-6.1.0.tar.gz
 edit Formula/miller.rb
 # Test the URL from the line like
 #   url "https://github.com/johnkerl/miller/releases/download/v6.1.0/mlr-6.1.0.tar.gz"
-# in a browser for typos
-# A '@BrewTestBot Test this please' comment within the homebrew-core pull request will restart the homebrew travis build
+# in a browser for typos.
+# A '@BrewTestBot Test this please' comment within the homebrew-core pull request
+# will restart the homebrew travis build.
 git add Formula/miller.rb
 git commit -m 'miller 6.1.0'
 git push -u origin miller-6.1.0
@@ -98,9 +99,9 @@ git push -u origin miller-6.1.0
 
 * Afterwork:
 
-    * Edit ``go/src/version/version.go`` and ``docs6/conf.py`` to change version from ``6.2.0`` to ``6.2.0-dev``.
-    * ``cd go``
-    * ``./build``
+    * Edit `go/src/version/version.go` and `docs6/conf.py` to change version from `6.2.0` to `6.2.0-dev`.
+    * `cd go`
+    * `./build`
     * Commit and push.
 
 ## Misc. development notes
diff --git a/docs6b/docs/build.md.in b/docs6b/docs/build.md.in
index e18b627ff..49ce8c5ba 100644
--- a/docs6b/docs/build.md.in
+++ b/docs6b/docs/build.md.in
@@ -4,40 +4,40 @@ Please also see [Installation](installation.md) for information about pre-built
 
 ## Miller license
 
-Two-clause BSD license https://github.com/johnkerl/miller/blob/master/LICENSE.txt.
+Two-clause BSD license [https://github.com/johnkerl/miller/blob/master/LICENSE.txt](https://github.com/johnkerl/miller/blob/master/LICENSE.txt).
 
 ## From release tarball
 
-* Obtain ``mlr-i.j.k.tar.gz`` from https://github.com/johnkerl/miller/tags, replacing ``i.j.k`` with the desired release, e.g. ``6.1.0``.
-* ``tar zxvf mlr-i.j.k.tar.gz``
-* ``cd mlr-i.j.k``
-* ``cd go``
-* ``./build`` creates the ``go/mlr`` executable and runs regression tests
-* ``go build mlr.go`` creates the ``go/mlr`` executable without running regression tests
+* Obtain `mlr-i.j.k.tar.gz` from [https://github.com/johnkerl/miller/tags](https://github.com/johnkerl/miller/tags), replacing `i.j.k` with the desired release, e.g. `6.1.0`.
+* `tar zxvf mlr-i.j.k.tar.gz`
+* `cd mlr-i.j.k`
+* `cd go`
+* `./build` creates the `go/mlr` executable and runs regression tests
+* `go build mlr.go` creates the `go/mlr` executable without running regression tests
 
 ## From git clone
 
-* ``git clone https://github.com/johnkerl/miller``
-* ``cd miller/go``
-* ``./build`` creates the ``go/mlr`` executable and runs regression tests
-* ``go build mlr.go`` creates the ``go/mlr`` executable without running regression tests
+* `git clone https://github.com/johnkerl/miller`
+* `cd miller/go`
+* `./build` creates the `go/mlr` executable and runs regression tests
+* `go build mlr.go` creates the `go/mlr` executable without running regression tests
 
 ## In case of problems
 
-If you have any build errors, feel free to open an issue with "New Issue" at https://github.com/johnkerl/miller/issues.
+If you have any build errors, feel free to open an issue with "New Issue" at [https://github.com/johnkerl/miller/issues](https://github.com/johnkerl/miller/issues).
 
 ## Dependencies
 
 ### Required external dependencies
 
-These are necessary to produce the ``mlr`` executable.
+These are necessary to produce the `mlr` executable.
 
 * Go version 1.16 or higher
-* Others packaged within ``go.mod`` and ``go.sum`` which you don't need to deal with manually -- the Go build process handles them for us
+* Others packaged within `go.mod` and `go.sum` which you don't need to deal with manually -- the Go build process handles them for us
 
 ### Optional external dependencies
 
-This documentation pageset is built using Sphinx. Please see https://github.com/johnkerl/miller/blob/main/docs6/README.md for details.
+This documentation pageset is built using Sphinx. Please see [https://github.com/johnkerl/miller/blob/main/docs6/README.md](https://github.com/johnkerl/miller/blob/main/docs6/README.md) for details.
 
 ## Creating a new release: for developers
 
@@ -45,37 +45,37 @@ At present I'm the primary developer so this is just my checklist for making new
 
 In this example I am using version 6.1.0 to 6.2.0; of course that will change for subsequent revisions.
 
-* Update version found in ``mlr --version`` and ``man mlr``:
+* Update version found in `mlr --version` and `man mlr`:
 
-    * Edit ``go/src/version/version.go`` from ``6.1.0-dev`` to ``6.2.0``.
-    * Likewise ``docs6/conf.py``
-    * ``cd ../docs6``
-    * ``export PATH=../go:$PATH``
-    * ``make html``
-    * The ordering is important: the first build creates ``mlr``; the second runs ``mlr`` to create ``manpage.txt``; the third includes ``manpage.txt`` into one of its outputs.
+    * Edit `go/src/version/version.go` from `6.1.0-dev` to `6.2.0`.
+    * Likewise `docs6/conf.py`
+    * `cd ../docs6`
+    * `export PATH=../go:$PATH`
+    * `make html`
+    * The ordering is important: the first build creates `mlr`; the second runs `mlr` to create `manpage.txt`; the third includes `manpage.txt` into one of its outputs.
     * Commit and push.
 
 * Create the release tarball and SRPM:
 
     * TBD for the Go port ...
     * Linux/MacOS/Windows binaries from GitHub Actions ...
-    * Pull back release tarball ``mlr-6.2.0.tar.gz`` from buildbox, and ``mlr.{arch}`` binaries from whatever buildboxes.
+    * Pull back release tarball `mlr-6.2.0.tar.gz` from buildbox, and `mlr.{arch}` binaries from whatever buildboxes.
 
 * Create the Github release tag:
 
-    * Don't forget the ``v`` in ``v6.2.0``
+    * Don't forget the `v` in `v6.2.0`
     * Write the release notes
     * Attach the release tarball and binaries. Double-check assets were successfully uploaded.
     * Publish the release
 
 * Check the release-specific docs:
 
-    * Look at https://miller.readthedocs.io for new-version docs, after a few minutes' propagation time.
+    * Look at [https://miller.readthedocs.io](https://miller.readthedocs.io) for new-version docs, after a few minutes' propagation time.
 
 * Notify:
 
-    * Submit ``brew`` pull request; notify any other distros which don't appear to have autoupdated since the previous release (notes below)
-    * Similarly for ``macports``: https://github.com/macports/macports-ports/blob/master/textproc/miller/Portfile.
+    * Submit `brew` pull request; notify any other distros which don't appear to have autoupdated since the previous release (notes below)
+    * Similarly for `macports`: [https://github.com/macports/macports-ports/blob/master/textproc/miller/Portfile](https://github.com/macports/macports-ports/blob/master/textproc/miller/Portfile)
     * Social-media updates.
 
 GENMD_CARDIFY
@@ -87,8 +87,9 @@ shasum -a 256 /path/to/mlr-6.1.0.tar.gz
 edit Formula/miller.rb
 # Test the URL from the line like
 #   url "https://github.com/johnkerl/miller/releases/download/v6.1.0/mlr-6.1.0.tar.gz"
-# in a browser for typos
-# A '@BrewTestBot Test this please' comment within the homebrew-core pull request will restart the homebrew travis build
+# in a browser for typos.
+# A '@BrewTestBot Test this please' comment within the homebrew-core pull request
+# will restart the homebrew travis build.
 git add Formula/miller.rb
 git commit -m 'miller 6.1.0'
 git push -u origin miller-6.1.0
@@ -97,9 +98,9 @@ GENMD_EOF
 
 * Afterwork:
 
-    * Edit ``go/src/version/version.go`` and ``docs6/conf.py`` to change version from ``6.2.0`` to ``6.2.0-dev``.
-    * ``cd go``
-    * ``./build``
+    * Edit `go/src/version/version.go` and `docs6/conf.py` to change version from `6.2.0` to `6.2.0-dev`.
+    * `cd go`
+    * `./build`
     * Commit and push.
 
 ## Misc. development notes
diff --git a/docs6b/docs/community.md b/docs6b/docs/community.md
index 1a11eb405..018d93f0e 100644
--- a/docs6b/docs/community.md
+++ b/docs6b/docs/community.md
@@ -2,6 +2,6 @@
 # Community
 
 * See [Miller GitHub Discussions](https://github.com/johnkerl/miller/discussions) for general Q&A, advice, sharing success stories, etc.
-* See also [Miller-tagged questions on Stack Overflow](https://stackoverflow.com/questions/tagged/miller?tab=Newest)
-* See [Miller GitHub Issues](https://github.com/johnkerl/miller/issues) for bug reports and feature requests
-* Other correspondence: [mailto:kerl.john.r+miller@gmail.com](mailto:kerl.john.r+miller@gmail.com)
+* See also [Miller-tagged questions on Stack Overflow](https://stackoverflow.com/questions/tagged/miller?tab=Newest).
+* See [Miller GitHub Issues](https://github.com/johnkerl/miller/issues) for bug reports and feature requests.
+* Other correspondence: [mailto:kerl.john.r+miller@gmail.com](mailto:kerl.john.r+miller@gmail.com).
diff --git a/docs6b/docs/community.md.in b/docs6b/docs/community.md.in
index f9bd0b94f..3198b665e 100644
--- a/docs6b/docs/community.md.in
+++ b/docs6b/docs/community.md.in
@@ -1,6 +1,6 @@
 # Community
 
 * See [Miller GitHub Discussions](https://github.com/johnkerl/miller/discussions) for general Q&A, advice, sharing success stories, etc.
-* See also [Miller-tagged questions on Stack Overflow](https://stackoverflow.com/questions/tagged/miller?tab=Newest)
-* See [Miller GitHub Issues](https://github.com/johnkerl/miller/issues) for bug reports and feature requests
-* Other correspondence: [mailto:kerl.john.r+miller@gmail.com](mailto:kerl.john.r+miller@gmail.com)
+* See also [Miller-tagged questions on Stack Overflow](https://stackoverflow.com/questions/tagged/miller?tab=Newest).
+* See [Miller GitHub Issues](https://github.com/johnkerl/miller/issues) for bug reports and feature requests.
+* Other correspondence: [mailto:kerl.john.r+miller@gmail.com](mailto:kerl.john.r+miller@gmail.com).
diff --git a/docs6b/docs/contributing.md b/docs6b/docs/contributing.md
index 6bb80cf30..21bb6b203 100644
--- a/docs6b/docs/contributing.md
+++ b/docs6b/docs/contributing.md
@@ -11,15 +11,15 @@ Pre-release Miller documentation is at [https://github.com/johnkerl/miller/tree/
 
 Clone [https://github.com/johnkerl/miller](https://github.com/johnkerl/miller) and `cd` into `docs6`.
 
-After ``sudo pip install sphinx`` (or ``pip3``) you should be able to do ``make html``.
+After `sudo pip install sphinx` (or `pip3`) you should be able to do `make html`.
 
-Edit ``*.md.in`` files, then ``make html`` to generate ``*.md``, then run the Sphinx document-generator.
+Edit `*.md.in` files, then `make html` to generate `*.md`, then run the Sphinx document-generator.
 
-Open ``_build/html/index.html`` in your browser, e.g. ``file:////Users/yourname/git/miller/docs6/_build/html/contributing.html``, to verify.
+Open `_build/html/index.html` in your browser, e.g. `file:////Users/yourname/git/miller/docs6/_build/html/contributing.html`, to verify.
 
 PRs are welcome at [https://github.com/johnkerl/miller](https://github.com/johnkerl/miller).
 
-Once PRs are merged, readthedocs creates https://miller.readthedocs.io using the following configs:
+Once PRs are merged, readthedocs creates [https://miller.readthedocs.io](https://miller.readthedocs.io) using the following configs:
 
 * [https://readthedocs.org/projects/miller](https://readthedocs.org/projects/miller)
 * [https://readthedocs.org/projects/miller/builds](https://readthedocs.org/projects/miller/builds)
diff --git a/docs6b/docs/contributing.md.in b/docs6b/docs/contributing.md.in
index 781163bb4..4fdc97491 100644
--- a/docs6b/docs/contributing.md.in
+++ b/docs6b/docs/contributing.md.in
@@ -10,15 +10,15 @@ Pre-release Miller documentation is at [https://github.com/johnkerl/miller/tree/
 
 Clone [https://github.com/johnkerl/miller](https://github.com/johnkerl/miller) and `cd` into `docs6`.
 
-After ``sudo pip install sphinx`` (or ``pip3``) you should be able to do ``make html``.
+After `sudo pip install sphinx` (or `pip3`) you should be able to do `make html`.
 
-Edit ``*.md.in`` files, then ``make html`` to generate ``*.md``, then run the Sphinx document-generator.
+Edit `*.md.in` files, then `make html` to generate `*.md`, then run the Sphinx document-generator.
 
-Open ``_build/html/index.html`` in your browser, e.g. ``file:////Users/yourname/git/miller/docs6/_build/html/contributing.html``, to verify.
+Open `_build/html/index.html` in your browser, e.g. `file:////Users/yourname/git/miller/docs6/_build/html/contributing.html`, to verify.
 
 PRs are welcome at [https://github.com/johnkerl/miller](https://github.com/johnkerl/miller).
 
-Once PRs are merged, readthedocs creates https://miller.readthedocs.io using the following configs:
+Once PRs are merged, readthedocs creates [https://miller.readthedocs.io](https://miller.readthedocs.io) using the following configs:
 
 * [https://readthedocs.org/projects/miller](https://readthedocs.org/projects/miller)
 * [https://readthedocs.org/projects/miller/builds](https://readthedocs.org/projects/miller/builds)
diff --git a/docs6b/docs/csv-with-and-without-headers.md b/docs6b/docs/csv-with-and-without-headers.md
index 49d0b29c0..0e69c2b96 100644
--- a/docs6b/docs/csv-with-and-without-headers.md
+++ b/docs6b/docs/csv-with-and-without-headers.md
@@ -3,20 +3,24 @@
 
 ## Headerless CSV on input or output
 
-Sometimes we get CSV files which lack a header. For example (`data/headerless.csv <./data/headerless.csv>`_):
+Sometimes we get CSV files which lack a header. For example, [data/headerless.csv](./data/headerless.csv):
 
-
+
 cat data/headerless.csv
+
+
 John,23,present
 Fred,34,present
 Alice,56,missing
 Carol,45,present
 
-You can use Miller to add a header. The ``--implicit-csv-header`` applies positionally indexed labels: +You can use Miller to add a header. The `--implicit-csv-header` applies positionally indexed labels: -
+
 mlr --csv --implicit-csv-header cat data/headerless.csv
+
+
 1,2,3
 John,23,present
 Fred,34,present
@@ -26,8 +30,10 @@ Carol,45,present
 
 Following that, you can rename the positionally indexed labels to names with meaning for your context.  For example:
 
-
+
 mlr --csv --implicit-csv-header label name,age,status data/headerless.csv
+
+
 name,age,status
 John,23,present
 Fred,34,present
@@ -35,10 +41,12 @@ Alice,56,missing
 Carol,45,present
 
-Likewise, if you need to produce CSV which is lacking its header, you can pipe Miller's output to the system command ``sed 1d``, or you can use Miller's ``--headerless-csv-output`` option: +Likewise, if you need to produce CSV which is lacking its header, you can pipe Miller's output to the system command `sed 1d`, or you can use Miller's `--headerless-csv-output` option: -
+
 head -5 data/colored-shapes.dkvp | mlr --ocsv cat
+
+
 color,shape,flag,i,u,v,w,x
 yellow,triangle,1,11,0.6321695890307647,0.9887207810889004,0.4364983936735774,5.7981881667050565
 red,square,1,15,0.21966833570651523,0.001257332190235938,0.7927778364718627,2.944117399716207
@@ -47,8 +55,10 @@ red,square,0,48,0.9562743938458542,0.7467203085342884,0.7755423050923582,7.11783
 purple,triangle,0,51,0.4355354501763202,0.8591292672156728,0.8122903963006748,5.753094629505863
 
-
+
 head -5 data/colored-shapes.dkvp | mlr --ocsv --headerless-csv-output cat
+
+
 yellow,triangle,1,11,0.6321695890307647,0.9887207810889004,0.4364983936735774,5.7981881667050565
 red,square,1,15,0.21966833570651523,0.001257332190235938,0.7927778364718627,2.944117399716207
 red,circle,1,16,0.20901671281497636,0.29005231936593445,0.13810280912907674,5.065034003400998
@@ -58,8 +68,10 @@ purple,triangle,0,51,0.4355354501763202,0.8591292672156728,0.8122903963006748,5.
 
 Lastly, often we say "CSV" or "TSV" when we have positionally indexed data in columns which are separated by commas or tabs, respectively. In this case it's perhaps simpler to **just use NIDX format** which was designed for this purpose. (See also [File Formats](file-formats.md).) For example:
 
-
+
 mlr --inidx --ifs comma --oxtab cut -f 1,3 data/headerless.csv
+
+
 1 John
 3 present
 
@@ -75,18 +87,20 @@ Lastly, often we say "CSV" or "TSV" when we have positionally indexed data in co
 
 ## Headerless CSV with duplicate field values
 
-Miller is (by central design) a mapping from name to value, rather than integer position to value as in most tools in the Unix toolkit such as ``sort``, ``cut``, ``awk``, etc. So given input ``Yea=1,Yea=2`` on the same input line, first ``Yea=1`` is stored, then updated with ``Yea=2``. This is in the input-parser and the value ``Yea=1`` is unavailable to any further processing. The following example line comes from a headerless CSV file and includes 5 times the string (value) ``'NA'``:
+Miller is (by central design) a mapping from name to value, rather than integer position to value as in most tools in the Unix toolkit such as `sort`, `cut`, `awk`, etc. So given input `Yea=1,Yea=2` on the same input line, first `Yea=1` is stored, then updated with `Yea=2`. This is in the input-parser and the value `Yea=1` is unavailable to any further processing. The following example line comes from a headerless CSV file and includes 5 times the string (value) `'NA'`:
 
-
+
 ag '0.9' nas.csv | head -1
+
+
 2:-349801.10097848,4537221.43295653,2,1,NA,NA,NA,NA,NA
 
-The repeated ``'NA'`` strings (values) in the same line will be treated as fields (columns) with same name, thus only one is kept in the output. +The repeated `'NA'` strings (values) in the same line will be treated as fields (columns) with same name, thus only one is kept in the output. -This can be worked around by telling ``mlr`` that there is no header row by using ``--implicit-csv-header`` or changing the input format by using ``nidx`` like so: +This can be worked around by telling `mlr` that there is no header row by using `--implicit-csv-header` or changing the input format by using `nidx` like so: -
+
 ag '0.9' nas.csv | mlr --n2c --fs "," label xsn,ysn,x,y,t,a,e29,e31,e32 then head
 
@@ -94,15 +108,17 @@ ag '0.9' nas.csv | mlr --n2c --fs "," label xsn,ysn,x,y,t,a,e29,e31,e32 then hea Miller handles compliant CSV: in particular, it's an error if the number of data fields in a given data line don't match the number of header lines. But in the event that you have a CSV file in which some lines have less than the full number of fields, you can use Miller to pad them out. The trick is to use NIDX format, for which each line stands on its own without respect to a header line. -
+
 cat data/ragged.csv
+
+
 a,b,c
 1,2,3
 4,5
 6,7,8,9
 
-
+
 mlr --from data/ragged.csv --fs comma --nidx put '
   @maxnf = max(@maxnf, NF);
   @nf = NF;
@@ -111,6 +127,8 @@ a,b,c
     $[@nf] = ""
   }
 '
+
+
 a,b,c
 1,2,3
 4,5
@@ -119,13 +137,15 @@ a,b,c
 
 or, more simply,
 
-
+
 mlr --from data/ragged.csv --fs comma --nidx put '
   @maxnf = max(@maxnf, NF);
   while(NF < @maxnf) {
     $[NF+1] = "";
   }
 '
+
+
 a,b,c
 1,2,3
 4,5
diff --git a/docs6b/docs/csv-with-and-without-headers.md.in b/docs6b/docs/csv-with-and-without-headers.md.in
index 673a7fe18..b62fad564 100644
--- a/docs6b/docs/csv-with-and-without-headers.md.in
+++ b/docs6b/docs/csv-with-and-without-headers.md.in
@@ -2,13 +2,13 @@
 
 ## Headerless CSV on input or output
 
-Sometimes we get CSV files which lack a header. For example (`data/headerless.csv <./data/headerless.csv>`_):
+Sometimes we get CSV files which lack a header. For example, [data/headerless.csv](./data/headerless.csv):
 
 GENMD_RUN_COMMAND
 cat data/headerless.csv
 GENMD_EOF
 
-You can use Miller to add a header. The ``--implicit-csv-header`` applies positionally indexed labels:
+You can use Miller to add a header. The `--implicit-csv-header` applies positionally indexed labels:
 
 GENMD_RUN_COMMAND
 mlr --csv --implicit-csv-header cat data/headerless.csv
@@ -20,7 +20,7 @@ GENMD_RUN_COMMAND
 mlr --csv --implicit-csv-header label name,age,status data/headerless.csv
 GENMD_EOF
 
-Likewise, if you need to produce CSV which is lacking its header, you can pipe Miller's output to the system command ``sed 1d``, or you can use Miller's ``--headerless-csv-output`` option:
+Likewise, if you need to produce CSV which is lacking its header, you can pipe Miller's output to the system command `sed 1d`, or you can use Miller's `--headerless-csv-output` option:
 
 GENMD_RUN_COMMAND
 head -5 data/colored-shapes.dkvp | mlr --ocsv cat
@@ -38,16 +38,16 @@ GENMD_EOF
 
 ## Headerless CSV with duplicate field values
 
-Miller is (by central design) a mapping from name to value, rather than integer position to value as in most tools in the Unix toolkit such as ``sort``, ``cut``, ``awk``, etc. So given input ``Yea=1,Yea=2`` on the same input line, first ``Yea=1`` is stored, then updated with ``Yea=2``. This is in the input-parser and the value ``Yea=1`` is unavailable to any further processing. The following example line comes from a headerless CSV file and includes 5 times the string (value) ``'NA'``:
+Miller is (by central design) a mapping from name to value, rather than integer position to value as in most tools in the Unix toolkit such as `sort`, `cut`, `awk`, etc. So given input `Yea=1,Yea=2` on the same input line, first `Yea=1` is stored, then updated with `Yea=2`. This is in the input-parser and the value `Yea=1` is unavailable to any further processing. The following example line comes from a headerless CSV file and includes 5 times the string (value) `'NA'`:
 
 GENMD_CARDIFY_HIGHLIGHT_ONE
 ag '0.9' nas.csv | head -1
 2:-349801.10097848,4537221.43295653,2,1,NA,NA,NA,NA,NA
 GENMD_EOF
 
-The repeated ``'NA'`` strings (values) in the same line will be treated as fields (columns) with same name, thus only one is kept in the output.
+The repeated `'NA'` strings (values) in the same line will be treated as fields (columns) with same name, thus only one is kept in the output.
 
-This can be worked around by telling ``mlr`` that there is no header row by using ``--implicit-csv-header`` or changing the input format by using ``nidx`` like so:
+This can be worked around by telling `mlr` that there is no header row by using `--implicit-csv-header` or changing the input format by using `nidx` like so:
 
 GENMD_CARDIFY
 ag '0.9' nas.csv | mlr --n2c --fs "," label xsn,ysn,x,y,t,a,e29,e31,e32 then head
diff --git a/docs6b/docs/customization.md b/docs6b/docs/customization.md
index 4d559cf44..dc49c173e 100644
--- a/docs6b/docs/customization.md
+++ b/docs6b/docs/customization.md
@@ -3,51 +3,51 @@
 
 ## How to use .mlrrc
 
-Suppose you always use CSV files. Then instead of always having to type ``--csv`` as in
+Suppose you always use CSV files. Then instead of always having to type `--csv` as in
 
-
+
 mlr --csv cut -x -f extra mydata.csv
 
-
+
 mlr --csv sort -n id mydata.csv
 
-and so on, you can instead put the following into your ``$HOME/.mlrrc``: +and so on, you can instead put the following into your `$HOME/.mlrrc`: -
+
     --csv
 
Then you can just type things like -
+
 mlr cut -x -f extra mydata.csv
 
-
+
 mlr sort -n id mydata.csv
 
-and the ``--csv`` part will automatically be understood. (If you do want to process, say, a JSON file then ``mlr --json ...`` at the command line will override the default from your ``.mlrrc``.) +and the `--csv` part will automatically be understood. (If you do want to process, say, a JSON file then `mlr --json ...` at the command line will override the default from your `.mlrrc`.) ## What you can put in your .mlrrc -* You can include any command-line flags, except the "terminal" ones such as ``--help``. +* You can include any command-line flags, except the "terminal" ones such as `--help`. -* The ``--prepipe``, ``--load``, and ``--mload`` flags aren't allowed in ``.mlrrc`` as they control code execution, and could result in your scripts running things you don't expect if you receive data from someone with a ``.mlrrc`` in it. +* The `--prepipe`, `--load`, and `--mload` flags aren't allowed in `.mlrrc` as they control code execution, and could result in your scripts running things you don't expect if you receive data from someone with a `.mlrrc` in it. -* The formatting rule is you need to put one flag beginning with ``--`` per line: for example, ``--csv`` on one line and ``--nr-progress-mod 1000`` on a separate line. +* The formatting rule is you need to put one flag beginning with `--` per line: for example, `--csv` on one line and `--nr-progress-mod 1000` on a separate line. -* Since every line starts with a ``--`` option, you can leave off the initial ``--`` if you want. For example, ``ojson`` is the same as ``--ojson``, and ``nr-progress-mod 1000`` is the same as ``--nr-progress-mod 1000``. +* Since every line starts with a `--` option, you can leave off the initial `--` if you want. For example, `ojson` is the same as `--ojson`, and `nr-progress-mod 1000` is the same as `--nr-progress-mod 1000`. -* Comments are from a ``#`` to the end of the line. +* Comments are from a `#` to the end of the line. * Empty lines are ignored -- including lines which are empty after comments are removed. -Here is an example ``.mlrrc`` file: +Here is an example `.mlrrc` file: -
+
 # Input and output formats are CSV by default (unless otherwise specified
 # on the mlr command line):
 csv
@@ -67,20 +67,20 @@ skip-comments-with @
 
 ## Where to put your .mlrrc
 
-If the environment variable ``MLRRC`` is set:
+If the environment variable `MLRRC` is set:
 
-* If its value is ``__none__`` then no ``.mlrrc`` files are processed.  (This is nice for things like regression testing.)
+* If its value is `__none__` then no `.mlrrc` files are processed.  (This is nice for things like regression testing.)
 
-* Otherwise, its value (as a filename) is loaded and processed. If there are syntax errors, they abort ``mlr`` with a usage message (as if you had mistyped something on the command line). If the file can't be loaded at all, though, it is silently skipped.
+* Otherwise, its value (as a filename) is loaded and processed. If there are syntax errors, they abort `mlr` with a usage message (as if you had mistyped something on the command line). If the file can't be loaded at all, though, it is silently skipped.
 
-* Any ``.mlrrc`` in your home directory or current directory is ignored whenever ``MLRRC`` is set in the environment.
+* Any `.mlrrc` in your home directory or current directory is ignored whenever `MLRRC` is set in the environment.
 
-* Example line in your shell's rc file: ``export MLRRC=/path/to/my/mlrrc``
+* Example line in your shell's rc file: `export MLRRC=/path/to/my/mlrrc`
 
 Otherwise:
 
-* If ``$HOME/.mlrrc`` exists, it's processed as above.
+* If `$HOME/.mlrrc` exists, it's processed as above.
 
-* If ``./.mlrrc`` exists, it's then also processed as above.
+* If `./.mlrrc` exists, it's then also processed as above.
 
-* The idea is you can have all your settings in your ``$HOME/.mlrrc``, then override maybe one or two for your current directory if you like.
+* The idea is you can have all your settings in your `$HOME/.mlrrc`, then override maybe one or two for your current directory if you like.
diff --git a/docs6b/docs/customization.md.in b/docs6b/docs/customization.md.in
index 720052b74..ff3664fd4 100644
--- a/docs6b/docs/customization.md.in
+++ b/docs6b/docs/customization.md.in
@@ -2,7 +2,7 @@
 
 ## How to use .mlrrc
 
-Suppose you always use CSV files. Then instead of always having to type ``--csv`` as in
+Suppose you always use CSV files. Then instead of always having to type `--csv` as in
 
 GENMD_CARDIFY_HIGHLIGHT_ONE
 mlr --csv cut -x -f extra mydata.csv
@@ -12,7 +12,7 @@ GENMD_CARDIFY_HIGHLIGHT_ONE
 mlr --csv sort -n id mydata.csv
 GENMD_EOF
 
-and so on, you can instead put the following into your ``$HOME/.mlrrc``:
+and so on, you can instead put the following into your `$HOME/.mlrrc`:
 
 GENMD_CARDIFY
     --csv
@@ -28,42 +28,42 @@ GENMD_CARDIFY_HIGHLIGHT_ONE
 mlr sort -n id mydata.csv
 GENMD_EOF
 
-and the ``--csv`` part will automatically be understood. (If you do want to process, say, a JSON file then ``mlr --json ...`` at the command line will override the default from your ``.mlrrc``.)
+and the `--csv` part will automatically be understood. (If you do want to process, say, a JSON file then `mlr --json ...` at the command line will override the default from your `.mlrrc`.)
 
 ## What you can put in your .mlrrc
 
-* You can include any command-line flags, except the "terminal" ones such as ``--help``.
+* You can include any command-line flags, except the "terminal" ones such as `--help`.
 
-* The ``--prepipe``, ``--load``, and ``--mload`` flags aren't allowed in ``.mlrrc`` as they control code execution, and could result in your scripts running things you don't expect if you receive data from someone with a ``.mlrrc`` in it.
+* The `--prepipe`, `--load`, and `--mload` flags aren't allowed in `.mlrrc` as they control code execution, and could result in your scripts running things you don't expect if you receive data from someone with a `.mlrrc` in it.
 
-* The formatting rule is you need to put one flag beginning with ``--`` per line: for example, ``--csv`` on one line and ``--nr-progress-mod 1000`` on a separate line.
+* The formatting rule is you need to put one flag beginning with `--` per line: for example, `--csv` on one line and `--nr-progress-mod 1000` on a separate line.
 
-* Since every line starts with a ``--`` option, you can leave off the initial ``--`` if you want. For example, ``ojson`` is the same as ``--ojson``, and ``nr-progress-mod 1000`` is the same as ``--nr-progress-mod 1000``.
+* Since every line starts with a `--` option, you can leave off the initial `--` if you want. For example, `ojson` is the same as `--ojson`, and `nr-progress-mod 1000` is the same as `--nr-progress-mod 1000`.
 
-* Comments are from a ``#`` to the end of the line.
+* Comments are from a `#` to the end of the line.
 
 * Empty lines are ignored -- including lines which are empty after comments are removed.
 
-Here is an example ``.mlrrc`` file:
+Here is an example `.mlrrc` file:
 
 GENMD_INCLUDE_ESCAPED(sample_mlrrc)
 
 ## Where to put your .mlrrc
 
-If the environment variable ``MLRRC`` is set:
+If the environment variable `MLRRC` is set:
 
-* If its value is ``__none__`` then no ``.mlrrc`` files are processed.  (This is nice for things like regression testing.)
+* If its value is `__none__` then no `.mlrrc` files are processed.  (This is nice for things like regression testing.)
 
-* Otherwise, its value (as a filename) is loaded and processed. If there are syntax errors, they abort ``mlr`` with a usage message (as if you had mistyped something on the command line). If the file can't be loaded at all, though, it is silently skipped.
+* Otherwise, its value (as a filename) is loaded and processed. If there are syntax errors, they abort `mlr` with a usage message (as if you had mistyped something on the command line). If the file can't be loaded at all, though, it is silently skipped.
 
-* Any ``.mlrrc`` in your home directory or current directory is ignored whenever ``MLRRC`` is set in the environment.
+* Any `.mlrrc` in your home directory or current directory is ignored whenever `MLRRC` is set in the environment.
 
-* Example line in your shell's rc file: ``export MLRRC=/path/to/my/mlrrc``
+* Example line in your shell's rc file: `export MLRRC=/path/to/my/mlrrc`
 
 Otherwise:
 
-* If ``$HOME/.mlrrc`` exists, it's processed as above.
+* If `$HOME/.mlrrc` exists, it's processed as above.
 
-* If ``./.mlrrc`` exists, it's then also processed as above.
+* If `./.mlrrc` exists, it's then also processed as above.
 
-* The idea is you can have all your settings in your ``$HOME/.mlrrc``, then override maybe one or two for your current directory if you like.
+* The idea is you can have all your settings in your `$HOME/.mlrrc`, then override maybe one or two for your current directory if you like.
diff --git a/docs6b/docs/data-cleaning-examples.md b/docs6b/docs/data-cleaning-examples.md
index 6374fc300..6702ec765 100644
--- a/docs6b/docs/data-cleaning-examples.md
+++ b/docs6b/docs/data-cleaning-examples.md
@@ -1,10 +1,12 @@
 
 # Data-cleaning examples
 
-Here are some ways to use the type-checking options as described in :ref:`reference-dsl-type-tests-and-assertions` Suppose you have the following data file, with inconsistent typing for boolean. (Also imagine that, for the sake of discussion, we have a million-line file rather than a four-line file, so we can't see it all at once and some automation is called for.)
+Here are some ways to use the type-checking options as described in [Type-checking](reference-dsl-variables.md#type-checking).  Suppose you have the following data file, with inconsistent typing for boolean. (Also imagine that, for the sake of discussion, we have a million-line file rather than a four-line file, so we can't see it all at once and some automation is called for.)
 
-
+
 cat data/het-bool.csv
+
+
 name,reachable
 barney,false
 betty,true
@@ -14,8 +16,10 @@ wilma,1
 
 One option is to coerce everything to boolean, or integer:
 
-
+
 mlr --icsv --opprint put '$reachable = boolean($reachable)' data/het-bool.csv
+
+
 name   reachable
 barney false
 betty  true
@@ -23,8 +27,10 @@ fred   true
 wilma  true
 
-
+
 mlr --icsv --opprint put '$reachable = int(boolean($reachable))' data/het-bool.csv
+
+
 name   reachable
 barney 0
 betty  1
@@ -34,8 +40,10 @@ wilma  1
 
 A second option is to flag badly formatted data within the output stream:
 
-
+
 mlr --icsv --opprint put '$format_ok = is_string($reachable)' data/het-bool.csv
+
+
 name   reachable format_ok
 barney false     false
 betty  true      false
@@ -45,10 +53,12 @@ wilma  1         false
 
 Or perhaps to flag badly formatted data outside the output stream:
 
-
+
 mlr --icsv --opprint put '
   if (!is_string($reachable)) {eprint "Malformed at NR=".NR}
 ' data/het-bool.csv
+
+
 Malformed at NR=1
 Malformed at NR=2
 Malformed at NR=3
@@ -62,7 +72,9 @@ wilma  1
 
 A third way is to abort the process on fimd.instance of bad data:
 
-
+
 mlr --csv put '$reachable = asserting_string($reachable)' data/het-bool.csv
+
+
 Miller: is_string type-assertion failed at NR=1 FNR=1 FILENAME=data/het-bool.csv
 
diff --git a/docs6b/docs/data-cleaning-examples.md.in b/docs6b/docs/data-cleaning-examples.md.in index c05886d59..b4cfc3e96 100644 --- a/docs6b/docs/data-cleaning-examples.md.in +++ b/docs6b/docs/data-cleaning-examples.md.in @@ -1,6 +1,6 @@ # Data-cleaning examples -Here are some ways to use the type-checking options as described in :ref:`reference-dsl-type-tests-and-assertions` Suppose you have the following data file, with inconsistent typing for boolean. (Also imagine that, for the sake of discussion, we have a million-line file rather than a four-line file, so we can't see it all at once and some automation is called for.) +Here are some ways to use the type-checking options as described in [Type-checking](reference-dsl-variables.md#type-checking). Suppose you have the following data file, with inconsistent typing for boolean. (Also imagine that, for the sake of discussion, we have a million-line file rather than a four-line file, so we can't see it all at once and some automation is called for.) GENMD_RUN_COMMAND cat data/het-bool.csv diff --git a/docs6b/docs/data-diving-examples.md b/docs6b/docs/data-diving-examples.md index aa5f4dad5..5772b6539 100644 --- a/docs6b/docs/data-diving-examples.md +++ b/docs6b/docs/data-diving-examples.md @@ -7,8 +7,10 @@ The [flins.csv](data/flins.csv) file is some sample data obtained from [https:// Vertical-tabular format is good for a quick look at CSV data layout -- seeing what columns you have to work with: -
+
 head -n 2 data/flins.csv | mlr --icsv --oxtab cat
+
+
 county   Seminole
 tiv_2011 22890.55
 tiv_2012 20848.71
@@ -17,8 +19,10 @@ line     Residential
 
 A few simple queries:
 
-
+
 mlr --from data/flins.csv --icsv --opprint count-distinct -f county | head
+
+
 county     count
 Seminole   1
 Miami Dade 2
@@ -28,32 +32,36 @@ Duval      1
 St. Johns  1
 
-
+
 mlr --from data/flins.csv --icsv --opprint count-distinct -f construction,line
 
Categorization of total insured value: -
+
 mlr --from data/flins.csv --icsv --opprint stats1 -a min,mean,max -f tiv_2012
+
+
 tiv_2012_min tiv_2012_mean      tiv_2012_max
 19757.91     1061531.4637499999 2785551.63
 
-
+
 mlr --from data/flins.csv --icsv --opprint \
   stats1 -a min,mean,max -f tiv_2012 -g construction,line
 
-
+
 mlr --from data/flins.csv --icsv --oxtab \
   stats1 -a p0,p10,p50,p90,p95,p99,p100 -f hu_site_deductible
 
-
+
 mlr --from data/flins.csv --icsv --opprint \
   stats1 -a p95,p99,p100 -f hu_site_deductible -g county \
   then sort -f county | head
+
+
 county
 Duval
 Highlands
@@ -63,9 +71,11 @@ Seminole
 St. Johns
 
-
+
 mlr --from data/flins.csv --icsv --oxtab \
   stats2 -a corr,linreg-ols,r2 -f tiv_2011,tiv_2012
+
+
 tiv_2011_tiv_2012_corr  0.9353629581411828
 tiv_2011_tiv_2012_ols_m 1.0890905877734807
 tiv_2011_tiv_2012_ols_b 103095.52335638746
@@ -73,9 +83,11 @@ tiv_2011_tiv_2012_ols_n 8
 tiv_2011_tiv_2012_r2    0.8749038634626236
 
-
+
 mlr --from data/flins.csv --icsv --opprint \
   stats2 -a corr,linreg-ols,r2 -f tiv_2011,tiv_2012 -g county
+
+
 county     tiv_2011_tiv_2012_corr tiv_2011_tiv_2012_ols_m tiv_2011_tiv_2012_ols_b tiv_2011_tiv_2012_ols_n tiv_2011_tiv_2012_r2
 Seminole   -                      -                       -                       1                       -
 Miami Dade 1                      0.9306426512386247      -2311.1543275160047     2                       0.9999999999999999
@@ -87,25 +99,29 @@ St. Johns  -                      -                       -
 
 ## Color/shape data
 
-The [data/colored-shapes.dkvp](data/colored-shapes.dkvp) file is some sample data produced by the [mkdat2](data/mkdat2) script. The idea is:
+The [data/colored-shapes.dkvp](data/colored-shapes.dkvp) file is some sample data produced by the [mkdat2](../data/mkdat2) script. The idea is:
 
 * Produce some data with known distributions and correlations, and verify that Miller recovers those properties empirically.
 * Each record is labeled with one of a few colors and one of a few shapes.
-* The ``flag`` field is 0 or 1, with probability dependent on color
-* The ``u`` field is plain uniform on the unit interval.
-* The ``v`` field is the same, except tightly correlated with ``u`` for red circles.
-* The ``w`` field is autocorrelated for each color/shape pair.
-* The ``x`` field is boring Gaussian with mean 5 and standard deviation about 1.2, with no dependence on color or shape.
+* The `flag` field is 0 or 1, with probability dependent on color
+* The `u` field is plain uniform on the unit interval.
+* The `v` field is the same, except tightly correlated with `u` for red circles.
+* The `w` field is autocorrelated for each color/shape pair.
+* The `x` field is boring Gaussian with mean 5 and standard deviation about 1.2, with no dependence on color or shape.
 
 Peek at the data:
 
-
+
 wc -l data/colored-shapes.dkvp
+
+
    10078 data/colored-shapes.dkvp
 
-
+
 head -n 6 data/colored-shapes.dkvp | mlr --opprint cat
+
+
 color  shape    flag i  u                   v                    w                   x
 yellow triangle 1    11 0.6321695890307647  0.9887207810889004   0.4364983936735774  5.7981881667050565
 red    square   1    15 0.21966833570651523 0.001257332190235938 0.7927778364718627  2.944117399716207
@@ -117,10 +133,12 @@ red    square   0    64 0.2015510269821953  0.9531098083420033   0.7719912015786
 
 Look at uncategorized stats (using [creach](https://github.com/johnkerl/scripts/blob/master/fundam/creach) for spacing).
 
-Here it looks reasonable that ``u`` is unit-uniform; something's up with ``v`` but we can't yet see what:
+Here it looks reasonable that `u` is unit-uniform; something's up with `v` but we can't yet see what:
 
-
+
 mlr --oxtab stats1 -a min,mean,max -f flag,u,v data/colored-shapes.dkvp | creach 3
+
+
 flag_min  0
 flag_mean 0.39888866838658465
 flag_max  1
@@ -136,8 +154,10 @@ v_max     1.0724998185026013
 
 The histogram shows the different distribution of 0/1 flags:
 
-
+
 mlr --opprint histogram -f flag,u,v --lo -0.1 --hi 1.1 --nbins 12 data/colored-shapes.dkvp
+
+
 bin_lo                bin_hi              flag_count u_count v_count
 -0.010000000000000002 0.09000000000000002 6058       0       36
 0.09000000000000002   0.19000000000000003 0          1062    988
@@ -155,10 +175,12 @@ bin_lo                bin_hi              flag_count u_count v_count
 
 Look at univariate stats by color and shape. In particular, color-dependent flag probabilities pop out, aligning with their original Bernoulli probablities from the data-generator script:
 
-
+
 mlr --opprint stats1 -a min,mean,max -f flag,u,v -g color \
   then sort -f color \
   data/colored-shapes.dkvp
+
+
 color  flag_min flag_mean           flag_max u_min                   u_mean              u_max              v_min                 v_mean              v_max
 blue   0        0.5843537414965987  1        0.000043912454007477564 0.517717155039078   0.9999687954968421 0.0014886830387470518 0.49105642841387653 0.9995761761685742
 green  0        0.20919747520288548 1        0.00048750676198217047  0.5048610622924616  0.9999361779701204 0.0005012669003675585 0.49908475928072205 0.9996764373885353
@@ -168,28 +190,34 @@ red    0        0.3031674208144796  1        0.0006711367180041172   0.492559648
 yellow 0        0.8924274593064402  1        0.001300228762057487    0.49712912165196765 0.99992313390574   0.0007109695568577878 0.510626599360317   0.9999189897724752
 
-
+
 mlr --opprint stats1 -a min,mean,max -f flag,u,v -g shape \
   then sort -f shape \
   data/colored-shapes.dkvp
+
+
 shape    flag_min flag_mean           flag_max u_min                   u_mean              u_max              v_min                  v_mean              v_max
 circle   0        0.3998456194519491  1        0.000043912454007477564 0.49855450951394115 0.99992313390574   -0.09270905318501277   0.49552415740048406 1.0724998185026013
 square   0        0.39611178614823817 1        0.0001881939925673093   0.499385458061097   0.9999687954968421 0.00008930277299445954 0.49653825501903986 0.9999751864255598
 triangle 0        0.4015421115065243  1        0.000881025170573424    0.4968585405884252  0.9996614910922645 0.000716883409890845   0.501049532862137   0.9999946837499262
 
-Look at bivariate stats by color and shape. In particular, ``u,v`` pairwise correlation for red circles pops out: +Look at bivariate stats by color and shape. In particular, `u,v` pairwise correlation for red circles pops out: -
+
 mlr --opprint --right stats2 -a corr -f u,v,w,x data/colored-shapes.dkvp
+
+
            u_v_corr              w_x_corr 
 0.13341803768384553 -0.011319938208638764 
 
-
+
 mlr --opprint --right \
   stats2 -a corr -f u,v,w,x -g color,shape then sort -nr u_v_corr \
   data/colored-shapes.dkvp
+
+
  color    shape              u_v_corr               w_x_corr 
    red   circle    0.9807984157534667  -0.018565046320623148 
 orange   square   0.17685846147882145   -0.07104374629148885 
diff --git a/docs6b/docs/data-diving-examples.md.in b/docs6b/docs/data-diving-examples.md.in
index c05bd1ffc..f700af8e6 100644
--- a/docs6b/docs/data-diving-examples.md.in
+++ b/docs6b/docs/data-diving-examples.md.in
@@ -54,15 +54,15 @@ GENMD_EOF
 
 ## Color/shape data
 
-The [data/colored-shapes.dkvp](data/colored-shapes.dkvp) file is some sample data produced by the [mkdat2](data/mkdat2) script. The idea is:
+The [data/colored-shapes.dkvp](data/colored-shapes.dkvp) file is some sample data produced by the [mkdat2](../data/mkdat2) script. The idea is:
 
 * Produce some data with known distributions and correlations, and verify that Miller recovers those properties empirically.
 * Each record is labeled with one of a few colors and one of a few shapes.
-* The ``flag`` field is 0 or 1, with probability dependent on color
-* The ``u`` field is plain uniform on the unit interval.
-* The ``v`` field is the same, except tightly correlated with ``u`` for red circles.
-* The ``w`` field is autocorrelated for each color/shape pair.
-* The ``x`` field is boring Gaussian with mean 5 and standard deviation about 1.2, with no dependence on color or shape.
+* The `flag` field is 0 or 1, with probability dependent on color
+* The `u` field is plain uniform on the unit interval.
+* The `v` field is the same, except tightly correlated with `u` for red circles.
+* The `w` field is autocorrelated for each color/shape pair.
+* The `x` field is boring Gaussian with mean 5 and standard deviation about 1.2, with no dependence on color or shape.
 
 Peek at the data:
 
@@ -76,7 +76,7 @@ GENMD_EOF
 
 Look at uncategorized stats (using [creach](https://github.com/johnkerl/scripts/blob/master/fundam/creach) for spacing).
 
-Here it looks reasonable that ``u`` is unit-uniform; something's up with ``v`` but we can't yet see what:
+Here it looks reasonable that `u` is unit-uniform; something's up with `v` but we can't yet see what:
 
 GENMD_RUN_COMMAND
 mlr --oxtab stats1 -a min,mean,max -f flag,u,v data/colored-shapes.dkvp | creach 3
@@ -102,7 +102,7 @@ mlr --opprint stats1 -a min,mean,max -f flag,u,v -g shape \
   data/colored-shapes.dkvp
 GENMD_EOF
 
-Look at bivariate stats by color and shape. In particular, ``u,v`` pairwise correlation for red circles pops out:
+Look at bivariate stats by color and shape. In particular, `u,v` pairwise correlation for red circles pops out:
 
 GENMD_RUN_COMMAND
 mlr --opprint --right stats2 -a corr -f u,v,w,x data/colored-shapes.dkvp
diff --git a/docs6b/docs/data/sar.mlr b/docs6b/docs/data/sar.mlr
index 4b5d8ead3..166406b44 100644
--- a/docs6b/docs/data/sar.mlr
+++ b/docs6b/docs/data/sar.mlr
@@ -1,3 +1,3 @@
-  for (k in $*) {
-    $[k] = gsub($[k], "e", "X");
-  }
+for (k in $*) {
+  $[k] = gsub($[k], "e", "X");
+}
diff --git a/docs6b/docs/dates-and-times.md b/docs6b/docs/dates-and-times.md
index 80aa9d879..520558c75 100644
--- a/docs6b/docs/dates-and-times.md
+++ b/docs6b/docs/dates-and-times.md
@@ -5,32 +5,38 @@
 
 Given input like
 
-
+
 cat dates.csv
+
+
 date,event
 2018-02-03,initialization
 2018-03-07,discovery
 2018-02-03,allocation
 
-we can use ``strptime`` to parse the date field into seconds-since-epoch and then do numeric comparisons. Simply match your input dataset's date-formatting to the :ref:`reference-dsl-strptime` format-string. For example: +we can use [strptime](reference-verbs.md#strptime) to parse the date field into seconds-since-epoch and then do numeric comparisons. Simply match your input dataset's date-formatting to the [strptime](reference-verbs.md#strptime) format-string. For example: -
+
 mlr --csv filter '
   strptime($date, "%Y-%m-%d") > strptime("2018-03-03", "%Y-%m-%d")
 ' dates.csv
+
+
 date,event
 2018-03-07,discovery
 
-Caveat: localtime-handling in timezones with DST is still a work in progress; see https://github.com/johnkerl/miller/issues/170. See also https://github.com/johnkerl/miller/issues/208 -- thanks @aborruso! +Caveat: localtime-handling in timezones with DST is still a work in progress; see [https://github.com/johnkerl/miller/issues/170](https://github.com/johnkerl/miller/issues/170) . See also [https://github.com/johnkerl/miller/issues/208](https://github.com/johnkerl/miller/issues/208) -- thanks @aborruso! ## Finding missing dates Suppose you have some date-stamped data which may (or may not) be missing entries for one or more dates: -
+
 head -n 10 data/miss-date.csv
+
+
 date,qoh
 2012-03-05,10055
 2012-03-06,10486
@@ -43,19 +49,23 @@ date,qoh
 2012-03-13,11177
 
-
+
 wc -l data/miss-date.csv
+
+
     1372 data/miss-date.csv
 
-Since there are 1372 lines in the data file, some automation is called for. To find the missing dates, you can convert the dates to seconds since the epoch using ``strptime``, then compute adjacent differences (the ``cat -n`` simply inserts record-counters): +Since there are 1372 lines in the data file, some automation is called for. To find the missing dates, you can convert the dates to seconds since the epoch using `strptime`, then compute adjacent differences (the `cat -n` simply inserts record-counters): -
+
 mlr --from data/miss-date.csv --icsv \
   cat -n \
   then put '$datestamp = strptime($date, "%Y-%m-%d")' \
   then step -a delta -f datestamp \
 | head
+
+
 n=1,date=2012-03-05,qoh=10055,datestamp=1330905600,datestamp_delta=0
 n=2,date=2012-03-06,qoh=10486,datestamp=1330992000,datestamp_delta=86400
 n=3,date=2012-03-07,qoh=10430,datestamp=1331078400,datestamp_delta=86400
@@ -70,20 +80,24 @@ n=10,date=2012-03-14,qoh=11498,datestamp=1331683200,datestamp_delta=86400
 
 Then, filter for adjacent difference not being 86400 (the number of seconds in a day):
 
-
+
 mlr --from data/miss-date.csv --icsv \
   cat -n \
   then put '$datestamp = strptime($date, "%Y-%m-%d")' \
   then step -a delta -f datestamp \
   then filter '$datestamp_delta != 86400 && $n != 1'
+
+
 n=774,date=2014-04-19,qoh=130140,datestamp=1397865600,datestamp_delta=259200
 n=1119,date=2015-03-31,qoh=181625,datestamp=1427760000,datestamp_delta=172800
 
Given this, it's now easy to see where the gaps are: -
+
 mlr cat -n then filter '$n >= 770 && $n <= 780' data/miss-date.csv
+
+
 n=770,1=2014-04-12,2=129435
 n=771,1=2014-04-13,2=129868
 n=772,1=2014-04-14,2=129797
@@ -97,8 +111,10 @@ n=779,1=2014-04-23,2=130849
 n=780,1=2014-04-24,2=131026
 
-
+
 mlr cat -n then filter '$n >= 1115 && $n <= 1125' data/miss-date.csv
+
+
 n=1115,1=2015-03-25,2=181006
 n=1116,1=2015-03-26,2=180995
 n=1117,1=2015-03-27,2=181043
diff --git a/docs6b/docs/dates-and-times.md.in b/docs6b/docs/dates-and-times.md.in
index 9a9f950e7..cf959cb09 100644
--- a/docs6b/docs/dates-and-times.md.in
+++ b/docs6b/docs/dates-and-times.md.in
@@ -8,7 +8,7 @@ GENMD_RUN_COMMAND
 cat dates.csv
 GENMD_EOF
 
-we can use ``strptime`` to parse the date field into seconds-since-epoch and then do numeric comparisons.  Simply match your input dataset's date-formatting to the :ref:`reference-dsl-strptime` format-string.  For example:
+we can use [strptime](reference-verbs.md#strptime) to parse the date field into seconds-since-epoch and then do numeric comparisons.  Simply match your input dataset's date-formatting to the [strptime](reference-verbs.md#strptime) format-string.  For example:
 
 GENMD_RUN_COMMAND
 mlr --csv filter '
@@ -16,7 +16,7 @@ mlr --csv filter '
 ' dates.csv
 GENMD_EOF
 
-Caveat: localtime-handling in timezones with DST is still a work in progress; see https://github.com/johnkerl/miller/issues/170. See also https://github.com/johnkerl/miller/issues/208 -- thanks @aborruso!
+Caveat: localtime-handling in timezones with DST is still a work in progress; see [https://github.com/johnkerl/miller/issues/170](https://github.com/johnkerl/miller/issues/170) . See also [https://github.com/johnkerl/miller/issues/208](https://github.com/johnkerl/miller/issues/208) -- thanks @aborruso!
 
 ## Finding missing dates
 
@@ -30,7 +30,7 @@ GENMD_RUN_COMMAND
 wc -l data/miss-date.csv
 GENMD_EOF
 
-Since there are 1372 lines in the data file, some automation is called for. To find the missing dates, you can convert the dates to seconds since the epoch using ``strptime``, then compute adjacent differences (the ``cat -n`` simply inserts record-counters):
+Since there are 1372 lines in the data file, some automation is called for. To find the missing dates, you can convert the dates to seconds since the epoch using `strptime`, then compute adjacent differences (the `cat -n` simply inserts record-counters):
 
 GENMD_INCLUDE_AND_RUN_ESCAPED(data/miss-date-1.sh)
 
diff --git a/docs6b/docs/dkvp-examples.md b/docs6b/docs/dkvp-examples.md
index f0c6fcad1..2790490f2 100644
--- a/docs6b/docs/dkvp-examples.md
+++ b/docs6b/docs/dkvp-examples.md
@@ -5,7 +5,7 @@
 
 Here are the I/O routines:
 
-
+
 #!/usr/bin/env python
 
 # ================================================================
@@ -68,8 +68,10 @@ def map2dkvpline(map , ops, ofs):
 
 And here is an example using them:
 
-
+
 cat polyglot-dkvp-io/example.py
+
+
 #!/usr/bin/env python
 
 import sys
@@ -108,8 +110,10 @@ while True:
 
 Run as-is:
 
-
+
 python polyglot-dkvp-io/example.py < data/small
+
+
 a=pan,b=pan,i=1,y=0.7268028627434533,ab=panpan,iy=1.7268028627434533,ta=str,tb=str,ti=int,ty=float,tab=str,tiy=float
 a=eks,b=pan,i=2,y=0.5221511083334797,ab=ekspan,iy=2.5221511083334796,ta=str,tb=str,ti=int,ty=float,tab=str,tiy=float
 a=wye,b=wye,i=3,y=0.33831852551664776,ab=wyewye,iy=3.3383185255166477,ta=str,tb=str,ti=int,ty=float,tab=str,tiy=float
@@ -119,8 +123,10 @@ a=wye,b=pan,i=5,y=0.8636244699032729,ab=wyepan,iy=5.863624469903273,ta=str,tb=st
 
 Run as-is, then pipe to Miller for pretty-printing:
 
-
+
 python polyglot-dkvp-io/example.py < data/small | mlr --opprint cat
+
+
 a   b   i y                   ab     iy                 ta  tb  ti  ty    tab tiy
 pan pan 1 0.7268028627434533  panpan 1.7268028627434533 str str int float str float
 eks pan 2 0.5221511083334797  ekspan 2.5221511083334796 str str int float str float
@@ -133,7 +139,7 @@ wye pan 5 0.8636244699032729  wyepan 5.863624469903273  str str int float str fl
 
 Here are the I/O routines:
 
-
+
 #!/usr/bin/env ruby
 
 # ================================================================
@@ -190,8 +196,10 @@ end
 
 And here is an example using them:
 
-
+
 cat polyglot-dkvp-io/example.rb
+
+
 #!/usr/bin/env ruby
 
 require 'dkvp_io'
@@ -220,8 +228,10 @@ end
 
 Run as-is:
 
-
+
 ruby -I./polyglot-dkvp-io polyglot-dkvp-io/example.rb data/small
+
+
 a=pan,b=pan,i=1,y=0.7268028627434533,ab=panpan,iy=1.7268028627434533,ta=String,tb=String,ti=Integer,ty=Float,tab=String,tiy=Float
 a=eks,b=pan,i=2,y=0.5221511083334797,ab=ekspan,iy=2.5221511083334796,ta=String,tb=String,ti=Integer,ty=Float,tab=String,tiy=Float
 a=wye,b=wye,i=3,y=0.33831852551664776,ab=wyewye,iy=3.3383185255166477,ta=String,tb=String,ti=Integer,ty=Float,tab=String,tiy=Float
@@ -231,8 +241,10 @@ a=wye,b=pan,i=5,y=0.8636244699032729,ab=wyepan,iy=5.863624469903273,ta=String,tb
 
 Run as-is, then pipe to Miller for pretty-printing:
 
-
+
 ruby -I./polyglot-dkvp-io polyglot-dkvp-io/example.rb data/small | mlr --opprint cat
+
+
 a   b   i y                   ab     iy                 ta     tb     ti      ty    tab    tiy
 pan pan 1 0.7268028627434533  panpan 1.7268028627434533 String String Integer Float String Float
 eks pan 2 0.5221511083334797  ekspan 2.5221511083334796 String String Integer Float String Float
diff --git a/docs6b/docs/etymology.md b/docs6b/docs/etymology.md
index 5fc98a6c5..d1a5470b3 100644
--- a/docs6b/docs/etymology.md
+++ b/docs6b/docs/etymology.md
@@ -1,6 +1,6 @@
 
 # Why call it Miller?
 
-The Unix toolkit was created in the **1970s** and is a mainstay to this day.  Miller's look and feel adheres closely to the [classic toolkit style](http://en.wikipedia.org/wiki/Unix_philosophy): if this were music, Miller would be a **tribute album**. Likewise, since commands are subcommands of the ``mlr`` executable, the result is a **band**, if you will, of command-line tools. Put these together and the namesake is another classic product of the 1970s: the [Steve Miller Band](http://en.wikipedia.org/wiki/Steve%5fMiller%5fBand).
+The Unix toolkit was created in the **1970s** and is a mainstay to this day.  Miller's look and feel adheres closely to the [classic toolkit style](http://en.wikipedia.org/wiki/Unix_philosophy): if this were music, Miller would be a **tribute album**. Likewise, since commands are subcommands of the `mlr` executable, the result is a **band**, if you will, of command-line tools. Put these together and the namesake is another classic product of the 1970s: the [Steve Miller Band](http://en.wikipedia.org/wiki/Steve%5fMiller%5fBand).
 
 (Additionally, and far more prosaically ... just as a miller is someone who grinds and mixes grain into flour to extend its usefulness, Miller grinds and mixes data for you.)
diff --git a/docs6b/docs/etymology.md.in b/docs6b/docs/etymology.md.in
index 32db8be73..348588db4 100644
--- a/docs6b/docs/etymology.md.in
+++ b/docs6b/docs/etymology.md.in
@@ -1,5 +1,5 @@
 # Why call it Miller?
 
-The Unix toolkit was created in the **1970s** and is a mainstay to this day.  Miller's look and feel adheres closely to the [classic toolkit style](http://en.wikipedia.org/wiki/Unix_philosophy): if this were music, Miller would be a **tribute album**. Likewise, since commands are subcommands of the ``mlr`` executable, the result is a **band**, if you will, of command-line tools. Put these together and the namesake is another classic product of the 1970s: the [Steve Miller Band](http://en.wikipedia.org/wiki/Steve%5fMiller%5fBand).
+The Unix toolkit was created in the **1970s** and is a mainstay to this day.  Miller's look and feel adheres closely to the [classic toolkit style](http://en.wikipedia.org/wiki/Unix_philosophy): if this were music, Miller would be a **tribute album**. Likewise, since commands are subcommands of the `mlr` executable, the result is a **band**, if you will, of command-line tools. Put these together and the namesake is another classic product of the 1970s: the [Steve Miller Band](http://en.wikipedia.org/wiki/Steve%5fMiller%5fBand).
 
 (Additionally, and far more prosaically ... just as a miller is someone who grinds and mixes grain into flour to extend its usefulness, Miller grinds and mixes data for you.)
diff --git a/docs6b/docs/extra.css b/docs6b/docs/extra.css
index bd0e7c0cb..06f6d18aa 100644
--- a/docs6b/docs/extra.css
+++ b/docs6b/docs/extra.css
@@ -1,11 +1,17 @@
+/* Top left, desktop */
 .wy-menu {
   background-color: #c0c0c0;
-  /*background-color: #404040;*/
 }
+/* Top left, mobile */
+.wy-nav-top {
+  background-color: #c0c0c0;
+}
+/* Left navbar */
 .wy-side-nav-search {
   background-color: #c0c0c0;
-  /*background-color: #404040;*/
 }
+
+/* Code samples without 
 (none currently in use) */
 .rst-content code {
   color: #000000;
   background-color: #e0e0e0;
@@ -18,11 +24,27 @@
   color: #0FF1CE;
 }
 .caption-text {
-  /*background-color: #808080; */
   color: maroon;
   padding: 4px;
 }
 
+/* Code samples using 
 (which is currently in use) */
+.pre-highlight {
+  color: #000000;
+  background-color: #c5b690;
+  font-size: 12px;
+  padding: 10px;
+  margin-bottom: 0px;
+}
+.pre-non-highlight {
+  color: #000000;
+  background-color: #eae2cb;
+  font-size: 12px;
+  padding: 10px;
+  margin-top: 0px;
+}
+
+/* Section titles in content pages */
 h1 {
   color: maroon;
   background-color: #e0e0e0;
@@ -36,31 +58,7 @@ h3 {
   background-color: #e0e0e0;
 }
 
-pre {
-  color: #000000;
-  background-color: #eae2cb;
-  font-size: 12px;
-  padding: 10px;
-}
-
-pre b {
-  background-color: #c5b690;
-  padding: 2px;
-}
-
-a {
-  color: orange;
-}
-a:visited {
-  color: green;
-}
-a:hover {
-  color: blue;
-}
-a .internal {
-  color: red;
-}
-
+/* Hyperlinks in content pages */
 a {
   color: maroon;
 }
@@ -72,6 +70,7 @@ a:hover {
   color: maroon;
 }
 
+/* Hyperlinks in navbar */
 .wy-menu-vertical li ul li a {
     margin-bottom:0;
     color:#000000;
diff --git a/docs6b/docs/feature-comparison.md b/docs6b/docs/feature-comparison.md
index f8dda8ff6..83348726e 100644
--- a/docs6b/docs/feature-comparison.md
+++ b/docs6b/docs/feature-comparison.md
@@ -1,61 +1,69 @@
 
 # Unix-toolkit context
 
-How does Miller fit within the Unix toolkit (``grep``, ``sed``, ``awk``, etc.)?
+How does Miller fit within the Unix toolkit (`grep`, `sed`, `awk`, etc.)?
 
 ## File-format awareness
 
-Miller respects CSV headers. If you do ``mlr --csv cat *.csv`` then the header line is written once:
+Miller respects CSV headers. If you do `mlr --csv cat *.csv` then the header line is written once:
 
-
+
 cat data/a.csv
+
+
 a,b,c
 1,2,3
 4,5,6
 
-
+
 cat data/b.csv
+
+
 a,b,c
 7,8,9
 
-
+
 mlr --csv cat data/a.csv data/b.csv
+
+
 a,b,c
 1,2,3
 4,5,6
 7,8,9
 
-
+
 mlr --csv sort -nr b data/a.csv data/b.csv
+
+
 a,b,c
 7,8,9
 4,5,6
 1,2,3
 
-Likewise with ``mlr sort``, ``mlr tac``, and so on. +Likewise with `mlr sort`, `mlr tac`, and so on. ## awk-like features: mlr filter and mlr put -* ``mlr filter`` includes/excludes records based on a filter expression, e.g. ``mlr filter '$count > 10'``. +* `mlr filter` includes/excludes records based on a filter expression, e.g. `mlr filter '$count > 10'`. -* ``mlr put`` adds a new field as a function of others, e.g. ``mlr put '$xy = $x * $y'`` or ``mlr put '$counter = NR'``. +* `mlr put` adds a new field as a function of others, e.g. `mlr put '$xy = $x * $y'` or `mlr put '$counter = NR'`. -* The ``$name`` syntax is straight from ``awk``'s ``$1 $2 $3`` (adapted to name-based indexing), as are the variables ``FS``, ``OFS``, ``RS``, ``ORS``, ``NF``, ``NR``, and ``FILENAME``. The ``ENV[...]`` syntax is from Ruby. +* The `$name` syntax is straight from `awk`'s `$1 $2 $3` (adapted to name-based indexing), as are the variables `FS`, `OFS`, `RS`, `ORS`, `NF`, `NR`, and `FILENAME`. The `ENV[...]` syntax is from Ruby. -* While ``awk`` functions are record-based, Miller subcommands (or *verbs*) are stream-based: each of them maps a stream of records into another stream of records. +* While `awk` functions are record-based, Miller subcommands (or *verbs*) are stream-based: each of them maps a stream of records into another stream of records. -* Like ``awk``, Miller (as of v5.0.0) allows you to define new functions within its ``put`` and ``filter`` expression language. Further programmability comes from chaining with ``then``. +* Like `awk`, Miller (as of v5.0.0) allows you to define new functions within its `put` and `filter` expression language. Further programmability comes from chaining with `then`. -* As with ``awk``, ``$``-variables are stream variables and all verbs (such as ``cut``, ``stats1``, ``put``, etc.) as well as ``put``/``filter`` statements operate on streams. This means that you define actions to be done on each record and then stream your data through those actions. The built-in variables ``NF``, ``NR``, etc. change from one line to another, ``$x`` is a label for field ``x`` in the current record, and the input to ``sqrt($x)`` changes from one record to the next. The expression language for the ``put`` and ``filter`` verbs additionally allows you to define ``begin {...}`` and ``end {...}`` blocks for actions to be taken before and after records are processed, respectively. +* As with `awk`, `$`-variables are stream variables and all verbs (such as `cut`, `stats1`, `put`, etc.) as well as `put`/`filter` statements operate on streams. This means that you define actions to be done on each record and then stream your data through those actions. The built-in variables `NF`, `NR`, etc. change from one line to another, `$x` is a label for field `x` in the current record, and the input to `sqrt($x)` changes from one record to the next. The expression language for the `put` and `filter` verbs additionally allows you to define `begin {...}` and `end {...}` blocks for actions to be taken before and after records are processed, respectively. -* As with ``awk``, Miller's ``put``/``filter`` language lets you set ``@sum=0`` before records are read, then update that sum on each record, then print its value at the end. Unlike ``awk``, Miller makes syntactically explicit the difference between variables with extent across all records (names starting with ``@``, such as ``@sum``) and variables which are local to the current expression (names starting without ``@``, such as ``sum``). +* As with `awk`, Miller's `put`/`filter` language lets you set `@sum=0` before records are read, then update that sum on each record, then print its value at the end. Unlike `awk`, Miller makes syntactically explicit the difference between variables with extent across all records (names starting with `@`, such as `@sum`) and variables which are local to the current expression (names starting without `@`, such as `sum`). -* Miller can be faster than ``awk``, ``cut``, and so on, depending on platform; see also [Performance](performance.md). In particular, Miller's DSL syntax is parsed into Go control structures at startup time, with the bulk data-stream processing all done in Go. +* Miller can be faster than `awk`, `cut`, and so on, depending on platform; see also [Performance](performance.md). In particular, Miller's DSL syntax is parsed into Go control structures at startup time, with the bulk data-stream processing all done in Go. ## See also -See [Verbs Reference](reference-verbs.md) for more on Miller's subcommands ``cat``, ``cut``, ``head``, ``sort``, ``tac``, ``tail``, ``top``, and ``uniq``, as well as [DSL reference](reference-dsl.md) for more on the awk-like ``mlr filter`` and ``mlr put``. +See [Verbs Reference](reference-verbs.md) for more on Miller's subcommands `cat`, `cut`, `head`, `sort`, `tac`, `tail`, `top`, and `uniq`, as well as [DSL reference](reference-dsl.md) for more on the awk-like `mlr filter` and `mlr put`. diff --git a/docs6b/docs/feature-comparison.md.in b/docs6b/docs/feature-comparison.md.in index 5e8cafed3..6c87b065f 100644 --- a/docs6b/docs/feature-comparison.md.in +++ b/docs6b/docs/feature-comparison.md.in @@ -1,10 +1,10 @@ # Unix-toolkit context -How does Miller fit within the Unix toolkit (``grep``, ``sed``, ``awk``, etc.)? +How does Miller fit within the Unix toolkit (`grep`, `sed`, `awk`, etc.)? ## File-format awareness -Miller respects CSV headers. If you do ``mlr --csv cat *.csv`` then the header line is written once: +Miller respects CSV headers. If you do `mlr --csv cat *.csv` then the header line is written once: GENMD_RUN_COMMAND cat data/a.csv @@ -22,26 +22,26 @@ GENMD_RUN_COMMAND mlr --csv sort -nr b data/a.csv data/b.csv GENMD_EOF -Likewise with ``mlr sort``, ``mlr tac``, and so on. +Likewise with `mlr sort`, `mlr tac`, and so on. ## awk-like features: mlr filter and mlr put -* ``mlr filter`` includes/excludes records based on a filter expression, e.g. ``mlr filter '$count > 10'``. +* `mlr filter` includes/excludes records based on a filter expression, e.g. `mlr filter '$count > 10'`. -* ``mlr put`` adds a new field as a function of others, e.g. ``mlr put '$xy = $x * $y'`` or ``mlr put '$counter = NR'``. +* `mlr put` adds a new field as a function of others, e.g. `mlr put '$xy = $x * $y'` or `mlr put '$counter = NR'`. -* The ``$name`` syntax is straight from ``awk``'s ``$1 $2 $3`` (adapted to name-based indexing), as are the variables ``FS``, ``OFS``, ``RS``, ``ORS``, ``NF``, ``NR``, and ``FILENAME``. The ``ENV[...]`` syntax is from Ruby. +* The `$name` syntax is straight from `awk`'s `$1 $2 $3` (adapted to name-based indexing), as are the variables `FS`, `OFS`, `RS`, `ORS`, `NF`, `NR`, and `FILENAME`. The `ENV[...]` syntax is from Ruby. -* While ``awk`` functions are record-based, Miller subcommands (or *verbs*) are stream-based: each of them maps a stream of records into another stream of records. +* While `awk` functions are record-based, Miller subcommands (or *verbs*) are stream-based: each of them maps a stream of records into another stream of records. -* Like ``awk``, Miller (as of v5.0.0) allows you to define new functions within its ``put`` and ``filter`` expression language. Further programmability comes from chaining with ``then``. +* Like `awk`, Miller (as of v5.0.0) allows you to define new functions within its `put` and `filter` expression language. Further programmability comes from chaining with `then`. -* As with ``awk``, ``$``-variables are stream variables and all verbs (such as ``cut``, ``stats1``, ``put``, etc.) as well as ``put``/``filter`` statements operate on streams. This means that you define actions to be done on each record and then stream your data through those actions. The built-in variables ``NF``, ``NR``, etc. change from one line to another, ``$x`` is a label for field ``x`` in the current record, and the input to ``sqrt($x)`` changes from one record to the next. The expression language for the ``put`` and ``filter`` verbs additionally allows you to define ``begin {...}`` and ``end {...}`` blocks for actions to be taken before and after records are processed, respectively. +* As with `awk`, `$`-variables are stream variables and all verbs (such as `cut`, `stats1`, `put`, etc.) as well as `put`/`filter` statements operate on streams. This means that you define actions to be done on each record and then stream your data through those actions. The built-in variables `NF`, `NR`, etc. change from one line to another, `$x` is a label for field `x` in the current record, and the input to `sqrt($x)` changes from one record to the next. The expression language for the `put` and `filter` verbs additionally allows you to define `begin {...}` and `end {...}` blocks for actions to be taken before and after records are processed, respectively. -* As with ``awk``, Miller's ``put``/``filter`` language lets you set ``@sum=0`` before records are read, then update that sum on each record, then print its value at the end. Unlike ``awk``, Miller makes syntactically explicit the difference between variables with extent across all records (names starting with ``@``, such as ``@sum``) and variables which are local to the current expression (names starting without ``@``, such as ``sum``). +* As with `awk`, Miller's `put`/`filter` language lets you set `@sum=0` before records are read, then update that sum on each record, then print its value at the end. Unlike `awk`, Miller makes syntactically explicit the difference between variables with extent across all records (names starting with `@`, such as `@sum`) and variables which are local to the current expression (names starting without `@`, such as `sum`). -* Miller can be faster than ``awk``, ``cut``, and so on, depending on platform; see also [Performance](performance.md). In particular, Miller's DSL syntax is parsed into Go control structures at startup time, with the bulk data-stream processing all done in Go. +* Miller can be faster than `awk`, `cut`, and so on, depending on platform; see also [Performance](performance.md). In particular, Miller's DSL syntax is parsed into Go control structures at startup time, with the bulk data-stream processing all done in Go. ## See also -See [Verbs Reference](reference-verbs.md) for more on Miller's subcommands ``cat``, ``cut``, ``head``, ``sort``, ``tac``, ``tail``, ``top``, and ``uniq``, as well as [DSL reference](reference-dsl.md) for more on the awk-like ``mlr filter`` and ``mlr put``. +See [Verbs Reference](reference-verbs.md) for more on Miller's subcommands `cat`, `cut`, `head`, `sort`, `tac`, `tail`, `top`, and `uniq`, as well as [DSL reference](reference-dsl.md) for more on the awk-like `mlr filter` and `mlr put`. diff --git a/docs6b/docs/features.md b/docs6b/docs/features.md index 4dcd197ec..476db199a 100644 --- a/docs6b/docs/features.md +++ b/docs6b/docs/features.md @@ -19,11 +19,11 @@ including but not limited to the familiar CSV, TSV, and JSON. (Miller can handl * Miller complements **data-analysis tools** such as **R**, **pandas**, etc.: you can use Miller to **clean** and **prepare** your data. While you can do **basic statistics** entirely in Miller, its streaming-data feature and single-pass algorithms enable you to **reduce very large data sets**. -* Miller complements SQL **databases**: you can slice, dice, and reformat data on the client side on its way into or out of a database. (Examples :ref:`here ` and :ref:`here `.) You can also reap some of the benefits of databases for quick, setup-free one-off tasks when you just need to query some data in disk files in a hurry. +* Miller complements SQL **databases**: you can slice, dice, and reformat data on the client side on its way into or out of a database. (See [SQL Examples](sql-examples.md).) You can also reap some of the benefits of databases for quick, setup-free one-off tasks when you just need to query some data in disk files in a hurry. * Miller also goes beyond the classic Unix tools by stepping fully into our modern, **no-SQL** world: its essential record-heterogeneity property allows Miller to operate on data where records with different schema (field names) are interleaved. -* Miller is **streaming**: most operations need only a single record in memory at a time, rather than ingesting all input before producing any output. For those operations which require deeper retention (``sort``, ``tac``, ``stats1``), Miller retains only as much data as needed. This means that whenever functionally possible, you can operate on files which are larger than your system's available RAM, and you can use Miller in **tail -f** contexts. +* Miller is **streaming**: most operations need only a single record in memory at a time, rather than ingesting all input before producing any output. For those operations which require deeper retention (`sort`, `tac`, `stats1`), Miller retains only as much data as needed. This means that whenever functionally possible, you can operate on files which are larger than your system's available RAM, and you can use Miller in **tail -f** contexts. * Miller is **pipe-friendly** and interoperates with the Unix toolkit @@ -31,10 +31,10 @@ including but not limited to the familiar CSV, TSV, and JSON. (Miller can handl * Miller does **conversion** between formats -* Miller's **processing is format-aware**: e.g. CSV ``sort`` and ``tac`` keep header lines first +* Miller's **processing is format-aware**: e.g. CSV `sort` and `tac` keep header lines first * Miller has high-throughput **performance** on par with the Unix toolkit -* Not unlike [jq](https://stedolan.github.io/jq/) (for JSON), Miller is written in Go which is a portable, modern language, and Miller has no runtime dependencies. You can download or compile a single binary, ``scp`` it to a faraway machine, and expect it to work. +* Not unlike [jq](https://stedolan.github.io/jq/) (for JSON), Miller is written in Go which is a portable, modern language, and Miller has no runtime dependencies. You can download or compile a single binary, `scp` it to a faraway machine, and expect it to work. -Releases and release notes: https://github.com/johnkerl/miller/releases. +Releases and release notes: [https://github.com/johnkerl/miller/releases](https://github.com/johnkerl/miller/releases). diff --git a/docs6b/docs/features.md.in b/docs6b/docs/features.md.in index 68b333e1b..c914a464f 100644 --- a/docs6b/docs/features.md.in +++ b/docs6b/docs/features.md.in @@ -18,11 +18,11 @@ including but not limited to the familiar CSV, TSV, and JSON. (Miller can handl * Miller complements **data-analysis tools** such as **R**, **pandas**, etc.: you can use Miller to **clean** and **prepare** your data. While you can do **basic statistics** entirely in Miller, its streaming-data feature and single-pass algorithms enable you to **reduce very large data sets**. -* Miller complements SQL **databases**: you can slice, dice, and reformat data on the client side on its way into or out of a database. (Examples :ref:`here ` and :ref:`here `.) You can also reap some of the benefits of databases for quick, setup-free one-off tasks when you just need to query some data in disk files in a hurry. +* Miller complements SQL **databases**: you can slice, dice, and reformat data on the client side on its way into or out of a database. (See [SQL Examples](sql-examples.md).) You can also reap some of the benefits of databases for quick, setup-free one-off tasks when you just need to query some data in disk files in a hurry. * Miller also goes beyond the classic Unix tools by stepping fully into our modern, **no-SQL** world: its essential record-heterogeneity property allows Miller to operate on data where records with different schema (field names) are interleaved. -* Miller is **streaming**: most operations need only a single record in memory at a time, rather than ingesting all input before producing any output. For those operations which require deeper retention (``sort``, ``tac``, ``stats1``), Miller retains only as much data as needed. This means that whenever functionally possible, you can operate on files which are larger than your system's available RAM, and you can use Miller in **tail -f** contexts. +* Miller is **streaming**: most operations need only a single record in memory at a time, rather than ingesting all input before producing any output. For those operations which require deeper retention (`sort`, `tac`, `stats1`), Miller retains only as much data as needed. This means that whenever functionally possible, you can operate on files which are larger than your system's available RAM, and you can use Miller in **tail -f** contexts. * Miller is **pipe-friendly** and interoperates with the Unix toolkit @@ -30,10 +30,10 @@ including but not limited to the familiar CSV, TSV, and JSON. (Miller can handl * Miller does **conversion** between formats -* Miller's **processing is format-aware**: e.g. CSV ``sort`` and ``tac`` keep header lines first +* Miller's **processing is format-aware**: e.g. CSV `sort` and `tac` keep header lines first * Miller has high-throughput **performance** on par with the Unix toolkit -* Not unlike [jq](https://stedolan.github.io/jq/) (for JSON), Miller is written in Go which is a portable, modern language, and Miller has no runtime dependencies. You can download or compile a single binary, ``scp`` it to a faraway machine, and expect it to work. +* Not unlike [jq](https://stedolan.github.io/jq/) (for JSON), Miller is written in Go which is a portable, modern language, and Miller has no runtime dependencies. You can download or compile a single binary, `scp` it to a faraway machine, and expect it to work. -Releases and release notes: https://github.com/johnkerl/miller/releases. +Releases and release notes: [https://github.com/johnkerl/miller/releases](https://github.com/johnkerl/miller/releases). diff --git a/docs6b/docs/file-formats.md b/docs6b/docs/file-formats.md index 07005b33f..7ef37c8f0 100644 --- a/docs6b/docs/file-formats.md +++ b/docs6b/docs/file-formats.md @@ -7,8 +7,10 @@ Additionally, Miller gives you the option of including comments within your data ## Examples -
+
 mlr help data-formats
+
+
 CSV/CSV-lite: comma-separated values with separate header line
 TSV: same but with tabs in places of commas
 +---------------------+
@@ -71,28 +73,26 @@ NIDX: implicitly numerically indexed (Unix-toolkit style)
 +---------------------+
 
-.. _file-formats-csv: - ## CSV/TSV/ASV/USV/etc. -When ``mlr`` is invoked with the ``--csv`` or ``--csvlite`` option, key names are found on the first record and values are taken from subsequent records. This includes the case of CSV-formatted files. See [Record Heterogeneity](record-heterogeneity.md) for how Miller handles changes of field names within a single data stream. +When `mlr` is invoked with the `--csv` or `--csvlite` option, key names are found on the first record and values are taken from subsequent records. This includes the case of CSV-formatted files. See [Record Heterogeneity](record-heterogeneity.md) for how Miller handles changes of field names within a single data stream. -Miller has record separator ``RS`` and field separator ``FS``, just as ``awk`` does. For TSV, use ``--fs tab``; to convert TSV to CSV, use ``--ifs tab --ofs comma``, etc. (See also :ref:`reference-separators`.) +Miller has record separator `RS` and field separator `FS`, just as `awk` does. For TSV, use `--fs tab`; to convert TSV to CSV, use `--ifs tab --ofs comma`, etc. (See also [Record/field/pair separators](reference-main-io-options.md#recordfieldpair-separators).) **TSV (tab-separated values):** the following are synonymous pairs: -* ``--tsv`` and ``--csv --fs tab`` -* ``--itsv`` and ``--icsv --ifs tab`` -* ``--otsv`` and ``--ocsv --ofs tab`` -* ``--tsvlite`` and ``--csvlite --fs tab`` -* ``--itsvlite`` and ``--icsvlite --ifs tab`` -* ``--otsvlite`` and ``--ocsvlite --ofs tab`` +* `--tsv` and `--csv --fs tab` +* `--itsv` and `--icsv --ifs tab` +* `--otsv` and `--ocsv --ofs tab` +* `--tsvlite` and `--csvlite --fs tab` +* `--itsvlite` and `--icsvlite --ifs tab` +* `--otsvlite` and `--ocsvlite --ofs tab` -**ASV (ASCII-separated values):** the flags ``--asv``, ``--iasv``, ``--oasv``, ``--asvlite``, ``--iasvlite``, and ``--oasvlite`` are analogous except they use ASCII FS and RS 0x1f and 0x1e, respectively. +**ASV (ASCII-separated values):** the flags `--asv`, `--iasv`, `--oasv`, `--asvlite`, `--iasvlite`, and `--oasvlite` are analogous except they use ASCII FS and RS 0x1f and 0x1e, respectively. -**USV (Unicode-separated values):** likewise, the flags ``--usv``, ``--iusv``, ``--ousv``, ``--usvlite``, ``--iusvlite``, and ``--ousvlite`` use Unicode FS and RS U+241F (UTF-8 0x0xe2909f) and U+241E (UTF-8 0xe2909e), respectively. +**USV (Unicode-separated values):** likewise, the flags `--usv`, `--iusv`, `--ousv`, `--usvlite`, `--iusvlite`, and `--ousvlite` use Unicode FS and RS U+241F (UTF-8 0x0xe2909f) and U+241E (UTF-8 0xe2909e), respectively. -Miller's ``--csv`` flag supports [RFC-4180 CSV](https://tools.ietf.org/html/rfc4180). This includes CRLF line-terminators by default, regardless of platform. +Miller's `--csv` flag supports [RFC-4180 CSV](https://tools.ietf.org/html/rfc4180). This includes CRLF line-terminators by default, regardless of platform. Here are the differences between CSV and CSV-lite: @@ -106,16 +106,16 @@ Here are things they have in common: * The ability to specify record/field separators other than the default, e.g. CR-LF vs. LF, or tab instead of comma for TSV, and so on. -* The ``--implicit-csv-header`` flag for input and the ``--headerless-csv-output`` flag for output. - -.. _file-formats-dkvp: +* The `--implicit-csv-header` flag for input and the `--headerless-csv-output` flag for output. ## DKVP: Key-value pairs Miller's default file format is DKVP, for **delimited key-value pairs**. Example: -
+
 mlr cat data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -125,51 +125,51 @@ a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
 Such data are easy to generate, e.g. in Ruby with
 
-
+
 puts "host=#{hostname},seconds=#{t2-t1},message=#{msg}"
 
-
+
 puts mymap.collect{|k,v| "#{k}=#{v}"}.join(',')
 
-or ``print`` statements in various languages, e.g. +or `print` statements in various languages, e.g. -
+
 echo "type=3,user=$USER,date=$date\n";
 
-
+
 logger.log("type=3,user=$USER,date=$date\n");
 
-Fields lacking an IPS will have positional index (starting at 1) used as the key, as in NIDX format. For example, ``dish=7,egg=8,flint`` is parsed as ``"dish" => "7", "egg" => "8", "3" => "flint"`` and ``dish,egg,flint`` is parsed as ``"1" => "dish", "2" => "egg", "3" => "flint"``. +Fields lacking an IPS will have positional index (starting at 1) used as the key, as in NIDX format. For example, `dish=7,egg=8,flint` is parsed as `"dish" => "7", "egg" => "8", "3" => "flint"` and `dish,egg,flint` is parsed as `"1" => "dish", "2" => "egg", "3" => "flint"`. As discussed in [Record Heterogeneity](record-heterogeneity.md), Miller handles changes of field names within the same data stream. But using DKVP format this is particularly natural. One of my favorite use-cases for Miller is in application/server logs, where I log all sorts of lines such as -
+
 resource=/path/to/file,loadsec=0.45,ok=true
 record_count=100, resource=/path/to/file
 resource=/some/other/path,loadsec=0.97,ok=false
 
-etc. and I just log them as needed. Then later, I can use ``grep``, ``mlr --opprint group-like``, etc. +etc. and I just log them as needed. Then later, I can use `grep`, `mlr --opprint group-like`, etc. to analyze my logs. -See :doc:`reference-main-io-options` regarding how to specify separators other than the default equals-sign and comma. - -.. _file-formats-nidx: +See [Reference: I/O options](reference-main-io-options.md) regarding how to specify separators other than the default equals-sign and comma. ## NIDX: Index-numbered (toolkit style) -With ``--inidx --ifs ' ' --repifs``, Miller splits lines on whitespace and assigns integer field names starting with 1. +With `--inidx --ifs ' ' --repifs`, Miller splits lines on whitespace and assigns integer field names starting with 1. This recapitulates Unix-toolkit behavior. Example with index-numbered output: -
+
 cat data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -177,8 +177,10 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
-
+
 mlr --onidx --ofs ' ' cat data/small
+
+
 pan pan 1 0.3467901443380824 0.7268028627434533
 eks pan 2 0.7586799647899636 0.5221511083334797
 wye wye 3 0.20460330576630303 0.33831852551664776
@@ -188,15 +190,19 @@ wye pan 5 0.5732889198020006 0.8636244699032729
 
 Example with index-numbered input:
 
-
+
 cat data/mydata.txt
+
+
 oh say can you see
 by the dawn's
 early light
 
-
+
 mlr --inidx --ifs ' ' --odkvp cat data/mydata.txt
+
+
 1=oh,2=say,3=can,4=you,5=see
 1=by,2=the,3=dawn's
 1=early,2=light
@@ -204,23 +210,25 @@ early light
 
 Example with index-numbered input and output:
 
-
+
 cat data/mydata.txt
+
+
 oh say can you see
 by the dawn's
 early light
 
-
+
 mlr --nidx --fs ' ' --repifs cut -f 2,3 data/mydata.txt
+
+
 say can
 the dawn's
 light
 
-.. _file-formats-json: - -## Tabular JSON +## JSON JSON is a format which supports arbitrarily deep nesting of "objects" (hashmaps) and "arrays" (lists), while Miller is a tool for handling **tabular data** only. This means Miller cannot (and should not) handle arbitrary JSON. (Check out [jq](https://stedolan.github.io/jq/).) @@ -232,8 +240,10 @@ By *tabular JSON* I mean the data is either a sequence of one or more objects, o An **array of single-level objects** is, quite simply, **a table**: -
+
 mlr --json head -n 2 then cut -f color,shape data/json-example-1.json
+
+
 {
   "color": "yellow",
   "shape": "triangle"
@@ -244,8 +254,10 @@ An **array of single-level objects** is, quite simply, **a table**:
 }
 
-
+
 mlr --json --jvstack head -n 2 then cut -f color,u,v data/json-example-1.json
+
+
 {
   "color": "yellow",
   "u": 0.6321695890307647,
@@ -258,8 +270,10 @@ An **array of single-level objects** is, quite simply, **a table**:
 }
 
-
+
 mlr --ijson --opprint stats1 -a mean,stddev,count -f u -g shape data/json-example-1.json
+
+
 shape    u_mean              u_stddev            u_count
 triangle 0.5839952367477192  0.13118354465618046 3
 square   0.409355036804889   0.3654281755508655  4
@@ -270,8 +284,10 @@ circle   0.36601268553826866 0.2090944565900053  3
 
 Additionally, Miller can **tabularize nested objects by concatentating keys**:
 
-
+
 mlr --json --jvstack head -n 2 data/json-example-2.json
+
+
 {
   "flag": 1,
   "i": 11,
@@ -302,8 +318,10 @@ Additionally, Miller can **tabularize nested objects by concatentating keys**:
 }
 
-
+
 mlr --ijson --opprint head -n 4 data/json-example-2.json
+
+
 flag i  attributes.color attributes.shape values.u values.v values.w values.x
 1    11 yellow           triangle         0.632170 0.988721 0.436498 5.798188
 1    15 red              square           0.219668 0.001257 0.792778 2.944117
@@ -311,12 +329,14 @@ flag i  attributes.color attributes.shape values.u values.v values.w values.x
 0    48 red              square           0.956274 0.746720 0.775542 7.117831
 
-Note in particular that as far as Miller's ``put`` and ``filter``, as well as other I/O formats, are concerned, these are simply field names with colons in them: +Note in particular that as far as Miller's `put` and `filter`, as well as other I/O formats, are concerned, these are simply field names with colons in them: -
+
 mlr --json --jvstack head -n 1 \
   then put '${values:uv} = ${values:u} * ${values:v}' \
   data/json-example-2.json
+
+
 {
   "flag": 1,
   "i": 11,
@@ -335,12 +355,14 @@ Note in particular that as far as Miller's ``put`` and ``filter``, as well as ot
 
 ### Arrays
 
-Arrays (TODO: update for Miller6) aren't supported in Miller's ``put``/``filter`` DSL. By default, JSON arrays are read in as integer-keyed maps.
+Arrays (TODO: update for Miller6) aren't supported in Miller's `put`/`filter` DSL. By default, JSON arrays are read in as integer-keyed maps.
 
 Suppose we have arrays like this in our input data:
 
-
+
 cat data/json-example-3.json
+
+
 {
   "label": "orange",
   "values": [12.2, 13.8, 17.2]
@@ -353,8 +375,10 @@ Suppose we have arrays like this in our input data:
 
 Then integer indices (starting from 0 and counting up) are used as map keys:
 
-
+
 mlr --ijson --oxtab cat data/json-example-3.json
+
+
 label    orange
 values.1 12.2
 values.2 13.8
@@ -367,8 +391,10 @@ values.2 32.4
 
 When the data are written back out as JSON, field names are re-expanded as above, but what were arrays on input are now maps on output:
 
-
+
 mlr --json --jvstack cat data/json-example-3.json
+
+
 {
   "label": "orange",
   "values": [12.2, 13.8, 17.2]
@@ -381,34 +407,34 @@ When the data are written back out as JSON, field names are re-expanded as above
 
 This is non-ideal, but it allows Miller (5.x release being latest as of this writing) to handle JSON arrays at all.
 
-You might also use ``mlr --json-skip-arrays-on-input`` or ``mlr --json-fatal-arrays-on-input``.
+You might also use `mlr --json-skip-arrays-on-input` or `mlr --json-fatal-arrays-on-input`.
 
 To truly handle JSON, please use a JSON-processing tool such as [jq](https://stedolan.github.io/jq/).
 
 ### Formatting JSON options
 
-JSON isn't a parameterized format, so ``RS``, ``FS``, ``PS`` aren't specifiable. Nonetheless, you can do the following:
+JSON isn't a parameterized format, so `RS`, `FS`, `PS` aren't specifiable. Nonetheless, you can do the following:
 
-* Use ``--jvstack`` to pretty-print JSON objects with multi-line (vertically stacked) spacing. By default, each Miller record (JSON object) is one per line.
+* Use `--jvstack` to pretty-print JSON objects with multi-line (vertically stacked) spacing. By default, each Miller record (JSON object) is one per line.
 
-* Keystroke-savers: ``--jsonx`` simply means ``--json --jvstack``, and ``--ojsonx`` simply means ``--ojson --jvstack``.
+* Keystroke-savers: `--jsonx` simply means `--json --jvstack`, and `--ojsonx` simply means `--ojson --jvstack`.
 
-* Use ``--jlistwrap`` to print the sequence of JSON objects wrapped in an outermost ``[`` and ``]``. By default, these aren't printed.
+* Use `--jlistwrap` to print the sequence of JSON objects wrapped in an outermost `[` and `]`. By default, these aren't printed.
 
-* Use ``--jquoteall`` to double-quote all object values. By default, integers, floating-point numbers, and booleans ``true`` and ``false`` are not double-quoted when they appear as JSON-object keys.
+* Use `--jquoteall` to double-quote all object values. By default, integers, floating-point numbers, and booleans `true` and `false` are not double-quoted when they appear as JSON-object keys.
 
-* Use ``--jflatsep youmd.inghere`` to specify the string used for key concatenation: this defaults to a single colon.
+* Use `--jflatsep youmd.inghere` to specify the string used for key concatenation: this defaults to a single colon.
 
 Again, please see [jq](https://stedolan.github.io/jq/) for a truly powerful, JSON-specific tool.
 
-.. _file-formats-pprint:
-
 ## PPRINT: Pretty-printed tabular
 
 Miller's pretty-print format is like CSV, but column-aligned.  For example, compare
 
-
+
 mlr --ocsv cat data/small
+
+
 a,b,i,x,y
 pan,pan,1,0.3467901443380824,0.7268028627434533
 eks,pan,2,0.7586799647899636,0.5221511083334797
@@ -417,8 +443,10 @@ eks,wye,4,0.38139939387114097,0.13418874328430463
 wye,pan,5,0.5732889198020006,0.8636244699032729
 
-
+
 mlr --opprint cat data/small
+
+
 a   b   i x                   y
 pan pan 1 0.3467901443380824  0.7268028627434533
 eks pan 2 0.7586799647899636  0.5221511083334797
@@ -427,14 +455,16 @@ eks wye 4 0.38139939387114097 0.13418874328430463
 wye pan 5 0.5732889198020006  0.8636244699032729
 
-Note that while Miller is a line-at-a-time processor and retains input lines in memory only where necessary (e.g. for sort), pretty-print output requires it to accumulate all input lines (so that it can compute maximum column widths) before producing any output. This has two consequences: (a) pretty-print output won't work on ``tail -f`` contexts, where Miller will be waiting for an end-of-file marker which never arrives; (b) pretty-print output for large files is constrained by available machine memory. +Note that while Miller is a line-at-a-time processor and retains input lines in memory only where necessary (e.g. for sort), pretty-print output requires it to accumulate all input lines (so that it can compute maximum column widths) before producing any output. This has two consequences: (a) pretty-print output won't work on `tail -f` contexts, where Miller will be waiting for an end-of-file marker which never arrives; (b) pretty-print output for large files is constrained by available machine memory. See [Record Heterogeneity](record-heterogeneity.md) for how Miller handles changes of field names within a single data stream. -For output only (this isn't supported in the input-scanner as of 5.0.0) you can use ``--barred`` with pprint output format: +For output only (this isn't supported in the input-scanner as of 5.0.0) you can use `--barred` with pprint output format: -
+
 mlr --opprint --barred cat data/small
+
+
 +-----+-----+---+---------------------+---------------------+
 | a   | b   | i | x                   | y                   |
 +-----+-----+---+---------------------+---------------------+
@@ -446,14 +476,12 @@ For output only (this isn't supported in the input-scanner as of 5.0.0) you can
 +-----+-----+---+---------------------+---------------------+
 
-.. _file-formats-xtab: - ## XTAB: Vertical tabular This is perhaps most useful for looking a very wide and/or multi-column data which causes line-wraps on the screen (but see also [ngrid](https://github.com/twosigma/ngrid/) for an entirely different, very powerful option). Namely: -
+
 $ grep -v '^#' /etc/passwd | head -n 6 | mlr --nidx --fs : --opprint cat
 1          2 3  4  5                          6               7
 nobody     * -2 -2 Unprivileged User          /var/empty      /usr/bin/false
@@ -464,7 +492,7 @@ _taskgated * 13 13 Task Gate Daemon           /var/empty      /usr/bin/false
 _networkd  * 24 24 Network Services           /var/networkd   /usr/bin/false
 
-
+
 $ grep -v '^#' /etc/passwd | head -n 2 | mlr --nidx --fs : --oxtab cat
 1 nobody
 2 *
@@ -483,7 +511,7 @@ $ grep -v '^#' /etc/passwd | head -n 2 | mlr --nidx --fs : --oxtab cat
 7 /bin/sh
 
-
+
 $ grep -v '^#' /etc/passwd | head -n 2 | \
   mlr --nidx --fs : --ojson --jvstack --jlistwrap label name,password,uid,gid,gecos,home_dir,shell
 [
@@ -512,8 +540,10 @@ $ grep -v '^#' /etc/passwd | head -n 2 | \
 
 Markdown format looks like this:
 
-
+
 mlr --omd cat data/small
+
+
 | a | b | i | x | y |
 | --- | --- | --- | --- | --- |
 | pan | pan | 1 | 0.3467901443380824 | 0.7268028627434533 |
@@ -531,10 +561,12 @@ As of Miller 4.3.0, markdown format is supported only for output, not input.
 
 ## Data-conversion keystroke-savers
 
-While you can do format conversion using ``mlr --icsv --ojson cat myfile.csv``, there are also keystroke-savers for this purpose, such as ``mlr --c2j cat myfile.csv``.  For a complete list:
+While you can do format conversion using `mlr --icsv --ojson cat myfile.csv`, there are also keystroke-savers for this purpose, such as `mlr --c2j cat myfile.csv`.  For a complete list:
 
-
+
 mlr help format-conversion
+
+
 As keystroke-savers for format-conversion you may use the following:
 --c2t --c2d --c2n --c2j --c2x --c2p --c2m
 --t2c       --t2d --t2n --t2j --t2x --t2p --t2m
@@ -550,24 +582,26 @@ output only.
 
 ## Autodetect of line endings
 
-Default line endings (``--irs`` and ``--ors``) are ``'auto'`` which means **autodetect from the input file format**, as long as the input file(s) have lines ending in either LF (also known as linefeed, ``'\n'``, ``0x0a``, Unix-style) or CRLF (also known as carriage-return/linefeed pairs, ``'\r\n'``, ``0x0d 0x0a``, Windows style).
+Default line endings (`--irs` and `--ors`) are `'auto'` which means **autodetect from the input file format**, as long as the input file(s) have lines ending in either LF (also known as linefeed, `'\n'`, `0x0a`, Unix-style) or CRLF (also known as carriage-return/linefeed pairs, `'\r\n'`, `0x0d 0x0a`, Windows style).
 
 **If both IRS and ORS are auto (which is the default) then LF input will lead to LF output and CRLF input will lead to CRLF output, regardless of the platform you're running on.**
 
 The line-ending autodetector triggers on the first line ending detected in the input stream. E.g. if you specify a CRLF-terminated file on the command line followed by an LF-terminated file then autodetected line endings will be CRLF.
 
-If you use ``--ors {something else}`` with (default or explicitly specified) ``--irs auto`` then line endings are autodetected on input and set to what you specify on output.
+If you use `--ors {something else}` with (default or explicitly specified) `--irs auto` then line endings are autodetected on input and set to what you specify on output.
 
-If you use ``--irs {something else}`` with (default or explicitly specified) ``--ors auto`` then the output line endings used are LF on Unix/Linux/BSD/MacOSX, and CRLF on Windows.
+If you use `--irs {something else}` with (default or explicitly specified) `--ors auto` then the output line endings used are LF on Unix/Linux/BSD/MacOSX, and CRLF on Windows.
 
-See also :ref:`reference-separators` for more information about record/field/pair separators.
+See also [Record/field/pair separators](reference-main-io-options.md#recordfieldpair-separators) for more information about record/field/pair separators.
 
 ## Comments in data
 
 You can include comments within your data files, and either have them ignored, or passed directly through to the standard output as soon as they are encountered:
 
-
+
 mlr help comments-in-data
+
+
 --skip-comments                 Ignore commented lines (prefixed by "#")
                                 within the input.
 --skip-comments-with {string}   Ignore commented lines within input, with
@@ -589,8 +623,10 @@ Notes:
 
 Examples:
 
-
+
 cat data/budget.csv
+
+
 # Asana -- here are the budget figures you asked for!
 type,quantity
 purple,456.78
@@ -598,16 +634,20 @@ green,678.12
 orange,123.45
 
-
+
 mlr --skip-comments --icsv --opprint sort -nr quantity data/budget.csv
+
+
 type   quantity
 green  678.12
 purple 456.78
 orange 123.45
 
-
+
 mlr --pass-comments --icsv --opprint sort -nr quantity data/budget.csv
+
+
 # Asana -- here are the budget figures you asked for!
 type   quantity
 green  678.12
diff --git a/docs6b/docs/file-formats.md.in b/docs6b/docs/file-formats.md.in
index 8ed7cc355..fa2bf9d26 100644
--- a/docs6b/docs/file-formats.md.in
+++ b/docs6b/docs/file-formats.md.in
@@ -10,28 +10,26 @@ GENMD_RUN_COMMAND
 mlr help data-formats
 GENMD_EOF
 
-.. _file-formats-csv:
-
 ## CSV/TSV/ASV/USV/etc.
 
-When ``mlr`` is invoked with the ``--csv`` or ``--csvlite`` option, key names are found on the first record and values are taken from subsequent records.  This includes the case of CSV-formatted files.  See [Record Heterogeneity](record-heterogeneity.md) for how Miller handles changes of field names within a single data stream.
+When `mlr` is invoked with the `--csv` or `--csvlite` option, key names are found on the first record and values are taken from subsequent records.  This includes the case of CSV-formatted files.  See [Record Heterogeneity](record-heterogeneity.md) for how Miller handles changes of field names within a single data stream.
 
-Miller has record separator ``RS`` and field separator ``FS``, just as ``awk`` does.  For TSV, use ``--fs tab``; to convert TSV to CSV, use ``--ifs tab --ofs comma``, etc.  (See also :ref:`reference-separators`.)
+Miller has record separator `RS` and field separator `FS`, just as `awk` does.  For TSV, use `--fs tab`; to convert TSV to CSV, use `--ifs tab --ofs comma`, etc.  (See also [Record/field/pair separators](reference-main-io-options.md#recordfieldpair-separators).)
 
 **TSV (tab-separated values):** the following are synonymous pairs:
 
-* ``--tsv`` and ``--csv --fs tab``
-* ``--itsv`` and ``--icsv --ifs tab``
-* ``--otsv`` and ``--ocsv --ofs tab``
-* ``--tsvlite`` and ``--csvlite --fs tab``
-* ``--itsvlite`` and ``--icsvlite --ifs tab``
-* ``--otsvlite`` and ``--ocsvlite --ofs tab``
+* `--tsv` and `--csv --fs tab`
+* `--itsv` and `--icsv --ifs tab`
+* `--otsv` and `--ocsv --ofs tab`
+* `--tsvlite` and `--csvlite --fs tab`
+* `--itsvlite` and `--icsvlite --ifs tab`
+* `--otsvlite` and `--ocsvlite --ofs tab`
 
-**ASV (ASCII-separated values):** the flags ``--asv``, ``--iasv``, ``--oasv``, ``--asvlite``, ``--iasvlite``, and ``--oasvlite`` are analogous except they use ASCII FS and RS 0x1f and 0x1e, respectively.
+**ASV (ASCII-separated values):** the flags `--asv`, `--iasv`, `--oasv`, `--asvlite`, `--iasvlite`, and `--oasvlite` are analogous except they use ASCII FS and RS 0x1f and 0x1e, respectively.
 
-**USV (Unicode-separated values):** likewise, the flags ``--usv``, ``--iusv``, ``--ousv``, ``--usvlite``, ``--iusvlite``, and ``--ousvlite`` use Unicode FS and RS U+241F (UTF-8 0x0xe2909f) and U+241E (UTF-8 0xe2909e), respectively.
+**USV (Unicode-separated values):** likewise, the flags `--usv`, `--iusv`, `--ousv`, `--usvlite`, `--iusvlite`, and `--ousvlite` use Unicode FS and RS U+241F (UTF-8 0x0xe2909f) and U+241E (UTF-8 0xe2909e), respectively.
 
-Miller's ``--csv`` flag supports [RFC-4180 CSV](https://tools.ietf.org/html/rfc4180). This includes CRLF line-terminators by default, regardless of platform.
+Miller's `--csv` flag supports [RFC-4180 CSV](https://tools.ietf.org/html/rfc4180). This includes CRLF line-terminators by default, regardless of platform.
 
 Here are the differences between CSV and CSV-lite:
 
@@ -45,9 +43,7 @@ Here are things they have in common:
 
 * The ability to specify record/field separators other than the default, e.g. CR-LF vs. LF, or tab instead of comma for TSV, and so on.
 
-* The ``--implicit-csv-header`` flag for input and the ``--headerless-csv-output`` flag for output.
-
-.. _file-formats-dkvp:
+* The `--implicit-csv-header` flag for input and the `--headerless-csv-output` flag for output.
 
 ## DKVP: Key-value pairs
 
@@ -67,7 +63,7 @@ GENMD_CARDIFY
 puts mymap.collect{|k,v| "#{k}=#{v}"}.join(',')
 GENMD_EOF
 
-or ``print`` statements in various languages, e.g.
+or `print` statements in various languages, e.g.
 
 GENMD_CARDIFY
 echo "type=3,user=$USER,date=$date\n";
@@ -77,7 +73,7 @@ GENMD_CARDIFY
 logger.log("type=3,user=$USER,date=$date\n");
 GENMD_EOF
 
-Fields lacking an IPS will have positional index (starting at 1) used as the key, as in NIDX format. For example, ``dish=7,egg=8,flint`` is parsed as ``"dish" => "7", "egg" => "8", "3" => "flint"`` and ``dish,egg,flint`` is parsed as ``"1" => "dish", "2" => "egg", "3" => "flint"``.
+Fields lacking an IPS will have positional index (starting at 1) used as the key, as in NIDX format. For example, `dish=7,egg=8,flint` is parsed as `"dish" => "7", "egg" => "8", "3" => "flint"` and `dish,egg,flint` is parsed as `"1" => "dish", "2" => "egg", "3" => "flint"`.
 
 As discussed in [Record Heterogeneity](record-heterogeneity.md), Miller handles changes of field names within the same data stream. But using DKVP format this is particularly natural. One of my favorite use-cases for Miller is in application/server logs, where I log all sorts of lines such as
 
@@ -87,16 +83,14 @@ record_count=100, resource=/path/to/file
 resource=/some/other/path,loadsec=0.97,ok=false
 GENMD_EOF
 
-etc. and I just log them as needed. Then later, I can use ``grep``, ``mlr --opprint group-like``, etc.
+etc. and I just log them as needed. Then later, I can use `grep`, `mlr --opprint group-like`, etc.
 to analyze my logs.
 
-See :doc:`reference-main-io-options` regarding how to specify separators other than the default equals-sign and comma.
-
-.. _file-formats-nidx:
+See [Reference: I/O options](reference-main-io-options.md) regarding how to specify separators other than the default equals-sign and comma.
 
 ## NIDX: Index-numbered (toolkit style)
 
-With ``--inidx --ifs ' ' --repifs``, Miller splits lines on whitespace and assigns integer field names starting with 1.
+With `--inidx --ifs ' ' --repifs`, Miller splits lines on whitespace and assigns integer field names starting with 1.
 
 This recapitulates Unix-toolkit behavior.
 
@@ -130,9 +124,7 @@ GENMD_RUN_COMMAND
 mlr --nidx --fs ' ' --repifs cut -f 2,3 data/mydata.txt
 GENMD_EOF
 
-.. _file-formats-json:
-
-## Tabular JSON
+## JSON
 
 JSON is a format which supports arbitrarily deep nesting of "objects" (hashmaps) and "arrays" (lists), while Miller is a tool for handling **tabular data** only. This means Miller cannot (and should not) handle arbitrary JSON. (Check out [jq](https://stedolan.github.io/jq/).)
 
@@ -168,7 +160,7 @@ GENMD_RUN_COMMAND
 mlr --ijson --opprint head -n 4 data/json-example-2.json
 GENMD_EOF
 
-Note in particular that as far as Miller's ``put`` and ``filter``, as well as other I/O formats, are concerned, these are simply field names with colons in them:
+Note in particular that as far as Miller's `put` and `filter`, as well as other I/O formats, are concerned, these are simply field names with colons in them:
 
 GENMD_RUN_COMMAND
 mlr --json --jvstack head -n 1 \
@@ -178,7 +170,7 @@ GENMD_EOF
 
 ### Arrays
 
-Arrays (TODO: update for Miller6) aren't supported in Miller's ``put``/``filter`` DSL. By default, JSON arrays are read in as integer-keyed maps.
+Arrays (TODO: update for Miller6) aren't supported in Miller's `put`/`filter` DSL. By default, JSON arrays are read in as integer-keyed maps.
 
 Suppose we have arrays like this in our input data:
 
@@ -200,28 +192,26 @@ GENMD_EOF
 
 This is non-ideal, but it allows Miller (5.x release being latest as of this writing) to handle JSON arrays at all.
 
-You might also use ``mlr --json-skip-arrays-on-input`` or ``mlr --json-fatal-arrays-on-input``.
+You might also use `mlr --json-skip-arrays-on-input` or `mlr --json-fatal-arrays-on-input`.
 
 To truly handle JSON, please use a JSON-processing tool such as [jq](https://stedolan.github.io/jq/).
 
 ### Formatting JSON options
 
-JSON isn't a parameterized format, so ``RS``, ``FS``, ``PS`` aren't specifiable. Nonetheless, you can do the following:
+JSON isn't a parameterized format, so `RS`, `FS`, `PS` aren't specifiable. Nonetheless, you can do the following:
 
-* Use ``--jvstack`` to pretty-print JSON objects with multi-line (vertically stacked) spacing. By default, each Miller record (JSON object) is one per line.
+* Use `--jvstack` to pretty-print JSON objects with multi-line (vertically stacked) spacing. By default, each Miller record (JSON object) is one per line.
 
-* Keystroke-savers: ``--jsonx`` simply means ``--json --jvstack``, and ``--ojsonx`` simply means ``--ojson --jvstack``.
+* Keystroke-savers: `--jsonx` simply means `--json --jvstack`, and `--ojsonx` simply means `--ojson --jvstack`.
 
-* Use ``--jlistwrap`` to print the sequence of JSON objects wrapped in an outermost ``[`` and ``]``. By default, these aren't printed.
+* Use `--jlistwrap` to print the sequence of JSON objects wrapped in an outermost `[` and `]`. By default, these aren't printed.
 
-* Use ``--jquoteall`` to double-quote all object values. By default, integers, floating-point numbers, and booleans ``true`` and ``false`` are not double-quoted when they appear as JSON-object keys.
+* Use `--jquoteall` to double-quote all object values. By default, integers, floating-point numbers, and booleans `true` and `false` are not double-quoted when they appear as JSON-object keys.
 
-* Use ``--jflatsep youmd.inghere`` to specify the string used for key concatenation: this defaults to a single colon.
+* Use `--jflatsep youmd.inghere` to specify the string used for key concatenation: this defaults to a single colon.
 
 Again, please see [jq](https://stedolan.github.io/jq/) for a truly powerful, JSON-specific tool.
 
-.. _file-formats-pprint:
-
 ## PPRINT: Pretty-printed tabular
 
 Miller's pretty-print format is like CSV, but column-aligned.  For example, compare
@@ -234,18 +224,16 @@ GENMD_RUN_COMMAND
 mlr --opprint cat data/small
 GENMD_EOF
 
-Note that while Miller is a line-at-a-time processor and retains input lines in memory only where necessary (e.g. for sort), pretty-print output requires it to accumulate all input lines (so that it can compute maximum column widths) before producing any output. This has two consequences: (a) pretty-print output won't work on ``tail -f`` contexts, where Miller will be waiting for an end-of-file marker which never arrives; (b) pretty-print output for large files is constrained by available machine memory.
+Note that while Miller is a line-at-a-time processor and retains input lines in memory only where necessary (e.g. for sort), pretty-print output requires it to accumulate all input lines (so that it can compute maximum column widths) before producing any output. This has two consequences: (a) pretty-print output won't work on `tail -f` contexts, where Miller will be waiting for an end-of-file marker which never arrives; (b) pretty-print output for large files is constrained by available machine memory.
 
 See [Record Heterogeneity](record-heterogeneity.md) for how Miller handles changes of field names within a single data stream.
 
-For output only (this isn't supported in the input-scanner as of 5.0.0) you can use ``--barred`` with pprint output format:
+For output only (this isn't supported in the input-scanner as of 5.0.0) you can use `--barred` with pprint output format:
 
 GENMD_RUN_COMMAND
 mlr --opprint --barred cat data/small
 GENMD_EOF
 
-.. _file-formats-xtab:
-
 ## XTAB: Vertical tabular
 
 This is perhaps most useful for looking a very wide and/or multi-column data which causes line-wraps on the screen (but see also
@@ -273,7 +261,7 @@ As of Miller 4.3.0, markdown format is supported only for output, not input.
 
 ## Data-conversion keystroke-savers
 
-While you can do format conversion using ``mlr --icsv --ojson cat myfile.csv``, there are also keystroke-savers for this purpose, such as ``mlr --c2j cat myfile.csv``.  For a complete list:
+While you can do format conversion using `mlr --icsv --ojson cat myfile.csv`, there are also keystroke-savers for this purpose, such as `mlr --c2j cat myfile.csv`.  For a complete list:
 
 GENMD_RUN_COMMAND
 mlr help format-conversion
@@ -281,17 +269,17 @@ GENMD_EOF
 
 ## Autodetect of line endings
 
-Default line endings (``--irs`` and ``--ors``) are ``'auto'`` which means **autodetect from the input file format**, as long as the input file(s) have lines ending in either LF (also known as linefeed, ``'\n'``, ``0x0a``, Unix-style) or CRLF (also known as carriage-return/linefeed pairs, ``'\r\n'``, ``0x0d 0x0a``, Windows style).
+Default line endings (`--irs` and `--ors`) are `'auto'` which means **autodetect from the input file format**, as long as the input file(s) have lines ending in either LF (also known as linefeed, `'\n'`, `0x0a`, Unix-style) or CRLF (also known as carriage-return/linefeed pairs, `'\r\n'`, `0x0d 0x0a`, Windows style).
 
 **If both IRS and ORS are auto (which is the default) then LF input will lead to LF output and CRLF input will lead to CRLF output, regardless of the platform you're running on.**
 
 The line-ending autodetector triggers on the first line ending detected in the input stream. E.g. if you specify a CRLF-terminated file on the command line followed by an LF-terminated file then autodetected line endings will be CRLF.
 
-If you use ``--ors {something else}`` with (default or explicitly specified) ``--irs auto`` then line endings are autodetected on input and set to what you specify on output.
+If you use `--ors {something else}` with (default or explicitly specified) `--irs auto` then line endings are autodetected on input and set to what you specify on output.
 
-If you use ``--irs {something else}`` with (default or explicitly specified) ``--ors auto`` then the output line endings used are LF on Unix/Linux/BSD/MacOSX, and CRLF on Windows.
+If you use `--irs {something else}` with (default or explicitly specified) `--ors auto` then the output line endings used are LF on Unix/Linux/BSD/MacOSX, and CRLF on Windows.
 
-See also :ref:`reference-separators` for more information about record/field/pair separators.
+See also [Record/field/pair separators](reference-main-io-options.md#recordfieldpair-separators) for more information about record/field/pair separators.
 
 ## Comments in data
 
diff --git a/docs6b/docs/genmd-filter b/docs6b/docs/genmd-filter
index dae5e8ef4..e8c04de5b 100755
--- a/docs6b/docs/genmd-filter
+++ b/docs6b/docs/genmd-filter
@@ -132,16 +132,22 @@ end
 
 # ----------------------------------------------------------------
 def write_card(command_lines, output_lines, output_handle)
-  output_handle.puts("
")
 
-  command_lines.each do |command_line|
-    output_handle.puts(""+command_line+"")
+  if command_lines.length > 0
+    output_handle.puts('
')
+    command_lines.each do |command_line|
+      output_handle.puts(""+command_line+"")
+    end
+    output_handle.puts("
") end - output_lines.each do |output_line| - output_handle.puts(output_line) + if output_lines.length > 0 + output_handle.puts('
')
+    output_lines.each do |output_line|
+      output_handle.puts(output_line)
+    end
+    output_handle.puts("
") end - output_handle.puts("
") end diff --git a/docs6b/docs/index.md b/docs6b/docs/index.md index 5a1895ddd..f6583e235 100644 --- a/docs6b/docs/index.md +++ b/docs6b/docs/index.md @@ -8,11 +8,11 @@ In several senses, Miller is more than one tool: **Format conversion:** You can convert CSV files to JSON, or vice versa, or pretty-print your data horizontally or vertically to make it easier to read. -**Data manipulation:** With a few keystrokes you can remove columns you don't care about -- or, make new ones using expressions like ``$rate = $units / $seconds``. +**Data manipulation:** With a few keystrokes you can remove columns you don't care about -- or, make new ones using expressions like `$rate = $units / $seconds`. **Pre-processing/post-processing vs standalone use:** You can use Miller to clean data files and put them into standard formats, perhaps in preparation to load them into a database or a hands-off data-processing pipeline. Or you can use it post-process and summary database-query output. As well, you can use Miller to explore and analyze your data interactively. -**Compact verbs vs programming language:** For low-keystroking you can do things like ``mlr --csv sort -f name input.csv`` or ``mlr --json head -n 1 myfile.json``. The ``sort``, ``head``, etc are called *verbs*. They're analogs of familiar command-line tools like ``sort``, ``head``, and so on -- but they're aware of name-indexed, multi-line file formats like CSV and JSON. In addition, though, using Miller's ``put`` verb you can use programming-language statements for expressions like ``$rate = $units / $seconds`` which allow you to succintly express your own logic. +**Compact verbs vs programming language:** For low-keystroking you can do things like `mlr --csv sort -f name input.csv` or `mlr --json head -n 1 myfile.json`. The `sort`, `head`, etc are called *verbs*. They're analogs of familiar command-line tools like `sort`, `head`, and so on -- but they're aware of name-indexed, multi-line file formats like CSV and JSON. In addition, though, using Miller's `put` verb you can use programming-language statements for expressions like `$rate = $units / $seconds` which allow you to succintly express your own logic. **Multiple domains:** People use Miller for data analysis, data science, software engineering, devops/system-administration, journalism, scientific research, and more. diff --git a/docs6b/docs/index.md.in b/docs6b/docs/index.md.in index a3fe96de3..8475c7879 100644 --- a/docs6b/docs/index.md.in +++ b/docs6b/docs/index.md.in @@ -7,11 +7,11 @@ In several senses, Miller is more than one tool: **Format conversion:** You can convert CSV files to JSON, or vice versa, or pretty-print your data horizontally or vertically to make it easier to read. -**Data manipulation:** With a few keystrokes you can remove columns you don't care about -- or, make new ones using expressions like ``$rate = $units / $seconds``. +**Data manipulation:** With a few keystrokes you can remove columns you don't care about -- or, make new ones using expressions like `$rate = $units / $seconds`. **Pre-processing/post-processing vs standalone use:** You can use Miller to clean data files and put them into standard formats, perhaps in preparation to load them into a database or a hands-off data-processing pipeline. Or you can use it post-process and summary database-query output. As well, you can use Miller to explore and analyze your data interactively. -**Compact verbs vs programming language:** For low-keystroking you can do things like ``mlr --csv sort -f name input.csv`` or ``mlr --json head -n 1 myfile.json``. The ``sort``, ``head``, etc are called *verbs*. They're analogs of familiar command-line tools like ``sort``, ``head``, and so on -- but they're aware of name-indexed, multi-line file formats like CSV and JSON. In addition, though, using Miller's ``put`` verb you can use programming-language statements for expressions like ``$rate = $units / $seconds`` which allow you to succintly express your own logic. +**Compact verbs vs programming language:** For low-keystroking you can do things like `mlr --csv sort -f name input.csv` or `mlr --json head -n 1 myfile.json`. The `sort`, `head`, etc are called *verbs*. They're analogs of familiar command-line tools like `sort`, `head`, and so on -- but they're aware of name-indexed, multi-line file formats like CSV and JSON. In addition, though, using Miller's `put` verb you can use programming-language statements for expressions like `$rate = $units / $seconds` which allow you to succintly express your own logic. **Multiple domains:** People use Miller for data analysis, data science, software engineering, devops/system-administration, journalism, scientific research, and more. diff --git a/docs6b/docs/installation.md b/docs6b/docs/installation.md index 9c21ae45c..6a327572d 100644 --- a/docs6b/docs/installation.md +++ b/docs6b/docs/installation.md @@ -9,33 +9,33 @@ Until then, please see the following sections for how to get Miller 6.* [Homebrew](https://brew.sh/) installation support for OSX is available via -
+
 brew update && brew install miller
 
...and also via [MacPorts](https://www.macports.org/): -
+
 sudo port selfupdate && sudo port install miller
 
-You may already have the ``mlr`` executable available in your platform's package manager on NetBSD, Debian Linux, Ubuntu Xenial and upward, Arch Linux, or perhaps other distributions. For example, on various Linux distributions you might do one of the following: +You may already have the `mlr` executable available in your platform's package manager on NetBSD, Debian Linux, Ubuntu Xenial and upward, Arch Linux, or perhaps other distributions. For example, on various Linux distributions you might do one of the following: -
+
 sudo apt-get install miller
 
-
+
 sudo apt install miller
 
-
+
 sudo yum install miller
 
On Windows, Miller is available via [Chocolatey](https://chocolatey.org/): -
+
 choco install miller
 
diff --git a/docs6b/docs/installation.md.in b/docs6b/docs/installation.md.in index e6920d9c5..e63440c23 100644 --- a/docs6b/docs/installation.md.in +++ b/docs6b/docs/installation.md.in @@ -18,7 +18,7 @@ GENMD_CARDIFY_HIGHLIGHT_ONE sudo port selfupdate && sudo port install miller GENMD_EOF -You may already have the ``mlr`` executable available in your platform's package manager on NetBSD, Debian Linux, Ubuntu Xenial and upward, Arch Linux, or perhaps other distributions. For example, on various Linux distributions you might do one of the following: +You may already have the `mlr` executable available in your platform's package manager on NetBSD, Debian Linux, Ubuntu Xenial and upward, Arch Linux, or perhaps other distributions. For example, on various Linux distributions you might do one of the following: GENMD_CARDIFY_HIGHLIGHT_ONE sudo apt-get install miller diff --git a/docs6b/docs/internationalization.md b/docs6b/docs/internationalization.md index a8079d03d..210bfbb20 100644 --- a/docs6b/docs/internationalization.md +++ b/docs6b/docs/internationalization.md @@ -6,7 +6,7 @@ Miller handles ASCII and UTF-8 strings. (I have no plans to support UTF-16 or IS Support for internationalization includes: * Tabular output formats such pprint and xtab (see [File Formats](file-formats.md)) are aligned correctly. -* The :ref:`reference-dsl-strlen` function correctly counts UTF-8 codepoints rather than bytes. -* The :ref:`reference-dsl-toupper`, :ref:`reference-dsl-tolower`, and :ref:`reference-dsl-capitalize` DSL functions operate within the capabilities of the Go libraries. +* The [strlen](reference-dsl-builtin-functions.md#strlen) function correctly counts UTF-8 codepoints rather than bytes. +* The [toupper](reference-dsl-builtin-functions.md#toupper), [tolower](reference-dsl-builtin-functions.md#tolower), and [capitalize](reference-dsl-builtin-functions.md#capitalize) DSL functions operate within the capabilities of the Go libraries. -Please file an issue at https://github.com/johnkerl/miller if you encounter bugs related to internationalization (or anything else for that matter). +Please file an issue at [https://github.com/johnkerl/miller](https://github.com/johnkerl/miller) if you encounter bugs related to internationalization (or anything else for that matter). diff --git a/docs6b/docs/internationalization.md.in b/docs6b/docs/internationalization.md.in index 332736c9a..f14f7cfd1 100644 --- a/docs6b/docs/internationalization.md.in +++ b/docs6b/docs/internationalization.md.in @@ -5,7 +5,7 @@ Miller handles ASCII and UTF-8 strings. (I have no plans to support UTF-16 or IS Support for internationalization includes: * Tabular output formats such pprint and xtab (see [File Formats](file-formats.md)) are aligned correctly. -* The :ref:`reference-dsl-strlen` function correctly counts UTF-8 codepoints rather than bytes. -* The :ref:`reference-dsl-toupper`, :ref:`reference-dsl-tolower`, and :ref:`reference-dsl-capitalize` DSL functions operate within the capabilities of the Go libraries. +* The [strlen](reference-dsl-builtin-functions.md#strlen) function correctly counts UTF-8 codepoints rather than bytes. +* The [toupper](reference-dsl-builtin-functions.md#toupper), [tolower](reference-dsl-builtin-functions.md#tolower), and [capitalize](reference-dsl-builtin-functions.md#capitalize) DSL functions operate within the capabilities of the Go libraries. -Please file an issue at https://github.com/johnkerl/miller if you encounter bugs related to internationalization (or anything else for that matter). +Please file an issue at [https://github.com/johnkerl/miller](https://github.com/johnkerl/miller) if you encounter bugs related to internationalization (or anything else for that matter). diff --git a/docs6b/docs/joins.md b/docs6b/docs/joins.md index 72936caea..2798a6e35 100644 --- a/docs6b/docs/joins.md +++ b/docs6b/docs/joins.md @@ -5,18 +5,22 @@ **This section describes behavior before Miller 5.1.0. As of 5.1.0, -u is the default.** -For example, the right file here has nine records, and the left file should add in the ``hostname`` column -- so the join output should also have 9 records: +For example, the right file here has nine records, and the left file should add in the `hostname` column -- so the join output should also have 9 records: -
+
 mlr --icsvlite --opprint cat data/join-u-left.csv
+
+
 hostname              ipaddr
 nadir.east.our.org    10.3.1.18
 zenith.west.our.org   10.3.1.27
 apoapsis.east.our.org 10.4.5.94
 
-
+
 mlr --icsvlite --opprint cat data/join-u-right.csv
+
+
 ipaddr    timestamp  bytes
 10.3.1.27 1448762579 4568
 10.3.1.18 1448762578 8729
@@ -29,8 +33,10 @@ ipaddr    timestamp  bytes
 10.4.5.94 1448762599 12200
 
-
+
 mlr --icsvlite --opprint join -s -j ipaddr -f data/join-u-left.csv data/join-u-right.csv
+
+
 ipaddr    hostname              timestamp  bytes
 10.3.1.27 zenith.west.our.org   1448762579 4568
 10.4.5.94 apoapsis.east.our.org 1448762579 17445
@@ -38,12 +44,14 @@ ipaddr    hostname              timestamp  bytes
 10.4.5.94 apoapsis.east.our.org 1448762599 12200
 
-The issue is that Miller's ``join``, by default (before 5.1.0), took input sorted (lexically ascending) by the sort keys on both the left and right files. This design decision was made intentionally to parallel the Unix/Linux system ``join`` command, which has the same semantics. The benefit of this default is that the joiner program can stream through the left and right files, needing to load neither entirely into memory. The drawback, of course, is that is requires sorted input. +The issue is that Miller's `join`, by default (before 5.1.0), took input sorted (lexically ascending) by the sort keys on both the left and right files. This design decision was made intentionally to parallel the Unix/Linux system `join` command, which has the same semantics. The benefit of this default is that the joiner program can stream through the left and right files, needing to load neither entirely into memory. The drawback, of course, is that is requires sorted input. The solution (besides pre-sorting the input files on the join keys) is to simply use **mlr join -u** (which is now the default). This loads the left file entirely into memory (while the right file is still streamed one line at a time) and does all possible joins without requiring sorted input: -
+
 mlr --icsvlite --opprint join -u -j ipaddr -f data/join-u-left.csv data/join-u-right.csv
+
+
 ipaddr    hostname              timestamp  bytes
 10.3.1.27 zenith.west.our.org   1448762579 4568
 10.3.1.18 nadir.east.our.org    1448762578 8729
@@ -62,14 +70,14 @@ General advice is to make sure the left-file is relatively small, e.g. containin
 
 Suppose you have the following two data files:
 
-
+
 id,code
 3,0000ff
 2,00ff00
 4,ff0000
 
-
+
 id,color
 4,red
 2,green
@@ -77,17 +85,21 @@ id,color
 
 Joining on color the results are as expected:
 
-
+
 mlr --csv join -j id -f data/color-codes.csv data/color-names.csv
+
+
 id,code,color
 4,ff0000,red
 2,00ff00,green
 
-However, if we ask for left-unpaireds, since there's no ``color`` column, we get a row not having the same column names as the other: +However, if we ask for left-unpaireds, since there's no `color` column, we get a row not having the same column names as the other: -
+
 mlr --csv join --ul -j id -f data/color-codes.csv data/color-names.csv
+
+
 id,code,color
 4,ff0000,red
 2,00ff00,green
@@ -98,10 +110,12 @@ id,code
 
 To fix this, we can use **unsparsify**:
 
-
+
 mlr --csv join --ul -j id -f data/color-codes.csv \
   then unsparsify --fill-with "" \
   data/color-names.csv
+
+
 id,code,color
 4,ff0000,red
 2,00ff00,green
@@ -114,8 +128,10 @@ Thanks to @aborruso for the tip!
 
 Suppose we have the following data:
 
-
+
 cat multi-join/input.csv
+
+
 id,task
 10,chop
 20,puree
@@ -127,30 +143,36 @@ id,task
 30,clean
 
-And we want to augment the ``id`` column with lookups from the following data files: +And we want to augment the `id` column with lookups from the following data files: -
+
 cat multi-join/name-lookup.csv
+
+
 id,name
 30,Alice
 10,Bob
 20,Carol
 
-
+
 cat multi-join/status-lookup.csv
+
+
 id,status
 30,occupied
 10,idle
 20,idle
 
-We can run the input file through multiple ``join`` commands in a ``then``-chain: +We can run the input file through multiple `join` commands in a `then`-chain: -
+
 mlr --icsv --opprint join -f multi-join/name-lookup.csv -j id \
   then join -f multi-join/status-lookup.csv -j id \
   multi-join/input.csv
+
+
 id status   name  task
 10 idle     Bob   chop
 20 idle     Carol puree
diff --git a/docs6b/docs/joins.md.in b/docs6b/docs/joins.md.in
index ba5e98e45..63ef05878 100644
--- a/docs6b/docs/joins.md.in
+++ b/docs6b/docs/joins.md.in
@@ -4,7 +4,7 @@
 
 **This section describes behavior before Miller 5.1.0. As of 5.1.0, -u is the default.**
 
-For example, the right file here has nine records, and the left file should add in the ``hostname`` column -- so the join output should also have 9 records:
+For example, the right file here has nine records, and the left file should add in the `hostname` column -- so the join output should also have 9 records:
 
 GENMD_RUN_COMMAND
 mlr --icsvlite --opprint cat data/join-u-left.csv
@@ -18,7 +18,7 @@ GENMD_RUN_COMMAND
 mlr --icsvlite --opprint join -s -j ipaddr -f data/join-u-left.csv data/join-u-right.csv
 GENMD_EOF
 
-The issue is that Miller's ``join``, by default (before 5.1.0), took input sorted (lexically ascending) by the sort keys on both the left and right files.  This design decision was made intentionally to parallel the Unix/Linux system ``join`` command, which has the same semantics. The benefit of this default is that the joiner program can stream through the left and right files, needing to load neither entirely into memory. The drawback, of course, is that is requires sorted input.
+The issue is that Miller's `join`, by default (before 5.1.0), took input sorted (lexically ascending) by the sort keys on both the left and right files.  This design decision was made intentionally to parallel the Unix/Linux system `join` command, which has the same semantics. The benefit of this default is that the joiner program can stream through the left and right files, needing to load neither entirely into memory. The drawback, of course, is that is requires sorted input.
 
 The solution (besides pre-sorting the input files on the join keys) is to simply use **mlr join -u** (which is now the default). This loads the left file entirely into memory (while the right file is still streamed one line at a time) and does all possible joins without requiring sorted input:
 
@@ -42,7 +42,7 @@ GENMD_RUN_COMMAND
 mlr --csv join -j id -f data/color-codes.csv data/color-names.csv
 GENMD_EOF
 
-However, if we ask for left-unpaireds, since there's no ``color`` column, we get a row not having the same column names as the other:
+However, if we ask for left-unpaireds, since there's no `color` column, we get a row not having the same column names as the other:
 
 GENMD_RUN_COMMAND
 mlr --csv join --ul -j id -f data/color-codes.csv data/color-names.csv
@@ -66,7 +66,7 @@ GENMD_RUN_COMMAND
 cat multi-join/input.csv
 GENMD_EOF
 
-And we want to augment the ``id`` column with lookups from the following data files:
+And we want to augment the `id` column with lookups from the following data files:
 
 GENMD_RUN_COMMAND
 cat multi-join/name-lookup.csv
@@ -76,7 +76,7 @@ GENMD_RUN_COMMAND
 cat multi-join/status-lookup.csv
 GENMD_EOF
 
-We can run the input file through multiple ``join`` commands in a ``then``-chain:
+We can run the input file through multiple `join` commands in a `then`-chain:
 
 GENMD_RUN_COMMAND
 mlr --icsv --opprint join -f multi-join/name-lookup.csv -j id \
diff --git a/docs6b/docs/keystroke-savers.md b/docs6b/docs/keystroke-savers.md
index 9ef87551d..e7bf57ce7 100644
--- a/docs6b/docs/keystroke-savers.md
+++ b/docs6b/docs/keystroke-savers.md
@@ -3,17 +3,21 @@
 
 ## Short format specifiers
 
-In our examples so far we've often made use of ``mlr --icsv --opprint`` or ``mlr --icsv --ojson``. These are such frequently occurring patterns that they have short options like **--c2p** and **--c2j**:
+In our examples so far we've often made use of `mlr --icsv --opprint` or `mlr --icsv --ojson`. These are such frequently occurring patterns that they have short options like **--c2p** and **--c2j**:
 
-
+
 mlr --c2p head -n 2 example.csv
+
+
 color  shape    flag index quantity rate
 yellow triangle true 11    43.6498  9.8870
 red    square   true 15    79.2778  0.0130
 
-
+
 mlr --c2j head -n 2 example.csv
+
+
 {
   "color": "yellow",
   "shape": "triangle",
@@ -36,32 +40,36 @@ You can get the full list here (TODO:linkify).
 
 ## File names up front
 
-Already we saw that you can put the filename first using ``--from``. When you're interacting with your data at the command line, this makes it easier to up-arrow and append to the previous command:
+Already we saw that you can put the filename first using `--from`. When you're interacting with your data at the command line, this makes it easier to up-arrow and append to the previous command:
 
-
+
 mlr --c2p --from example.csv sort -nr index then head -n 3
+
+
 color  shape  flag  index quantity rate
 purple square false 91    72.3735  8.2430
 yellow circle true  87    63.5058  8.3350
 yellow circle true  73    63.9785  4.2370
 
-
+
 mlr --c2p --from example.csv sort -nr index then head -n 3 then cut -f shape,quantity
+
+
 shape  quantity
 square 72.3735
 circle 63.5058
 circle 63.9785
 
-If there's more than one input file, you can use ``--mfrom``, then however many file names, then ``--`` to indicate the end of your input-file-name list: +If there's more than one input file, you can use `--mfrom`, then however many file names, then `--` to indicate the end of your input-file-name list: -
+
 mlr --c2p --mfrom data/*.csv -- sort -n index
 
## .mlrrc file -If you want the default file format for Miller to be CSV you can simply put ``--csv`` on a line by itself in your ``~/.mlrrc`` file. Then instead of ``mlr --csv cat example.csv`` you can just do ``mlr cat example.csv``. This is just the default, though, so ``mlr --opprint cat example.csv`` will still use default CSV format for input, and PPRINT (tabular) for output. +If you want the default file format for Miller to be CSV you can simply put `--csv` on a line by itself in your `~/.mlrrc` file. Then instead of `mlr --csv cat example.csv` you can just do `mlr cat example.csv`. This is just the default, though, so `mlr --opprint cat example.csv` will still use default CSV format for input, and PPRINT (tabular) for output. You can read more about this at the [Customization](customization.md) page. diff --git a/docs6b/docs/keystroke-savers.md.in b/docs6b/docs/keystroke-savers.md.in index c13ea4790..aa97133b1 100644 --- a/docs6b/docs/keystroke-savers.md.in +++ b/docs6b/docs/keystroke-savers.md.in @@ -2,7 +2,7 @@ ## Short format specifiers -In our examples so far we've often made use of ``mlr --icsv --opprint`` or ``mlr --icsv --ojson``. These are such frequently occurring patterns that they have short options like **--c2p** and **--c2j**: +In our examples so far we've often made use of `mlr --icsv --opprint` or `mlr --icsv --ojson`. These are such frequently occurring patterns that they have short options like **--c2p** and **--c2j**: GENMD_RUN_COMMAND mlr --c2p head -n 2 example.csv @@ -16,7 +16,7 @@ You can get the full list here (TODO:linkify). ## File names up front -Already we saw that you can put the filename first using ``--from``. When you're interacting with your data at the command line, this makes it easier to up-arrow and append to the previous command: +Already we saw that you can put the filename first using `--from`. When you're interacting with your data at the command line, this makes it easier to up-arrow and append to the previous command: GENMD_RUN_COMMAND mlr --c2p --from example.csv sort -nr index then head -n 3 @@ -26,7 +26,7 @@ GENMD_RUN_COMMAND mlr --c2p --from example.csv sort -nr index then head -n 3 then cut -f shape,quantity GENMD_EOF -If there's more than one input file, you can use ``--mfrom``, then however many file names, then ``--`` to indicate the end of your input-file-name list: +If there's more than one input file, you can use `--mfrom`, then however many file names, then `--` to indicate the end of your input-file-name list: GENMD_SHOW_COMMAND mlr --c2p --mfrom data/*.csv -- sort -n index @@ -34,6 +34,6 @@ GENMD_EOF ## .mlrrc file -If you want the default file format for Miller to be CSV you can simply put ``--csv`` on a line by itself in your ``~/.mlrrc`` file. Then instead of ``mlr --csv cat example.csv`` you can just do ``mlr cat example.csv``. This is just the default, though, so ``mlr --opprint cat example.csv`` will still use default CSV format for input, and PPRINT (tabular) for output. +If you want the default file format for Miller to be CSV you can simply put `--csv` on a line by itself in your `~/.mlrrc` file. Then instead of `mlr --csv cat example.csv` you can just do `mlr cat example.csv`. This is just the default, though, so `mlr --opprint cat example.csv` will still use default CSV format for input, and PPRINT (tabular) for output. You can read more about this at the [Customization](customization.md) page. diff --git a/docs6b/docs/log-processing-examples.md b/docs6b/docs/log-processing-examples.md index b4498f477..0a1b0dd8b 100644 --- a/docs6b/docs/log-processing-examples.md +++ b/docs6b/docs/log-processing-examples.md @@ -1,7 +1,7 @@ # Log-processing examples -Another of my favorite use-cases for Miller is doing ad-hoc processing of log-file data. Here's where DKVP format really shines: one, since the field names and field values are present on every line, every line stands on its own. That means you can ``grep`` or what have you. Also it means not every line needs to have the same list of field names ("schema"). +Another of my favorite use-cases for Miller is doing ad-hoc processing of log-file data. Here's where DKVP format really shines: one, since the field names and field values are present on every line, every line stands on its own. That means you can `grep` or what have you. Also it means not every line needs to have the same list of field names ("schema"). ## Generating and aggregating log-file output @@ -11,8 +11,10 @@ Writing a program -- in any language whatsoever -- you can have it print out log Suppose your program has printed something like this [log.txt](./log.txt): -
+
 cat log.txt
+
+
 op=enter,time=1472819681
 op=cache,type=A9,hit=0
 op=cache,type=A4,hit=1
@@ -58,22 +60,26 @@ op=cache,type=A9,hit=0
 time=1472819742,batch_size=100,num_filtered=728
 
-Each print statement simply contains local information: the current timestamp, whether a particular cache was hit or not, etc. Then using either the system ``grep`` command, or Miller's ``having-fields``, or ``is_present``, we can pick out the parts we want and analyze them: +Each print statement simply contains local information: the current timestamp, whether a particular cache was hit or not, etc. Then using either the system `grep` command, or Miller's `having-fields`, or `is_present`, we can pick out the parts we want and analyze them: -
+
 grep op=cache log.txt \
   | mlr --idkvp --opprint stats1 -a mean -f hit -g type then sort -f type
+
+
 type hit_mean
 A1   0.8571428571428571
 A4   0.7142857142857143
 A9   0.09090909090909091
 
-
+
 mlr --from log.txt --opprint \
   filter 'is_present($batch_size)' \
   then step -a delta -f time,num_filtered \
   then sec2gmt time
+
+
 time                 batch_size num_filtered time_delta num_filtered_delta
 2016-09-02T12:34:50Z 100        237          0          0
 2016-09-02T12:35:05Z 100        348          15         111
@@ -85,8 +91,10 @@ time                 batch_size num_filtered time_delta num_filtered_delta
 
 Alternatively, we can simply group the similar data for a better look:
 
-
+
 mlr --opprint group-like log.txt
+
+
 op    time
 enter 1472819681
 
@@ -137,8 +145,10 @@ time       batch_size num_filtered
 1472819742 100        728
 
-
+
 mlr --opprint group-like then sec2gmt time log.txt
+
+
 op    time
 enter 2016-09-02T12:34:41Z
 
@@ -193,13 +203,13 @@ time                 batch_size num_filtered
 
 This, of course, depends highly on what's in your log files. But, as an example, suppose you have log-file lines such as
 
-
+
 2015-10-08 08:29:09,445 INFO com.company.path.to.ClassName @ [sometext] various/sorts/of data {& punctuation} hits=1 status=0 time=2.378
 
-I prefer to pre-filter with ``grep`` and/or ``sed`` to extract the structured text, then hand that to Miller. Example: +I prefer to pre-filter with `grep` and/or `sed` to extract the structured text, then hand that to Miller. Example: -
+
 grep 'various sorts' *.log \
   | sed 's/.*} //' \
   | mlr --fs space --repifs --oxtab stats1 -a min,p10,p50,p90,max -f time -g status
diff --git a/docs6b/docs/log-processing-examples.md.in b/docs6b/docs/log-processing-examples.md.in
index 987aeb823..a0c245f31 100644
--- a/docs6b/docs/log-processing-examples.md.in
+++ b/docs6b/docs/log-processing-examples.md.in
@@ -1,6 +1,6 @@
 # Log-processing examples
 
-Another of my favorite use-cases for Miller is doing ad-hoc processing of log-file data.  Here's where DKVP format really shines: one, since the field names and field values are present on every line, every line stands on its own. That means you can ``grep`` or what have you. Also it means not every line needs to have the same list of field names ("schema").
+Another of my favorite use-cases for Miller is doing ad-hoc processing of log-file data.  Here's where DKVP format really shines: one, since the field names and field values are present on every line, every line stands on its own. That means you can `grep` or what have you. Also it means not every line needs to have the same list of field names ("schema").
 
 ## Generating and aggregating log-file output
 
@@ -14,7 +14,7 @@ GENMD_RUN_COMMAND
 cat log.txt
 GENMD_EOF
 
-Each print statement simply contains local information: the current timestamp, whether a particular cache was hit or not, etc. Then using either the system ``grep`` command, or Miller's ``having-fields``, or ``is_present``, we can pick out the parts we want and analyze them:
+Each print statement simply contains local information: the current timestamp, whether a particular cache was hit or not, etc. Then using either the system `grep` command, or Miller's `having-fields`, or `is_present`, we can pick out the parts we want and analyze them:
 
 GENMD_INCLUDE_AND_RUN_ESCAPED(10-1.sh)
 
@@ -38,7 +38,7 @@ GENMD_CARDIFY
 2015-10-08 08:29:09,445 INFO com.company.path.to.ClassName @ [sometext] various/sorts/of data {& punctuation} hits=1 status=0 time=2.378
 GENMD_EOF
 
-I prefer to pre-filter with ``grep`` and/or ``sed`` to extract the structured text, then hand that to Miller. Example:
+I prefer to pre-filter with `grep` and/or `sed` to extract the structured text, then hand that to Miller. Example:
 
 GENMD_SHOW_COMMAND
 grep 'various sorts' *.log \
diff --git a/docs6b/docs/manpage.md b/docs6b/docs/manpage.md
index b796701a2..0bb01ffc0 100644
--- a/docs6b/docs/manpage.md
+++ b/docs6b/docs/manpage.md
@@ -3,7 +3,7 @@
 
 This is simply a copy of what you should see on running **man mlr** at a command prompt, once Miller is installed on your system.
 
-
+
 MILLER(1)							     MILLER(1)
 
 
diff --git a/docs6b/docs/miller-on-windows.md b/docs6b/docs/miller-on-windows.md
index b7c690da7..efa2b8a29 100644
--- a/docs6b/docs/miller-on-windows.md
+++ b/docs6b/docs/miller-on-windows.md
@@ -7,17 +7,17 @@ Miller was originally developed for Unix-like operating systems including Linux
 
 The experience is now almost the same as on Linux, NetBSD/FreeBSD, and MacOS.
 
-MSYS2 is no longer required, although you can use Miller from within MSYS2 if you like. There is now simply a single ``mlr.exe``, with no ``msys2.dll`` alongside anymore.
+MSYS2 is no longer required, although you can use Miller from within MSYS2 if you like. There is now simply a single `mlr.exe`, with no `msys2.dll` alongside anymore.
 
-See [Installation](installation.md) for how to get a copy of ``mlr.exe``.
+See [Installation](installation.md) for how to get a copy of `mlr.exe`.
 
 ## Setup
 
-Simply place ``mlr.exe`` somewhere within your ``PATH`` variable.
+Simply place `mlr.exe` somewhere within your `PATH` variable.
 
 ![pix/miller-windows.png](pix/miller-windows.png)
 
-To use Miller from within MSYS2/Cygwin, also make sure ``mlr.exe`` is within the ``PATH`` variable.
+To use Miller from within MSYS2/Cygwin, also make sure `mlr.exe` is within the `PATH` variable.
 
 ![pix/miller-msys.png](pix/miller-msys.png)
 
@@ -25,18 +25,18 @@ To use Miller from within MSYS2/Cygwin, also make sure ``mlr.exe`` is within the
 
 [Output Colorization](output-colorization.md) doesn't work on Windows, outside of MSYS2.
 
-The Windows-support code within Miller makes effort to support Linux/Unix/MacOS-like command-line syntax including single-quoting of expressions for ``mlr put`` and ``mlr filter`` -- and in the examples above, this often works. However, there are still some cases where more complex expressions aren't successfully parsed from the Windows prompt, even though they are from MSYS2:
+The Windows-support code within Miller makes effort to support Linux/Unix/MacOS-like command-line syntax including single-quoting of expressions for `mlr put` and `mlr filter` -- and in the examples above, this often works. However, there are still some cases where more complex expressions aren't successfully parsed from the Windows prompt, even though they are from MSYS2:
 
 ![pix/miller-windows-complex.png](pix/miller-windows-complex.png)
 
 ![pix/miller-msys-complex.png](pix/miller-msys-complex.png)
 
-Single quotes with ``&&`` or ``||`` inside are fundamentally unhandleable within Windows; there is nothing Miller can do here as the Windows command line is split before Miller ever receives it.
+Single quotes with `&&` or `||` inside are fundamentally unhandleable within Windows; there is nothing Miller can do here as the Windows command line is split before Miller ever receives it.
 
 One workaround is to use MSYS2. Another workaround is to put more complex DSL expressions into a file:
 
 ![pix/miller-windows-complex-workaround.png](pix/miller-windows-complex-workaround.png)
 
-A third workaround is to replace ``"`` with ``"""``, then ``'`` with ``"``:
+A third workaround is to replace `"` with `"""`, then `'` with `"`:
 
 ![pix/miller-windows-triple-double-quote.png](pix/miller-windows-triple-double-quote.png)
diff --git a/docs6b/docs/miller-on-windows.md.in b/docs6b/docs/miller-on-windows.md.in
index 42f62d10e..f1e1f2d30 100644
--- a/docs6b/docs/miller-on-windows.md.in
+++ b/docs6b/docs/miller-on-windows.md.in
@@ -6,17 +6,17 @@ Miller was originally developed for Unix-like operating systems including Linux
 
 The experience is now almost the same as on Linux, NetBSD/FreeBSD, and MacOS.
 
-MSYS2 is no longer required, although you can use Miller from within MSYS2 if you like. There is now simply a single ``mlr.exe``, with no ``msys2.dll`` alongside anymore.
+MSYS2 is no longer required, although you can use Miller from within MSYS2 if you like. There is now simply a single `mlr.exe`, with no `msys2.dll` alongside anymore.
 
-See [Installation](installation.md) for how to get a copy of ``mlr.exe``.
+See [Installation](installation.md) for how to get a copy of `mlr.exe`.
 
 ## Setup
 
-Simply place ``mlr.exe`` somewhere within your ``PATH`` variable.
+Simply place `mlr.exe` somewhere within your `PATH` variable.
 
 ![pix/miller-windows.png](pix/miller-windows.png)
 
-To use Miller from within MSYS2/Cygwin, also make sure ``mlr.exe`` is within the ``PATH`` variable.
+To use Miller from within MSYS2/Cygwin, also make sure `mlr.exe` is within the `PATH` variable.
 
 ![pix/miller-msys.png](pix/miller-msys.png)
 
@@ -24,18 +24,18 @@ To use Miller from within MSYS2/Cygwin, also make sure ``mlr.exe`` is within the
 
 [Output Colorization](output-colorization.md) doesn't work on Windows, outside of MSYS2.
 
-The Windows-support code within Miller makes effort to support Linux/Unix/MacOS-like command-line syntax including single-quoting of expressions for ``mlr put`` and ``mlr filter`` -- and in the examples above, this often works. However, there are still some cases where more complex expressions aren't successfully parsed from the Windows prompt, even though they are from MSYS2:
+The Windows-support code within Miller makes effort to support Linux/Unix/MacOS-like command-line syntax including single-quoting of expressions for `mlr put` and `mlr filter` -- and in the examples above, this often works. However, there are still some cases where more complex expressions aren't successfully parsed from the Windows prompt, even though they are from MSYS2:
 
 ![pix/miller-windows-complex.png](pix/miller-windows-complex.png)
 
 ![pix/miller-msys-complex.png](pix/miller-msys-complex.png)
 
-Single quotes with ``&&`` or ``||`` inside are fundamentally unhandleable within Windows; there is nothing Miller can do here as the Windows command line is split before Miller ever receives it.
+Single quotes with `&&` or `||` inside are fundamentally unhandleable within Windows; there is nothing Miller can do here as the Windows command line is split before Miller ever receives it.
 
 One workaround is to use MSYS2. Another workaround is to put more complex DSL expressions into a file:
 
 ![pix/miller-windows-complex-workaround.png](pix/miller-windows-complex-workaround.png)
 
-A third workaround is to replace ``"`` with ``"""``, then ``'`` with ``"``:
+A third workaround is to replace `"` with `"""`, then `'` with `"`:
 
 ![pix/miller-windows-triple-double-quote.png](pix/miller-windows-triple-double-quote.png)
diff --git a/docs6b/docs/misc-examples.md b/docs6b/docs/misc-examples.md
index a69082e31..cd7eb1ddc 100644
--- a/docs6b/docs/misc-examples.md
+++ b/docs6b/docs/misc-examples.md
@@ -3,62 +3,64 @@
 
 Column select:
 
-
+
 mlr --csv cut -f hostname,uptime mydata.csv
 
Add new columns as function of other columns: -
+
 mlr --nidx put '$sum = $7 < 0.0 ? 3.5 : $7 + 2.1*$8' *.dat
 
Row filter: -
+
 mlr --csv filter '$status != "down" && $upsec >= 10000' *.csv
 
Apply column labels and pretty-print: -
+
 grep -v '^#' /etc/group | mlr --ifs : --nidx --opprint label group,pass,gid,member then sort -f group
 
Join multiple data sources on key columns: -
+
 mlr join -j account_id -f accounts.dat then group-by account_name balances.dat
 
Mulltiple formats including JSON: -
+
 mlr --json put '$attr = sub($attr, "([0-9]+)_([0-9]+)_.*", "\1:\2")' data/*.json
 
Aggregate per-column statistics: -
+
 mlr stats1 -a min,mean,max,p10,p50,p90 -f flag,u,v data/*
 
Linear regression: -
+
 mlr stats2 -a linreg-pca -f u,v -g shape data/*
 
Aggregate custom per-column statistics: -
+
 mlr put -q '@sum[$a][$b] += $x; end {emit @sum, "a", "b"}' data/*
 
Iterate over data using DSL expressions: -
+
 mlr --from estimates.tbl put '
+
+
   for (k,v in $*) {
     if (is_numeric(v) && k =~ "^[t-z].*$") {
       $sum += v; $count += 1
@@ -70,39 +72,39 @@ Iterate over data using DSL expressions:
 
 Run DSL expressions from a script file:
 
-
+
 mlr --from infile.dat put -f analyze.mlr
 
Split/reduce output to multiple filenames: -
+
 mlr --from infile.dat put 'tee > "./taps/data-".$a."-".$b, $*'
 
Compressed I/O: -
+
 mlr --from infile.dat put 'tee | "gzip > ./taps/data-".$a."-".$b.".gz", $*'
 
Interoperate with other data-processing tools using standard pipes: -
+
 mlr --from infile.dat put -q '@v=$*; dump | "jq .[]"'
 
Tap/trace: -
+
 mlr --from infile.dat put  '(NR % 1000 == 0) { print > stderr, "Checkpoint ".NR}'
 
## Program timing -This admittedly artificial example demonstrates using Miller time and stats functions to introspectively acquire some information about Miller's own runtime. The ``delta`` function computes the difference between successive timestamps. +This admittedly artificial example demonstrates using Miller time and stats functions to introspectively acquire some information about Miller's own runtime. The `delta` function computes the difference between successive timestamps. -
+
 $ ruby -e '10000.times{|i|puts "i=#{i+1}"}' > lines.txt
 
 $ head -n 5 lines.txt
@@ -136,8 +138,10 @@ t_delta_max  5.388259888e-05
 
 Suppose you have a database query which you run at one point in time, producing the output on the left, then again later producing the output on the right:
 
-
+
 cat data/previous_counters.csv
+
+
 color,count
 red,3472
 blue,6838
@@ -145,8 +149,10 @@ orange,694
 purple,12
 
-
+
 cat data/current_counters.csv
+
+
 color,count
 red,3467
 orange,670
@@ -158,12 +164,14 @@ And, suppose you want to compute the differences in the counters between adjacen
 
 First, rename counter columns to make them distinct:
 
-
+
 mlr --csv rename count,previous_count data/previous_counters.csv > data/prevtemp.csv
 
-
+
 cat data/prevtemp.csv
+
+
 color,previous_count
 red,3472
 blue,6838
@@ -171,12 +179,14 @@ orange,694
 purple,12
 
-
+
 mlr --csv rename count,current_count data/current_counters.csv > data/currtemp.csv
 
-
+
 cat data/currtemp.csv
+
+
 color,current_count
 red,3467
 orange,670
@@ -184,14 +194,16 @@ yellow,27
 blue,6944
 
-Then, join on the key field(s), and use unsparsify to zero-fill counters absent on one side but present on the other. Use ``--ul`` and ``--ur`` to emit unpaired records (namely, purple on the left and yellow on the right): +Then, join on the key field(s), and use unsparsify to zero-fill counters absent on one side but present on the other. Use `--ul` and `--ur` to emit unpaired records (namely, purple on the left and yellow on the right): -
+
 mlr --icsv --opprint \
   join -j color --ul --ur -f data/prevtemp.csv \
   then unsparsify --fill-with 0 \
   then put '$count_delta = $current_count - $previous_count' \
   data/currtemp.csv
+
+
 color  previous_count current_count count_delta
 red    3472           3467          -5
 orange 694            670           -24
@@ -200,13 +212,11 @@ blue   6838           6944          106
 purple 12             0             (error)
 
-.. _cookbook-memoization-with-oosvars: - ## Memoization with out-of-stream variables The recursive function for the Fibonacci sequence is famous for its computational complexity. Namely, using f(0)=1, f(1)=1, f(n)=f(n-1)+f(n-2) for n>=2, the evaluation tree branches left as well as right at each non-trivial level, resulting in millions or more paths to the root 0/1 nodes for larger n. This program -
+
 mlr --ofmt '%.9lf' --opprint seqgen --start 1 --stop 28 then put '
   func f(n) {
       @fcount += 1;              # count number of calls to the function
@@ -226,7 +236,7 @@ mlr --ofmt '%.9lf' --opprint seqgen --start 1 --stop 28 then put '
 
 produces output like this:
 
-
+
 i  o      fcount  seconds_delta
 1  1      1       0
 2  2      3       0.000039101
@@ -258,9 +268,9 @@ i  o      fcount  seconds_delta
 28 514229 1028457 0.971235037
 
-Note that the time it takes to evaluate the function is blowing up exponentially as the input argument increases. Using ``@``-variables, which persist across records, we can cache and reuse the results of previous computations: +Note that the time it takes to evaluate the function is blowing up exponentially as the input argument increases. Using `@`-variables, which persist across records, we can cache and reuse the results of previous computations: -
+
 mlr --ofmt '%.9lf' --opprint seqgen --start 1 --stop 28 then put '
   func f(n) {
     @fcount += 1;                 # count number of calls to the function
@@ -283,7 +293,7 @@ mlr --ofmt '%.9lf' --opprint seqgen --start 1 --stop 28 then put '
 
 with output like this:
 
-
+
 i  o      fcount seconds_delta
 1  1      1      0
 2  2      3      0.000053883
diff --git a/docs6b/docs/misc-examples.md.in b/docs6b/docs/misc-examples.md.in
index addc054d3..5efe1d74f 100644
--- a/docs6b/docs/misc-examples.md.in
+++ b/docs6b/docs/misc-examples.md.in
@@ -99,7 +99,7 @@ GENMD_EOF
 
 ## Program timing
 
-This admittedly artificial example demonstrates using Miller time and stats functions to introspectively acquire some information about Miller's own runtime. The ``delta`` function computes the difference between successive timestamps.
+This admittedly artificial example demonstrates using Miller time and stats functions to introspectively acquire some information about Miller's own runtime. The `delta` function computes the difference between successive timestamps.
 
 GENMD_INCLUDE_ESCAPED(data/timing-example.txt)
 
@@ -135,12 +135,10 @@ GENMD_RUN_COMMAND
 cat data/currtemp.csv
 GENMD_EOF
 
-Then, join on the key field(s), and use unsparsify to zero-fill counters absent on one side but present on the other. Use ``--ul`` and ``--ur`` to emit unpaired records (namely, purple on the left and yellow on the right):
+Then, join on the key field(s), and use unsparsify to zero-fill counters absent on one side but present on the other. Use `--ul` and `--ur` to emit unpaired records (namely, purple on the left and yellow on the right):
 
 GENMD_INCLUDE_AND_RUN_ESCAPED(data/previous-to-current.sh)
 
-.. _cookbook-memoization-with-oosvars:
-
 ## Memoization with out-of-stream variables
 
 The recursive function for the Fibonacci sequence is famous for its computational complexity.  Namely, using f(0)=1, f(1)=1, f(n)=f(n-1)+f(n-2) for n>=2, the evaluation tree branches left as well as right at each non-trivial level, resulting in millions or more paths to the root 0/1 nodes for larger n. This program
@@ -181,7 +179,7 @@ i  o      fcount  seconds_delta
 28 514229 1028457 0.971235037
 GENMD_EOF
 
-Note that the time it takes to evaluate the function is blowing up exponentially as the input argument increases. Using ``@``-variables, which persist across records, we can cache and reuse the results of previous computations:
+Note that the time it takes to evaluate the function is blowing up exponentially as the input argument increases. Using `@`-variables, which persist across records, we can cache and reuse the results of previous computations:
 
 GENMD_INCLUDE_ESCAPED(data/fibo-cached.sh)
 
diff --git a/docs6b/docs/mk-func-h2s.sh b/docs6b/docs/mk-func-h2s.sh
index d7965d84f..394cc966e 100755
--- a/docs6b/docs/mk-func-h2s.sh
+++ b/docs6b/docs/mk-func-h2s.sh
@@ -47,16 +47,20 @@ mlr help list-functions | grep -v '^[a-zA-Z]' | uniq | while read funcname; do
     linkname='ursheq'
   fi
 
-  echo ""
-  echo ".. _reference-dsl-${linkname}:"
-  echo ""
-  echo "$displayname"
-  echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
-  echo ""
-  echo '.. code-block:: none'
+  # TODO: fix section-links for mnkdocs
+  #echo ''
+  #echo ".. _reference-dsl-${linkname}:"
   echo ''
-  mlr help function "$funcname" | sed 's/^/    /'
+  if [ "$linkname" = "$displayname" ]; then
+    echo "## $displayname"
+  else
+    echo ""
+    echo "## $displayname"
+  fi
   echo ''
+  echo '
'
+  mlr help function "$funcname"
+  echo '
' echo '' done @@ -101,15 +105,19 @@ mlr help list-functions | grep '^[a-zA-Z]' | sort -u | while read funcname; do linkname='ursheq' fi - echo "" - echo ".. _reference-dsl-${linkname}:" - echo "" - echo "$displayname" - echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" - echo "" - echo '.. code-block:: none' + # TODO: fix section-links for mnkdocs + #echo '' + #echo ".. _reference-dsl-${linkname}:" echo '' - mlr help function "$funcname" | sed 's/^/ /' + if [ "$linkname" = "$displayname" ]; then + echo "## $displayname" + else + echo "
" + echo "## $displayname" + fi echo '' + echo '
'
+  mlr help function "$funcname"
+  echo '
' echo '' done diff --git a/docs6b/docs/new-in-miller-6.md b/docs6b/docs/new-in-miller-6.md index 156b86ae8..6429ec5ec 100644 --- a/docs6b/docs/new-in-miller-6.md +++ b/docs6b/docs/new-in-miller-6.md @@ -5,10 +5,10 @@ See also the [list of issues tagged with go-port](https://github.com/johnkerl/mi ## Documentation improvements -Documentation (what you're reading here) and on-line help (``mlr --help``) have been completely reworked. +Documentation (what you're reading here) and on-line help (`mlr --help`) have been completely reworked. In the initial release, the focus was convincing users already familiar with -``awk``/``grep``/``cut`` that Miller was a viable option. Over time it's become +`awk`/`grep`/`cut` that Miller was a viable option. Over time it's become clear that many users aren't expert with these. The focus has shifted toward a higher quantity of more introductory/accessible material for command-line data processing. @@ -25,14 +25,14 @@ now discussed first, and more examples use CSV. ## JSON support, and arrays -Arrays are now supported in Miller's ``put``/``filter`` programming language, -as described at :doc:`reference-dsl-arrays`. Also, ``array`` is now a keyword +Arrays are now supported in Miller's `put`/`filter` programming language, +as described at [Reference: arrays](reference-dsl-arrays.md). Also, `array` is now a keyword so this is no longer usable as a local-variable or UDF name. JSON support is improved: * Direct support for arrays means that you can now use Miller to process more JSON files. -* Streamable JSON parsing: Miller's internal record-processing pipeline starts as soon as the first record is read (which was already the case for other file formats). This means that, unless records are wrapped with outermost ``[...]``, Miller now handles JSON in ``tail -f`` contexts like it does for other file formats. +* Streamable JSON parsing: Miller's internal record-processing pipeline starts as soon as the first record is read (which was already the case for other file formats). This means that, unless records are wrapped with outermost `[...]`, Miller now handles JSON in `tail -f` contexts like it does for other file formats. * Flatten/unflatten -- TODO pick a name and link to a separate page/section ## Improved Windows experience @@ -44,11 +44,7 @@ Binaries are reliably available using GitHub Actions: see also [Installation](in ## In-process support for compressed input -In addition to ``--prepipe gunzip``, you can now use the ``--gzin`` flag. In -fact, if your files end in ``.gz`` you don't even need to do that -- Miller -will autodetect by file extension and automatically uncompress ``mlr --csv cat -foo.csv.gz``. Similarly for ``.z`` and ``.bz2`` files. Please see section -[TODO:linkify] for more information. +In addition to `--prepipe gunzip`, you can now use the `--gzin` flag. In fact, if your files end in `.gz` you don't even need to do that -- Miller will autodetect by file extension and automatically uncompress `mlr --csv cat foo.csv.gz`. Similarly for `.z` and `.bz2` files. Please see section [TODO:linkify] for more information. ## Output colorization @@ -60,12 +56,12 @@ The most central part of Miller 6 is a deep refactor of how data values are pars from file contents, how types are inferred, and how they're converted back to text into output files. -This was all initiated by https://github.com/johnkerl/miller/issues/151. +This was all initiated by [https://github.com/johnkerl/miller/issues/151](https://github.com/johnkerl/miller/issues/151). In Miller 5 and below, all values were stored as strings, then only converted to int/float as-needed, for example when a particular field was referenced in -the ``stats1`` or ``put`` verbs. This led to awkwardnesses such as the ``-S`` -and ``-F`` flags for ``put`` and ``filter``. +the `stats1` or `put` verbs. This led to awkwardnesses such as the `-S` +and `-F` flags for `put` and `filter`. In Miller 6, things parseable as int/float are treated as such from the moment the input data is read, and these are passed along through the verb chain. All @@ -75,17 +71,21 @@ a numeric field isn't modified during the processing chain, it's printed out the way it arrived. Also, quoted values in JSON strings are flagged as being strings throughout the processing chain. -For example (see https://github.com/johnkerl/miller/issues/178) you can now do +For example (see [https://github.com/johnkerl/miller/issues/178](https://github.com/johnkerl/miller/issues/178)) you can now do -
+
 echo '{ "a": "0123" }' | mlr --json cat
+
+
 {
   "a": "0123"
 }
 
-
+
 echo '{ "x": 1.230, "y": 1.230000000 }' | mlr --json cat
+
+
 {
   "x": 1.230,
   "y": 1.230000000
@@ -98,28 +98,30 @@ Miller now has a read-evaluate-print-loop ([REPL](repl.md)) where you can single
 
 ## New DSL functions / operators
 
-* String-hashing functions :ref:`reference-dsl-md5`, :ref:`reference-dsl-sha1`, :ref:`reference-dsl-sha256`, and :ref:`reference-dsl-sha512`.
-* Platform-property functions :ref:`reference-dsl-hostname`, :ref:`reference-dsl-os`, and :ref:`reference-dsl-version`.
-* Unsigned right-shift :ref:`reference-dsl-ursh` along with ``>>>=``.
+* String-hashing functions [md5](reference-dsl-builtin-functions.md#md5), [sha1](reference-dsl-builtin-functions.md#sha1), [sha256](reference-dsl-builtin-functions.md#sha256), and [sha512](reference-dsl-builtin-functions.md#sha512).
+
+* Platform-property functions [hostname](reference-dsl-builtin-functions.md#hostname), [os](reference-dsl-builtin-functions.md#os), and [version](reference-dsl-builtin-functions.md#version).
+
+* Unsigned right-shift [`>>>`](reference-dsl-builtin-functions.md#ursh) along with `>>>=`.
 
 ## Improved command-line parsing
 
 Miller 6 has getoptish command-line parsing (https://github.com/johnkerl/miller/pull/467):
 
-* ``-xyz`` expands automatically to ``-x -y -z``, so (for example) ``mlr cut -of shape,flag`` is the same as ``mlr cut -o -f shape,flag``.
-* ``--foo=bar`` expands automatically to  ``--foo bar``, so (for example) ``mlr --ifs=comma`` is the same as ``mlr --ifs comma``.
-* ``--mfrom``, ``--load``, ``--mload`` as described at [TODO:linkify].
+* `-xyz` expands automatically to `-x -y -z`, so (for example) `mlr cut -of shape,flag` is the same as `mlr cut -o -f shape,flag`.
+* `--foo=bar` expands automatically to  `--foo bar`, so (for example) `mlr --ifs=comma` is the same as `mlr --ifs comma`.
+* `--mfrom`, `--load`, `--mload` as described at [TODO:linkify].
 
 ## Improved error messages for DSL parsing
 
-For ``mlr put`` and ``mlr filter``, parse-error messages now include location information:
+For `mlr put` and `mlr filter`, parse-error messages now include location information:
 
-
+
 mlr: cannot parse DSL expression.
 Parse error on token ">" at line 63 columnn 7.
 
## Developer-specific aspects -* Miller has been ported from C to Go. Developer notes: https://github.com/johnkerl/miller/blob/main/go/README.md -* Completely reworked regression testing, including running on Windows +* Miller has been ported from C to Go. Developer notes: [https://github.com/johnkerl/miller/blob/main/go/README.md](https://github.com/johnkerl/miller/blob/main/go/README.md). +* Completely reworked regression testing, including regresstion-testing now running on Windows. diff --git a/docs6b/docs/new-in-miller-6.md.in b/docs6b/docs/new-in-miller-6.md.in index d9a8843d9..28cfdf756 100644 --- a/docs6b/docs/new-in-miller-6.md.in +++ b/docs6b/docs/new-in-miller-6.md.in @@ -4,10 +4,10 @@ See also the [list of issues tagged with go-port](https://github.com/johnkerl/mi ## Documentation improvements -Documentation (what you're reading here) and on-line help (``mlr --help``) have been completely reworked. +Documentation (what you're reading here) and on-line help (`mlr --help`) have been completely reworked. In the initial release, the focus was convincing users already familiar with -``awk``/``grep``/``cut`` that Miller was a viable option. Over time it's become +`awk`/`grep`/`cut` that Miller was a viable option. Over time it's become clear that many users aren't expert with these. The focus has shifted toward a higher quantity of more introductory/accessible material for command-line data processing. @@ -24,14 +24,14 @@ now discussed first, and more examples use CSV. ## JSON support, and arrays -Arrays are now supported in Miller's ``put``/``filter`` programming language, -as described at :doc:`reference-dsl-arrays`. Also, ``array`` is now a keyword +Arrays are now supported in Miller's `put`/`filter` programming language, +as described at [Reference: arrays](reference-dsl-arrays.md). Also, `array` is now a keyword so this is no longer usable as a local-variable or UDF name. JSON support is improved: * Direct support for arrays means that you can now use Miller to process more JSON files. -* Streamable JSON parsing: Miller's internal record-processing pipeline starts as soon as the first record is read (which was already the case for other file formats). This means that, unless records are wrapped with outermost ``[...]``, Miller now handles JSON in ``tail -f`` contexts like it does for other file formats. +* Streamable JSON parsing: Miller's internal record-processing pipeline starts as soon as the first record is read (which was already the case for other file formats). This means that, unless records are wrapped with outermost `[...]`, Miller now handles JSON in `tail -f` contexts like it does for other file formats. * Flatten/unflatten -- TODO pick a name and link to a separate page/section ## Improved Windows experience @@ -43,11 +43,7 @@ Binaries are reliably available using GitHub Actions: see also [Installation](in ## In-process support for compressed input -In addition to ``--prepipe gunzip``, you can now use the ``--gzin`` flag. In -fact, if your files end in ``.gz`` you don't even need to do that -- Miller -will autodetect by file extension and automatically uncompress ``mlr --csv cat -foo.csv.gz``. Similarly for ``.z`` and ``.bz2`` files. Please see section -[TODO:linkify] for more information. +In addition to `--prepipe gunzip`, you can now use the `--gzin` flag. In fact, if your files end in `.gz` you don't even need to do that -- Miller will autodetect by file extension and automatically uncompress `mlr --csv cat foo.csv.gz`. Similarly for `.z` and `.bz2` files. Please see section [TODO:linkify] for more information. ## Output colorization @@ -59,12 +55,12 @@ The most central part of Miller 6 is a deep refactor of how data values are pars from file contents, how types are inferred, and how they're converted back to text into output files. -This was all initiated by https://github.com/johnkerl/miller/issues/151. +This was all initiated by [https://github.com/johnkerl/miller/issues/151](https://github.com/johnkerl/miller/issues/151). In Miller 5 and below, all values were stored as strings, then only converted to int/float as-needed, for example when a particular field was referenced in -the ``stats1`` or ``put`` verbs. This led to awkwardnesses such as the ``-S`` -and ``-F`` flags for ``put`` and ``filter``. +the `stats1` or `put` verbs. This led to awkwardnesses such as the `-S` +and `-F` flags for `put` and `filter`. In Miller 6, things parseable as int/float are treated as such from the moment the input data is read, and these are passed along through the verb chain. All @@ -74,7 +70,7 @@ a numeric field isn't modified during the processing chain, it's printed out the way it arrived. Also, quoted values in JSON strings are flagged as being strings throughout the processing chain. -For example (see https://github.com/johnkerl/miller/issues/178) you can now do +For example (see [https://github.com/johnkerl/miller/issues/178](https://github.com/johnkerl/miller/issues/178)) you can now do GENMD_RUN_COMMAND echo '{ "a": "0123" }' | mlr --json cat @@ -90,21 +86,23 @@ Miller now has a read-evaluate-print-loop ([REPL](repl.md)) where you can single ## New DSL functions / operators -* String-hashing functions :ref:`reference-dsl-md5`, :ref:`reference-dsl-sha1`, :ref:`reference-dsl-sha256`, and :ref:`reference-dsl-sha512`. -* Platform-property functions :ref:`reference-dsl-hostname`, :ref:`reference-dsl-os`, and :ref:`reference-dsl-version`. -* Unsigned right-shift :ref:`reference-dsl-ursh` along with ``>>>=``. +* String-hashing functions [md5](reference-dsl-builtin-functions.md#md5), [sha1](reference-dsl-builtin-functions.md#sha1), [sha256](reference-dsl-builtin-functions.md#sha256), and [sha512](reference-dsl-builtin-functions.md#sha512). + +* Platform-property functions [hostname](reference-dsl-builtin-functions.md#hostname), [os](reference-dsl-builtin-functions.md#os), and [version](reference-dsl-builtin-functions.md#version). + +* Unsigned right-shift [`>>>`](reference-dsl-builtin-functions.md#ursh) along with `>>>=`. ## Improved command-line parsing Miller 6 has getoptish command-line parsing (https://github.com/johnkerl/miller/pull/467): -* ``-xyz`` expands automatically to ``-x -y -z``, so (for example) ``mlr cut -of shape,flag`` is the same as ``mlr cut -o -f shape,flag``. -* ``--foo=bar`` expands automatically to ``--foo bar``, so (for example) ``mlr --ifs=comma`` is the same as ``mlr --ifs comma``. -* ``--mfrom``, ``--load``, ``--mload`` as described at [TODO:linkify]. +* `-xyz` expands automatically to `-x -y -z`, so (for example) `mlr cut -of shape,flag` is the same as `mlr cut -o -f shape,flag`. +* `--foo=bar` expands automatically to `--foo bar`, so (for example) `mlr --ifs=comma` is the same as `mlr --ifs comma`. +* `--mfrom`, `--load`, `--mload` as described at [TODO:linkify]. ## Improved error messages for DSL parsing -For ``mlr put`` and ``mlr filter``, parse-error messages now include location information: +For `mlr put` and `mlr filter`, parse-error messages now include location information: GENMD_CARDIFY mlr: cannot parse DSL expression. @@ -113,5 +111,5 @@ GENMD_EOF ## Developer-specific aspects -* Miller has been ported from C to Go. Developer notes: https://github.com/johnkerl/miller/blob/main/go/README.md -* Completely reworked regression testing, including running on Windows +* Miller has been ported from C to Go. Developer notes: [https://github.com/johnkerl/miller/blob/main/go/README.md](https://github.com/johnkerl/miller/blob/main/go/README.md). +* Completely reworked regression testing, including regresstion-testing now running on Windows. diff --git a/docs6b/docs/operating-on-all-fields.md b/docs6b/docs/operating-on-all-fields.md index e489db565..275276d76 100644 --- a/docs6b/docs/operating-on-all-fields.md +++ b/docs6b/docs/operating-on-all-fields.md @@ -5,26 +5,32 @@ Suppose you want to replace spaces with underscores in your column names: -
+
 cat data/spaces.csv
+
+
 a b c,def,g h i
 123,4567,890
 2468,1357,3579
 9987,3312,4543
 
-The simplest way is to use ``mlr rename`` with ``-g`` (for global replace, not just first occurrence of space within each field) and ``-r`` for pattern-matching (rather than explicit single-column renames): +The simplest way is to use `mlr rename` with `-g` (for global replace, not just first occurrence of space within each field) and `-r` for pattern-matching (rather than explicit single-column renames): -
+
 mlr --csv rename -g -r ' ,_'  data/spaces.csv
+
+
 a_b_c,def,g_h_i
 123,4567,890
 2468,1357,3579
 9987,3312,4543
 
-
+
 mlr --csv --opprint rename -g -r ' ,_'  data/spaces.csv
+
+
 a_b_c def  g_h_i
 123   4567 890
 2468  1357 3579
@@ -33,8 +39,10 @@ a_b_c def  g_h_i
 
 You can also do this with a for-loop:
 
-
+
 cat data/bulk-rename-for-loop.mlr
+
+
 map newrec = {};
 for (oldk, v in $*) {
     newrec[gsub(oldk, " ", "_")] = v;
@@ -42,8 +50,10 @@ for (oldk, v in $*) {
 $* = newrec
 
-
+
 mlr --icsv --opprint put -f data/bulk-rename-for-loop.mlr data/spaces.csv
+
+
 a_b_c def  g_h_i
 123   4567 890
 2468  1357 3579
@@ -52,24 +62,30 @@ a_b_c def  g_h_i
 
 ## Search-and-replace over all fields
 
-How to do ``$name = gsub($name, "old", "new")`` for all fields?
+How to do `$name = gsub($name, "old", "new")` for all fields?
 
-
+
 cat data/sar.csv
+
+
 a,b,c
 the quick,brown fox,jumped
 over,the,lazy dogs
 
-
+
 cat data/sar.mlr
-  for (k in $*) {
-    $[k] = gsub($[k], "e", "X");
-  }
+
+
+for (k in $*) {
+  $[k] = gsub($[k], "e", "X");
+}
 
-
+
 mlr --csv put -f data/sar.mlr data/sar.csv
+
+
 a,b,c
 thX quick,brown fox,jumpXd
 ovXr,thX,lazy dogs
@@ -77,10 +93,12 @@ ovXr,thX,lazy dogs
 
 ## Full field renames and reassigns
 
-Using Miller 5.0.0's map literals and assigning to ``$*``, you can fully generalize :ref:`mlr rename `, :ref:`mlr reorder `, etc.
+Using Miller 5.0.0's map literals and assigning to `$*`, you can fully generalize [rename](reference-verbs.md#rename), [reorder](reference-verbs.md#reorder), etc.
 
-
+
 cat data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -88,7 +106,7 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
-
+
 mlr put '
   begin {
     @i_cumu = 0;
@@ -104,6 +122,8 @@ a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
     "x": $y,
   };
 ' data/small
+
+
 z=0.3467901443380824,KEYFIELD=pan,i=1,b=pan,y=0.3467901443380824,x=0.7268028627434533
 z=0.7586799647899636,KEYFIELD=eks,i=3,b=pan,y=0.7586799647899636,x=0.5221511083334797
 z=0.20460330576630303,KEYFIELD=wye,i=6,b=wye,y=0.20460330576630303,x=0.33831852551664776
diff --git a/docs6b/docs/operating-on-all-fields.md.in b/docs6b/docs/operating-on-all-fields.md.in
index 2af4c6c92..8a6eb09f2 100644
--- a/docs6b/docs/operating-on-all-fields.md.in
+++ b/docs6b/docs/operating-on-all-fields.md.in
@@ -8,7 +8,7 @@ GENMD_RUN_COMMAND
 cat data/spaces.csv
 GENMD_EOF
 
-The simplest way is to use ``mlr rename`` with ``-g`` (for global replace, not just first occurrence of space within each field) and ``-r`` for pattern-matching (rather than explicit single-column renames):
+The simplest way is to use `mlr rename` with `-g` (for global replace, not just first occurrence of space within each field) and `-r` for pattern-matching (rather than explicit single-column renames):
 
 GENMD_RUN_COMMAND
 mlr --csv rename -g -r ' ,_'  data/spaces.csv
@@ -30,7 +30,7 @@ GENMD_EOF
 
 ## Search-and-replace over all fields
 
-How to do ``$name = gsub($name, "old", "new")`` for all fields?
+How to do `$name = gsub($name, "old", "new")` for all fields?
 
 GENMD_RUN_COMMAND
 cat data/sar.csv
@@ -46,7 +46,7 @@ GENMD_EOF
 
 ## Full field renames and reassigns
 
-Using Miller 5.0.0's map literals and assigning to ``$*``, you can fully generalize :ref:`mlr rename `, :ref:`mlr reorder `, etc.
+Using Miller 5.0.0's map literals and assigning to `$*`, you can fully generalize [rename](reference-verbs.md#rename), [reorder](reference-verbs.md#reorder), etc.
 
 GENMD_RUN_COMMAND
 cat data/small
diff --git a/docs6b/docs/originality.md b/docs6b/docs/originality.md
index 5c984d7d2..ce42d40a8 100644
--- a/docs6b/docs/originality.md
+++ b/docs6b/docs/originality.md
@@ -1,40 +1,40 @@
 
 # How original is Miller?
 
-It isn't. Miller is one of many, many participants in the online-analytical-processing culture. Other key participants include ``awk``, SQL, spreadsheets, etc. etc.  etc.  Far from being an original concept, Miller explicitly strives to imitate several existing tools:
+It isn't. Miller is one of many, many participants in the online-analytical-processing culture. Other key participants include `awk`, SQL, spreadsheets, etc. etc.  etc.  Far from being an original concept, Miller explicitly strives to imitate several existing tools:
 
 **The Unix toolkit**: Intentional similarities as described in [Unix-toolkit Context](feature-comparison.md).
 
 Recipes abound for command-line data analysis using the Unix toolkit. Here are just a couple of my favorites:
 
-* http://en.wikibooks.org/wiki/Ad_Hoc_Data_Analysis_From_The_Unix_Command_Line
-* http://www.gregreda.com/2013/07/15/unix-commands-for-data-science
-* https://github.com/dbohdan/structured-text-tools
+* [http://en.wikibooks.org/wiki/Ad_Hoc_Data_Analysis_From_The_Unix_Command_Line](http://en.wikibooks.org/wiki/Ad_Hoc_Data_Analysis_From_The_Unix_Command_Line)
+* [http://www.gregreda.com/2013/07/15/unix-commands-for-data-science](http://www.gregreda.com/2013/07/15/unix-commands-for-data-science)
+* [https://github.com/dbohdan/structured-text-tools](https://github.com/dbohdan/structured-text-tools)
 
-**RecordStream**: Miller owes particular inspiration to [RecordStream](https://github.com/benbernard/RecordStream). The key difference is that RecordStream is a Perl-based tool for manipulating JSON (including requiring it to separately manipulate other formats such as CSV into and out of JSON), while Miller is fast Go which handles its formats natively.  The similarities include the ``sort``, ``stats1`` (analog of RecordStream's ``collate``), and ``delta`` operations, as well as ``filter`` and ``put``, and pretty-print formatting.
+**RecordStream**: Miller owes particular inspiration to [RecordStream](https://github.com/benbernard/RecordStream). The key difference is that RecordStream is a Perl-based tool for manipulating JSON (including requiring it to separately manipulate other formats such as CSV into and out of JSON), while Miller is fast Go which handles its formats natively.  The similarities include the `sort`, `stats1` (analog of RecordStream's `collate`), and `delta` operations, as well as `filter` and `put`, and pretty-print formatting.
 
-**stats_m**: A third source of lineage is my Python [stats_m](https://github.com/johnkerl/scripts-math/tree/master/stats) module.  This includes simple single-pass algorithms which form Miller's ``stats1`` and ``stats2`` subcommands.
+**stats_m**: A third source of lineage is my Python [stats_m](https://github.com/johnkerl/scripts-math/tree/master/stats) module.  This includes simple single-pass algorithms which form Miller's `stats1` and `stats2` subcommands.
 
-**SQL**: Fourthly, Miller's ``group-by`` command name is from SQL, as is the term ``aggregate``.
+**SQL**: Fourthly, Miller's `group-by` command name is from SQL, as is the term `aggregate`.
 
 **Added value**: Miller's added values include:
 
 * Name-indexing, compared to the Unix toolkit's positional indexing.
-* Raw speed, compared to ``awk``, RecordStream, ``stats_m``, or various other kinds of Python/Ruby/etc. scripts one can easily create.
+* Raw speed, compared to `awk`, RecordStream, `stats_m`, or various other kinds of Python/Ruby/etc. scripts one can easily create.
 * Compact keystroking for many common tasks, with a decent amount of flexibility.
 * Ability to handle text files on the Unix pipe, without need for creating database tables, compared to SQL databases.
 * Various file formats, and on-the-fly format conversion.
 
-**jq**: Miller does for name-indexed text what [jq](https://stedolan.github.io/jq/) does for JSON. If you're not already familiar with ``jq``, please check it out!.
+**jq**: Miller does for name-indexed text what [jq](https://stedolan.github.io/jq/) does for JSON. If you're not already familiar with `jq`, please check it out!.
 
 **What about similar tools?**
 
-Here's a comprehensive list: https://github.com/dbohdan/structured-text-tools.  Last I knew it doesn't mention [rows](https://github.com/turicas/rows) so here's a plug for that as well.  As it turns out, I learned about most of these after writing Miller.
+Here's a comprehensive list: [https://github.com/dbohdan/structured-text-tools](https://github.com/dbohdan/structured-text-tools).  Last I knew it doesn't mention [rows](https://github.com/turicas/rows) so here's a plug for that as well.  As it turns out, I learned about most of these after writing Miller.
 
-**What about DOTADIW?** One of the key points of the [Unix philosophy](http://en.wikipedia.org/wiki/Unix_philosophy) is that a tool should do one thing and do it well.  Hence ``sort`` and ``cut`` do just one thing. Why does Miller put ``awk``-like processing, a few SQL-like operations, and statistical reduction all into one tool?  This is a fair question. First note that many standard tools, such as ``awk`` and ``perl``, do quite a few things -- as does ``jq``.  But I could have pushed for putting format awareness and name-indexing options into ``cut``, ``awk``, and so on (so you could do ``cut -f hostname,uptime`` or ``awk '{sum += $x*$y}END{print sum}'``).  Patching ``cut``, ``sort``, etc. on multiple operating systems is a non-starter in terms of uptake.  Moreover, it makes sense for me to have Miller be a tool which collects together format-aware record-stream processing into one place, with good reuse of Miller-internal library code for its various features.
+**What about DOTADIW?** One of the key points of the [Unix philosophy](http://en.wikipedia.org/wiki/Unix_philosophy) is that a tool should do one thing and do it well.  Hence `sort` and `cut` do just one thing. Why does Miller put `awk`-like processing, a few SQL-like operations, and statistical reduction all into one tool?  This is a fair question. First note that many standard tools, such as `awk` and `perl`, do quite a few things -- as does `jq`.  But I could have pushed for putting format awareness and name-indexing options into `cut`, `awk`, and so on (so you could do `cut -f hostname,uptime` or `awk '{sum += $x*$y}END{print sum}'`).  Patching `cut`, `sort`, etc. on multiple operating systems is a non-starter in terms of uptake.  Moreover, it makes sense for me to have Miller be a tool which collects together format-aware record-stream processing into one place, with good reuse of Miller-internal library code for its various features.
 
 **Why not use Perl/Python/Ruby etc.?** Maybe you should. With those tools you'll get far more expressive power, and sufficiently quick turnaround time for small-to-medium-sized data.  Using Miller you'll get something less than a complete programming language, but which is fast, with moderate amounts of flexibility and much less keystroking.
 
-When I was first developing Miller I made a survey of several languages. Using low-level implementation languages like C, Go, Rust, and Nim, I'd need to create my own domain-specific language (DSL) which would always be less featured than a full programming language, but I'd get better performance.  Using high-level interpreted languages such as Perl/Python/Ruby I'd get the language's ``eval`` for free and I wouldn't need a DSL; Miller would have mainly been a set of format-specific I/O hooks. If I'd gotten good enough performance from the latter I'd have done it without question and Miller would be far more flexible.  But high-level languages win the performance criteria by a landslide so we have Miller in Go with a custom DSL.
+When I was first developing Miller I made a survey of several languages. Using low-level implementation languages like C, Go, Rust, and Nim, I'd need to create my own domain-specific language (DSL) which would always be less featured than a full programming language, but I'd get better performance.  Using high-level interpreted languages such as Perl/Python/Ruby I'd get the language's `eval` for free and I wouldn't need a DSL; Miller would have mainly been a set of format-specific I/O hooks. If I'd gotten good enough performance from the latter I'd have done it without question and Miller would be far more flexible.  But high-level languages win the performance criteria by a landslide so we have Miller in Go with a custom DSL.
 
-**No, really, why one more command-line data-manipulation tool?** I wrote Miller because I was frustrated with tools like ``grep``, ``sed``, and so on being *line-aware* without being *format-aware*. The single most poignant example I can think of is seeing people grep data lines out of their CSV files and sadly losing their header lines.  While some lighter-than-SQL processing is very nice to have, at core I wanted the format-awareness of [RecordStream](https://github.com/benbernard/RecordStream) combined with the raw speed of the Unix toolkit. Miller does precisely that.
+**No, really, why one more command-line data-manipulation tool?** I wrote Miller because I was frustrated with tools like `grep`, `sed`, and so on being *line-aware* without being *format-aware*. The single most poignant example I can think of is seeing people grep data lines out of their CSV files and sadly losing their header lines.  While some lighter-than-SQL processing is very nice to have, at core I wanted the format-awareness of [RecordStream](https://github.com/benbernard/RecordStream) combined with the raw speed of the Unix toolkit. Miller does precisely that.
diff --git a/docs6b/docs/originality.md.in b/docs6b/docs/originality.md.in
index c327e3ec9..9c32d9e1d 100644
--- a/docs6b/docs/originality.md.in
+++ b/docs6b/docs/originality.md.in
@@ -1,39 +1,39 @@
 # How original is Miller?
 
-It isn't. Miller is one of many, many participants in the online-analytical-processing culture. Other key participants include ``awk``, SQL, spreadsheets, etc. etc.  etc.  Far from being an original concept, Miller explicitly strives to imitate several existing tools:
+It isn't. Miller is one of many, many participants in the online-analytical-processing culture. Other key participants include `awk`, SQL, spreadsheets, etc. etc.  etc.  Far from being an original concept, Miller explicitly strives to imitate several existing tools:
 
 **The Unix toolkit**: Intentional similarities as described in [Unix-toolkit Context](feature-comparison.md).
 
 Recipes abound for command-line data analysis using the Unix toolkit. Here are just a couple of my favorites:
 
-* http://en.wikibooks.org/wiki/Ad_Hoc_Data_Analysis_From_The_Unix_Command_Line
-* http://www.gregreda.com/2013/07/15/unix-commands-for-data-science
-* https://github.com/dbohdan/structured-text-tools
+* [http://en.wikibooks.org/wiki/Ad_Hoc_Data_Analysis_From_The_Unix_Command_Line](http://en.wikibooks.org/wiki/Ad_Hoc_Data_Analysis_From_The_Unix_Command_Line)
+* [http://www.gregreda.com/2013/07/15/unix-commands-for-data-science](http://www.gregreda.com/2013/07/15/unix-commands-for-data-science)
+* [https://github.com/dbohdan/structured-text-tools](https://github.com/dbohdan/structured-text-tools)
 
-**RecordStream**: Miller owes particular inspiration to [RecordStream](https://github.com/benbernard/RecordStream). The key difference is that RecordStream is a Perl-based tool for manipulating JSON (including requiring it to separately manipulate other formats such as CSV into and out of JSON), while Miller is fast Go which handles its formats natively.  The similarities include the ``sort``, ``stats1`` (analog of RecordStream's ``collate``), and ``delta`` operations, as well as ``filter`` and ``put``, and pretty-print formatting.
+**RecordStream**: Miller owes particular inspiration to [RecordStream](https://github.com/benbernard/RecordStream). The key difference is that RecordStream is a Perl-based tool for manipulating JSON (including requiring it to separately manipulate other formats such as CSV into and out of JSON), while Miller is fast Go which handles its formats natively.  The similarities include the `sort`, `stats1` (analog of RecordStream's `collate`), and `delta` operations, as well as `filter` and `put`, and pretty-print formatting.
 
-**stats_m**: A third source of lineage is my Python [stats_m](https://github.com/johnkerl/scripts-math/tree/master/stats) module.  This includes simple single-pass algorithms which form Miller's ``stats1`` and ``stats2`` subcommands.
+**stats_m**: A third source of lineage is my Python [stats_m](https://github.com/johnkerl/scripts-math/tree/master/stats) module.  This includes simple single-pass algorithms which form Miller's `stats1` and `stats2` subcommands.
 
-**SQL**: Fourthly, Miller's ``group-by`` command name is from SQL, as is the term ``aggregate``.
+**SQL**: Fourthly, Miller's `group-by` command name is from SQL, as is the term `aggregate`.
 
 **Added value**: Miller's added values include:
 
 * Name-indexing, compared to the Unix toolkit's positional indexing.
-* Raw speed, compared to ``awk``, RecordStream, ``stats_m``, or various other kinds of Python/Ruby/etc. scripts one can easily create.
+* Raw speed, compared to `awk`, RecordStream, `stats_m`, or various other kinds of Python/Ruby/etc. scripts one can easily create.
 * Compact keystroking for many common tasks, with a decent amount of flexibility.
 * Ability to handle text files on the Unix pipe, without need for creating database tables, compared to SQL databases.
 * Various file formats, and on-the-fly format conversion.
 
-**jq**: Miller does for name-indexed text what [jq](https://stedolan.github.io/jq/) does for JSON. If you're not already familiar with ``jq``, please check it out!.
+**jq**: Miller does for name-indexed text what [jq](https://stedolan.github.io/jq/) does for JSON. If you're not already familiar with `jq`, please check it out!.
 
 **What about similar tools?**
 
-Here's a comprehensive list: https://github.com/dbohdan/structured-text-tools.  Last I knew it doesn't mention [rows](https://github.com/turicas/rows) so here's a plug for that as well.  As it turns out, I learned about most of these after writing Miller.
+Here's a comprehensive list: [https://github.com/dbohdan/structured-text-tools](https://github.com/dbohdan/structured-text-tools).  Last I knew it doesn't mention [rows](https://github.com/turicas/rows) so here's a plug for that as well.  As it turns out, I learned about most of these after writing Miller.
 
-**What about DOTADIW?** One of the key points of the [Unix philosophy](http://en.wikipedia.org/wiki/Unix_philosophy) is that a tool should do one thing and do it well.  Hence ``sort`` and ``cut`` do just one thing. Why does Miller put ``awk``-like processing, a few SQL-like operations, and statistical reduction all into one tool?  This is a fair question. First note that many standard tools, such as ``awk`` and ``perl``, do quite a few things -- as does ``jq``.  But I could have pushed for putting format awareness and name-indexing options into ``cut``, ``awk``, and so on (so you could do ``cut -f hostname,uptime`` or ``awk '{sum += $x*$y}END{print sum}'``).  Patching ``cut``, ``sort``, etc. on multiple operating systems is a non-starter in terms of uptake.  Moreover, it makes sense for me to have Miller be a tool which collects together format-aware record-stream processing into one place, with good reuse of Miller-internal library code for its various features.
+**What about DOTADIW?** One of the key points of the [Unix philosophy](http://en.wikipedia.org/wiki/Unix_philosophy) is that a tool should do one thing and do it well.  Hence `sort` and `cut` do just one thing. Why does Miller put `awk`-like processing, a few SQL-like operations, and statistical reduction all into one tool?  This is a fair question. First note that many standard tools, such as `awk` and `perl`, do quite a few things -- as does `jq`.  But I could have pushed for putting format awareness and name-indexing options into `cut`, `awk`, and so on (so you could do `cut -f hostname,uptime` or `awk '{sum += $x*$y}END{print sum}'`).  Patching `cut`, `sort`, etc. on multiple operating systems is a non-starter in terms of uptake.  Moreover, it makes sense for me to have Miller be a tool which collects together format-aware record-stream processing into one place, with good reuse of Miller-internal library code for its various features.
 
 **Why not use Perl/Python/Ruby etc.?** Maybe you should. With those tools you'll get far more expressive power, and sufficiently quick turnaround time for small-to-medium-sized data.  Using Miller you'll get something less than a complete programming language, but which is fast, with moderate amounts of flexibility and much less keystroking.
 
-When I was first developing Miller I made a survey of several languages. Using low-level implementation languages like C, Go, Rust, and Nim, I'd need to create my own domain-specific language (DSL) which would always be less featured than a full programming language, but I'd get better performance.  Using high-level interpreted languages such as Perl/Python/Ruby I'd get the language's ``eval`` for free and I wouldn't need a DSL; Miller would have mainly been a set of format-specific I/O hooks. If I'd gotten good enough performance from the latter I'd have done it without question and Miller would be far more flexible.  But high-level languages win the performance criteria by a landslide so we have Miller in Go with a custom DSL.
+When I was first developing Miller I made a survey of several languages. Using low-level implementation languages like C, Go, Rust, and Nim, I'd need to create my own domain-specific language (DSL) which would always be less featured than a full programming language, but I'd get better performance.  Using high-level interpreted languages such as Perl/Python/Ruby I'd get the language's `eval` for free and I wouldn't need a DSL; Miller would have mainly been a set of format-specific I/O hooks. If I'd gotten good enough performance from the latter I'd have done it without question and Miller would be far more flexible.  But high-level languages win the performance criteria by a landslide so we have Miller in Go with a custom DSL.
 
-**No, really, why one more command-line data-manipulation tool?** I wrote Miller because I was frustrated with tools like ``grep``, ``sed``, and so on being *line-aware* without being *format-aware*. The single most poignant example I can think of is seeing people grep data lines out of their CSV files and sadly losing their header lines.  While some lighter-than-SQL processing is very nice to have, at core I wanted the format-awareness of [RecordStream](https://github.com/benbernard/RecordStream) combined with the raw speed of the Unix toolkit. Miller does precisely that.
+**No, really, why one more command-line data-manipulation tool?** I wrote Miller because I was frustrated with tools like `grep`, `sed`, and so on being *line-aware* without being *format-aware*. The single most poignant example I can think of is seeing people grep data lines out of their CSV files and sadly losing their header lines.  While some lighter-than-SQL processing is very nice to have, at core I wanted the format-awareness of [RecordStream](https://github.com/benbernard/RecordStream) combined with the raw speed of the Unix toolkit. Miller does precisely that.
diff --git a/docs6b/docs/output-colorization.md b/docs6b/docs/output-colorization.md
index 1b6d02fd1..910d32215 100644
--- a/docs6b/docs/output-colorization.md
+++ b/docs6b/docs/output-colorization.md
@@ -3,7 +3,7 @@
 
 As of version 6.0.0, Miller supports output-colorization. Here are examples using side-by-side black-background and white-background terminals:
 
-.. image:: pix/colorization.png
+![pix/colorization.png](pix/colorization.png)
 
 Things having colors:
 
@@ -16,38 +16,37 @@ Rules for colorization:
 
 * By default, colorize output only if writing to stdout and stdout is a TTY.
 
-    * Example: color: ``mlr --csv cat foo.csv``
-    * Example: no color: ``mlr --csv cat foo.csv > bar.csv``
-    * Example: no color: ``mlr --csv cat foo.csv | less``
+    * Example: color: `mlr --csv cat foo.csv`
+    * Example: no color: `mlr --csv cat foo.csv > bar.csv`
+    * Example: no color: `mlr --csv cat foo.csv | less`
 
 * The default colors were chosen since they look OK with white or black terminal background, and are differentiable with common varieties of human color vision.
 
 Mechanisms for colorization:
 
 * Miller uses ANSI escape sequences only. This does not work on Windows except on Cygwin.
-* Requires ``TERM`` environment variable to be set to non-empty string.
+* Requires `TERM` environment variable to be set to non-empty string.
 * Doesn't try to check to see whether the terminal is capable of 256-color ANSI vs 16-color ANSI. Note that if colors are in the range 0..15 then 16-color ANSI escapes are used, so this is in the user's control.
 
 How you can control colorization:
 
 * Suppression/unsuppression:
 
-    * ``export MLR_NO_COLOR=true`` means Miller won't color even when it normally would.
-    * ``export MLR_ALWAYS_COLOR=true`` means Miller will color even when it normally would not. For example, you might want to use this when piping ``mlr`` output to ``less -r``.
-    * Command-line flags ``--no-color`` or ``-M``, ``--always-color`` or ``-C``.
+    * `export MLR_NO_COLOR=true` means Miller won't color even when it normally would.
+    * `export MLR_ALWAYS_COLOR=true` means Miller will color even when it normally would not. For example, you might want to use this when piping `mlr` output to `less -r`.
+    * Command-line flags `--no-color` or `-M`, `--always-color` or `-C`.
 
 
 * Color choices can be specified by using environment variables or command-line flags, with values 0..255:
 
-    * ``export MLR_KEY_COLOR=208``
-    * ``export MLR_VALUE_COLOR=33``
-    * Likewise for ``MLR_PASS_COLOR``, ``MLR_FAIL_COLOR``, ``MLR_HELP_COLOR``, ``MLR_REPL_PS1_COLOR``, and ``MLR_REPL_PS2_COLOR``.
-    * Command-line flags ``--key-color 208``, ``--value-color 33``, etc., and likewise for ``--pass-color``, ``--fail-color``, ``--repl-ps1-color``, ``--repl-ps2-color``, and ``--help-color``.
+    * `export MLR_KEY_COLOR=208`
+    * `export MLR_VALUE_COLOR=33`
+    * Likewise for `MLR_PASS_COLOR`, `MLR_FAIL_COLOR`, `MLR_HELP_COLOR`, `MLR_REPL_PS1_COLOR`, and `MLR_REPL_PS2_COLOR`.
+    * Command-line flags `--key-color 208`, `--value-color 33`, etc., and likewise for `--pass-color`, `--fail-color`, `--repl-ps1-color`, `--repl-ps2-color`, and `--help-color`.
     * This is particularly useful if your terminal's background color clashes with current settings.
 
 If environment-variable settings and command-line flags are both provided, the latter take precedence.
 
-Please do ``mlr --list-color-codes`` to see the available color codes (like ``170``), and ``mlr --list-color-names`` to see available names (like ``orchid``).
-
-.. image:: pix/colorization2.png
+Please do `mlr --list-color-codes` to see the available color codes (like `170`), and `mlr --list-color-names` to see available names (like `orchid`).
 
+![pix/colorization2.png](pix/colorization2.png)
diff --git a/docs6b/docs/output-colorization.md.in b/docs6b/docs/output-colorization.md.in
index 2a3df54fe..f0a118f7a 100644
--- a/docs6b/docs/output-colorization.md.in
+++ b/docs6b/docs/output-colorization.md.in
@@ -2,7 +2,7 @@
 
 As of version 6.0.0, Miller supports output-colorization. Here are examples using side-by-side black-background and white-background terminals:
 
-.. image:: pix/colorization.png
+![pix/colorization.png](pix/colorization.png)
 
 Things having colors:
 
@@ -15,38 +15,37 @@ Rules for colorization:
 
 * By default, colorize output only if writing to stdout and stdout is a TTY.
 
-    * Example: color: ``mlr --csv cat foo.csv``
-    * Example: no color: ``mlr --csv cat foo.csv > bar.csv``
-    * Example: no color: ``mlr --csv cat foo.csv | less``
+    * Example: color: `mlr --csv cat foo.csv`
+    * Example: no color: `mlr --csv cat foo.csv > bar.csv`
+    * Example: no color: `mlr --csv cat foo.csv | less`
 
 * The default colors were chosen since they look OK with white or black terminal background, and are differentiable with common varieties of human color vision.
 
 Mechanisms for colorization:
 
 * Miller uses ANSI escape sequences only. This does not work on Windows except on Cygwin.
-* Requires ``TERM`` environment variable to be set to non-empty string.
+* Requires `TERM` environment variable to be set to non-empty string.
 * Doesn't try to check to see whether the terminal is capable of 256-color ANSI vs 16-color ANSI. Note that if colors are in the range 0..15 then 16-color ANSI escapes are used, so this is in the user's control.
 
 How you can control colorization:
 
 * Suppression/unsuppression:
 
-    * ``export MLR_NO_COLOR=true`` means Miller won't color even when it normally would.
-    * ``export MLR_ALWAYS_COLOR=true`` means Miller will color even when it normally would not. For example, you might want to use this when piping ``mlr`` output to ``less -r``.
-    * Command-line flags ``--no-color`` or ``-M``, ``--always-color`` or ``-C``.
+    * `export MLR_NO_COLOR=true` means Miller won't color even when it normally would.
+    * `export MLR_ALWAYS_COLOR=true` means Miller will color even when it normally would not. For example, you might want to use this when piping `mlr` output to `less -r`.
+    * Command-line flags `--no-color` or `-M`, `--always-color` or `-C`.
 
 
 * Color choices can be specified by using environment variables or command-line flags, with values 0..255:
 
-    * ``export MLR_KEY_COLOR=208``
-    * ``export MLR_VALUE_COLOR=33``
-    * Likewise for ``MLR_PASS_COLOR``, ``MLR_FAIL_COLOR``, ``MLR_HELP_COLOR``, ``MLR_REPL_PS1_COLOR``, and ``MLR_REPL_PS2_COLOR``.
-    * Command-line flags ``--key-color 208``, ``--value-color 33``, etc., and likewise for ``--pass-color``, ``--fail-color``, ``--repl-ps1-color``, ``--repl-ps2-color``, and ``--help-color``.
+    * `export MLR_KEY_COLOR=208`
+    * `export MLR_VALUE_COLOR=33`
+    * Likewise for `MLR_PASS_COLOR`, `MLR_FAIL_COLOR`, `MLR_HELP_COLOR`, `MLR_REPL_PS1_COLOR`, and `MLR_REPL_PS2_COLOR`.
+    * Command-line flags `--key-color 208`, `--value-color 33`, etc., and likewise for `--pass-color`, `--fail-color`, `--repl-ps1-color`, `--repl-ps2-color`, and `--help-color`.
     * This is particularly useful if your terminal's background color clashes with current settings.
 
 If environment-variable settings and command-line flags are both provided, the latter take precedence.
 
-Please do ``mlr --list-color-codes`` to see the available color codes (like ``170``), and ``mlr --list-color-names`` to see available names (like ``orchid``).
-
-.. image:: pix/colorization2.png
+Please do `mlr --list-color-codes` to see the available color codes (like `170`), and `mlr --list-color-names` to see available names (like `orchid`).
 
+![pix/colorization2.png](pix/colorization2.png)
diff --git a/docs6b/docs/performance.md b/docs6b/docs/performance.md
index 3fbb64ae9..faaacfbcb 100644
--- a/docs6b/docs/performance.md
+++ b/docs6b/docs/performance.md
@@ -5,14 +5,14 @@
 
 In a previous version of this page, I compared Miller to some items in the Unix toolkit in terms of run time. But such comparisons are very much not apples-to-apples:
 
-* Miller's principal strength is that it handles **key-value data in various formats** while the system tools **do not**. So if you time ``mlr sort`` on a CSV file against system ``sort``, it's not relevant to say which is faster by how many percent -- Miller will respect the header line, leaving it in place, while the system sort will move it, sorting it along with all the other header lines. This would be comparing the run times of two programs produce different outputs.  Likewise, ``awk`` doesn't respect header lines, although you can code up some CSV-handling using ``if (NR==1) { ... } else { ... }``. And that's just CSV: I don't know any simple way to get ``sort``, ``awk``, etc. to handle DKVP, JSON, etc. -- which is the main reason I wrote Miller.
+* Miller's principal strength is that it handles **key-value data in various formats** while the system tools **do not**. So if you time `mlr sort` on a CSV file against system `sort`, it's not relevant to say which is faster by how many percent -- Miller will respect the header line, leaving it in place, while the system sort will move it, sorting it along with all the other header lines. This would be comparing the run times of two programs produce different outputs.  Likewise, `awk` doesn't respect header lines, although you can code up some CSV-handling using `if (NR==1) { ... } else { ... }`. And that's just CSV: I don't know any simple way to get `sort`, `awk`, etc. to handle DKVP, JSON, etc. -- which is the main reason I wrote Miller.
 
-* **Implementations differ by platform**: one ``awk`` may be fundamentally faster than another, and ``mawk`` has a very efficient bytecode implementation -- which handles positionally indexed data far faster than Miller does.
+* **Implementations differ by platform**: one `awk` may be fundamentally faster than another, and `mawk` has a very efficient bytecode implementation -- which handles positionally indexed data far faster than Miller does.
 
-* The system ``sort`` command will, on some systems, handle too-large-for-RAM datasets by spilling to disk; Miller (as of version 5.2.0, mid-2017) does not. Miller sorts are always stable; GNU supports stable and unstable variants.
+* The system `sort` command will, on some systems, handle too-large-for-RAM datasets by spilling to disk; Miller (as of version 5.2.0, mid-2017) does not. Miller sorts are always stable; GNU supports stable and unstable variants.
 
 * Etc.
 
 ## Summary
 
-Miller can do many kinds of processing on key-value-pair data using elapsed time roughly of the same order of magnitude as items in the Unix toolkit can handle positionally indexed data. Specific results vary widely by platform, implementation details, multi-core use (or not). Lastly, specific special-purpose non-record-aware processing will run far faster in ``grep``, ``sed``, etc.
+Miller can do many kinds of processing on key-value-pair data using elapsed time roughly of the same order of magnitude as items in the Unix toolkit can handle positionally indexed data. Specific results vary widely by platform, implementation details, multi-core use (or not). Lastly, specific special-purpose non-record-aware processing will run far faster in `grep`, `sed`, etc.
diff --git a/docs6b/docs/performance.md.in b/docs6b/docs/performance.md.in
index 593dca56c..a319ee8a4 100644
--- a/docs6b/docs/performance.md.in
+++ b/docs6b/docs/performance.md.in
@@ -4,14 +4,14 @@
 
 In a previous version of this page, I compared Miller to some items in the Unix toolkit in terms of run time. But such comparisons are very much not apples-to-apples:
 
-* Miller's principal strength is that it handles **key-value data in various formats** while the system tools **do not**. So if you time ``mlr sort`` on a CSV file against system ``sort``, it's not relevant to say which is faster by how many percent -- Miller will respect the header line, leaving it in place, while the system sort will move it, sorting it along with all the other header lines. This would be comparing the run times of two programs produce different outputs.  Likewise, ``awk`` doesn't respect header lines, although you can code up some CSV-handling using ``if (NR==1) { ... } else { ... }``. And that's just CSV: I don't know any simple way to get ``sort``, ``awk``, etc. to handle DKVP, JSON, etc. -- which is the main reason I wrote Miller.
+* Miller's principal strength is that it handles **key-value data in various formats** while the system tools **do not**. So if you time `mlr sort` on a CSV file against system `sort`, it's not relevant to say which is faster by how many percent -- Miller will respect the header line, leaving it in place, while the system sort will move it, sorting it along with all the other header lines. This would be comparing the run times of two programs produce different outputs.  Likewise, `awk` doesn't respect header lines, although you can code up some CSV-handling using `if (NR==1) { ... } else { ... }`. And that's just CSV: I don't know any simple way to get `sort`, `awk`, etc. to handle DKVP, JSON, etc. -- which is the main reason I wrote Miller.
 
-* **Implementations differ by platform**: one ``awk`` may be fundamentally faster than another, and ``mawk`` has a very efficient bytecode implementation -- which handles positionally indexed data far faster than Miller does.
+* **Implementations differ by platform**: one `awk` may be fundamentally faster than another, and `mawk` has a very efficient bytecode implementation -- which handles positionally indexed data far faster than Miller does.
 
-* The system ``sort`` command will, on some systems, handle too-large-for-RAM datasets by spilling to disk; Miller (as of version 5.2.0, mid-2017) does not. Miller sorts are always stable; GNU supports stable and unstable variants.
+* The system `sort` command will, on some systems, handle too-large-for-RAM datasets by spilling to disk; Miller (as of version 5.2.0, mid-2017) does not. Miller sorts are always stable; GNU supports stable and unstable variants.
 
 * Etc.
 
 ## Summary
 
-Miller can do many kinds of processing on key-value-pair data using elapsed time roughly of the same order of magnitude as items in the Unix toolkit can handle positionally indexed data. Specific results vary widely by platform, implementation details, multi-core use (or not). Lastly, specific special-purpose non-record-aware processing will run far faster in ``grep``, ``sed``, etc.
+Miller can do many kinds of processing on key-value-pair data using elapsed time roughly of the same order of magnitude as items in the Unix toolkit can handle positionally indexed data. Specific results vary widely by platform, implementation details, multi-core use (or not). Lastly, specific special-purpose non-record-aware processing will run far faster in `grep`, `sed`, etc.
diff --git a/docs6b/docs/programming-examples.md b/docs6b/docs/programming-examples.md
index 644da2304..47fc25a21 100644
--- a/docs6b/docs/programming-examples.md
+++ b/docs6b/docs/programming-examples.md
@@ -5,10 +5,12 @@ Here are a few things focusing on Miller's DSL as a programming language per se,
 
 ## Sieve of Eratosthenes
 
-The [Sieve of Eratosthenes](http://en.wikipedia.org/wiki/Sieve_of_Eratosthenes) is a standard introductory programming topic. The idea is to find all primes up to some *N* by making a list of the numbers 1 to *N*, then striking out all multiples of 2 except 2 itself, all multiples of 3 except 3 itself, all multiples of 4 except 4 itself, and so on. Whatever survives that without getting marked is a prime. This is easy enough in Miller. Notice that here all the work is in ``begin`` and ``end`` statements; there is no file input (so we use ``mlr -n`` to keep Miller from waiting for input data).
+The [Sieve of Eratosthenes](http://en.wikipedia.org/wiki/Sieve_of_Eratosthenes) is a standard introductory programming topic. The idea is to find all primes up to some *N* by making a list of the numbers 1 to *N*, then striking out all multiples of 2 except 2 itself, all multiples of 3 except 3 itself, all multiples of 4 except 4 itself, and so on. Whatever survives that without getting marked is a prime. This is easy enough in Miller. Notice that here all the work is in `begin` and `end` statements; there is no file input (so we use `mlr -n` to keep Miller from waiting for input data).
 
-
+
 cat programs/sieve.mlr
+
+
 # ================================================================
 # Sieve of Eratosthenes: simple example of Miller DSL as programming language.
 # ================================================================
@@ -43,8 +45,10 @@ end {
 }
 
-
+
 mlr -n put -f programs/sieve.mlr
+
+
 2
 3
 5
@@ -78,8 +82,10 @@ The [Mandelbrot set](http://en.wikipedia.org/wiki/Mandelbrot_set) is also easily
 
 The (approximate) computation of points in the complex plane which are and aren't members is just a few lines of complex arithmetic (see the Wikipedia article); how to render them is another task.  Using graphics libraries you can create PNG or JPEG files, but another fun way to do this is by printing various characters to the screen:
 
-
+
 cat programs/mand.mlr
+
+
 # Mandelbrot set generator: simple example of Miller DSL as programming language.
 begin {
   # Set defaults
@@ -184,8 +190,10 @@ func get_point_plot(pr, pi, maxits, do_julia, jr, ji) {
 
 At standard resolution this makes a nice little ASCII plot:
 
-
+
 mlr -n put -f ./programs/mand.mlr
+
+
 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@@ -240,7 +248,7 @@ At standard resolution this makes a nice little ASCII plot:
 
 But using a very small font size (as small as my Mac will let me go), and by choosing the coordinates to zoom in on a particular part of the complex plane, we can get a nice little picture:
 
-
+
 #!/bin/bash
 # Get the number of rows and columns from the terminal window dimensions
 iheight=$(stty size | mlr --nidx --fs space cut -f 1)
diff --git a/docs6b/docs/programming-examples.md.in b/docs6b/docs/programming-examples.md.in
index 5aedd31de..ff7a1c40f 100644
--- a/docs6b/docs/programming-examples.md.in
+++ b/docs6b/docs/programming-examples.md.in
@@ -4,7 +4,7 @@ Here are a few things focusing on Miller's DSL as a programming language per se,
 
 ## Sieve of Eratosthenes
 
-The [Sieve of Eratosthenes](http://en.wikipedia.org/wiki/Sieve_of_Eratosthenes) is a standard introductory programming topic. The idea is to find all primes up to some *N* by making a list of the numbers 1 to *N*, then striking out all multiples of 2 except 2 itself, all multiples of 3 except 3 itself, all multiples of 4 except 4 itself, and so on. Whatever survives that without getting marked is a prime. This is easy enough in Miller. Notice that here all the work is in ``begin`` and ``end`` statements; there is no file input (so we use ``mlr -n`` to keep Miller from waiting for input data).
+The [Sieve of Eratosthenes](http://en.wikipedia.org/wiki/Sieve_of_Eratosthenes) is a standard introductory programming topic. The idea is to find all primes up to some *N* by making a list of the numbers 1 to *N*, then striking out all multiples of 2 except 2 itself, all multiples of 3 except 3 itself, all multiples of 4 except 4 itself, and so on. Whatever survives that without getting marked is a prime. This is easy enough in Miller. Notice that here all the work is in `begin` and `end` statements; there is no file input (so we use `mlr -n` to keep Miller from waiting for input data).
 
 GENMD_RUN_COMMAND
 cat programs/sieve.mlr
diff --git a/docs6b/docs/programming-language.md b/docs6b/docs/programming-language.md
index bdacbdc9a..bde9003e4 100644
--- a/docs6b/docs/programming-language.md
+++ b/docs6b/docs/programming-language.md
@@ -1,18 +1,20 @@
 
 # Intro to Miller's programming language
 
-In the [Miller in 10 minutes](10min.md) page we took a tour of some of Miller's most-used verbs including ``cat``, ``head``, ``tail``, ``cut``, and ``sort``. These are analogs of familiar system commands, but empowered by field-name indexing and file-format awareness: the system ``sort`` command only knows about lines and column names like ``1,2,3,4``, while ``mlr sort`` knows about CSV/TSV/JSON/etc records, and field names like ``color,shape,flag,index``.
+In the [Miller in 10 minutes](10min.md) page we took a tour of some of Miller's most-used verbs including `cat`, `head`, `tail`, `cut`, and `sort`. These are analogs of familiar system commands, but empowered by field-name indexing and file-format awareness: the system `sort` command only knows about lines and column names like `1,2,3,4`, while `mlr sort` knows about CSV/TSV/JSON/etc records, and field names like `color,shape,flag,index`.
 
-We also caught a glimpse of Miller's ``put`` and ``filter`` verbs. These two are special since they let you express statements using Miller's programming language. It's a *embedded domain-specific language* since it's inside Miller: often referred to simply as the *Miller DSL*.
+We also caught a glimpse of Miller's `put` and `filter` verbs. These two are special since they let you express statements using Miller's programming language. It's a *embedded domain-specific language* since it's inside Miller: often referred to simply as the *Miller DSL*.
 
 In the [DSL reference](reference-dsl.md) page we have a complete reference to Miller's programming language. For now, let's take a quick look at key features -- you can use as few or as many features as you like.
 
 ## Records and fields
 
-Let's keep using the sample [example.csv](./example.csv). When we type
+Let's keep using the sample [example.csv](./example.csv). For example:
 
-
+
 mlr --c2p put '$cost = $quantity * $rate' example.csv
+
+
 color  shape    flag  index quantity rate   cost
 yellow triangle true  11    43.6498  9.8870 431.5655726
 red    square   true  15    79.2778  0.0130 1.0306114
@@ -26,19 +28,21 @@ yellow circle   true  87    63.5058  8.3350 529.3208430000001
 purple square   false 91    72.3735  8.2430 596.5747605000001
 
-a few things are happening: +When we type that, a few things are happening: -* We refer to fields in the input data using a dollar sign and then the field name, e.g. ``$quantity``. (If a field name has special characters like a dot or slash, just use curly braces: ``${field.name}``.) -* The expression ``$cost = $quantity * $rate`` is executed once per record of the data file. Our [example.csv](./example.csv) has 10 records so this expression was executed 10 times, with the field names ``$quantity`` and ``$rate`` bound to the current record's values for those fields. -* On the left-hand side we have the new field name ``$cost`` which didn't come from the input data. Assignments to new variables result in a new field being placed after all the other ones. If we'd assigned to an existing field name, it would have been updated in-place. +* We refer to fields in the input data using a dollar sign and then the field name, e.g. `$quantity`. (If a field name contains special characters like a dot or slash, just use curly braces: `${field.name}`.) +* The expression `$cost = $quantity * $rate` is executed once per record of the data file. Our [example.csv](./example.csv) has 10 records so this expression was executed 10 times, with the field names `$quantity` and `$rate` bound to the current record's values for those fields. +* On the left-hand side we have the new field name `$cost` which didn't come from the input data. Assignments to new variables result in a new field being placed after all the other ones. If we'd assigned to an existing field name, it would have been updated in-place. * The entire expression is surrounded by single quotes, to get it past the system shell. Inside those, only double quotes have meaning in Miller's programming language. ## Multi-line statements, and statements-from-file You can use more than one statement, separating them with semicolons, and optionally putting them on lines of their own: -
+
 mlr --c2p put '$cost = $quantity * $rate; $index = $index * 100'  example.csv
+
+
 color  shape    flag  index quantity rate   cost
 yellow triangle true  1100  43.6498  9.8870 431.5655726
 red    square   true  1500  79.2778  0.0130 1.0306114
@@ -52,11 +56,13 @@ yellow circle   true  8700  63.5058  8.3350 529.3208430000001
 purple square   false 9100  72.3735  8.2430 596.5747605000001
 
-
+
 mlr --c2p put '
   $cost = $quantity * $rate;
   $index *= 100
 ' example.csv
+
+
 color  shape    flag  index quantity rate   cost
 yellow triangle true  1100  43.6498  9.8870 431.5655726
 red    square   true  1500  79.2778  0.0130 1.0306114
@@ -70,16 +76,20 @@ yellow circle   true  8700  63.5058  8.3350 529.3208430000001
 purple square   false 9100  72.3735  8.2430 596.5747605000001
 
-One of Miller's key features is the ability to express data-transformation right there at the keyboard, interactively. But if you find yourself using expressions repeatedly, you can put everything between the single quotes into a file and refer to that using ``put -f``: +One of Miller's key features is the ability to express data-transformation right there at the keyboard, interactively. But if you find yourself using expressions repeatedly, you can put everything between the single quotes into a file and refer to that using `put -f`: -
+
 cat dsl-example.mlr
+
+
 $cost = $quantity * $rate;
 $index *= 100
 
-
+
 mlr --c2p put -f dsl-example.mlr example.csv
+
+
 color  shape    flag  index quantity rate   cost
 yellow triangle true  1100  43.6498  9.8870 431.5655726
 red    square   true  1500  79.2778  0.0130 1.0306114
@@ -93,18 +103,20 @@ yellow circle   true  8700  63.5058  8.3350 529.3208430000001
 purple square   false 9100  72.3735  8.2430 596.5747605000001
 
-This becomes particularly important on Windows. Quite a bit of effort was put into making Miller on Windows be able to handle the kinds of single-quoted expressions we're showing here, but if you get syntax-error messages on Windows using examples in this documentation, you can put the parts between single quotes into a file and refer to that using ``mlr put -f``. +This becomes particularly important on Windows. Quite a bit of effort was put into making Miller on Windows be able to handle the kinds of single-quoted expressions we're showing here, but if you get syntax-error messages on Windows using examples in this documentation, you can put the parts between single quotes into a file and refer to that using `mlr put -f`. ## Out-of-stream variables, begin, and end -Above we saw that your expression is executed once per record -- if a file has a million records, your expression will be executed a million times, once for each record. But you can mark statements to only be executed once, either before the record stream begins, or after the record stream is ended. If you know about [AWK](https://en.wikipedia.org/wiki/AWK), you might have noticed that Miller's programming language is loosely inspired by it, including the ``begin`` and ``end`` statements. +Above we saw that your expression is executed once per record -- if a file has a million records, your expression will be executed a million times, once for each record. But you can mark statements to only be executed once, either before the record stream begins, or after the record stream is ended. If you know about [AWK](https://en.wikipedia.org/wiki/AWK), you might have noticed that Miller's programming language is loosely inspired by it, including the `begin` and `end` statements. -Above we also saw that names like ``$quantity`` are bound to each record in turn. +Above we also saw that names like `$quantity` are bound to each record in turn. -To make ``begin`` and ``end`` statements useful, we need somewhere to put things that persist across the duration of the record stream, and a way to emit them. Miller uses **out-of-stream variables** (or **oosvars** for short) whose names start with an ``@`` sigil, and the **emit** keyword to write them into the output record stream: +To make `begin` and `end` statements useful, we need somewhere to put things that persist across the duration of the record stream, and a way to emit them. Miller uses **out-of-stream variables** (or **oosvars** for short) whose names start with an `@` sigil, and the **emit** keyword to write them into the output record stream: -
+
 mlr --c2p --from example.csv put 'begin { @sum = 0 } @sum += $quantity; end {emit @sum}'
+
+
 color  shape    flag  index quantity rate
 yellow triangle true  11    43.6498  9.8870
 red    square   true  15    79.2778  0.0130
@@ -121,48 +133,56 @@ sum
 652.7185
 
-If you want the end-block output to be the only output, and not include the input data, you can use ``mlr put -q``: +If you want the end-block output to be the only output, and not include the input data, you can use `mlr put -q`: -
+
 mlr --c2p --from example.csv put -q 'begin { @sum = 0 } @sum += $quantity; end {emit @sum}'
+
+
 sum
 652.7185
 
-
+
 mlr --c2j --from example.csv put -q 'begin { @sum = 0 } @sum += $quantity; end {emit @sum}'
+
+
 {
   "sum": 652.7185
 }
 
-
+
 mlr --c2j --from example.csv put -q '
   begin { @count = 0; @sum = 0 }
   @count += 1;
   @sum += $quantity;
   end {emit (@count, @sum)}
 '
+
+
 {
   "count": 10,
   "sum": 652.7185
 }
 
-We'll see in the documentation for :ref:`reference-verbs-stats1` that there's a lower-keystroking way to get counts and sums of things -- so, take this sum/count example as an indication of the kinds of things you can do using Miller's programming language. +We'll see in the documentation for [stats1](reference-verbs.md#stats1) that there's a lower-keystroking way to get counts and sums of things -- so, take this sum/count example as an indication of the kinds of things you can do using Miller's programming language. ## Context variables Also inspired by [AWK](https://en.wikipedia.org/wiki/AWK), the Miller DSL has the following special **context variables**: -* ``FILENAME`` -- the filename the current record came from. Especially useful in things like ``mlr ... *.csv``. -* ``FILENUM`` -- similarly, but integer 1,2,3,... rather than filenam.e -* ``NF`` -- the number of fields in the current record. Note that if you assign ``$newcolumn = some value`` then ``NF`` will increment. -* ``NR`` -- starting from 1, counter of how many records processed so far. -* ``FNR`` -- similar, but resets to 1 at the start of each file. +* `FILENAME` -- the filename the current record came from. Especially useful in things like `mlr ... *.csv`. +* `FILENUM` -- similarly, but integer 1,2,3,... rather than filenam.e +* `NF` -- the number of fields in the current record. Note that if you assign `$newcolumn = some value` then `NF` will increment. +* `NR` -- starting from 1, counter of how many records processed so far. +* `FNR` -- similar, but resets to 1 at the start of each file. -
+
 cat context-example.mlr
+
+
 $nf       = NF;
 $nr       = NR;
 $fnr      = FNR;
@@ -171,8 +191,10 @@ $filenum  = FILENUM;
 $newnf    = NF;
 
-
+
 mlr --c2p put -f context-example.mlr data/a.csv data/b.csv
+
+
 a b c nf nr fnr filename   filenum newnf
 1 2 3 3  1  1   data/a.csv 1       8
 4 5 6 3  2  2   data/a.csv 1       8
@@ -183,8 +205,10 @@ a b c nf nr fnr filename   filenum newnf
 
 You can define your own functions:
 
-
+
 cat factorial-example.mlr
+
+
 func factorial(n) {
   if (n <= 1) {
     return n
@@ -194,8 +218,10 @@ func factorial(n) {
 }
 
-
+
 mlr --c2p --from example.csv put -f factorial-example.mlr -e '$fact = factorial(NR)'
+
+
 color  shape    flag  index quantity rate   fact
 yellow triangle true  11    43.6498  9.8870 1
 red    square   true  15    79.2778  0.0130 2
@@ -209,18 +235,20 @@ yellow circle   true  87    63.5058  8.3350 362880
 purple square   false 91    72.3735  8.2430 3628800
 
-Note that here we used the ``-f`` flag to ``put`` to load our function -definition, and also the ``-e`` flag to add another statement on the command -line. (We could have also put ``$fact = factorial(NR)`` inside -``factorial-example.mlr`` but that would have made that file less flexible for our +Note that here we used the `-f` flag to `put` to load our function +definition, and also the `-e` flag to add another statement on the command +line. (We could have also put `$fact = factorial(NR)` inside +`factorial-example.mlr` but that would have made that file less flexible for our future use.) ## If-statements, loops, and local variables -Suppose you want to only compute sums conditionally -- you can use an ``if`` statement: +Suppose you want to only compute sums conditionally -- you can use an `if` statement: -
+
 cat if-example.mlr
+
+
 begin {
   @count_of_red = 0;
   @sum_of_red = 0
@@ -236,38 +264,46 @@ end {
 }
 
-
+
 mlr --c2p --from example.csv put -q -f if-example.mlr
+
+
 count_of_red sum_of_red
 4            247.84139999999996
 
-Miller's else-if is spelled ``elif``. +Miller's else-if is spelled `elif`. As we'll see more of in section (TODO:linkify), Miller has a few kinds of -for-loops. In addition to the usual 3-part ``for (i = 0; i < 10; i += 1)`` kind +for-loops. In addition to the usual 3-part `for (i = 0; i < 10; i += 1)` kind that many programming languages have, Miller also lets you loop over arrays and hashmaps. We haven't encountered arrays and hashmaps yet in this introduction, -but for now it suffices to know that ``$*`` is a special variable holding the +but for now it suffices to know that `$*` is a special variable holding the current record as a hashmap: -
+
 cat for-example.mlr
+
+
 for (k, v in $*) {
   print "KEY IS ". k . " VALUE IS ". v;
 }
 print
 
-
+
 mlr --csv cat data/a.csv
+
+
 a,b,c
 1,2,3
 4,5,6
 
-
+
 mlr --csv --from data/a.csv put -qf for-example.mlr
+
+
 KEY IS a VALUE IS 1
 KEY IS b VALUE IS 2
 KEY IS c VALUE IS 3
@@ -277,12 +313,12 @@ KEY IS b VALUE IS 5
 KEY IS c VALUE IS 6
 
-Here we used the local variables ``k`` and ``v``. Now we've seen four kinds of variables: +Here we used the local variables `k` and `v`. Now we've seen four kinds of variables: -* Record fields like ``$shape`` -* Out-of-stream variables like ``@sum`` -* Local variables like ``k`` -* Built-in context variables like ``NF`` and ``NR`` +* Record fields like `$shape` +* Out-of-stream variables like `@sum` +* Local variables like `k` +* Built-in context variables like `NF` and `NR` If you're curious about scope and extent of local variables, you can read more in (TODO:linkify) the section on variables. @@ -291,8 +327,8 @@ If you're curious about scope and extent of local variables, you can read more i Numbers in Miller's programming language are intended to operate with the principle of least surprise: * Internally, numbers are either 64-bit signed integers or double-precision floating-point. -* Sums, differences, and products of integers are also integers (so ``2*3=6`` not ``6.0``) -- unless the result of the operation would overflow a 64-bit signed integer in which case the result is automatically converted to float. (If you ever want integer-to-integer arithmetic, use ``x .+ y``, ``x .* y``, etc.) -* Quotients of integers are integers if the division is exact, else floating-point: so ``6/2=3`` but ``7/2=3.5``. +* Sums, differences, and products of integers are also integers (so `2*3=6` not `6.0`) -- unless the result of the operation would overflow a 64-bit signed integer in which case the result is automatically converted to float. (If you ever want integer-to-integer arithmetic, use `x .+ y`, `x .* y`, etc.) +* Quotients of integers are integers if the division is exact, else floating-point: so `6/2=3` but `7/2=3.5`. You can read more about this at (TODO:linkify). @@ -300,13 +336,15 @@ You can read more about this at (TODO:linkify). In addition to types including string, number (int/float), arrays, and hashmaps, Miller varibles can also be **absent**. This is when a variable never had a value assigned to it. Miller's treatment of absent data is intended to make it easy for you to handle non-heterogeneous data. We'll see more in section (TODO:linkify) but the basic idea is: -* Adding a number to absent gives the number back. This means you don't have to put ``@sum = 0`` in your ``begin`` blocks. +* Adding a number to absent gives the number back. This means you don't have to put `@sum = 0` in your `begin` blocks. * Any variable which has the absent value is not assigned. This means you don't have to check presence of things from one record to the next. -For example, you can sum up all the ``$a`` values across records without having to check whether they're present or not: +For example, you can sum up all the `$a` values across records without having to check whether they're present or not: -
+
 mlr --json cat absent-example.json
+
+
 {
   "a": 1,
   "b": 2
@@ -320,8 +358,10 @@ For example, you can sum up all the ``$a`` values across records without having
 }
 
-
+
 mlr --json put '@sum_of_a += $a; end {emit @sum_of_a}' absent-example.json
+
+
 {
   "a": 1,
   "b": 2
diff --git a/docs6b/docs/programming-language.md.in b/docs6b/docs/programming-language.md.in
index 2a577413e..8847cf5ca 100644
--- a/docs6b/docs/programming-language.md.in
+++ b/docs6b/docs/programming-language.md.in
@@ -1,24 +1,24 @@
 # Intro to Miller's programming language
 
-In the [Miller in 10 minutes](10min.md) page we took a tour of some of Miller's most-used verbs including ``cat``, ``head``, ``tail``, ``cut``, and ``sort``. These are analogs of familiar system commands, but empowered by field-name indexing and file-format awareness: the system ``sort`` command only knows about lines and column names like ``1,2,3,4``, while ``mlr sort`` knows about CSV/TSV/JSON/etc records, and field names like ``color,shape,flag,index``.
+In the [Miller in 10 minutes](10min.md) page we took a tour of some of Miller's most-used verbs including `cat`, `head`, `tail`, `cut`, and `sort`. These are analogs of familiar system commands, but empowered by field-name indexing and file-format awareness: the system `sort` command only knows about lines and column names like `1,2,3,4`, while `mlr sort` knows about CSV/TSV/JSON/etc records, and field names like `color,shape,flag,index`.
 
-We also caught a glimpse of Miller's ``put`` and ``filter`` verbs. These two are special since they let you express statements using Miller's programming language. It's a *embedded domain-specific language* since it's inside Miller: often referred to simply as the *Miller DSL*.
+We also caught a glimpse of Miller's `put` and `filter` verbs. These two are special since they let you express statements using Miller's programming language. It's a *embedded domain-specific language* since it's inside Miller: often referred to simply as the *Miller DSL*.
 
 In the [DSL reference](reference-dsl.md) page we have a complete reference to Miller's programming language. For now, let's take a quick look at key features -- you can use as few or as many features as you like.
 
 ## Records and fields
 
-Let's keep using the sample [example.csv](./example.csv). When we type
+Let's keep using the sample [example.csv](./example.csv). For example:
 
 GENMD_RUN_COMMAND
 mlr --c2p put '$cost = $quantity * $rate' example.csv
 GENMD_EOF
 
-a few things are happening:
+When we type that, a few things are happening:
 
-* We refer to fields in the input data using a dollar sign and then the field name, e.g. ``$quantity``. (If a field name has special characters like a dot or slash, just use curly braces: ``${field.name}``.)
-* The expression ``$cost = $quantity * $rate`` is executed once per record of the data file. Our [example.csv](./example.csv) has 10 records so this expression was executed 10 times, with the field names ``$quantity`` and ``$rate`` bound to the current record's values for those fields.
-* On the left-hand side we have the new field name ``$cost`` which didn't come from the input data. Assignments to new variables result in a new field being placed after all the other ones. If we'd assigned to an existing field name, it would have been updated in-place.
+* We refer to fields in the input data using a dollar sign and then the field name, e.g. `$quantity`. (If a field name contains special characters like a dot or slash, just use curly braces: `${field.name}`.)
+* The expression `$cost = $quantity * $rate` is executed once per record of the data file. Our [example.csv](./example.csv) has 10 records so this expression was executed 10 times, with the field names `$quantity` and `$rate` bound to the current record's values for those fields.
+* On the left-hand side we have the new field name `$cost` which didn't come from the input data. Assignments to new variables result in a new field being placed after all the other ones. If we'd assigned to an existing field name, it would have been updated in-place.
 * The entire expression is surrounded by single quotes, to get it past the system shell. Inside those, only double quotes have meaning in Miller's programming language.
 
 ## Multi-line statements, and statements-from-file
@@ -31,7 +31,7 @@ GENMD_EOF
 
 GENMD_INCLUDE_AND_RUN_ESCAPED(dsl-example-multiline.sh)
 
-One of Miller's key features is the ability to express data-transformation right there at the keyboard, interactively. But if you find yourself using expressions repeatedly, you can put everything between the single quotes into a file and refer to that using ``put -f``:
+One of Miller's key features is the ability to express data-transformation right there at the keyboard, interactively. But if you find yourself using expressions repeatedly, you can put everything between the single quotes into a file and refer to that using `put -f`:
 
 GENMD_RUN_COMMAND
 cat dsl-example.mlr
@@ -41,21 +41,21 @@ GENMD_RUN_COMMAND
 mlr --c2p put -f dsl-example.mlr example.csv
 GENMD_EOF
 
-This becomes particularly important on Windows. Quite a bit of effort was put into making Miller on Windows be able to handle the kinds of single-quoted expressions we're showing here, but if you get syntax-error messages on Windows using examples in this documentation, you can put the parts between single quotes into a file and refer to that using ``mlr put -f``.
+This becomes particularly important on Windows. Quite a bit of effort was put into making Miller on Windows be able to handle the kinds of single-quoted expressions we're showing here, but if you get syntax-error messages on Windows using examples in this documentation, you can put the parts between single quotes into a file and refer to that using `mlr put -f`.
 
 ## Out-of-stream variables, begin, and end
 
-Above we saw that your expression is executed once per record -- if a file has a million records, your expression will be executed a million times, once for each record. But you can mark statements to only be executed once, either before the record stream begins, or after the record stream is ended. If you know about [AWK](https://en.wikipedia.org/wiki/AWK), you might have noticed that Miller's programming language is loosely inspired by it, including the ``begin`` and ``end`` statements.
+Above we saw that your expression is executed once per record -- if a file has a million records, your expression will be executed a million times, once for each record. But you can mark statements to only be executed once, either before the record stream begins, or after the record stream is ended. If you know about [AWK](https://en.wikipedia.org/wiki/AWK), you might have noticed that Miller's programming language is loosely inspired by it, including the `begin` and `end` statements.
 
-Above we also saw that names like ``$quantity`` are bound to each record in turn.
+Above we also saw that names like `$quantity` are bound to each record in turn.
 
-To make ``begin`` and ``end`` statements useful, we need somewhere to put things that persist across the duration of the record stream, and a way to emit them. Miller uses **out-of-stream variables** (or **oosvars** for short) whose names start with an ``@`` sigil, and the **emit** keyword to write them into the output record stream:
+To make `begin` and `end` statements useful, we need somewhere to put things that persist across the duration of the record stream, and a way to emit them. Miller uses **out-of-stream variables** (or **oosvars** for short) whose names start with an `@` sigil, and the **emit** keyword to write them into the output record stream:
 
 GENMD_RUN_COMMAND
 mlr --c2p --from example.csv put 'begin { @sum = 0 } @sum += $quantity; end {emit @sum}'
 GENMD_EOF
 
-If you want the end-block output to be the only output, and not include the input data, you can use ``mlr put -q``:
+If you want the end-block output to be the only output, and not include the input data, you can use `mlr put -q`:
 
 GENMD_RUN_COMMAND
 mlr --c2p --from example.csv put -q 'begin { @sum = 0 } @sum += $quantity; end {emit @sum}'
@@ -74,17 +74,17 @@ mlr --c2j --from example.csv put -q '
 '
 GENMD_EOF
 
-We'll see in the documentation for :ref:`reference-verbs-stats1` that there's a lower-keystroking way to get counts and sums of things -- so, take this sum/count example as an indication of the kinds of things you can do using Miller's programming language.
+We'll see in the documentation for [stats1](reference-verbs.md#stats1) that there's a lower-keystroking way to get counts and sums of things -- so, take this sum/count example as an indication of the kinds of things you can do using Miller's programming language.
 
 ## Context variables
 
 Also inspired by [AWK](https://en.wikipedia.org/wiki/AWK), the Miller DSL has the following special **context variables**:
 
-* ``FILENAME`` -- the filename the current record came from. Especially useful in things like ``mlr ... *.csv``.
-* ``FILENUM`` -- similarly, but integer 1,2,3,... rather than filenam.e
-* ``NF`` -- the number of fields in the current record. Note that if you assign ``$newcolumn = some value`` then ``NF`` will increment.
-* ``NR`` -- starting from 1, counter of how many records processed so far.
-* ``FNR`` -- similar, but resets to 1 at the start of each file.
+* `FILENAME` -- the filename the current record came from. Especially useful in things like `mlr ... *.csv`.
+* `FILENUM` -- similarly, but integer 1,2,3,... rather than filenam.e
+* `NF` -- the number of fields in the current record. Note that if you assign `$newcolumn = some value` then `NF` will increment.
+* `NR` -- starting from 1, counter of how many records processed so far.
+* `FNR` -- similar, but resets to 1 at the start of each file.
 
 GENMD_RUN_COMMAND
 cat context-example.mlr
@@ -106,15 +106,15 @@ GENMD_RUN_COMMAND
 mlr --c2p --from example.csv put -f factorial-example.mlr -e '$fact = factorial(NR)'
 GENMD_EOF
 
-Note that here we used the ``-f`` flag to ``put`` to load our function
-definition, and also the ``-e`` flag to add another statement on the command
-line. (We could have also put ``$fact = factorial(NR)`` inside
-``factorial-example.mlr`` but that would have made that file less flexible for our
+Note that here we used the `-f` flag to `put` to load our function
+definition, and also the `-e` flag to add another statement on the command
+line. (We could have also put `$fact = factorial(NR)` inside
+`factorial-example.mlr` but that would have made that file less flexible for our
 future use.)
 
 ## If-statements, loops, and local variables
 
-Suppose you want to only compute sums conditionally -- you can use an ``if`` statement:
+Suppose you want to only compute sums conditionally -- you can use an `if` statement:
 
 GENMD_RUN_COMMAND
 cat if-example.mlr
@@ -124,13 +124,13 @@ GENMD_RUN_COMMAND
 mlr --c2p --from example.csv put -q -f if-example.mlr
 GENMD_EOF
 
-Miller's else-if is spelled ``elif``.
+Miller's else-if is spelled `elif`.
 
 As we'll see more of in section (TODO:linkify), Miller has a few kinds of
-for-loops. In addition to the usual 3-part ``for (i = 0; i < 10; i += 1)`` kind
+for-loops. In addition to the usual 3-part `for (i = 0; i < 10; i += 1)` kind
 that many programming languages have, Miller also lets you loop over arrays and
 hashmaps. We haven't encountered arrays and hashmaps yet in this introduction,
-but for now it suffices to know that ``$*`` is a special variable holding the
+but for now it suffices to know that `$*` is a special variable holding the
 current record as a hashmap:
 
 GENMD_RUN_COMMAND
@@ -145,12 +145,12 @@ GENMD_RUN_COMMAND
 mlr --csv --from data/a.csv put -qf for-example.mlr
 GENMD_EOF
 
-Here we used the local variables ``k`` and ``v``. Now we've seen four kinds of variables:
+Here we used the local variables `k` and `v`. Now we've seen four kinds of variables:
 
-* Record fields like ``$shape``
-* Out-of-stream variables like ``@sum``
-* Local variables like ``k``
-* Built-in context variables like ``NF`` and ``NR``
+* Record fields like `$shape`
+* Out-of-stream variables like `@sum`
+* Local variables like `k`
+* Built-in context variables like `NF` and `NR`
 
 If you're curious about scope and extent of local variables, you can read more in (TODO:linkify) the section on variables.
 
@@ -159,8 +159,8 @@ If you're curious about scope and extent of local variables, you can read more i
 Numbers in Miller's programming language are intended to operate with the principle of least surprise:
 
 * Internally, numbers are either 64-bit signed integers or double-precision floating-point.
-* Sums, differences, and products of integers are also integers (so ``2*3=6`` not ``6.0``) -- unless the result of the operation would overflow a 64-bit signed integer in which case the result is automatically converted to float. (If you ever want integer-to-integer arithmetic, use ``x .+ y``, ``x .* y``, etc.)
-* Quotients of integers are integers if the division is exact, else floating-point:  so ``6/2=3`` but ``7/2=3.5``.
+* Sums, differences, and products of integers are also integers (so `2*3=6` not `6.0`) -- unless the result of the operation would overflow a 64-bit signed integer in which case the result is automatically converted to float. (If you ever want integer-to-integer arithmetic, use `x .+ y`, `x .* y`, etc.)
+* Quotients of integers are integers if the division is exact, else floating-point:  so `6/2=3` but `7/2=3.5`.
 
 You can read more about this at (TODO:linkify).
 
@@ -168,10 +168,10 @@ You can read more about this at (TODO:linkify).
 
 In addition to types including string, number (int/float), arrays, and hashmaps, Miller varibles can also be **absent**. This is when a variable never had a value assigned to it. Miller's treatment of absent data is intended to make it easy for you to handle non-heterogeneous data. We'll see more in section (TODO:linkify) but the basic idea is:
 
-* Adding a number to absent gives the number back. This means you don't have to put ``@sum = 0`` in your ``begin`` blocks.
+* Adding a number to absent gives the number back. This means you don't have to put `@sum = 0` in your `begin` blocks.
 * Any variable which has the absent value is not assigned. This means you don't have to check presence of things from one record to the next.
 
-For example, you can sum up all the ``$a`` values across records without having to check whether they're present or not:
+For example, you can sum up all the `$a` values across records without having to check whether they're present or not:
 
 GENMD_RUN_COMMAND
 mlr --json cat absent-example.json
diff --git a/docs6b/docs/randomizing-examples.md b/docs6b/docs/randomizing-examples.md
index fe707a4c5..0ea2353ad 100644
--- a/docs6b/docs/randomizing-examples.md
+++ b/docs6b/docs/randomizing-examples.md
@@ -5,8 +5,10 @@
 
 Here we can chain together a few simple building blocks:
 
-
+
 cat expo-sample.sh
+
+
 # Generate 100,000 pairs of independent and identically distributed
 # exponentially distributed random variables with the same rate parameter
 # (namely, 2.5). Then compute histograms of one of them, along with
@@ -40,14 +42,16 @@ Namely:
 * Set the Miller random-number seed so this webdoc looks the same every time I regenerate it.
 * Use pretty-printed tabular output.
 * Use pretty-printed tabular output.
-* Use ``seqgen`` to produce 100,000 records ``i=0``, ``i=1``, etc.
-* Send those to a ``put`` step which defines an inverse-transform-sampling function and calls it twice, then computes the sum and product of samples.
+* Use `seqgen` to produce 100,000 records `i=0`, `i=1`, etc.
+* Send those to a `put` step which defines an inverse-transform-sampling function and calls it twice, then computes the sum and product of samples.
 * Send those to a histogram, and from there to a bar-plotter. This is just for visualization; you could just as well output CSV and send that off to your own plotting tool, etc.
 
 The output is as follows:
 
-
+
 sh expo-sample.sh
+
+
 bin_lo bin_hi u_count                        s_count
 0      0.04   [64]*******************#[9554] [326]#...................[3703]
 0.04   0.08   [64]*****************...[9554] [326]*****...............[3703]
@@ -105,8 +109,10 @@ bin_lo bin_hi u_count                        s_count
 
 Given this [word list](./data/english-words.txt), first take a look to see what the first few lines look like:
 
-
+
 head data/english-words.txt
+
+
 a
 aa
 aal
@@ -121,8 +127,10 @@ abaca
 
 Then the following will randomly sample ten words with four to eight characters in them:
 
-
+
 mlr --from data/english-words.txt --nidx filter -S 'n=strlen($1);4<=n&&n<=8' then sample -k 10
+
+
 thionine
 birchman
 mildewy
@@ -141,8 +149,10 @@ These are simple *n*-grams as [described here](http://johnkerl.org/randspell/ran
 
 The idea is that words from the input file are consumed, then taken apart and pasted back together in ways which imitate the letter-to-letter transitions found in the word list -- giving us automatically generated words in the same vein as *bromance* and *spork*:
 
-
+
 mlr --nidx --from ./ngrams/gsl-2000.txt put -q -f ./ngrams/ngfuncs.mlr -f ./ngrams/ng5.mlr
+
+
 beard
 plastinguish
 politicially
diff --git a/docs6b/docs/randomizing-examples.md.in b/docs6b/docs/randomizing-examples.md.in
index 07bc5df92..e8563a139 100644
--- a/docs6b/docs/randomizing-examples.md.in
+++ b/docs6b/docs/randomizing-examples.md.in
@@ -13,8 +13,8 @@ Namely:
 * Set the Miller random-number seed so this webdoc looks the same every time I regenerate it.
 * Use pretty-printed tabular output.
 * Use pretty-printed tabular output.
-* Use ``seqgen`` to produce 100,000 records ``i=0``, ``i=1``, etc.
-* Send those to a ``put`` step which defines an inverse-transform-sampling function and calls it twice, then computes the sum and product of samples.
+* Use `seqgen` to produce 100,000 records `i=0`, `i=1`, etc.
+* Send those to a `put` step which defines an inverse-transform-sampling function and calls it twice, then computes the sum and product of samples.
 * Send those to a histogram, and from there to a bar-plotter. This is just for visualization; you could just as well output CSV and send that off to your own plotting tool, etc.
 
 The output is as follows:
diff --git a/docs6b/docs/record-heterogeneity.md b/docs6b/docs/record-heterogeneity.md
index 3362be9aa..ae28c7703 100644
--- a/docs6b/docs/record-heterogeneity.md
+++ b/docs6b/docs/record-heterogeneity.md
@@ -11,8 +11,10 @@ But heterogeneous data abound (today's no-SQL databases for example). Miller han
 
 Miller simply prints a newline and a new header when there is a schema change. When there is no schema change, you get CSV per se as a special case. Likewise, Miller reads heterogeneous CSV or pretty-print input the same way. The difference between CSV and CSV-lite is that the former is RFC4180-compliant, while the latter readily handles heterogeneous data (which is non-compliant). For example:
 
-
+
 cat data/het.dkvp
+
+
 resource=/path/to/file,loadsec=0.45,ok=true
 record_count=100,resource=/path/to/file
 resource=/path/to/second/file,loadsec=0.32,ok=true
@@ -20,8 +22,10 @@ record_count=150,resource=/path/to/second/file
 resource=/some/other/path,loadsec=0.97,ok=false
 
-
+
 mlr --ocsvlite cat data/het.dkvp
+
+
 resource,loadsec,ok
 /path/to/file,0.45,true
 
@@ -38,8 +42,10 @@ resource,loadsec,ok
 /some/other/path,0.97,false
 
-
+
 mlr --opprint cat data/het.dkvp
+
+
 resource      loadsec ok
 /path/to/file 0.45    true
 
@@ -56,18 +62,22 @@ resource         loadsec ok
 /some/other/path 0.97    false
 
-Miller handles explicit header changes as just shown. If your CSV input contains ragged data -- if there are implicit header changes -- you can use ``--allow-ragged-csv-input`` (or keystroke-saver ``--ragged``). For too-short data lines, values are filled with empty string; for too-long data lines, missing field names are replaced with positional indices (counting up from 1, not 0), as follows: +Miller handles explicit header changes as just shown. If your CSV input contains ragged data -- if there are implicit header changes -- you can use `--allow-ragged-csv-input` (or keystroke-saver `--ragged`). For too-short data lines, values are filled with empty string; for too-long data lines, missing field names are replaced with positional indices (counting up from 1, not 0), as follows: -
+
 cat data/ragged.csv
+
+
 a,b,c
 1,2,3
 4,5
 6,7,8,9
 
-
+
 mlr --icsv --oxtab --allow-ragged-csv-input cat data/ragged.csv
+
+
 a 1
 b 2
 c 3
@@ -82,10 +92,12 @@ c 8
 4 9
 
-You may also find Miller's ``group-like`` feature handy (see also [Verbs Reference](reference-verbs.md)): +You may also find Miller's `group-like` feature handy (see also [Verbs Reference](reference-verbs.md)): -
+
 mlr --ocsvlite group-like data/het.dkvp
+
+
 resource,loadsec,ok
 /path/to/file,0.45,true
 /path/to/second/file,0.32,true
@@ -96,8 +108,10 @@ record_count,resource
 150,/path/to/second/file
 
-
+
 mlr --opprint group-like data/het.dkvp
+
+
 resource             loadsec ok
 /path/to/file        0.45    true
 /path/to/second/file 0.32    true
@@ -112,8 +126,10 @@ record_count resource
 
 For these formats, record-heterogeneity comes naturally:
 
-
+
 cat data/het.dkvp
+
+
 resource=/path/to/file,loadsec=0.45,ok=true
 record_count=100,resource=/path/to/file
 resource=/path/to/second/file,loadsec=0.32,ok=true
@@ -121,8 +137,10 @@ record_count=150,resource=/path/to/second/file
 resource=/some/other/path,loadsec=0.97,ok=false
 
-
+
 mlr --onidx --ofs ' ' cat data/het.dkvp
+
+
 /path/to/file 0.45 true
 100 /path/to/file
 /path/to/second/file 0.32 true
@@ -130,8 +148,10 @@ resource=/some/other/path,loadsec=0.97,ok=false
 /some/other/path 0.97 false
 
-
+
 mlr --oxtab cat data/het.dkvp
+
+
 resource /path/to/file
 loadsec  0.45
 ok       true
@@ -151,8 +171,10 @@ loadsec  0.97
 ok       false
 
-
+
 mlr --oxtab group-like data/het.dkvp
+
+
 resource /path/to/file
 loadsec  0.45
 ok       true
@@ -174,10 +196,12 @@ resource     /path/to/second/file
 
 ## For processing
 
-Miller operates on specified fields and takes the rest along: for example, if you are sorting on the ``count`` field then all records in the input stream must have a ``count`` field but the other fields can vary, and moreover the sorted-on field name(s) don't need to be in the same position on each line:
+Miller operates on specified fields and takes the rest along: for example, if you are sorting on the `count` field then all records in the input stream must have a `count` field but the other fields can vary, and moreover the sorted-on field name(s) don't need to be in the same position on each line:
 
-
+
 cat data/sort-het.dkvp
+
+
 count=500,color=green
 count=600
 status=ok,count=250,hours=0.22
@@ -187,8 +211,10 @@ count=100,color=green
 count=450
 
-
+
 mlr sort -n count data/sort-het.dkvp
+
+
 count=100,color=green
 status=ok,count=200,hours=3.4
 status=ok,count=250,hours=0.22
diff --git a/docs6b/docs/record-heterogeneity.md.in b/docs6b/docs/record-heterogeneity.md.in
index 26f71a28c..64ee60b27 100644
--- a/docs6b/docs/record-heterogeneity.md.in
+++ b/docs6b/docs/record-heterogeneity.md.in
@@ -22,7 +22,7 @@ GENMD_RUN_COMMAND
 mlr --opprint cat data/het.dkvp
 GENMD_EOF
 
-Miller handles explicit header changes as just shown. If your CSV input contains ragged data -- if there are implicit header changes -- you can use ``--allow-ragged-csv-input`` (or keystroke-saver ``--ragged``). For too-short data lines, values are filled with empty string; for too-long data lines, missing field names are replaced with positional indices (counting up from 1, not 0), as follows:
+Miller handles explicit header changes as just shown. If your CSV input contains ragged data -- if there are implicit header changes -- you can use `--allow-ragged-csv-input` (or keystroke-saver `--ragged`). For too-short data lines, values are filled with empty string; for too-long data lines, missing field names are replaced with positional indices (counting up from 1, not 0), as follows:
 
 GENMD_RUN_COMMAND
 cat data/ragged.csv
@@ -32,7 +32,7 @@ GENMD_RUN_COMMAND
 mlr --icsv --oxtab --allow-ragged-csv-input cat data/ragged.csv
 GENMD_EOF
 
-You may also find Miller's ``group-like`` feature handy (see also [Verbs Reference](reference-verbs.md)):
+You may also find Miller's `group-like` feature handy (see also [Verbs Reference](reference-verbs.md)):
 
 GENMD_RUN_COMMAND
 mlr --ocsvlite group-like data/het.dkvp
@@ -64,7 +64,7 @@ GENMD_EOF
 
 ## For processing
 
-Miller operates on specified fields and takes the rest along: for example, if you are sorting on the ``count`` field then all records in the input stream must have a ``count`` field but the other fields can vary, and moreover the sorted-on field name(s) don't need to be in the same position on each line:
+Miller operates on specified fields and takes the rest along: for example, if you are sorting on the `count` field then all records in the input stream must have a `count` field but the other fields can vary, and moreover the sorted-on field name(s) don't need to be in the same position on each line:
 
 GENMD_RUN_COMMAND
 cat data/sort-het.dkvp
diff --git a/docs6b/docs/reference-dsl-arrays.md b/docs6b/docs/reference-dsl-arrays.md
index ced2bd785..38ee52123 100644
--- a/docs6b/docs/reference-dsl-arrays.md
+++ b/docs6b/docs/reference-dsl-arrays.md
@@ -3,8 +3,10 @@
 
 TODO
 
-
+
 mlr --json cat data/array-example.json
+
+
 {
   "key": "ax04",
   "samples": [45, 67, 22]
diff --git a/docs6b/docs/reference-dsl-builtin-functions.md b/docs6b/docs/reference-dsl-builtin-functions.md
index 43e36d0a4..0d32048c6 100644
--- a/docs6b/docs/reference-dsl-builtin-functions.md
+++ b/docs6b/docs/reference-dsl-builtin-functions.md
@@ -8,2165 +8,1418 @@ Please run "mlr --help" for usage information.
 
 ## List of functions
 
-Each function takes a specific number of arguments, as shown below, except for functions marked as variadic such as ``min`` and ``max``. (The latter compute min and max of any number of numerical arguments.) There is no notion of optional or default-on-absent arguments. All argument-passing is positional rather than by name; arguments are passed by value, not by reference.
+Each function takes a specific number of arguments, as shown below, except for functions marked as variadic such as `min` and `max`. (The latter compute min and max of any number of numerical arguments.) There is no notion of optional or default-on-absent arguments. All argument-passing is positional rather than by name; arguments are passed by value, not by reference.
 
 You can get a list of all functions using **mlr -f**, with details using **mlr -F**.
 
 
-.. _reference-dsl-colon:
+
+## \!
 
-\!
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+!  (class=boolean #args=1) Logical negation.
+
-.. code-block:: none - ! (class=boolean #args=1) Logical negation. +## != +
+!=  (class=boolean #args=2) String/numeric inequality. Mixing number and string results in string compare.
+
-.. _reference-dsl-!=: +## !=~ -!= -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+!=~  (class=boolean #args=2) String (left-hand side) does not match regex (right-hand side), e.g. '$name !=~ "^a.*b$"'.
+
-.. code-block:: none - != (class=boolean #args=2) String/numeric inequality. Mixing number and string results in string compare. +## % +
+%  (class=arithmetic #args=2) Remainder; never negative-valued (pythonic).
+
-.. _reference-dsl-!=~: +## & -!=~ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+&  (class=arithmetic #args=2) Bitwise AND.
+
-.. code-block:: none - !=~ (class=boolean #args=2) String (left-hand side) does not match regex (right-hand side), e.g. '$name !=~ "^a.*b$"'. +## && +
+&&  (class=boolean #args=2) Logical AND.
+
-.. _reference-dsl-%: +
+## \* -% -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+*  (class=arithmetic #args=2) Multiplication, with integer*integer overflow to float.
+
-.. code-block:: none - % (class=arithmetic #args=2) Remainder; never negative-valued (pythonic). +
+## \** +
+**  (class=arithmetic #args=2) Exponentiation. Same as pow, but as an infix operator.
+
-.. _reference-dsl-&: +
+## \+ -& -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
++  (class=arithmetic #args=1,2) Addition as binary operator; unary plus operator.
+
-.. code-block:: none - & (class=arithmetic #args=2) Bitwise AND. +
+## \- +
+-  (class=arithmetic #args=1,2) Subtraction as binary operator; unary negation operator.
+
-.. _reference-dsl-&&: +## . -&& -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+.  (class=string #args=2) String concatenation.
+
-.. code-block:: none - && (class=boolean #args=2) Logical AND. +## .* +
+.*  (class=arithmetic #args=2) Multiplication, with integer-to-integer overflow.
+
-.. _reference-dsl-times: +## .+ -\* -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+.+  (class=arithmetic #args=2) Addition, with integer-to-integer overflow.
+
-.. code-block:: none - * (class=arithmetic #args=2) Multiplication, with integer*integer overflow to float. +## .- +
+.-  (class=arithmetic #args=2) Subtraction, with integer-to-integer overflow.
+
-.. _reference-dsl-exponentiation: +## ./ -\** -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+./  (class=arithmetic #args=2) Integer division; not pythonic.
+
-.. code-block:: none - ** (class=arithmetic #args=2) Exponentiation. Same as pow, but as an infix operator. +## / +
+/  (class=arithmetic #args=2) Division. Integer / integer is floating-point.
+
-.. _reference-dsl-plus: +## // -\+ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+//  (class=arithmetic #args=2) Pythonic integer division, rounding toward negative.
+
-.. code-block:: none - + (class=arithmetic #args=1,2) Addition as binary operator; unary plus operator. +## < +
+<  (class=boolean #args=2) String/numeric less-than. Mixing number and string results in string compare.
+
-.. _reference-dsl-minus: +## << -\- -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+<<  (class=arithmetic #args=2) Bitwise left-shift.
+
-.. code-block:: none - - (class=arithmetic #args=1,2) Subtraction as binary operator; unary negation operator. +## <= +
+<=  (class=boolean #args=2) String/numeric less-than-or-equals. Mixing number and string results in string compare.
+
-.. _reference-dsl-.: +## == -. -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+==  (class=boolean #args=2) String/numeric equality. Mixing number and string results in string compare.
+
-.. code-block:: none - . (class=string #args=2) String concatenation. +## =~ +
+=~  (class=boolean #args=2) String (left-hand side) matches regex (right-hand side), e.g. '$name =~ "^a.*b$"'.
+
-.. _reference-dsl-.*: +## > -.* -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+>  (class=boolean #args=2) String/numeric greater-than. Mixing number and string results in string compare.
+
-.. code-block:: none - .* (class=arithmetic #args=2) Multiplication, with integer-to-integer overflow. +## >= +
+>=  (class=boolean #args=2) String/numeric greater-than-or-equals. Mixing number and string results in string compare.
+
-.. _reference-dsl-.+: +
+## \>\> -.+ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+>>  (class=arithmetic #args=2) Bitwise signed right-shift.
+
-.. code-block:: none - .+ (class=arithmetic #args=2) Addition, with integer-to-integer overflow. +
+## \>\>\> +
+>>>  (class=arithmetic #args=2) Bitwise unsigned right-shift.
+
-.. _reference-dsl-.-: +
+## \? -.- -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+?:  (class=boolean #args=3) Standard ternary operator.
+
-.. code-block:: none - .- (class=arithmetic #args=2) Subtraction, with integer-to-integer overflow. +## ?? +
+??  (class=boolean #args=2) Absent-coalesce operator. $a ?? 1 evaluates to 1 if $a isn't defined in the current record.
+
-.. _reference-dsl-./: +## ??? -./ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+???  (class=boolean #args=2) Absent-coalesce operator. $a ?? 1 evaluates to 1 if $a isn't defined in the current record, or has empty value.
+
-.. code-block:: none - ./ (class=arithmetic #args=2) Integer division; not pythonic. +## ^ +
+^  (class=arithmetic #args=2) Bitwise XOR.
+
-.. _reference-dsl-/: +## ^^ -/ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+^^  (class=boolean #args=2) Logical XOR.
+
-.. code-block:: none - / (class=arithmetic #args=2) Division. Integer / integer is floating-point. +
+## \| +
+|  (class=arithmetic #args=2) Bitwise OR.
+
-.. _reference-dsl-//: +## || -// -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+||  (class=boolean #args=2) Logical OR.
+
-.. code-block:: none - // (class=arithmetic #args=2) Pythonic integer division, rounding toward negative. +## ~ +
+~  (class=arithmetic #args=1) Bitwise NOT. Beware '$y=~$x' since =~ is the
+regex-match operator: try '$y = ~$x'.
+
-.. _reference-dsl-<: +## abs -< -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+abs  (class=math #args=1) Absolute value.
+
-.. code-block:: none - < (class=boolean #args=2) String/numeric less-than. Mixing number and string results in string compare. +## acos +
+acos  (class=math #args=1) Inverse trigonometric cosine.
+
-.. _reference-dsl-<<: +## acosh -<< -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+acosh  (class=math #args=1) Inverse hyperbolic cosine.
+
-.. code-block:: none - << (class=arithmetic #args=2) Bitwise left-shift. +## append +
+append  (class=maps/arrays #args=2) Appends second argument to end of first argument, which must be an array.
+
-.. _reference-dsl-<=: +## arrayify -<= -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+arrayify  (class=maps/arrays #args=1) Walks through a nested map/array, converting any map with consecutive keys
+"1", "2", ... into an array. Useful to wrap the output of unflatten.
+
-.. code-block:: none - <= (class=boolean #args=2) String/numeric less-than-or-equals. Mixing number and string results in string compare. +## asin +
+asin  (class=math #args=1) Inverse trigonometric sine.
+
-.. _reference-dsl-==: +## asinh -== -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+asinh  (class=math #args=1) Inverse hyperbolic sine.
+
-.. code-block:: none - == (class=boolean #args=2) String/numeric equality. Mixing number and string results in string compare. +## asserting_absent +
+asserting_absent  (class=typing #args=1) Aborts with an error if is_absent on the argument returns false,
+else returns its argument.
+
-.. _reference-dsl-=~: +## asserting_array -=~ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+asserting_array  (class=typing #args=1) Aborts with an error if is_array on the argument returns false,
+else returns its argument.
+
-.. code-block:: none - =~ (class=boolean #args=2) String (left-hand side) matches regex (right-hand side), e.g. '$name =~ "^a.*b$"'. +## asserting_bool +
+asserting_bool  (class=typing #args=1) Aborts with an error if is_bool on the argument returns false,
+else returns its argument.
+
-.. _reference-dsl->: +## asserting_boolean -> -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+asserting_boolean  (class=typing #args=1) Aborts with an error if is_boolean on the argument returns false,
+else returns its argument.
+
-.. code-block:: none - > (class=boolean #args=2) String/numeric greater-than. Mixing number and string results in string compare. +## asserting_empty +
+asserting_empty  (class=typing #args=1) Aborts with an error if is_empty on the argument returns false,
+else returns its argument.
+
-.. _reference-dsl->=: +## asserting_empty_map ->= -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+asserting_empty_map  (class=typing #args=1) Aborts with an error if is_empty_map on the argument returns false,
+else returns its argument.
+
-.. code-block:: none - >= (class=boolean #args=2) String/numeric greater-than-or-equals. Mixing number and string results in string compare. +## asserting_error +
+asserting_error  (class=typing #args=1) Aborts with an error if is_error on the argument returns false,
+else returns its argument.
+
-.. _reference-dsl-srsh: +## asserting_float -\>\> -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+asserting_float  (class=typing #args=1) Aborts with an error if is_float on the argument returns false,
+else returns its argument.
+
-.. code-block:: none - >> (class=arithmetic #args=2) Bitwise signed right-shift. +## asserting_int +
+asserting_int  (class=typing #args=1) Aborts with an error if is_int on the argument returns false,
+else returns its argument.
+
-.. _reference-dsl-ursh: +## asserting_map -\>\>\> -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+asserting_map  (class=typing #args=1) Aborts with an error if is_map on the argument returns false,
+else returns its argument.
+
-.. code-block:: none - >>> (class=arithmetic #args=2) Bitwise unsigned right-shift. +## asserting_nonempty_map +
+asserting_nonempty_map  (class=typing #args=1) Aborts with an error if is_nonempty_map on the argument returns false,
+else returns its argument.
+
-.. _reference-dsl-question-mark-colon: +## asserting_not_array -\? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+asserting_not_array  (class=typing #args=1) Aborts with an error if is_not_array on the argument returns false,
+else returns its argument.
+
-.. code-block:: none - ?: (class=boolean #args=3) Standard ternary operator. +## asserting_not_empty +
+asserting_not_empty  (class=typing #args=1) Aborts with an error if is_not_empty on the argument returns false,
+else returns its argument.
+
-.. _reference-dsl-??: +## asserting_not_map -?? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+asserting_not_map  (class=typing #args=1) Aborts with an error if is_not_map on the argument returns false,
+else returns its argument.
+
-.. code-block:: none - ?? (class=boolean #args=2) Absent-coalesce operator. $a ?? 1 evaluates to 1 if $a isn't defined in the current record. +## asserting_not_null +
+asserting_not_null  (class=typing #args=1) Aborts with an error if is_not_null on the argument returns false,
+else returns its argument.
+
-.. _reference-dsl-???: +## asserting_null -??? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+asserting_null  (class=typing #args=1) Aborts with an error if is_null on the argument returns false,
+else returns its argument.
+
-.. code-block:: none - ??? (class=boolean #args=2) Absent-coalesce operator. $a ?? 1 evaluates to 1 if $a isn't defined in the current record, or has empty value. +## asserting_numeric +
+asserting_numeric  (class=typing #args=1) Aborts with an error if is_numeric on the argument returns false,
+else returns its argument.
+
-.. _reference-dsl-^: +## asserting_present -^ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+asserting_present  (class=typing #args=1) Aborts with an error if is_present on the argument returns false,
+else returns its argument.
+
-.. code-block:: none - ^ (class=arithmetic #args=2) Bitwise XOR. +## asserting_string +
+asserting_string  (class=typing #args=1) Aborts with an error if is_string on the argument returns false,
+else returns its argument.
+
-.. _reference-dsl-^^: +## atan -^^ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+atan  (class=math #args=1) One-argument arctangent.
+
-.. code-block:: none - ^^ (class=boolean #args=2) Logical XOR. +## atan2 +
+atan2  (class=math #args=2) Two-argument arctangent.
+
-.. _reference-dsl-bitwise-or: +## atanh -\| -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+atanh  (class=math #args=1) Inverse hyperbolic tangent.
+
-.. code-block:: none - | (class=arithmetic #args=2) Bitwise OR. +## bitcount +
+bitcount  (class=arithmetic #args=1) Count of 1-bits.
+
-.. _reference-dsl-||: +## boolean -|| -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+boolean  (class=conversion #args=1) Convert int/float/bool/string to boolean.
+
-.. code-block:: none - || (class=boolean #args=2) Logical OR. +## capitalize +
+capitalize  (class=string #args=1) Convert string's first character to uppercase.
+
-.. _reference-dsl-~: +## cbrt -~ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+cbrt  (class=math #args=1) Cube root.
+
-.. code-block:: none - ~ (class=arithmetic #args=1) Bitwise NOT. Beware '$y=~$x' since =~ is the - regex-match operator: try '$y = ~$x'. +## ceil +
+ceil  (class=math #args=1) Ceiling: nearest integer at or above.
+
-.. _reference-dsl-abs: +## clean_whitespace -abs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+clean_whitespace  (class=string #args=1) Same as collapse_whitespace and strip.
+
-.. code-block:: none - abs (class=math #args=1) Absolute value. +## collapse_whitespace +
+collapse_whitespace  (class=string #args=1) Strip repeated whitespace from string.
+
-.. _reference-dsl-acos: +## cos -acos -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+cos  (class=math #args=1) Trigonometric cosine.
+
-.. code-block:: none - acos (class=math #args=1) Inverse trigonometric cosine. +## cosh +
+cosh  (class=math #args=1) Hyperbolic cosine.
+
-.. _reference-dsl-acosh: +## depth -acosh -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+depth  (class=maps/arrays #args=1) Prints maximum depth of map/array. Scalars have depth 0.
+
-.. code-block:: none - acosh (class=math #args=1) Inverse hyperbolic cosine. +## dhms2fsec +
+dhms2fsec  (class=time #args=1) Recovers floating-point seconds as in dhms2fsec("5d18h53m20.250000s") = 500000.250000
+
-.. _reference-dsl-append: +## dhms2sec -append -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+dhms2sec  (class=time #args=1) Recovers integer seconds as in dhms2sec("5d18h53m20s") = 500000
+
-.. code-block:: none - append (class=maps/arrays #args=2) Appends second argument to end of first argument, which must be an array. +## erf +
+erf  (class=math #args=1) Error function.
+
-.. _reference-dsl-arrayify: +## erfc -arrayify -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+erfc  (class=math #args=1) Complementary error function.
+
-.. code-block:: none - arrayify (class=maps/arrays #args=1) Walks through a nested map/array, converting any map with consecutive keys - "1", "2", ... into an array. Useful to wrap the output of unflatten. +## exp +
+exp  (class=math #args=1) Exponential function e**x.
+
-.. _reference-dsl-asin: +## expm1 -asin -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+expm1  (class=math #args=1) e**x - 1.
+
-.. code-block:: none - asin (class=math #args=1) Inverse trigonometric sine. +## flatten +
+flatten  (class=maps/arrays #args=3) Flattens multi-level maps to single-level ones. Examples:
+flatten("a", ".", {"b": { "c": 4 }}) is {"a.b.c" : 4}.
+flatten("", ".", {"a": { "b": 3 }}) is {"a.b" : 3}.
+Two-argument version: flatten($*, ".") is the same as flatten("", ".", $*).
+Useful for nested JSON-like structures for non-JSON file formats like CSV.
+
-.. _reference-dsl-asinh: +## float -asinh -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+float  (class=conversion #args=1) Convert int/float/bool/string to float.
+
-.. code-block:: none - asinh (class=math #args=1) Inverse hyperbolic sine. +## floor +
+floor  (class=math #args=1) Floor: nearest integer at or below.
+
-.. _reference-dsl-asserting_absent: +## fmtnum -asserting_absent -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+fmtnum  (class=conversion #args=2) Convert int/float/bool to string using
+printf-style format string, e.g. '$s = fmtnum($n, "%06lld")'.
+
-.. code-block:: none - asserting_absent (class=typing #args=1) Aborts with an error if is_absent on the argument returns false, - else returns its argument. +## fsec2dhms +
+fsec2dhms  (class=time #args=1) Formats floating-point seconds as in fsec2dhms(500000.25) = "5d18h53m20.250000s"
+
-.. _reference-dsl-asserting_array: +## fsec2hms -asserting_array -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+fsec2hms  (class=time #args=1) Formats floating-point seconds as in fsec2hms(5000.25) = "01:23:20.250000"
+
-.. code-block:: none - asserting_array (class=typing #args=1) Aborts with an error if is_array on the argument returns false, - else returns its argument. +## get_keys +
+get_keys  (class=maps/arrays #args=1) Returns array of keys of map or array
+
-.. _reference-dsl-asserting_bool: +## get_values -asserting_bool -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+get_values  (class=maps/arrays #args=1) Returns array of keys of map or array -- in the latter case, returns a copy of the array
+
-.. code-block:: none - asserting_bool (class=typing #args=1) Aborts with an error if is_bool on the argument returns false, - else returns its argument. +## gmt2sec +
+gmt2sec  (class=time #args=1) Parses GMT timestamp as integer seconds since the epoch.
+
-.. _reference-dsl-asserting_boolean: +## gsub -asserting_boolean -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+gsub  (class=string #args=3) Example: '$name=gsub($name, "old", "new")' (replace all).
+
-.. code-block:: none - asserting_boolean (class=typing #args=1) Aborts with an error if is_boolean on the argument returns false, - else returns its argument. +## haskey +
+haskey  (class=maps/arrays #args=2) True/false if map has/hasn't key, e.g. 'haskey($*, "a")' or
+'haskey(mymap, mykey)', or true/false if array index is in bounds / out of bounds.
+Error if 1st argument is not a map or array. Note -n..-1 alias to 1..n in Miller arrays.
+
-.. _reference-dsl-asserting_empty: +## hexfmt -asserting_empty -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+hexfmt  (class=conversion #args=1) Convert int to hex string, e.g. 255 to "0xff".
+
-.. code-block:: none - asserting_empty (class=typing #args=1) Aborts with an error if is_empty on the argument returns false, - else returns its argument. +## hms2fsec +
+hms2fsec  (class=time #args=1) Recovers floating-point seconds as in hms2fsec("01:23:20.250000") = 5000.250000
+
-.. _reference-dsl-asserting_empty_map: +## hms2sec -asserting_empty_map -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+hms2sec  (class=time #args=1) Recovers integer seconds as in hms2sec("01:23:20") = 5000
+
-.. code-block:: none - asserting_empty_map (class=typing #args=1) Aborts with an error if is_empty_map on the argument returns false, - else returns its argument. +## hostname +
+hostname  (class=system #args=0) Returns the hostname as a string.
+
-.. _reference-dsl-asserting_error: +## int -asserting_error -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+int  (class=conversion #args=1) Convert int/float/bool/string to int.
+
-.. code-block:: none - asserting_error (class=typing #args=1) Aborts with an error if is_error on the argument returns false, - else returns its argument. +## invqnorm +
+invqnorm  (class=math #args=1) Inverse of normal cumulative distribution function.
+Note that invqorm(urand()) is normally distributed.
+
-.. _reference-dsl-asserting_float: +## is_absent -asserting_float -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+is_absent  (class=typing #args=1) False if field is present in input, true otherwise
+
-.. code-block:: none - asserting_float (class=typing #args=1) Aborts with an error if is_float on the argument returns false, - else returns its argument. +## is_array +
+is_array  (class=typing #args=1) True if argument is an array.
+
-.. _reference-dsl-asserting_int: +## is_bool -asserting_int -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+is_bool  (class=typing #args=1) True if field is present with boolean value. Synonymous with is_boolean.
+
-.. code-block:: none - asserting_int (class=typing #args=1) Aborts with an error if is_int on the argument returns false, - else returns its argument. +## is_boolean +
+is_boolean  (class=typing #args=1) True if field is present with boolean value. Synonymous with is_bool.
+
-.. _reference-dsl-asserting_map: +## is_empty -asserting_map -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+is_empty  (class=typing #args=1) True if field is present in input with empty string value, false otherwise.
+
-.. code-block:: none - asserting_map (class=typing #args=1) Aborts with an error if is_map on the argument returns false, - else returns its argument. +## is_empty_map +
+is_empty_map  (class=typing #args=1) True if argument is a map which is empty.
+
-.. _reference-dsl-asserting_nonempty_map: +## is_error -asserting_nonempty_map -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+is_error  (class=typing #args=1) True if if argument is an error, such as taking string length of an integer.
+
-.. code-block:: none - asserting_nonempty_map (class=typing #args=1) Aborts with an error if is_nonempty_map on the argument returns false, - else returns its argument. +## is_float +
+is_float  (class=typing #args=1) True if field is present with value inferred to be float
+
-.. _reference-dsl-asserting_not_array: +## is_int -asserting_not_array -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+is_int  (class=typing #args=1) True if field is present with value inferred to be int
+
-.. code-block:: none - asserting_not_array (class=typing #args=1) Aborts with an error if is_not_array on the argument returns false, - else returns its argument. +## is_map +
+is_map  (class=typing #args=1) True if argument is a map.
+
-.. _reference-dsl-asserting_not_empty: +## is_nonempty_map -asserting_not_empty -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+is_nonempty_map  (class=typing #args=1) True if argument is a map which is non-empty.
+
-.. code-block:: none - asserting_not_empty (class=typing #args=1) Aborts with an error if is_not_empty on the argument returns false, - else returns its argument. +## is_not_array +
+is_not_array  (class=typing #args=1) True if argument is not an array.
+
-.. _reference-dsl-asserting_not_map: +## is_not_empty -asserting_not_map -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+is_not_empty  (class=typing #args=1) False if field is present in input with empty value, true otherwise
+
-.. code-block:: none - asserting_not_map (class=typing #args=1) Aborts with an error if is_not_map on the argument returns false, - else returns its argument. +## is_not_map +
+is_not_map  (class=typing #args=1) True if argument is not a map.
+
-.. _reference-dsl-asserting_not_null: +## is_not_null -asserting_not_null -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+is_not_null  (class=typing #args=1) False if argument is null (empty or absent), true otherwise.
+
-.. code-block:: none - asserting_not_null (class=typing #args=1) Aborts with an error if is_not_null on the argument returns false, - else returns its argument. +## is_null +
+is_null  (class=typing #args=1) True if argument is null (empty or absent), false otherwise.
+
-.. _reference-dsl-asserting_null: +## is_numeric -asserting_null -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+is_numeric  (class=typing #args=1) True if field is present with value inferred to be int or float
+
-.. code-block:: none - asserting_null (class=typing #args=1) Aborts with an error if is_null on the argument returns false, - else returns its argument. +## is_present +
+is_present  (class=typing #args=1) True if field is present in input, false otherwise.
+
-.. _reference-dsl-asserting_numeric: +## is_string -asserting_numeric -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+is_string  (class=typing #args=1) True if field is present with string (including empty-string) value
+
-.. code-block:: none - asserting_numeric (class=typing #args=1) Aborts with an error if is_numeric on the argument returns false, - else returns its argument. +## joink +
+joink  (class=conversion #args=2) Makes string from map/array keys. Examples:
+joink({"a":3,"b":4,"c":5}, ",") = "a,b,c"
+joink([1,2,3], ",") = "1,2,3".
+
-.. _reference-dsl-asserting_present: +## joinkv -asserting_present -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+joinkv  (class=conversion #args=3) Makes string from map/array key-value pairs. Examples:
+joinkv([3,4,5], "=", ",") = "1=3,2=4,3=5"
+joinkv({"a":3,"b":4,"c":5}, "=", ",") = "a=3,b=4,c=5"
+
-.. code-block:: none - asserting_present (class=typing #args=1) Aborts with an error if is_present on the argument returns false, - else returns its argument. +## joinv +
+joinv  (class=conversion #args=2) Makes string from map/array values.
+joinv([3,4,5], ",") = "3,4,5"
+joinv({"a":3,"b":4,"c":5}, ",") = "3,4,5"
+
-.. _reference-dsl-asserting_string: +## json_parse -asserting_string -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+json_parse  (class=maps/arrays #args=1) Converts value from JSON-formatted string.
+
-.. code-block:: none - asserting_string (class=typing #args=1) Aborts with an error if is_string on the argument returns false, - else returns its argument. +## json_stringify +
+json_stringify  (class=maps/arrays #args=1,2) Converts value to JSON-formatted string. Default output is single-line.
+With optional second boolean argument set to true, produces multiline output.
+
-.. _reference-dsl-atan: +## leafcount -atan -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+leafcount  (class=maps/arrays #args=1) Counts total number of terminal values in map/array. For single-level
+map/array, same as length.
+
-.. code-block:: none - atan (class=math #args=1) One-argument arctangent. +## length +
+length  (class=maps/arrays #args=1) Counts number of top-level entries in array/map. Scalars have length 1.
+
-.. _reference-dsl-atan2: +## log -atan2 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+log  (class=math #args=1) Natural (base-e) logarithm.
+
-.. code-block:: none - atan2 (class=math #args=2) Two-argument arctangent. +## log10 +
+log10  (class=math #args=1) Base-10 logarithm.
+
-.. _reference-dsl-atanh: +## log1p -atanh -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+log1p  (class=math #args=1) log(1-x).
+
-.. code-block:: none - atanh (class=math #args=1) Inverse hyperbolic tangent. +## logifit +
+logifit  (class=math #args=3)  Given m and b from logistic regression, compute fit:
+$yhat=logifit($x,$m,$b).
+
-.. _reference-dsl-bitcount: +## lstrip -bitcount -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+lstrip  (class=string #args=1) Strip leading whitespace from string.
+
-.. code-block:: none - bitcount (class=arithmetic #args=1) Count of 1-bits. +## madd +
+madd  (class=arithmetic #args=3) a + b mod m (integers)
+
-.. _reference-dsl-boolean: +## mapdiff -boolean -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+mapdiff  (class=maps/arrays #args=variadic) With 0 args, returns empty map. With 1 arg, returns copy of arg.
+With 2 or more, returns copy of arg 1 with all keys from any of remaining
+argument maps removed.
+
-.. code-block:: none - boolean (class=conversion #args=1) Convert int/float/bool/string to boolean. +## mapexcept +
+mapexcept  (class=maps/arrays #args=variadic) Returns a map with keys from remaining arguments, if any, unset.
+Remaining arguments can be strings or arrays of string.
+E.g. 'mapexcept({1:2,3:4,5:6}, 1, 5, 7)' is '{3:4}'
+and  'mapexcept({1:2,3:4,5:6}, [1, 5, 7])' is '{3:4}'.
+
-.. _reference-dsl-capitalize: +## mapselect -capitalize -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+mapselect  (class=maps/arrays #args=variadic) Returns a map with only keys from remaining arguments set.
+Remaining arguments can be strings or arrays of string.
+E.g. 'mapselect({1:2,3:4,5:6}, 1, 5, 7)' is '{1:2,5:6}'
+and  'mapselect({1:2,3:4,5:6}, [1, 5, 7])' is '{1:2,5:6}'.
+
-.. code-block:: none - capitalize (class=string #args=1) Convert string's first character to uppercase. +## mapsum +
+mapsum  (class=maps/arrays #args=variadic) With 0 args, returns empty map. With >= 1 arg, returns a map with
+key-value pairs from all arguments. Rightmost collisions win, e.g.
+'mapsum({1:2,3:4},{1:5})' is '{1:5,3:4}'.
+
-.. _reference-dsl-cbrt: +## max -cbrt -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+max  (class=math #args=variadic) Max of n numbers; null loses.
+
-.. code-block:: none - cbrt (class=math #args=1) Cube root. +## md5 +
+md5  (class=hashing #args=1) MD5 hash.
+
-.. _reference-dsl-ceil: +## mexp -ceil -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+mexp  (class=arithmetic #args=3) a ** b mod m (integers)
+
-.. code-block:: none - ceil (class=math #args=1) Ceiling: nearest integer at or above. +## min +
+min  (class=math #args=variadic) Min of n numbers; null loses.
+
-.. _reference-dsl-clean_whitespace: +## mmul -clean_whitespace -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+mmul  (class=arithmetic #args=3) a * b mod m (integers)
+
-.. code-block:: none - clean_whitespace (class=string #args=1) Same as collapse_whitespace and strip. +## msub +
+msub  (class=arithmetic #args=3) a - b mod m (integers)
+
-.. _reference-dsl-collapse_whitespace: +## os -collapse_whitespace -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+os  (class=system #args=0) Returns the operating-system name as a string.
+
-.. code-block:: none - collapse_whitespace (class=string #args=1) Strip repeated whitespace from string. +## pow +
+pow  (class=arithmetic #args=2) Exponentiation. Same as **, but as a function.
+
-.. _reference-dsl-cos: +## qnorm -cos -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+qnorm  (class=math #args=1) Normal cumulative distribution function.
+
-.. code-block:: none - cos (class=math #args=1) Trigonometric cosine. +## regextract +
+regextract  (class=string #args=2) Example: '$name=regextract($name, "[A-Z]{3}[0-9]{2}")'
+
-.. _reference-dsl-cosh: +## regextract_or_else -cosh -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+regextract_or_else  (class=string #args=3) Example: '$name=regextract_or_else($name, "[A-Z]{3}[0-9]{2}", "default")'
+
-.. code-block:: none - cosh (class=math #args=1) Hyperbolic cosine. +## round +
+round  (class=math #args=1) Round to nearest integer.
+
-.. _reference-dsl-depth: +## roundm -depth -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+roundm  (class=math #args=2) Round to nearest multiple of m: roundm($x,$m) is
+the same as round($x/$m)*$m.
+
-.. code-block:: none - depth (class=maps/arrays #args=1) Prints maximum depth of map/array. Scalars have depth 0. +## rstrip +
+rstrip  (class=string #args=1) Strip trailing whitespace from string.
+
-.. _reference-dsl-dhms2fsec: +## sec2dhms -dhms2fsec -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+sec2dhms  (class=time #args=1) Formats integer seconds as in sec2dhms(500000) = "5d18h53m20s"
+
-.. code-block:: none - dhms2fsec (class=time #args=1) Recovers floating-point seconds as in dhms2fsec("5d18h53m20.250000s") = 500000.250000 +## sec2gmt +
+sec2gmt  (class=time #args=1,2) Formats seconds since epoch (integer part)
+as GMT timestamp, e.g. sec2gmt(1440768801.7) = "2015-08-28T13:33:21Z".
+Leaves non-numbers as-is. With second integer argument n, includes n decimal places
+for the seconds part
+
-.. _reference-dsl-dhms2sec: +## sec2gmtdate -dhms2sec -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+sec2gmtdate  (class=time #args=1) Formats seconds since epoch (integer part)
+as GMT timestamp with year-month-date, e.g. sec2gmtdate(1440768801.7) = "2015-08-28".
+Leaves non-numbers as-is.
+
-.. code-block:: none - dhms2sec (class=time #args=1) Recovers integer seconds as in dhms2sec("5d18h53m20s") = 500000 +## sec2hms +
+sec2hms  (class=time #args=1) Formats integer seconds as in sec2hms(5000) = "01:23:20"
+
-.. _reference-dsl-erf: +## sgn -erf -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+sgn  (class=math #args=1)  +1, 0, -1 for positive, zero, negative input respectively.
+
-.. code-block:: none - erf (class=math #args=1) Error function. +## sha1 +
+sha1  (class=hashing #args=1) SHA1 hash.
+
-.. _reference-dsl-erfc: +## sha256 -erfc -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+sha256  (class=hashing #args=1) SHA256 hash.
+
-.. code-block:: none - erfc (class=math #args=1) Complementary error function. +## sha512 +
+sha512  (class=hashing #args=1) SHA512 hash.
+
-.. _reference-dsl-exp: +## sin -exp -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+sin  (class=math #args=1) Trigonometric sine.
+
-.. code-block:: none - exp (class=math #args=1) Exponential function e**x. +## sinh +
+sinh  (class=math #args=1) Hyperbolic sine.
+
-.. _reference-dsl-expm1: +## splita -expm1 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+splita  (class=conversion #args=2) Splits string into array with type inference. Example:
+splita("3,4,5", ",") = [3,4,5]
+
-.. code-block:: none - expm1 (class=math #args=1) e**x - 1. +## splitax +
+splitax  (class=conversion #args=2) Splits string into array without type inference. Example:
+splita("3,4,5", ",") = ["3","4","5"]
+
-.. _reference-dsl-flatten: +## splitkv -flatten -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+splitkv  (class=conversion #args=3) Splits string by separators into map with type inference. Example:
+splitkv("a=3,b=4,c=5", "=", ",") = {"a":3,"b":4,"c":5}
+
-.. code-block:: none - flatten (class=maps/arrays #args=3) Flattens multi-level maps to single-level ones. Examples: - flatten("a", ".", {"b": { "c": 4 }}) is {"a.b.c" : 4}. - flatten("", ".", {"a": { "b": 3 }}) is {"a.b" : 3}. - Two-argument version: flatten($*, ".") is the same as flatten("", ".", $*). - Useful for nested JSON-like structures for non-JSON file formats like CSV. +## splitkvx +
+splitkvx  (class=conversion #args=3) Splits string by separators into map without type inference (keys and
+values are strings). Example:
+splitkvx("a=3,b=4,c=5", "=", ",") = {"a":"3","b":"4","c":"5"}
+
-.. _reference-dsl-float: +## splitnv -float -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+splitnv  (class=conversion #args=2) Splits string by separator into integer-indexed map with type inference. Example:
+splitnv("a,b,c", ",") = {"1":"a","2":"b","3":"c"}
+
-.. code-block:: none - float (class=conversion #args=1) Convert int/float/bool/string to float. +## splitnvx +
+splitnvx  (class=conversion #args=2) Splits string by separator into integer-indexed map without type
+inference (values are strings). Example:
+splitnvx("3,4,5", ",") = {"1":"3","2":"4","3":"5"}
+
-.. _reference-dsl-floor: +## sqrt -floor -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+sqrt  (class=math #args=1) Square root.
+
-.. code-block:: none - floor (class=math #args=1) Floor: nearest integer at or below. +## ssub +
+ssub  (class=string #args=3) Like sub but does no regexing. No characters are special.
+
-.. _reference-dsl-fmtnum: +## strftime -fmtnum -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+strftime  (class=time #args=2)  Formats seconds since the epoch as timestamp, e.g.
+	strftime(1440768801.7,"%Y-%m-%dT%H:%M:%SZ") = "2015-08-28T13:33:21Z", and
+	strftime(1440768801.7,"%Y-%m-%dT%H:%M:%3SZ") = "2015-08-28T13:33:21.700Z".
+	Format strings are as in the C library (please see "man strftime" on your system),
+	with the Miller-specific addition of "%1S" through "%9S" which format the seconds
+	with 1 through 9 decimal places, respectively. ("%S" uses no decimal places.)
+	See also strftime_local.
+
-.. code-block:: none - fmtnum (class=conversion #args=2) Convert int/float/bool to string using - printf-style format string, e.g. '$s = fmtnum($n, "%06lld")'. +## string +
+string  (class=conversion #args=1) Convert int/float/bool/string/array/map to string.
+
-.. _reference-dsl-fsec2dhms: +## strip -fsec2dhms -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+strip  (class=string #args=1) Strip leading and trailing whitespace from string.
+
-.. code-block:: none - fsec2dhms (class=time #args=1) Formats floating-point seconds as in fsec2dhms(500000.25) = "5d18h53m20.250000s" +## strlen +
+strlen  (class=string #args=1) String length.
+
-.. _reference-dsl-fsec2hms: +## strptime -fsec2hms -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+strptime  (class=time #args=2) strptime: Parses timestamp as floating-point seconds since the epoch,
+	e.g. strptime("2015-08-28T13:33:21Z","%Y-%m-%dT%H:%M:%SZ") = 1440768801.000000,
+	and  strptime("2015-08-28T13:33:21.345Z","%Y-%m-%dT%H:%M:%SZ") = 1440768801.345000.
+	See also strptime_local.
+
-.. code-block:: none - fsec2hms (class=time #args=1) Formats floating-point seconds as in fsec2hms(5000.25) = "01:23:20.250000" +## sub +
+sub  (class=string #args=3) Example: '$name=sub($name, "old", "new")' (replace once).
+
-.. _reference-dsl-get_keys: +## substr -get_keys -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+substr  (class=string #args=3) substr is an alias for substr0. See also substr1. Miller is generally 1-up
+with all array indices, but, this is a backward-compatibility issue with Miller 5 and below.
+Arrays are new in Miller 6; the substr function is older.
+
-.. code-block:: none - get_keys (class=maps/arrays #args=1) Returns array of keys of map or array +## substr0 +
+substr0  (class=string #args=3) substr0(s,m,n) gives substring of s from 0-up position m to n
+inclusive. Negative indices -len .. -1 alias to 0 .. len-1.
+
-.. _reference-dsl-get_values: +## substr1 -get_values -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+substr1  (class=string #args=3) substr1(s,m,n) gives substring of s from 1-up position m to n
+inclusive. Negative indices -len .. -1 alias to 1 .. len.
+
-.. code-block:: none - get_values (class=maps/arrays #args=1) Returns array of keys of map or array -- in the latter case, returns a copy of the array +## system +
+system  (class=system #args=1) Run command string, yielding its stdout minus final carriage return.
+
-.. _reference-dsl-gmt2sec: +## systime -gmt2sec -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+systime  (class=time #args=0) help string will go here
+
-.. code-block:: none - gmt2sec (class=time #args=1) Parses GMT timestamp as integer seconds since the epoch. +## systimeint +
+systimeint  (class=time #args=0) help string will go here
+
-.. _reference-dsl-gsub: +## tan -gsub -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+tan  (class=math #args=1) Trigonometric tangent.
+
-.. code-block:: none - gsub (class=string #args=3) Example: '$name=gsub($name, "old", "new")' (replace all). +## tanh +
+tanh  (class=math #args=1) Hyperbolic tangent.
+
-.. _reference-dsl-haskey: +## tolower -haskey -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+tolower  (class=string #args=1) Convert string to lowercase.
+
-.. code-block:: none - haskey (class=maps/arrays #args=2) True/false if map has/hasn't key, e.g. 'haskey($*, "a")' or - 'haskey(mymap, mykey)', or true/false if array index is in bounds / out of bounds. - Error if 1st argument is not a map or array. Note -n..-1 alias to 1..n in Miller arrays. +## toupper +
+toupper  (class=string #args=1) Convert string to uppercase.
+
-.. _reference-dsl-hexfmt: +## truncate -hexfmt -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+truncate  (class=string #args=2) Truncates string first argument to max length of int second argument.
+
-.. code-block:: none - hexfmt (class=conversion #args=1) Convert int to hex string, e.g. 255 to "0xff". +## typeof +
+typeof  (class=typing #args=1) Convert argument to type of argument (e.g. "str"). For debug.
+
-.. _reference-dsl-hms2fsec: +## unflatten -hms2fsec -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+unflatten  (class=maps/arrays #args=2) Reverses flatten. Example:
+unflatten({"a.b.c" : 4}, ".") is {"a": "b": { "c": 4 }}.
+Useful for nested JSON-like structures for non-JSON file formats like CSV.
+See also arrayify.
+
-.. code-block:: none - hms2fsec (class=time #args=1) Recovers floating-point seconds as in hms2fsec("01:23:20.250000") = 5000.250000 +## uptime +
+uptime  (class=time #args=0) help string will go here
+
-.. _reference-dsl-hms2sec: +## urand -hms2sec -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+urand  (class=math #args=0) Floating-point numbers uniformly distributed on the unit interval.
+Int-valued example: '$n=floor(20+urand()*11)'.
+
-.. code-block:: none - hms2sec (class=time #args=1) Recovers integer seconds as in hms2sec("01:23:20") = 5000 +## urand32 +
+urand32  (class=math #args=0) Integer uniformly distributed 0 and 2**32-1 inclusive.
+
-.. _reference-dsl-hostname: +## urandint -hostname -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
+urandint  (class=math #args=2) Integer uniformly distributed between inclusive integer endpoints.
+
-.. code-block:: none - hostname (class=system #args=0) Returns the hostname as a string. +## urandrange +
+urandrange  (class=math #args=2) Floating-point numbers uniformly distributed on the interval [a, b).
+
-.. _reference-dsl-int: - -int -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - int (class=conversion #args=1) Convert int/float/bool/string to int. - - - -.. _reference-dsl-invqnorm: - -invqnorm -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - invqnorm (class=math #args=1) Inverse of normal cumulative distribution function. - Note that invqorm(urand()) is normally distributed. - - - -.. _reference-dsl-is_absent: - -is_absent -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_absent (class=typing #args=1) False if field is present in input, true otherwise - - - -.. _reference-dsl-is_array: - -is_array -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_array (class=typing #args=1) True if argument is an array. - - - -.. _reference-dsl-is_bool: - -is_bool -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_bool (class=typing #args=1) True if field is present with boolean value. Synonymous with is_boolean. - - - -.. _reference-dsl-is_boolean: - -is_boolean -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_boolean (class=typing #args=1) True if field is present with boolean value. Synonymous with is_bool. - - - -.. _reference-dsl-is_empty: - -is_empty -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_empty (class=typing #args=1) True if field is present in input with empty string value, false otherwise. - - - -.. _reference-dsl-is_empty_map: - -is_empty_map -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_empty_map (class=typing #args=1) True if argument is a map which is empty. - - - -.. _reference-dsl-is_error: - -is_error -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_error (class=typing #args=1) True if if argument is an error, such as taking string length of an integer. - - - -.. _reference-dsl-is_float: - -is_float -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_float (class=typing #args=1) True if field is present with value inferred to be float - - - -.. _reference-dsl-is_int: - -is_int -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_int (class=typing #args=1) True if field is present with value inferred to be int - - - -.. _reference-dsl-is_map: - -is_map -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_map (class=typing #args=1) True if argument is a map. - - - -.. _reference-dsl-is_nonempty_map: - -is_nonempty_map -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_nonempty_map (class=typing #args=1) True if argument is a map which is non-empty. - - - -.. _reference-dsl-is_not_array: - -is_not_array -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_not_array (class=typing #args=1) True if argument is not an array. - - - -.. _reference-dsl-is_not_empty: - -is_not_empty -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_not_empty (class=typing #args=1) False if field is present in input with empty value, true otherwise - - - -.. _reference-dsl-is_not_map: - -is_not_map -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_not_map (class=typing #args=1) True if argument is not a map. - - - -.. _reference-dsl-is_not_null: - -is_not_null -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_not_null (class=typing #args=1) False if argument is null (empty or absent), true otherwise. - - - -.. _reference-dsl-is_null: - -is_null -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_null (class=typing #args=1) True if argument is null (empty or absent), false otherwise. - - - -.. _reference-dsl-is_numeric: - -is_numeric -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_numeric (class=typing #args=1) True if field is present with value inferred to be int or float - - - -.. _reference-dsl-is_present: - -is_present -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_present (class=typing #args=1) True if field is present in input, false otherwise. - - - -.. _reference-dsl-is_string: - -is_string -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - is_string (class=typing #args=1) True if field is present with string (including empty-string) value - - - -.. _reference-dsl-joink: - -joink -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - joink (class=conversion #args=2) Makes string from map/array keys. Examples: - joink({"a":3,"b":4,"c":5}, ",") = "a,b,c" - joink([1,2,3], ",") = "1,2,3". - - - -.. _reference-dsl-joinkv: - -joinkv -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - joinkv (class=conversion #args=3) Makes string from map/array key-value pairs. Examples: - joinkv([3,4,5], "=", ",") = "1=3,2=4,3=5" - joinkv({"a":3,"b":4,"c":5}, "=", ",") = "a=3,b=4,c=5" - - - -.. _reference-dsl-joinv: - -joinv -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - joinv (class=conversion #args=2) Makes string from map/array values. - joinv([3,4,5], ",") = "3,4,5" - joinv({"a":3,"b":4,"c":5}, ",") = "3,4,5" - - - -.. _reference-dsl-json_parse: - -json_parse -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - json_parse (class=maps/arrays #args=1) Converts value from JSON-formatted string. - - - -.. _reference-dsl-json_stringify: - -json_stringify -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - json_stringify (class=maps/arrays #args=1,2) Converts value to JSON-formatted string. Default output is single-line. - With optional second boolean argument set to true, produces multiline output. - - - -.. _reference-dsl-leafcount: - -leafcount -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - leafcount (class=maps/arrays #args=1) Counts total number of terminal values in map/array. For single-level - map/array, same as length. - - - -.. _reference-dsl-length: - -length -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - length (class=maps/arrays #args=1) Counts number of top-level entries in array/map. Scalars have length 1. - - - -.. _reference-dsl-log: - -log -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - log (class=math #args=1) Natural (base-e) logarithm. - - - -.. _reference-dsl-log10: - -log10 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - log10 (class=math #args=1) Base-10 logarithm. - - - -.. _reference-dsl-log1p: - -log1p -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - log1p (class=math #args=1) log(1-x). - - - -.. _reference-dsl-logifit: - -logifit -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - logifit (class=math #args=3) Given m and b from logistic regression, compute fit: - $yhat=logifit($x,$m,$b). - - - -.. _reference-dsl-lstrip: - -lstrip -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - lstrip (class=string #args=1) Strip leading whitespace from string. - - - -.. _reference-dsl-madd: - -madd -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - madd (class=arithmetic #args=3) a + b mod m (integers) - - - -.. _reference-dsl-mapdiff: - -mapdiff -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - mapdiff (class=maps/arrays #args=variadic) With 0 args, returns empty map. With 1 arg, returns copy of arg. - With 2 or more, returns copy of arg 1 with all keys from any of remaining - argument maps removed. - - - -.. _reference-dsl-mapexcept: - -mapexcept -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - mapexcept (class=maps/arrays #args=variadic) Returns a map with keys from remaining arguments, if any, unset. - Remaining arguments can be strings or arrays of string. - E.g. 'mapexcept({1:2,3:4,5:6}, 1, 5, 7)' is '{3:4}' - and 'mapexcept({1:2,3:4,5:6}, [1, 5, 7])' is '{3:4}'. - - - -.. _reference-dsl-mapselect: - -mapselect -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - mapselect (class=maps/arrays #args=variadic) Returns a map with only keys from remaining arguments set. - Remaining arguments can be strings or arrays of string. - E.g. 'mapselect({1:2,3:4,5:6}, 1, 5, 7)' is '{1:2,5:6}' - and 'mapselect({1:2,3:4,5:6}, [1, 5, 7])' is '{1:2,5:6}'. - - - -.. _reference-dsl-mapsum: - -mapsum -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - mapsum (class=maps/arrays #args=variadic) With 0 args, returns empty map. With >= 1 arg, returns a map with - key-value pairs from all arguments. Rightmost collisions win, e.g. - 'mapsum({1:2,3:4},{1:5})' is '{1:5,3:4}'. - - - -.. _reference-dsl-max: - -max -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - max (class=math #args=variadic) Max of n numbers; null loses. - - - -.. _reference-dsl-md5: - -md5 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - md5 (class=hashing #args=1) MD5 hash. - - - -.. _reference-dsl-mexp: - -mexp -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - mexp (class=arithmetic #args=3) a ** b mod m (integers) - - - -.. _reference-dsl-min: - -min -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - min (class=math #args=variadic) Min of n numbers; null loses. - - - -.. _reference-dsl-mmul: - -mmul -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - mmul (class=arithmetic #args=3) a * b mod m (integers) - - - -.. _reference-dsl-msub: - -msub -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - msub (class=arithmetic #args=3) a - b mod m (integers) - - - -.. _reference-dsl-os: - -os -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - os (class=system #args=0) Returns the operating-system name as a string. - - - -.. _reference-dsl-pow: - -pow -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - pow (class=arithmetic #args=2) Exponentiation. Same as **, but as a function. - - - -.. _reference-dsl-qnorm: - -qnorm -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - qnorm (class=math #args=1) Normal cumulative distribution function. - - - -.. _reference-dsl-regextract: - -regextract -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - regextract (class=string #args=2) Example: '$name=regextract($name, "[A-Z]{3}[0-9]{2}")' - - - -.. _reference-dsl-regextract_or_else: - -regextract_or_else -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - regextract_or_else (class=string #args=3) Example: '$name=regextract_or_else($name, "[A-Z]{3}[0-9]{2}", "default")' - - - -.. _reference-dsl-round: - -round -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - round (class=math #args=1) Round to nearest integer. - - - -.. _reference-dsl-roundm: - -roundm -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - roundm (class=math #args=2) Round to nearest multiple of m: roundm($x,$m) is - the same as round($x/$m)*$m. - - - -.. _reference-dsl-rstrip: - -rstrip -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - rstrip (class=string #args=1) Strip trailing whitespace from string. - - - -.. _reference-dsl-sec2dhms: - -sec2dhms -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - sec2dhms (class=time #args=1) Formats integer seconds as in sec2dhms(500000) = "5d18h53m20s" - - - -.. _reference-dsl-sec2gmt: - -sec2gmt -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - sec2gmt (class=time #args=1,2) Formats seconds since epoch (integer part) - as GMT timestamp, e.g. sec2gmt(1440768801.7) = "2015-08-28T13:33:21Z". - Leaves non-numbers as-is. With second integer argument n, includes n decimal places - for the seconds part - - - -.. _reference-dsl-sec2gmtdate: - -sec2gmtdate -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - sec2gmtdate (class=time #args=1) Formats seconds since epoch (integer part) - as GMT timestamp with year-month-date, e.g. sec2gmtdate(1440768801.7) = "2015-08-28". - Leaves non-numbers as-is. - - - -.. _reference-dsl-sec2hms: - -sec2hms -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - sec2hms (class=time #args=1) Formats integer seconds as in sec2hms(5000) = "01:23:20" - - - -.. _reference-dsl-sgn: - -sgn -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - sgn (class=math #args=1) +1, 0, -1 for positive, zero, negative input respectively. - - - -.. _reference-dsl-sha1: - -sha1 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - sha1 (class=hashing #args=1) SHA1 hash. - - - -.. _reference-dsl-sha256: - -sha256 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - sha256 (class=hashing #args=1) SHA256 hash. - - - -.. _reference-dsl-sha512: - -sha512 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - sha512 (class=hashing #args=1) SHA512 hash. - - - -.. _reference-dsl-sin: - -sin -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - sin (class=math #args=1) Trigonometric sine. - - - -.. _reference-dsl-sinh: - -sinh -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - sinh (class=math #args=1) Hyperbolic sine. - - - -.. _reference-dsl-splita: - -splita -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - splita (class=conversion #args=2) Splits string into array with type inference. Example: - splita("3,4,5", ",") = [3,4,5] - - - -.. _reference-dsl-splitax: - -splitax -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - splitax (class=conversion #args=2) Splits string into array without type inference. Example: - splita("3,4,5", ",") = ["3","4","5"] - - - -.. _reference-dsl-splitkv: - -splitkv -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - splitkv (class=conversion #args=3) Splits string by separators into map with type inference. Example: - splitkv("a=3,b=4,c=5", "=", ",") = {"a":3,"b":4,"c":5} - - - -.. _reference-dsl-splitkvx: - -splitkvx -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - splitkvx (class=conversion #args=3) Splits string by separators into map without type inference (keys and - values are strings). Example: - splitkvx("a=3,b=4,c=5", "=", ",") = {"a":"3","b":"4","c":"5"} - - - -.. _reference-dsl-splitnv: - -splitnv -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - splitnv (class=conversion #args=2) Splits string by separator into integer-indexed map with type inference. Example: - splitnv("a,b,c", ",") = {"1":"a","2":"b","3":"c"} - - - -.. _reference-dsl-splitnvx: - -splitnvx -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - splitnvx (class=conversion #args=2) Splits string by separator into integer-indexed map without type - inference (values are strings). Example: - splitnvx("3,4,5", ",") = {"1":"3","2":"4","3":"5"} - - - -.. _reference-dsl-sqrt: - -sqrt -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - sqrt (class=math #args=1) Square root. - - - -.. _reference-dsl-ssub: - -ssub -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - ssub (class=string #args=3) Like sub but does no regexing. No characters are special. - - - -.. _reference-dsl-strftime: - -strftime -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - strftime (class=time #args=2) Formats seconds since the epoch as timestamp, e.g. - strftime(1440768801.7,"%Y-%m-%dT%H:%M:%SZ") = "2015-08-28T13:33:21Z", and - strftime(1440768801.7,"%Y-%m-%dT%H:%M:%3SZ") = "2015-08-28T13:33:21.700Z". - Format strings are as in the C library (please see "man strftime" on your system), - with the Miller-specific addition of "%1S" through "%9S" which format the seconds - with 1 through 9 decimal places, respectively. ("%S" uses no decimal places.) - See also strftime_local. - - - -.. _reference-dsl-string: - -string -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - string (class=conversion #args=1) Convert int/float/bool/string/array/map to string. - - - -.. _reference-dsl-strip: - -strip -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - strip (class=string #args=1) Strip leading and trailing whitespace from string. - - - -.. _reference-dsl-strlen: - -strlen -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - strlen (class=string #args=1) String length. - - - -.. _reference-dsl-strptime: - -strptime -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - strptime (class=time #args=2) strptime: Parses timestamp as floating-point seconds since the epoch, - e.g. strptime("2015-08-28T13:33:21Z","%Y-%m-%dT%H:%M:%SZ") = 1440768801.000000, - and strptime("2015-08-28T13:33:21.345Z","%Y-%m-%dT%H:%M:%SZ") = 1440768801.345000. - See also strptime_local. - - - -.. _reference-dsl-sub: - -sub -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - sub (class=string #args=3) Example: '$name=sub($name, "old", "new")' (replace once). - - - -.. _reference-dsl-substr: - -substr -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - substr (class=string #args=3) substr is an alias for substr0. See also substr1. Miller is generally 1-up - with all array indices, but, this is a backward-compatibility issue with Miller 5 and below. - Arrays are new in Miller 6; the substr function is older. - - - -.. _reference-dsl-substr0: - -substr0 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - substr0 (class=string #args=3) substr0(s,m,n) gives substring of s from 0-up position m to n - inclusive. Negative indices -len .. -1 alias to 0 .. len-1. - - - -.. _reference-dsl-substr1: - -substr1 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - substr1 (class=string #args=3) substr1(s,m,n) gives substring of s from 1-up position m to n - inclusive. Negative indices -len .. -1 alias to 1 .. len. - - - -.. _reference-dsl-system: - -system -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - system (class=system #args=1) Run command string, yielding its stdout minus final carriage return. - - - -.. _reference-dsl-systime: - -systime -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - systime (class=time #args=0) help string will go here - - - -.. _reference-dsl-systimeint: - -systimeint -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - systimeint (class=time #args=0) help string will go here - - - -.. _reference-dsl-tan: - -tan -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - tan (class=math #args=1) Trigonometric tangent. - - - -.. _reference-dsl-tanh: - -tanh -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - tanh (class=math #args=1) Hyperbolic tangent. - - - -.. _reference-dsl-tolower: - -tolower -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - tolower (class=string #args=1) Convert string to lowercase. - - - -.. _reference-dsl-toupper: - -toupper -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - toupper (class=string #args=1) Convert string to uppercase. - - - -.. _reference-dsl-truncate: - -truncate -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - truncate (class=string #args=2) Truncates string first argument to max length of int second argument. - - - -.. _reference-dsl-typeof: - -typeof -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - typeof (class=typing #args=1) Convert argument to type of argument (e.g. "str"). For debug. - - - -.. _reference-dsl-unflatten: - -unflatten -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - unflatten (class=maps/arrays #args=2) Reverses flatten. Example: - unflatten({"a.b.c" : 4}, ".") is {"a": "b": { "c": 4 }}. - Useful for nested JSON-like structures for non-JSON file formats like CSV. - See also arrayify. - - - -.. _reference-dsl-uptime: - -uptime -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - uptime (class=time #args=0) help string will go here - - - -.. _reference-dsl-urand: - -urand -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - urand (class=math #args=0) Floating-point numbers uniformly distributed on the unit interval. - Int-valued example: '$n=floor(20+urand()*11)'. - - - -.. _reference-dsl-urand32: - -urand32 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - urand32 (class=math #args=0) Integer uniformly distributed 0 and 2**32-1 inclusive. - - - -.. _reference-dsl-urandint: - -urandint -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - urandint (class=math #args=2) Integer uniformly distributed between inclusive integer endpoints. - - - -.. _reference-dsl-urandrange: - -urandrange -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - urandrange (class=math #args=2) Floating-point numbers uniformly distributed on the interval [a, b). - - - -.. _reference-dsl-version: - -version -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: none - - version (class=system #args=0) Returns the Miller version as a string. +## version +
+version  (class=system #args=0) Returns the Miller version as a string.
+
diff --git a/docs6b/docs/reference-dsl-builtin-functions.md.in b/docs6b/docs/reference-dsl-builtin-functions.md.in index 69eaa9bb8..13cb0dd9b 100644 --- a/docs6b/docs/reference-dsl-builtin-functions.md.in +++ b/docs6b/docs/reference-dsl-builtin-functions.md.in @@ -6,7 +6,7 @@ GENMD_RUN_CONTENT_GENERATOR(mk-func-table.rb) ## List of functions -Each function takes a specific number of arguments, as shown below, except for functions marked as variadic such as ``min`` and ``max``. (The latter compute min and max of any number of numerical arguments.) There is no notion of optional or default-on-absent arguments. All argument-passing is positional rather than by name; arguments are passed by value, not by reference. +Each function takes a specific number of arguments, as shown below, except for functions marked as variadic such as `min` and `max`. (The latter compute min and max of any number of numerical arguments.) There is no notion of optional or default-on-absent arguments. All argument-passing is positional rather than by name; arguments are passed by value, not by reference. You can get a list of all functions using **mlr -f**, with details using **mlr -F**. diff --git a/docs6b/docs/reference-dsl-complexity.md b/docs6b/docs/reference-dsl-complexity.md index 658e2955d..560132d33 100644 --- a/docs6b/docs/reference-dsl-complexity.md +++ b/docs6b/docs/reference-dsl-complexity.md @@ -1,9 +1,9 @@ # DSL reference: a note on the complexity of Miller's expression language -One of Miller's strengths is its brevity: it's much quicker -- and less error-prone -- to type ``mlr stats1 -a sum -f x,y -g a,b`` than having to track summation variables as in ``awk``, or using Miller's out-of-stream variables. And the more language features Miller's put-DSL has (for-loops, if-statements, nested control structures, user-defined functions, etc.) then the *less* powerful it begins to seem: because of the other programming-language features it *doesn't* have (classes, exceptions, and so on). +One of Miller's strengths is its brevity: it's much quicker -- and less error-prone -- to type `mlr stats1 -a sum -f x,y -g a,b` than having to track summation variables as in `awk`, or using Miller's out-of-stream variables. And the more language features Miller's put-DSL has (for-loops, if-statements, nested control structures, user-defined functions, etc.) then the *less* powerful it begins to seem: because of the other programming-language features it *doesn't* have (classes, exceptions, and so on). -When I was originally prototyping Miller in 2015, the decision I had was whether to hand-code in a low-level language like C or Rust or Go, with my own hand-rolled DSL, or whether to use a higher-level language (like Python or Lua or Nim) and let the ``put`` statements be handled by the implementation language's own ``eval``: the implementation language would take the place of a DSL. Multiple performance experiments showed me I could get better throughput using the former, by a wide margin. So Miller is Go under the hood with a hand-rolled DSL. +When I was originally prototyping Miller in 2015, the decision I had was whether to hand-code in a low-level language like C or Rust or Go, with my own hand-rolled DSL, or whether to use a higher-level language (like Python or Lua or Nim) and let the `put` statements be handled by the implementation language's own `eval`: the implementation language would take the place of a DSL. Multiple performance experiments showed me I could get better throughput using the former, by a wide margin. So Miller is Go under the hood with a hand-rolled DSL. -I do want to keep focusing on what Miller is good at -- concise notation, low latency, and high throughput -- and not add too much in terms of high-level-language features to the DSL. That said, some sort of customizability is a basic thing to want. As of 4.1.0 we have recursive for/while/if structures on about the same complexity level as ``awk``; as of 5.0.0 we have user-defined functions and map-valued variables, again on about the same complexity level as ``awk`` along with optional type-declaration syntax; as of Miller 6 we have full support for arrays. While I'm excited by these powerful language features, I hope to keep new features focused on Miller's sweet spot which is speed plus simplicity. +I do want to keep focusing on what Miller is good at -- concise notation, low latency, and high throughput -- and not add too much in terms of high-level-language features to the DSL. That said, some sort of customizability is a basic thing to want. As of 4.1.0 we have recursive for/while/if structures on about the same complexity level as `awk`; as of 5.0.0 we have user-defined functions and map-valued variables, again on about the same complexity level as `awk` along with optional type-declaration syntax; as of Miller 6 we have full support for arrays. While I'm excited by these powerful language features, I hope to keep new features focused on Miller's sweet spot which is speed plus simplicity. diff --git a/docs6b/docs/reference-dsl-complexity.md.in b/docs6b/docs/reference-dsl-complexity.md.in index fbf62a307..7ea4a390e 100644 --- a/docs6b/docs/reference-dsl-complexity.md.in +++ b/docs6b/docs/reference-dsl-complexity.md.in @@ -1,8 +1,8 @@ # DSL reference: a note on the complexity of Miller's expression language -One of Miller's strengths is its brevity: it's much quicker -- and less error-prone -- to type ``mlr stats1 -a sum -f x,y -g a,b`` than having to track summation variables as in ``awk``, or using Miller's out-of-stream variables. And the more language features Miller's put-DSL has (for-loops, if-statements, nested control structures, user-defined functions, etc.) then the *less* powerful it begins to seem: because of the other programming-language features it *doesn't* have (classes, exceptions, and so on). +One of Miller's strengths is its brevity: it's much quicker -- and less error-prone -- to type `mlr stats1 -a sum -f x,y -g a,b` than having to track summation variables as in `awk`, or using Miller's out-of-stream variables. And the more language features Miller's put-DSL has (for-loops, if-statements, nested control structures, user-defined functions, etc.) then the *less* powerful it begins to seem: because of the other programming-language features it *doesn't* have (classes, exceptions, and so on). -When I was originally prototyping Miller in 2015, the decision I had was whether to hand-code in a low-level language like C or Rust or Go, with my own hand-rolled DSL, or whether to use a higher-level language (like Python or Lua or Nim) and let the ``put`` statements be handled by the implementation language's own ``eval``: the implementation language would take the place of a DSL. Multiple performance experiments showed me I could get better throughput using the former, by a wide margin. So Miller is Go under the hood with a hand-rolled DSL. +When I was originally prototyping Miller in 2015, the decision I had was whether to hand-code in a low-level language like C or Rust or Go, with my own hand-rolled DSL, or whether to use a higher-level language (like Python or Lua or Nim) and let the `put` statements be handled by the implementation language's own `eval`: the implementation language would take the place of a DSL. Multiple performance experiments showed me I could get better throughput using the former, by a wide margin. So Miller is Go under the hood with a hand-rolled DSL. -I do want to keep focusing on what Miller is good at -- concise notation, low latency, and high throughput -- and not add too much in terms of high-level-language features to the DSL. That said, some sort of customizability is a basic thing to want. As of 4.1.0 we have recursive for/while/if structures on about the same complexity level as ``awk``; as of 5.0.0 we have user-defined functions and map-valued variables, again on about the same complexity level as ``awk`` along with optional type-declaration syntax; as of Miller 6 we have full support for arrays. While I'm excited by these powerful language features, I hope to keep new features focused on Miller's sweet spot which is speed plus simplicity. +I do want to keep focusing on what Miller is good at -- concise notation, low latency, and high throughput -- and not add too much in terms of high-level-language features to the DSL. That said, some sort of customizability is a basic thing to want. As of 4.1.0 we have recursive for/while/if structures on about the same complexity level as `awk`; as of 5.0.0 we have user-defined functions and map-valued variables, again on about the same complexity level as `awk` along with optional type-declaration syntax; as of Miller 6 we have full support for arrays. While I'm excited by these powerful language features, I hope to keep new features focused on Miller's sweet spot which is speed plus simplicity. diff --git a/docs6b/docs/reference-dsl-control-structures.md b/docs6b/docs/reference-dsl-control-structures.md index 6a786ffe6..1f97126c2 100644 --- a/docs6b/docs/reference-dsl-control-structures.md +++ b/docs6b/docs/reference-dsl-control-structures.md @@ -3,10 +3,12 @@ ## Pattern-action blocks -These are reminiscent of ``awk`` syntax. They can be used to allow assignments to be done only when appropriate -- e.g. for math-function domain restrictions, regex-matching, and so on: +These are reminiscent of `awk` syntax. They can be used to allow assignments to be done only when appropriate -- e.g. for math-function domain restrictions, regex-matching, and so on: -
+
 mlr cat data/put-gating-example-1.dkvp
+
+
 x=-1
 x=0
 x=1
@@ -14,8 +16,10 @@ x=2
 x=3
 
-
+
 mlr put '$x > 0.0 { $y = log10($x); $z = sqrt($y) }' data/put-gating-example-1.dkvp
+
+
 x=-1
 x=0
 x=1,y=0,z=0
@@ -23,58 +27,66 @@ x=2,y=0.3010299956639812,z=0.5486620049392715
 x=3,y=0.4771212547196624,z=0.6907396432228734
 
-
+
 mlr cat data/put-gating-example-2.dkvp
+
+
 a=abc_123
 a=some other name
 a=xyz_789
 
-
+
 mlr put '
   $a =~ "([a-z]+)_([0-9]+)" {
     $b = "left_\1"; $c = "right_\2"
   }' \
   data/put-gating-example-2.dkvp
+
+
 a=abc_123,b=left_\1,c=right_\2
 a=some other name
 a=xyz_789,b=left_\1,c=right_\2
 
-This produces heteregenous output which Miller, of course, has no problems with (see [Record Heterogeneity](record-heterogeneity.md)). But if you want homogeneous output, the curly braces can be replaced with a semicolon between the expression and the body statements. This causes ``put`` to evaluate the boolean expression (along with any side effects, namely, regex-captures ``\1``, ``\2``, etc.) but doesn't use it as a criterion for whether subsequent assignments should be executed. Instead, subsequent assignments are done unconditionally: +This produces heteregenous output which Miller, of course, has no problems with (see [Record Heterogeneity](record-heterogeneity.md)). But if you want homogeneous output, the curly braces can be replaced with a semicolon between the expression and the body statements. This causes `put` to evaluate the boolean expression (along with any side effects, namely, regex-captures `\1`, `\2`, etc.) but doesn't use it as a criterion for whether subsequent assignments should be executed. Instead, subsequent assignments are done unconditionally: -
+
 mlr put '$x > 0.0; $y = log10($x); $z = sqrt($y)' data/put-gating-example-1.dkvp
+
+
 x=1,y=0,z=0
 x=2,y=0.3010299956639812,z=0.5486620049392715
 x=3,y=0.4771212547196624,z=0.6907396432228734
 
-
+
 mlr put '
   $a =~ "([a-z]+)_([0-9]+)";
   $b = "left_\1";
   $c = "right_\2"
 ' data/put-gating-example-2.dkvp
+
+
 a=abc_123,b=left_\1,c=right_\2
 a=xyz_789,b=left_\1,c=right_\2
 
## If-statements -These are again reminiscent of ``awk``. Pattern-action blocks are a special case of ``if`` with no ``elif`` or ``else`` blocks, no ``if`` keyword, and parentheses optional around the boolean expression: +These are again reminiscent of `awk`. Pattern-action blocks are a special case of `if` with no `elif` or `else` blocks, no `if` keyword, and parentheses optional around the boolean expression: -
+
 mlr put 'NR == 4 {$foo = "bar"}'
 
-
+
 mlr put 'if (NR == 4) {$foo = "bar"}'
 
-Compound statements use ``elif`` (rather than ``elsif`` or ``else if``): +Compound statements use `elif` (rather than `elsif` or `else if`): -
+
 mlr put '
   if (NR == 2) {
     ...
@@ -90,19 +102,21 @@ mlr put '
 
 ## While and do-while loops
 
-Miller's ``while`` and ``do-while`` are unsurprising in comparison to various languages, as are ``break`` and ``continue``:
+Miller's `while` and `do-while` are unsurprising in comparison to various languages, as are `break` and `continue`:
 
-
+
 echo x=1,y=2 | mlr put '
   while (NF < 10) {
     $[NF+1] = ""
   }
   $foo = "bar"
 '
+
+
 x=1,y=2,3=,4=,5=,6=,7=,8=,9=,10=,foo=bar
 
-
+
 echo x=1,y=2 | mlr put '
   do {
     $[NF+1] = "";
@@ -112,24 +126,26 @@ x=1,y=2,3=,4=,5=,6=,7=,8=,9=,10=,foo=bar
   } while (NF < 10);
   $foo = "bar"
 '
+
+
 x=1,y=2,3=,4=,5=,foo=bar
 
-A ``break`` or ``continue`` within nested conditional blocks or if-statements will, of course, propagate to the innermost loop enclosing them, if any. A ``break`` or ``continue`` outside a loop is a syntax error that will be flagged as soon as the expression is parsed, before any input records are ingested. -The existence of ``while``, ``do-while``, and ``for`` loops in Miller's DSL means that you can create infinite-loop scenarios inadvertently. In particular, please recall that DSL statements are executed once if in ``begin`` or ``end`` blocks, and once *per record* otherwise. For example, **while (NR < 10) will never terminate as NR is only incremented between records**. +A `break` or `continue` within nested conditional blocks or if-statements will, of course, propagate to the innermost loop enclosing them, if any. A `break` or `continue` outside a loop is a syntax error that will be flagged as soon as the expression is parsed, before any input records are ingested. +The existence of `while`, `do-while`, and `for` loops in Miller's DSL means that you can create infinite-loop scenarios inadvertently. In particular, please recall that DSL statements are executed once if in `begin` or `end` blocks, and once *per record* otherwise. For example, **while (NR < 10) will never terminate as NR is only incremented between records**. ## For-loops -While Miller's ``while`` and ``do-while`` statements are much as in many other languages, ``for`` loops are more idiosyncratic to Miller. They are loops over key-value pairs, whether in stream records, out-of-stream variables, local variables, or map-literals: more reminiscent of ``foreach``, as in (for example) PHP. There are **for-loops over map keys** and **for-loops over key-value tuples**. Additionally, Miller has a **C-style triple-for loop** with initialize, test, and update statements. +While Miller's `while` and `do-while` statements are much as in many other languages, `for` loops are more idiosyncratic to Miller. They are loops over key-value pairs, whether in stream records, out-of-stream variables, local variables, or map-literals: more reminiscent of `foreach`, as in (for example) PHP. There are **for-loops over map keys** and **for-loops over key-value tuples**. Additionally, Miller has a **C-style triple-for loop** with initialize, test, and update statements. -As with ``while`` and ``do-while``, a ``break`` or ``continue`` within nested control structures will propagate to the innermost loop enclosing them, if any, and a ``break`` or ``continue`` outside a loop is a syntax error that will be flagged as soon as the expression is parsed, before any input records are ingested. +As with `while` and `do-while`, a `break` or `continue` within nested control structures will propagate to the innermost loop enclosing them, if any, and a `break` or `continue` outside a loop is a syntax error that will be flagged as soon as the expression is parsed, before any input records are ingested. Key-only for-loops ................................................................ -The ``key`` variable is always bound to the *key* of key-value pairs: +The `key` variable is always bound to the *key* of key-value pairs: -
+
 mlr --from data/small put '
   print "NR = ".NR;
   for (key in $*) {
@@ -138,6 +154,8 @@ The ``key`` variable is always bound to the *key* of key-value pairs:
   }
 
 '
+
+
 NR = 1
   key:a  value:pan
   key:b  value:pan
@@ -175,7 +193,7 @@ NR = 5
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
-
+
 mlr -n put '
   end {
     o = {1:2, 3:{4:5}};
@@ -184,26 +202,30 @@ a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
     }
   }
 '
+
+
   key:1  valuetype:int
   key:3  valuetype:map
 
-Note that the value corresponding to a given key may be gotten as through a **computed field name** using square brackets as in ``$[key]`` for stream records, or by indexing the looped-over variable using square brackets. +Note that the value corresponding to a given key may be gotten as through a **computed field name** using square brackets as in `$[key]` for stream records, or by indexing the looped-over variable using square brackets. Key-value for-loops ................................................................ -Single-level keys may be gotten at using either ``for(k,v)`` or ``for((k),v)``; multi-level keys may be gotten at using ``for((k1,k2,k3),v)`` and so on. The ``v`` variable will be bound to to a scalar value (a string or a number) if the map stops at that level, or to a map-valued variable if the map goes deeper. If the map isn't deep enough then the loop body won't be executed. +Single-level keys may be gotten at using either `for(k,v)` or `for((k),v)`; multi-level keys may be gotten at using `for((k1,k2,k3),v)` and so on. The `v` variable will be bound to to a scalar value (a string or a number) if the map stops at that level, or to a map-valued variable if the map goes deeper. If the map isn't deep enough then the loop body won't be executed. -
+
 cat data/for-srec-example.tbl
+
+
 label1 label2 f1  f2  f3
 blue   green  100 240 350
 red    green  120 11  195
 yellow blue   140 0   240
 
-
+
 mlr --pprint --from data/for-srec-example.tbl put '
   $sum1 = $f1 + $f2 + $f3;
   $sum2 = 0;
@@ -215,14 +237,18 @@ yellow blue   140 0   240
     }
   }
 '
+
+
 label1 label2 f1  f2  f3  sum1 sum2 sum3
 blue   green  100 240 350 690  690  690
 red    green  120 11  195 326  326  326
 yellow blue   140 0   240 380  380  380
 
-
+
 mlr --from data/small --opprint put 'for (k,v in $*) { $[k."_type"] = typeof(v) }'
+
+
 a   b   i x                   y                   a_type b_type i_type x_type y_type
 pan pan 1 0.3467901443380824  0.7268028627434533  string string int    float  float
 eks pan 2 0.7586799647899636  0.5221511083334797  string string int    float  float
@@ -231,11 +257,11 @@ eks wye 4 0.38139939387114097 0.13418874328430463 string string int    float  fl
 wye pan 5 0.5732889198020006  0.8636244699032729  string string int    float  float
 
-Note that the value of the current field in the for-loop can be gotten either using the bound variable ``value``, or through a **computed field name** using square brackets as in ``$[key]``. +Note that the value of the current field in the for-loop can be gotten either using the bound variable `value`, or through a **computed field name** using square brackets as in `$[key]`. Important note: to avoid inconsistent looping behavior in case you're setting new fields (and/or unsetting existing ones) while looping over the record, **Miller makes a copy of the record before the loop: loop variables are bound from the copy and all other reads/writes involve the record itself**: -
+
 mlr --from data/small --opprint put '
   $sum1 = 0;
   $sum2 = 0;
@@ -246,6 +272,8 @@ Important note: to avoid inconsistent looping behavior in case you're setting ne
     }
   }
 '
+
+
 a   b   i x                   y                   sum1               sum2
 pan pan 1 0.3467901443380824  0.7268028627434533  2.0735930070815356 8.294372028326142
 eks pan 2 0.7586799647899636  0.5221511083334797  3.280831073123443  13.123324292493772
@@ -256,7 +284,7 @@ wye pan 5 0.5732889198020006  0.8636244699032729  6.436913389705273  25.74765355
 
 It can be confusing to modify the stream record while iterating over a copy of it, so instead you might find it simpler to use a local variable in the loop and only update the stream record after the loop:
 
-
+
 mlr --from data/small --opprint put '
   sum = 0;
   for (k,v in $*) {
@@ -266,6 +294,8 @@ It can be confusing to modify the stream record while iterating over a copy of i
   }
   $sum = sum
 '
+
+
 a   b   i x                   y                   sum
 pan pan 1 0.3467901443380824  0.7268028627434533  2.0735930070815356
 eks pan 2 0.7586799647899636  0.5221511083334797  3.280831073123443
@@ -274,9 +304,9 @@ eks wye 4 0.38139939387114097 0.13418874328430463 4.515588137155445
 wye pan 5 0.5732889198020006  0.8636244699032729  6.436913389705273
 
-You can also start iterating on sub-hashmaps of an out-of-stream or local variable; you can loop over nested keys; you can loop over all out-of-stream variables. The bound variables are bound to a copy of the sub-hashmap as it was before the loop started. The sub-hashmap is specified by square-bracketed indices after ``in``, and additional deeper indices are bound to loop key-variables. The terminal values are bound to the loop value-variable whenever the keys are not too shallow. The value-variable may refer to a terminal (string, number) or it may be map-valued if the map goes deeper. Example indexing is as follows: +You can also start iterating on sub-hashmaps of an out-of-stream or local variable; you can loop over nested keys; you can loop over all out-of-stream variables. The bound variables are bound to a copy of the sub-hashmap as it was before the loop started. The sub-hashmap is specified by square-bracketed indices after `in`, and additional deeper indices are bound to loop key-variables. The terminal values are bound to the loop value-variable whenever the keys are not too shallow. The value-variable may refer to a terminal (string, number) or it may be map-valued if the map goes deeper. Example indexing is as follows: -
+
 # Parentheses are optional for single key:
 for (k1,           v in @a["b"]["c"]) { ... }
 for ((k1),         v in @a["b"]["c"]) { ... }
@@ -287,9 +317,9 @@ for ((k1, k2, k3), v in @a { ... }            # Loop over variable starting from
 for ((k1, k2, k3), v in @* { ... }            # Loop over all variables (k1 is bound to basename)
 
-That's confusing in the abstract, so a concrete example is in order. Suppose the out-of-stream variable ``@myvar`` is populated as follows: +That's confusing in the abstract, so a concrete example is in order. Suppose the out-of-stream variable `@myvar` is populated as follows: -
+
 mlr -n put --jknquoteint -q '
   begin {
     @myvar = {
@@ -300,6 +330,8 @@ That's confusing in the abstract, so a concrete example is in order. Suppose the
   }
   end { dump }
 '
+
+
 {
   "myvar": {
     "1": 2,
@@ -317,7 +349,7 @@ That's confusing in the abstract, so a concrete example is in order. Suppose the
 
 Then we can get at various values as follows:
 
-
+
 mlr -n put --jknquoteint -q '
   begin {
     @myvar = {
@@ -334,12 +366,14 @@ Then we can get at various values as follows:
     }
   }
 '
+
+
 key=1,valuetype=int
 key=3,valuetype=map
 key=6,valuetype=map
 
-
+
 mlr -n put --jknquoteint -q '
   begin {
     @myvar = {
@@ -357,11 +391,13 @@ key=6,valuetype=map
     }
   }
 '
+
+
 key1=3,key2=4,valuetype=int
 key1=6,key2=7,valuetype=map
 
-
+
 mlr -n put --jknquoteint -q '
   begin {
     @myvar = {
@@ -379,6 +415,8 @@ key1=6,key2=7,valuetype=map
     }
   }
 '
+
+
 key1=7,key2=8,valuetype=int
 
@@ -387,7 +425,7 @@ C-style triple-for loops These are supported as follows: -
+
 mlr --from data/small --opprint put '
   num suma = 0;
   for (a = 1; a <= NR; a += 1) {
@@ -395,6 +433,8 @@ These are supported as follows:
   }
   $suma = suma;
 '
+
+
 a   b   i x                   y                   suma
 pan pan 1 0.3467901443380824  0.7268028627434533  1
 eks pan 2 0.7586799647899636  0.5221511083334797  3
@@ -403,7 +443,7 @@ eks wye 4 0.38139939387114097 0.13418874328430463 10
 wye pan 5 0.5732889198020006  0.8636244699032729  15
 
-
+
 mlr --from data/small --opprint put '
   num suma = 0;
   num sumb = 0;
@@ -414,6 +454,8 @@ wye pan 5 0.5732889198020006  0.8636244699032729  15
   $suma = suma;
   $sumb = sumb;
 '
+
+
 a   b   i x                   y                   suma sumb
 pan pan 1 0.3467901443380824  0.7268028627434533  1    1
 eks pan 2 0.7586799647899636  0.5221511083334797  3    3
@@ -424,26 +466,28 @@ wye pan 5 0.5732889198020006  0.8636244699032729  15   31
 
 Notes:
 
-* In ``for (start; continuation; update) { body }``, the start, continuation, and update statements may be empty, single statements, or multiple comma-separated statements. If the continuation is empty (e.g. ``for(i=1;;i+=1)``) it defaults to true.
+* In `for (start; continuation; update) { body }`, the start, continuation, and update statements may be empty, single statements, or multiple comma-separated statements. If the continuation is empty (e.g. `for(i=1;;i+=1)`) it defaults to true.
 
-* In particular, you may use ``$``-variables and/or ``@``-variables in the start, continuation, and/or update steps (as well as the body, of course).
+* In particular, you may use `$`-variables and/or `@`-variables in the start, continuation, and/or update steps (as well as the body, of course).
 
-* The typedecls such as ``int`` or ``num`` are optional.  If a typedecl is provided (for a local variable), it binds a variable scoped to the for-loop regardless of whether a same-name variable is present in outer scope. If a typedecl is not provided, then the variable is scoped to the for-loop if no same-name variable is present in outer scope, or if a same-name variable is present in outer scope then it is modified.
+* The typedecls such as `int` or `num` are optional.  If a typedecl is provided (for a local variable), it binds a variable scoped to the for-loop regardless of whether a same-name variable is present in outer scope. If a typedecl is not provided, then the variable is scoped to the for-loop if no same-name variable is present in outer scope, or if a same-name variable is present in outer scope then it is modified.
 
-* Miller has no ``++`` or ``--`` operators.
+* Miller has no `++` or `--` operators.
 
 * As with all for/if/while statements in Miller, the curly braces are required even if the body is a single statement, or empty.
 
 ## Begin/end blocks
 
-Miller supports an ``awk``-like ``begin/end`` syntax.  The statements in the ``begin`` block are executed before any input records are read; the statements in the ``end`` block are executed after the last input record is read.  (If you want to execute some statement at the start of each file, not at the start of the first file as with ``begin``, you might use a pattern/action block of the form ``FNR == 1 { ... }``.) All statements outside of ``begin`` or ``end`` are, of course, executed on every input record. Semicolons separate statements inside or outside of begin/end blocks; semicolons are required between begin/end block bodies and any subsequent statement.  For example:
+Miller supports an `awk`-like `begin/end` syntax.  The statements in the `begin` block are executed before any input records are read; the statements in the `end` block are executed after the last input record is read.  (If you want to execute some statement at the start of each file, not at the start of the first file as with `begin`, you might use a pattern/action block of the form `FNR == 1 { ... }`.) All statements outside of `begin` or `end` are, of course, executed on every input record. Semicolons separate statements inside or outside of begin/end blocks; semicolons are required between begin/end block bodies and any subsequent statement.  For example:
 
-
+
 mlr put '
   begin { @sum = 0 };
   @x_sum += $x;
   end { emit @x_sum }
 ' ./data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -452,13 +496,15 @@ a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 x_sum=2.264761728567491
 
-Since uninitialized out-of-stream variables default to 0 for addition/substraction and 1 for multiplication when they appear on expression right-hand sides (not quite as in ``awk``, where they'd default to 0 either way), the above can be written more succinctly as +Since uninitialized out-of-stream variables default to 0 for addition/substraction and 1 for multiplication when they appear on expression right-hand sides (not quite as in `awk`, where they'd default to 0 either way), the above can be written more succinctly as -
+
 mlr put '
   @x_sum += $x;
   end { emit @x_sum }
 ' ./data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -467,19 +513,21 @@ a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 x_sum=2.264761728567491
 
-The **put -q** option is a shorthand which suppresses printing of each output record, with only ``emit`` statements being output. So to get only summary outputs, one could write +The **put -q** option is a shorthand which suppresses printing of each output record, with only `emit` statements being output. So to get only summary outputs, one could write -
+
 mlr put -q '
   @x_sum += $x;
   end { emit @x_sum }
 ' ./data/small
+
+
 x_sum=2.264761728567491
 
We can do similarly with multiple out-of-stream variables: -
+
 mlr put -q '
   @x_count += 1;
   @x_sum += $x;
@@ -488,16 +536,20 @@ We can do similarly with multiple out-of-stream variables:
     emit @x_sum;
   }
 ' ./data/small
+
+
 x_count=5
 x_sum=2.264761728567491
 
This is of course not much different than -
+
 mlr stats1 -a count,sum -f x ./data/small
+
+
 x_count=5,x_sum=2.264761728567491
 
-Note that it's a syntax error for begin/end blocks to refer to field names (beginning with ``$``), since these execute outside the context of input records. +Note that it's a syntax error for begin/end blocks to refer to field names (beginning with `$`), since these execute outside the context of input records. diff --git a/docs6b/docs/reference-dsl-control-structures.md.in b/docs6b/docs/reference-dsl-control-structures.md.in index cc28e0209..cd9d29a03 100644 --- a/docs6b/docs/reference-dsl-control-structures.md.in +++ b/docs6b/docs/reference-dsl-control-structures.md.in @@ -2,7 +2,7 @@ ## Pattern-action blocks -These are reminiscent of ``awk`` syntax. They can be used to allow assignments to be done only when appropriate -- e.g. for math-function domain restrictions, regex-matching, and so on: +These are reminiscent of `awk` syntax. They can be used to allow assignments to be done only when appropriate -- e.g. for math-function domain restrictions, regex-matching, and so on: GENMD_RUN_COMMAND mlr cat data/put-gating-example-1.dkvp @@ -24,7 +24,7 @@ mlr put ' data/put-gating-example-2.dkvp GENMD_EOF -This produces heteregenous output which Miller, of course, has no problems with (see [Record Heterogeneity](record-heterogeneity.md)). But if you want homogeneous output, the curly braces can be replaced with a semicolon between the expression and the body statements. This causes ``put`` to evaluate the boolean expression (along with any side effects, namely, regex-captures ``\1``, ``\2``, etc.) but doesn't use it as a criterion for whether subsequent assignments should be executed. Instead, subsequent assignments are done unconditionally: +This produces heteregenous output which Miller, of course, has no problems with (see [Record Heterogeneity](record-heterogeneity.md)). But if you want homogeneous output, the curly braces can be replaced with a semicolon between the expression and the body statements. This causes `put` to evaluate the boolean expression (along with any side effects, namely, regex-captures `\1`, `\2`, etc.) but doesn't use it as a criterion for whether subsequent assignments should be executed. Instead, subsequent assignments are done unconditionally: GENMD_RUN_COMMAND mlr put '$x > 0.0; $y = log10($x); $z = sqrt($y)' data/put-gating-example-1.dkvp @@ -40,7 +40,7 @@ GENMD_EOF ## If-statements -These are again reminiscent of ``awk``. Pattern-action blocks are a special case of ``if`` with no ``elif`` or ``else`` blocks, no ``if`` keyword, and parentheses optional around the boolean expression: +These are again reminiscent of `awk`. Pattern-action blocks are a special case of `if` with no `elif` or `else` blocks, no `if` keyword, and parentheses optional around the boolean expression: GENMD_SHOW_COMMAND mlr put 'NR == 4 {$foo = "bar"}' @@ -50,42 +50,40 @@ GENMD_SHOW_COMMAND mlr put 'if (NR == 4) {$foo = "bar"}' GENMD_EOF -Compound statements use ``elif`` (rather than ``elsif`` or ``else if``): +Compound statements use `elif` (rather than `elsif` or `else if`): GENMD_INCLUDE_ESCAPED(data/if-chain.sh) ## While and do-while loops -Miller's ``while`` and ``do-while`` are unsurprising in comparison to various languages, as are ``break`` and ``continue``: +Miller's `while` and `do-while` are unsurprising in comparison to various languages, as are `break` and `continue`: GENMD_INCLUDE_AND_RUN_ESCAPED(data/while-example-1.sh) GENMD_INCLUDE_AND_RUN_ESCAPED(data/while-example-2.sh) -A ``break`` or ``continue`` within nested conditional blocks or if-statements will, of course, propagate to the innermost loop enclosing them, if any. A ``break`` or ``continue`` outside a loop is a syntax error that will be flagged as soon as the expression is parsed, before any input records are ingested. -The existence of ``while``, ``do-while``, and ``for`` loops in Miller's DSL means that you can create infinite-loop scenarios inadvertently. In particular, please recall that DSL statements are executed once if in ``begin`` or ``end`` blocks, and once *per record* otherwise. For example, **while (NR < 10) will never terminate as NR is only incremented between records**. +A `break` or `continue` within nested conditional blocks or if-statements will, of course, propagate to the innermost loop enclosing them, if any. A `break` or `continue` outside a loop is a syntax error that will be flagged as soon as the expression is parsed, before any input records are ingested. +The existence of `while`, `do-while`, and `for` loops in Miller's DSL means that you can create infinite-loop scenarios inadvertently. In particular, please recall that DSL statements are executed once if in `begin` or `end` blocks, and once *per record* otherwise. For example, **while (NR < 10) will never terminate as NR is only incremented between records**. ## For-loops -While Miller's ``while`` and ``do-while`` statements are much as in many other languages, ``for`` loops are more idiosyncratic to Miller. They are loops over key-value pairs, whether in stream records, out-of-stream variables, local variables, or map-literals: more reminiscent of ``foreach``, as in (for example) PHP. There are **for-loops over map keys** and **for-loops over key-value tuples**. Additionally, Miller has a **C-style triple-for loop** with initialize, test, and update statements. +While Miller's `while` and `do-while` statements are much as in many other languages, `for` loops are more idiosyncratic to Miller. They are loops over key-value pairs, whether in stream records, out-of-stream variables, local variables, or map-literals: more reminiscent of `foreach`, as in (for example) PHP. There are **for-loops over map keys** and **for-loops over key-value tuples**. Additionally, Miller has a **C-style triple-for loop** with initialize, test, and update statements. -As with ``while`` and ``do-while``, a ``break`` or ``continue`` within nested control structures will propagate to the innermost loop enclosing them, if any, and a ``break`` or ``continue`` outside a loop is a syntax error that will be flagged as soon as the expression is parsed, before any input records are ingested. +As with `while` and `do-while`, a `break` or `continue` within nested control structures will propagate to the innermost loop enclosing them, if any, and a `break` or `continue` outside a loop is a syntax error that will be flagged as soon as the expression is parsed, before any input records are ingested. -Key-only for-loops -................................................................ +### Key-only for-loops -The ``key`` variable is always bound to the *key* of key-value pairs: +The `key` variable is always bound to the *key* of key-value pairs: GENMD_INCLUDE_AND_RUN_ESCAPED(data/single-for-example-1.sh) GENMD_INCLUDE_AND_RUN_ESCAPED(data/single-for-example-2.sh) -Note that the value corresponding to a given key may be gotten as through a **computed field name** using square brackets as in ``$[key]`` for stream records, or by indexing the looped-over variable using square brackets. +Note that the value corresponding to a given key may be gotten as through a **computed field name** using square brackets as in `$[key]` for stream records, or by indexing the looped-over variable using square brackets. -Key-value for-loops -................................................................ +### Key-value for-loops -Single-level keys may be gotten at using either ``for(k,v)`` or ``for((k),v)``; multi-level keys may be gotten at using ``for((k1,k2,k3),v)`` and so on. The ``v`` variable will be bound to to a scalar value (a string or a number) if the map stops at that level, or to a map-valued variable if the map goes deeper. If the map isn't deep enough then the loop body won't be executed. +Single-level keys may be gotten at using either `for(k,v)` or `for((k),v)`; multi-level keys may be gotten at using `for((k1,k2,k3),v)` and so on. The `v` variable will be bound to to a scalar value (a string or a number) if the map stops at that level, or to a map-valued variable if the map goes deeper. If the map isn't deep enough then the loop body won't be executed. GENMD_RUN_COMMAND cat data/for-srec-example.tbl @@ -97,7 +95,7 @@ GENMD_RUN_COMMAND mlr --from data/small --opprint put 'for (k,v in $*) { $[k."_type"] = typeof(v) }' GENMD_EOF -Note that the value of the current field in the for-loop can be gotten either using the bound variable ``value``, or through a **computed field name** using square brackets as in ``$[key]``. +Note that the value of the current field in the for-loop can be gotten either using the bound variable `value`, or through a **computed field name** using square brackets as in `$[key]`. Important note: to avoid inconsistent looping behavior in case you're setting new fields (and/or unsetting existing ones) while looping over the record, **Miller makes a copy of the record before the loop: loop variables are bound from the copy and all other reads/writes involve the record itself**: @@ -107,11 +105,11 @@ It can be confusing to modify the stream record while iterating over a copy of i GENMD_INCLUDE_AND_RUN_ESCAPED(data/for-srec-example-3.sh) -You can also start iterating on sub-hashmaps of an out-of-stream or local variable; you can loop over nested keys; you can loop over all out-of-stream variables. The bound variables are bound to a copy of the sub-hashmap as it was before the loop started. The sub-hashmap is specified by square-bracketed indices after ``in``, and additional deeper indices are bound to loop key-variables. The terminal values are bound to the loop value-variable whenever the keys are not too shallow. The value-variable may refer to a terminal (string, number) or it may be map-valued if the map goes deeper. Example indexing is as follows: +You can also start iterating on sub-hashmaps of an out-of-stream or local variable; you can loop over nested keys; you can loop over all out-of-stream variables. The bound variables are bound to a copy of the sub-hashmap as it was before the loop started. The sub-hashmap is specified by square-bracketed indices after `in`, and additional deeper indices are bound to loop key-variables. The terminal values are bound to the loop value-variable whenever the keys are not too shallow. The value-variable may refer to a terminal (string, number) or it may be map-valued if the map goes deeper. Example indexing is as follows: GENMD_INCLUDE_ESCAPED(data/for-oosvar-example-0a.txt) -That's confusing in the abstract, so a concrete example is in order. Suppose the out-of-stream variable ``@myvar`` is populated as follows: +That's confusing in the abstract, so a concrete example is in order. Suppose the out-of-stream variable `@myvar` is populated as follows: GENMD_INCLUDE_AND_RUN_ESCAPED(data/for-oosvar-example-0b.sh) @@ -123,8 +121,7 @@ GENMD_INCLUDE_AND_RUN_ESCAPED(data/for-oosvar-example-0d.sh) GENMD_INCLUDE_AND_RUN_ESCAPED(data/for-oosvar-example-0e.sh) -C-style triple-for loops -................................................................ +### C-style triple-for loops These are supported as follows: @@ -134,27 +131,27 @@ GENMD_INCLUDE_AND_RUN_ESCAPED(data/triple-for-example-2.sh) Notes: -* In ``for (start; continuation; update) { body }``, the start, continuation, and update statements may be empty, single statements, or multiple comma-separated statements. If the continuation is empty (e.g. ``for(i=1;;i+=1)``) it defaults to true. +* In `for (start; continuation; update) { body }`, the start, continuation, and update statements may be empty, single statements, or multiple comma-separated statements. If the continuation is empty (e.g. `for(i=1;;i+=1)`) it defaults to true. -* In particular, you may use ``$``-variables and/or ``@``-variables in the start, continuation, and/or update steps (as well as the body, of course). +* In particular, you may use `$`-variables and/or `@`-variables in the start, continuation, and/or update steps (as well as the body, of course). -* The typedecls such as ``int`` or ``num`` are optional. If a typedecl is provided (for a local variable), it binds a variable scoped to the for-loop regardless of whether a same-name variable is present in outer scope. If a typedecl is not provided, then the variable is scoped to the for-loop if no same-name variable is present in outer scope, or if a same-name variable is present in outer scope then it is modified. +* The typedecls such as `int` or `num` are optional. If a typedecl is provided (for a local variable), it binds a variable scoped to the for-loop regardless of whether a same-name variable is present in outer scope. If a typedecl is not provided, then the variable is scoped to the for-loop if no same-name variable is present in outer scope, or if a same-name variable is present in outer scope then it is modified. -* Miller has no ``++`` or ``--`` operators. +* Miller has no `++` or `--` operators. * As with all for/if/while statements in Miller, the curly braces are required even if the body is a single statement, or empty. ## Begin/end blocks -Miller supports an ``awk``-like ``begin/end`` syntax. The statements in the ``begin`` block are executed before any input records are read; the statements in the ``end`` block are executed after the last input record is read. (If you want to execute some statement at the start of each file, not at the start of the first file as with ``begin``, you might use a pattern/action block of the form ``FNR == 1 { ... }``.) All statements outside of ``begin`` or ``end`` are, of course, executed on every input record. Semicolons separate statements inside or outside of begin/end blocks; semicolons are required between begin/end block bodies and any subsequent statement. For example: +Miller supports an `awk`-like `begin/end` syntax. The statements in the `begin` block are executed before any input records are read; the statements in the `end` block are executed after the last input record is read. (If you want to execute some statement at the start of each file, not at the start of the first file as with `begin`, you might use a pattern/action block of the form `FNR == 1 { ... }`.) All statements outside of `begin` or `end` are, of course, executed on every input record. Semicolons separate statements inside or outside of begin/end blocks; semicolons are required between begin/end block bodies and any subsequent statement. For example: GENMD_INCLUDE_AND_RUN_ESCAPED(data/begin-end-example-1.sh) -Since uninitialized out-of-stream variables default to 0 for addition/substraction and 1 for multiplication when they appear on expression right-hand sides (not quite as in ``awk``, where they'd default to 0 either way), the above can be written more succinctly as +Since uninitialized out-of-stream variables default to 0 for addition/substraction and 1 for multiplication when they appear on expression right-hand sides (not quite as in `awk`, where they'd default to 0 either way), the above can be written more succinctly as GENMD_INCLUDE_AND_RUN_ESCAPED(data/begin-end-example-2.sh) -The **put -q** option is a shorthand which suppresses printing of each output record, with only ``emit`` statements being output. So to get only summary outputs, one could write +The **put -q** option is a shorthand which suppresses printing of each output record, with only `emit` statements being output. So to get only summary outputs, one could write GENMD_INCLUDE_AND_RUN_ESCAPED(data/begin-end-example-3.sh) @@ -166,5 +163,5 @@ This is of course not much different than GENMD_INCLUDE_AND_RUN_ESCAPED(data/begin-end-example-5.sh) -Note that it's a syntax error for begin/end blocks to refer to field names (beginning with ``$``), since these execute outside the context of input records. +Note that it's a syntax error for begin/end blocks to refer to field names (beginning with `$`), since these execute outside the context of input records. diff --git a/docs6b/docs/reference-dsl-errors.md b/docs6b/docs/reference-dsl-errors.md index 28603bcfe..19012b628 100644 --- a/docs6b/docs/reference-dsl-errors.md +++ b/docs6b/docs/reference-dsl-errors.md @@ -3,22 +3,22 @@ As soon as you have a programming language, you start having the problem *What is my code doing, and why?* This includes getting syntax errors -- which are always annoying -- as well as the even more annoying problem of a program which parses without syntax error but doesn't do what you expect. -The ``syntax error`` message is cryptic: it says ``syntax error at `` followed by the next symbol it couldn't parse. This is good, but (as of 5.0.0) it doesn't say things like ``syntax error at line 17, character 22``. Here are some common causes of syntax errors: +The `syntax error` message is cryptic: it says `syntax error at ` followed by the next symbol it couldn't parse. This is good, but (as of 5.0.0) it doesn't say things like `syntax error at line 17, character 22`. Here are some common causes of syntax errors: -* Don't forget ``;`` at end of line, before another statement on the next line. +* Don't forget `;` at end of line, before another statement on the next line. -* Miller's DSL lacks the ``++`` and ``--`` operators. +* Miller's DSL lacks the `++` and `--` operators. -* Curly braces are required for the bodies of ``if``/``while``/``for`` blocks, even when the body is a single statement. +* Curly braces are required for the bodies of `if`/`while`/`for` blocks, even when the body is a single statement. -Now for transparency: +As for transparency: -* As in any language, you can do (see :ref:`reference-dsl-print-statements`) ``print`` (or ``eprint`` to print to stderr). See also :ref:`reference-dsl-dump-statements` and :ref:`reference-dsl-emit-statements`. +* As in any language, you can do `print`, or `eprint` to print to stderr. See [Print statements](reference-dsl-output-statements.md#print-statements); see also [Dump statements](reference-dsl-output-statements.md#dump-statements) and [Emit statements](reference-dsl-output-statements.md#emit-statements). -* The ``-v`` option to ``mlr put`` and ``mlr filter`` prints abstract syntax trees for your code. While not all details here will be of interest to everyone, certainly this makes questions such as operator precedence completely unambiguous. +* The `-v` option to `mlr put` and `mlr filter` prints abstract syntax trees for your code. While not all details here will be of interest to everyone, certainly this makes questions such as operator precedence completely unambiguous. -* The ``-T`` option prints a trace of each statement executed. +* The `-T` option prints a trace of each statement executed. -* The ``-t`` and ``-a`` options show low-level details for the parsing process and for stack-variable-index allocation, respectively. These will likely be of interest to people who enjoy compilers, and probably less useful for a more general audience. +* The `-t` and `-a` options show low-level details for the parsing process and for stack-variable-index allocation, respectively. These will likely be of interest to people who enjoy compilers, and probably less useful for a more general audience. -* Please see :ref:`reference-dsl-type-checking` for type declarations and type-assertions you can use to make sure expressions and the data flowing them are evaluating as you expect. I made them optional because one of Miller's important use-cases is being able to say simple things like ``mlr put '$y = $x + 1' myfile.dat`` with a minimum of punctuational bric-a-brac -- but for programs over a few lines I generally find that the more type-specification, the better. +* Please see [type-checking](reference-dsl-variables.md#type-checking) for type declarations and type-assertions you can use to make sure expressions and the data flowing them are evaluating as you expect. I made them optional because one of Miller's important use-cases is being able to say simple things like `mlr put '$y = $x + 1' myfile.dat` with a minimum of punctuational bric-a-brac -- but for programs over a few lines I generally find that the more type-specification, the better. diff --git a/docs6b/docs/reference-dsl-errors.md.in b/docs6b/docs/reference-dsl-errors.md.in index 70976a801..91994fa94 100644 --- a/docs6b/docs/reference-dsl-errors.md.in +++ b/docs6b/docs/reference-dsl-errors.md.in @@ -2,22 +2,22 @@ As soon as you have a programming language, you start having the problem *What is my code doing, and why?* This includes getting syntax errors -- which are always annoying -- as well as the even more annoying problem of a program which parses without syntax error but doesn't do what you expect. -The ``syntax error`` message is cryptic: it says ``syntax error at `` followed by the next symbol it couldn't parse. This is good, but (as of 5.0.0) it doesn't say things like ``syntax error at line 17, character 22``. Here are some common causes of syntax errors: +The `syntax error` message is cryptic: it says `syntax error at ` followed by the next symbol it couldn't parse. This is good, but (as of 5.0.0) it doesn't say things like `syntax error at line 17, character 22`. Here are some common causes of syntax errors: -* Don't forget ``;`` at end of line, before another statement on the next line. +* Don't forget `;` at end of line, before another statement on the next line. -* Miller's DSL lacks the ``++`` and ``--`` operators. +* Miller's DSL lacks the `++` and `--` operators. -* Curly braces are required for the bodies of ``if``/``while``/``for`` blocks, even when the body is a single statement. +* Curly braces are required for the bodies of `if`/`while`/`for` blocks, even when the body is a single statement. -Now for transparency: +As for transparency: -* As in any language, you can do (see :ref:`reference-dsl-print-statements`) ``print`` (or ``eprint`` to print to stderr). See also :ref:`reference-dsl-dump-statements` and :ref:`reference-dsl-emit-statements`. +* As in any language, you can do `print`, or `eprint` to print to stderr. See [Print statements](reference-dsl-output-statements.md#print-statements); see also [Dump statements](reference-dsl-output-statements.md#dump-statements) and [Emit statements](reference-dsl-output-statements.md#emit-statements). -* The ``-v`` option to ``mlr put`` and ``mlr filter`` prints abstract syntax trees for your code. While not all details here will be of interest to everyone, certainly this makes questions such as operator precedence completely unambiguous. +* The `-v` option to `mlr put` and `mlr filter` prints abstract syntax trees for your code. While not all details here will be of interest to everyone, certainly this makes questions such as operator precedence completely unambiguous. -* The ``-T`` option prints a trace of each statement executed. +* The `-T` option prints a trace of each statement executed. -* The ``-t`` and ``-a`` options show low-level details for the parsing process and for stack-variable-index allocation, respectively. These will likely be of interest to people who enjoy compilers, and probably less useful for a more general audience. +* The `-t` and `-a` options show low-level details for the parsing process and for stack-variable-index allocation, respectively. These will likely be of interest to people who enjoy compilers, and probably less useful for a more general audience. -* Please see :ref:`reference-dsl-type-checking` for type declarations and type-assertions you can use to make sure expressions and the data flowing them are evaluating as you expect. I made them optional because one of Miller's important use-cases is being able to say simple things like ``mlr put '$y = $x + 1' myfile.dat`` with a minimum of punctuational bric-a-brac -- but for programs over a few lines I generally find that the more type-specification, the better. +* Please see [type-checking](reference-dsl-variables.md#type-checking) for type declarations and type-assertions you can use to make sure expressions and the data flowing them are evaluating as you expect. I made them optional because one of Miller's important use-cases is being able to say simple things like `mlr put '$y = $x + 1' myfile.dat` with a minimum of punctuational bric-a-brac -- but for programs over a few lines I generally find that the more type-specification, the better. diff --git a/docs6b/docs/reference-dsl-filter-statements.md b/docs6b/docs/reference-dsl-filter-statements.md index db61cb877..bbeb18539 100644 --- a/docs6b/docs/reference-dsl-filter-statements.md +++ b/docs6b/docs/reference-dsl-filter-statements.md @@ -1,31 +1,39 @@ # DSL reference: filter statements -You can use ``filter`` within ``put``. In fact, the following two are synonymous: +You can use `filter` within `put`. In fact, the following two are synonymous: -
+
 mlr filter 'NR==2 || NR==3' data/small
+
+
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
 
-
+
 mlr put 'filter NR==2 || NR==3' data/small
+
+
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
 
The former, of course, is much easier to type. But the latter allows you to define more complex expressions for the filter, and/or do other things in addition to the filter: -
+
 mlr put '@running_sum += $x; filter @running_sum > 1.3' data/small
+
+
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
 a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
-
+
 mlr put '$z = $x * $y; filter $z > 0.3' data/small
+
+
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,z=0.3961455844854848
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729,z=0.4951063394654227
 
diff --git a/docs6b/docs/reference-dsl-filter-statements.md.in b/docs6b/docs/reference-dsl-filter-statements.md.in index fa3dab546..10e387546 100644 --- a/docs6b/docs/reference-dsl-filter-statements.md.in +++ b/docs6b/docs/reference-dsl-filter-statements.md.in @@ -1,6 +1,6 @@ # DSL reference: filter statements -You can use ``filter`` within ``put``. In fact, the following two are synonymous: +You can use `filter` within `put`. In fact, the following two are synonymous: GENMD_RUN_COMMAND mlr filter 'NR==2 || NR==3' data/small diff --git a/docs6b/docs/reference-dsl-operators.md b/docs6b/docs/reference-dsl-operators.md index ebb777c6c..a3db993bc 100644 --- a/docs6b/docs/reference-dsl-operators.md +++ b/docs6b/docs/reference-dsl-operators.md @@ -5,7 +5,7 @@ Operators are listed in order of decreasing precedence, highest first. -
+
 Operators              Associativity
 ---------              -------------
 ()                     left to right
@@ -30,11 +30,11 @@ binary+ binary- .      left to right
 
 * Functions are often pass-throughs straight to the system-standard Go libraries.
 
-* The ``min`` and ``max`` functions are different from other multi-argument functions which return null if any of their inputs are null: for ``min`` and ``max``, by contrast, if one argument is absent-null, the other is returned. Empty-null loses min or max against numeric or boolean; empty-null is less than any other string.
+* The `min` and `max` functions are different from other multi-argument functions which return null if any of their inputs are null: for `min` and `max`, by contrast, if one argument is absent-null, the other is returned. Empty-null loses min or max against numeric or boolean; empty-null is less than any other string.
 
-* Symmetrically with respect to the bitwise OR, XOR, and AND operators ``|``, ``^``, ``&``, Miller has logical operators ``||``, ``^^``, ``&&``: the logical XOR not existing in Go.
+* Symmetrically with respect to the bitwise OR, XOR, and AND operators `|`, `^`, `&`, Miller has logical operators `||`, `^^`, `&&`: the logical XOR not existing in Go.
 
-* The exponentiation operator ``**`` is familiar from many languages.
+* The exponentiation operator `**` is familiar from many languages.
 
-* The regex-match and regex-not-match operators ``=~`` and ``!=~`` are similar to those in Ruby and Perl.
+* The regex-match and regex-not-match operators `=~` and `!=~` are similar to those in Ruby and Perl.
 
diff --git a/docs6b/docs/reference-dsl-operators.md.in b/docs6b/docs/reference-dsl-operators.md.in
index f7bf25990..a7761b0b1 100644
--- a/docs6b/docs/reference-dsl-operators.md.in
+++ b/docs6b/docs/reference-dsl-operators.md.in
@@ -29,11 +29,11 @@ GENMD_EOF
 
 * Functions are often pass-throughs straight to the system-standard Go libraries.
 
-* The ``min`` and ``max`` functions are different from other multi-argument functions which return null if any of their inputs are null: for ``min`` and ``max``, by contrast, if one argument is absent-null, the other is returned. Empty-null loses min or max against numeric or boolean; empty-null is less than any other string.
+* The `min` and `max` functions are different from other multi-argument functions which return null if any of their inputs are null: for `min` and `max`, by contrast, if one argument is absent-null, the other is returned. Empty-null loses min or max against numeric or boolean; empty-null is less than any other string.
 
-* Symmetrically with respect to the bitwise OR, XOR, and AND operators ``|``, ``^``, ``&``, Miller has logical operators ``||``, ``^^``, ``&&``: the logical XOR not existing in Go.
+* Symmetrically with respect to the bitwise OR, XOR, and AND operators `|`, `^`, `&`, Miller has logical operators `||`, `^^`, `&&`: the logical XOR not existing in Go.
 
-* The exponentiation operator ``**`` is familiar from many languages.
+* The exponentiation operator `**` is familiar from many languages.
 
-* The regex-match and regex-not-match operators ``=~`` and ``!=~`` are similar to those in Ruby and Perl.
+* The regex-match and regex-not-match operators `=~` and `!=~` are similar to those in Ruby and Perl.
 
diff --git a/docs6b/docs/reference-dsl-output-statements.md b/docs6b/docs/reference-dsl-output-statements.md
index 842a9247a..4eff2ca11 100644
--- a/docs6b/docs/reference-dsl-output-statements.md
+++ b/docs6b/docs/reference-dsl-output-statements.md
@@ -3,63 +3,57 @@
 
 You can **output** variable-values or expressions in **five ways**:
 
-* **Assign** them to stream-record fields. For example, ``$cumulative_sum = @sum``. For another example, ``$nr = NR`` adds a field named ``nr`` to each output record, containing the value of the built-in variable ``NR`` as of when that record was ingested.
+* **Assign** them to stream-record fields. For example, `$cumulative_sum = @sum`. For another example, `$nr = NR` adds a field named `nr` to each output record, containing the value of the built-in variable `NR` as of when that record was ingested.
 
-* Use the **print** or **eprint** keywords which immediately print an expression *directly to standard output or standard error*, respectively. Note that ``dump``, ``edump``, ``print``, and ``eprint`` don't output records which participate in ``then``-chaining; rather, they're just immediate prints to stdout/stderr. The ``printn`` and ``eprintn`` keywords are the same except that they don't print final newlines. Additionally, you can print to a specified file instead of stdout/stderr.
+* Use the **print** or **eprint** keywords which immediately print an expression *directly to standard output or standard error*, respectively. Note that `dump`, `edump`, `print`, and `eprint` don't output records which participate in `then`-chaining; rather, they're just immediate prints to stdout/stderr. The `printn` and `eprintn` keywords are the same except that they don't print final newlines. Additionally, you can print to a specified file instead of stdout/stderr.
 
 * Use the **dump** or **edump** keywords, which *immediately print all out-of-stream variables as a JSON data structure to the standard output or standard error* (respectively).
 
 * Use **tee** which formats the current stream record (not just an arbitrary string as with **print**) to a specific file.
 
-* Use **emit**/**emitp**/**emitf** to send out-of-stream variables' current values to the output record stream, e.g.  ``@sum += $x; emit @sum`` which produces an extra output record such as ``sum=3.1648382``.
+* Use **emit**/**emitp**/**emitf** to send out-of-stream variables' current values to the output record stream, e.g.  `@sum += $x; emit @sum` which produces an extra output record such as `sum=3.1648382`.
 
-For the first two options you are populating the output-records stream which feeds into the next verb in a ``then``-chain (if any), or which otherwise is formatted for output using ``--o...`` flags.
+For the first two options you are populating the output-records stream which feeds into the next verb in a `then`-chain (if any), or which otherwise is formatted for output using `--o...` flags.
 
 For the last three options you are sending output directly to standard output, standard error, or a file.
 
-.. _reference-dsl-print-statements:
-
 ## Print statements
 
-The ``print`` statement is perhaps self-explanatory, but with a few light caveats:
+The `print` statement is perhaps self-explanatory, but with a few light caveats:
 
-* There are four variants: ``print`` goes to stdout with final newline, ``printn`` goes to stdout without final newline (you can include one using "\n" in your output string), ``eprint`` goes to stderr with final newline, and ``eprintn`` goes to stderr without final newline.
+* There are four variants: `print` goes to stdout with final newline, `printn` goes to stdout without final newline (you can include one using "\n" in your output string), `eprint` goes to stderr with final newline, and `eprintn` goes to stderr without final newline.
 
-* Output goes directly to stdout/stderr, respectively: data produced this way do not go downstream to the next verb in a ``then``-chain. (Use ``emit`` for that.)
+* Output goes directly to stdout/stderr, respectively: data produced this way do not go downstream to the next verb in a `then`-chain. (Use `emit` for that.)
 
-* Print statements are for strings (``print "hello"``), or things which can be made into strings: numbers (``print 3``, ``print $a + $b``, or concatenations thereof (``print "a + b = " . ($a + $b)``). Maps (in ``$*``, map-valued out-of-stream or local variables, and map literals) aren't convertible into strings. If you print a map, you get ``{is-a-map}`` as output. Please use ``dump`` to print maps.
+* Print statements are for strings (`print "hello"`), or things which can be made into strings: numbers (`print 3`, `print $a + $b`, or concatenations thereof (`print "a + b = " . ($a + $b)`). Maps (in `$*`, map-valued out-of-stream or local variables, and map literals) aren't convertible into strings. If you print a map, you get `{is-a-map}` as output. Please use `dump` to print maps.
 
-* You can redirect print output to a file: ``mlr --from myfile.dat put 'print > "tap.txt", $x'`` ``mlr --from myfile.dat put 'o=$*; print > $a.".txt", $x'``.
+* You can redirect print output to a file: `mlr --from myfile.dat put 'print > "tap.txt", $x'` `mlr --from myfile.dat put 'o=$*; print > $a.".txt", $x'`.
 
-* See also :ref:`reference-dsl-redirected-output-statements` for examples.
-
-.. _reference-dsl-dump-statements:
+* See also [Redirected-output statements](reference-dsl-output-statements.md#redirected-output-statements) for examples.
 
 ## Dump statements
 
-The ``dump`` statement is for printing expressions, including maps, directly to stdout/stderr, respectively:
+The `dump` statement is for printing expressions, including maps, directly to stdout/stderr, respectively:
 
-* There are two variants: ``dump`` prints to stdout; ``edump`` prints to stderr.
+* There are two variants: `dump` prints to stdout; `edump` prints to stderr.
 
-* Output goes directly to stdout/stderr, respectively: data produced this way do not go downstream to the next verb in a ``then``-chain. (Use ``emit`` for that.)
+* Output goes directly to stdout/stderr, respectively: data produced this way do not go downstream to the next verb in a `then`-chain. (Use `emit` for that.)
 
-* You can use ``dump`` to output single strings, numbers, or expressions including map-valued data. Map-valued data are printed as JSON. Miller allows string and integer keys in its map literals while JSON allows only string keys, so use ``mlr put --jknquoteint`` if you want integer-valued map keys not double-quoted.
+* You can use `dump` to output single strings, numbers, or expressions including map-valued data. Map-valued data are printed as JSON. Miller allows string and integer keys in its map literals while JSON allows only string keys, so use `mlr put --jknquoteint` if you want integer-valued map keys not double-quoted.
 
-* If you use ``dump`` (or ``edump``) with no arguments, you get a JSON structure representing the current values of all out-of-stream variables.
+* If you use `dump` (or `edump`) with no arguments, you get a JSON structure representing the current values of all out-of-stream variables.
 
-* As with ``print``, you can redirect output to files.
+* As with `print`, you can redirect output to files.
 
-* See also :ref:`reference-dsl-redirected-output-statements` for examples.
+* See also [Redirected-output statements](reference-dsl-output-statements.md#redirected-output-statements) for examples.
 
 ## Tee statements
 
-Records produced by a ``mlr put`` go downstream to the next verb in your ``then``-chain, if any, or otherwise to standard output.  If you want to additionally copy out records to files, you can do that using ``tee``.
+Records produced by a `mlr put` go downstream to the next verb in your `then`-chain, if any, or otherwise to standard output.  If you want to additionally copy out records to files, you can do that using `tee`.
 
-The syntax is, by example, ``mlr --from myfile.dat put 'tee > "tap.dat", $*' then sort -n index``.  First is ``tee >``, then the filename expression (which can be an expression such as ``"tap.".$a.".dat"``), then a comma, then ``$*``. (Nothing else but ``$*`` is teeable.)
+The syntax is, by example, `mlr --from myfile.dat put 'tee > "tap.dat", $*' then sort -n index`.  First is `tee >`, then the filename expression (which can be an expression such as `"tap.".$a.".dat"`), then a comma, then `$*`. (Nothing else but `$*` is teeable.)
 
-See also :ref:`reference-dsl-redirected-output-statements` for examples.
-
-.. _reference-dsl-redirected-output-statements:
+See also [Redirected-output statements](reference-dsl-output-statements.md#redirected-output-statements) for examples.
 
 ## Redirected-output statements
 
@@ -67,10 +61,12 @@ The **print**, **dump** **tee**, **emitf**, **emit**, and **emitp** keywords all
 
 Details:
 
-* The ``print`` and ``dump`` keywords produce output immediately to standard output, or to specified file(s) or pipe-to command if present.
+* The `print` and `dump` keywords produce output immediately to standard output, or to specified file(s) or pipe-to command if present.
 
-
+
 mlr help keyword print
+
+
 print: prints expression immediately to stdout.
 
   Example: mlr --from f.dat put -q 'print "The sum of x and y is ".($x+$y)'
@@ -78,8 +74,10 @@ print: prints expression immediately to stdout.
   Example: mlr --from f.dat put  '(NR %% 1000 == 0) { print > stderr, "Checkpoint ".NR}'
 
-
+
 mlr help keyword dump
+
+
 dump: prints all currently defined out-of-stream variables immediately
 to stdout as JSON.
 
@@ -99,10 +97,12 @@ the main command line.
   Example: mlr --from f.dat put -q '@v[NR]=$*; end { dump | "jq .[]"}'
 
-* ``mlr put`` sends the current record (possibly modified by the ``put`` expression) to the output record stream. Records are then input to the following verb in a ``then``-chain (if any), else printed to standard output (unless ``put -q``). The **tee** keyword *additionally* writes the output record to specified file(s) or pipe-to command, or immediately to ``stdout``/``stderr``. +* `mlr put` sends the current record (possibly modified by the `put` expression) to the output record stream. Records are then input to the following verb in a `then`-chain (if any), else printed to standard output (unless `put -q`). The **tee** keyword *additionally* writes the output record to specified file(s) or pipe-to command, or immediately to `stdout`/`stderr`. -
+
 mlr help keyword tee
+
+
 tee: prints the current record to specified file.
 This is an immediate print to the specified file (except for pprint format
 which of course waits until the end of the input stream to format all output).
@@ -129,10 +129,12 @@ output $*.
   Example: mlr --from f.dat put -q --ojson 'tee | "gzip > /tmp/data-".$a.".gz", $*'
 
-* ``mlr put``'s ``emitf``, ``emitp``, and ``emit`` send out-of-stream variables to the output record stream. These are then input to the following verb in a ``then``-chain (if any), else printed to standard output. When redirected with ``>``, ``>>``, or ``|``, they *instead* write the out-of-stream variable(s) to specified file(s) or pipe-to command, or immediately to ``stdout``/``stderr``. +* `mlr put`'s `emitf`, `emitp`, and `emit` send out-of-stream variables to the output record stream. These are then input to the following verb in a `then`-chain (if any), else printed to standard output. When redirected with `>`, `>>`, or `|`, they *instead* write the out-of-stream variable(s) to specified file(s) or pipe-to command, or immediately to `stdout`/`stderr`. -
+
 mlr help keyword emitf
+
+
 emitf: inserts non-indexed out-of-stream variable(s) side-by-side into the
 output record stream.
 
@@ -161,8 +163,10 @@ etc., to control the format of the output if the output is redirected. See also
 Please see https://johnkerl.org/miller6://johnkerl.org/miller/doc for more information.
 
-
+
 mlr help keyword emitp
+
+
 emitp: inserts an out-of-stream variable into the output record stream.
 Hashmap indices present in the data but not slotted by emitp arguments are
 output concatenated with ":".
@@ -193,8 +197,10 @@ etc., to control the format of the output if the output is redirected. See also
 Please see https://johnkerl.org/miller6://johnkerl.org/miller/doc for more information.
 
-
+
 mlr help keyword emit
+
+
 emit: inserts an out-of-stream variable into the output record stream. Hashmap
 indices present in the data but not slotted by emit arguments are not output.
 
@@ -226,28 +232,30 @@ etc., to control the format of the output if the output is redirected. See also
 Please see https://johnkerl.org/miller6://johnkerl.org/miller/doc for more information.
 
-.. _reference-dsl-emit-statements: - ## Emit statements -There are three variants: ``emitf``, ``emit``, and ``emitp``. Keep in mind that out-of-stream variables are a nested, multi-level hashmap (directly viewable as JSON using ``dump``), whereas Miller output records are lists of single-level key-value pairs. The three emit variants allow you to control how the multilevel hashmaps are flatten down to output records. You can emit any map-valued expression, including ``$*``, map-valued out-of-stream variables, the entire out-of-stream-variable collection ``@*``, map-valued local variables, map literals, or map-valued function return values. +There are three variants: `emitf`, `emit`, and `emitp`. Keep in mind that out-of-stream variables are a nested, multi-level hashmap (directly viewable as JSON using `dump`), whereas Miller output records are lists of single-level key-value pairs. The three emit variants allow you to control how the multilevel hashmaps are flatten down to output records. You can emit any map-valued expression, including `$*`, map-valued out-of-stream variables, the entire out-of-stream-variable collection `@*`, map-valued local variables, map literals, or map-valued function return values. -Use **emitf** to output several out-of-stream variables side-by-side in the same output record. For ``emitf`` these mustn't have indexing using ``@name[...]``. Example: +Use **emitf** to output several out-of-stream variables side-by-side in the same output record. For `emitf` these mustn't have indexing using `@name[...]`. Example: -
+
 mlr put -q '
   @count += 1;
   @x_sum += $x;
   @y_sum += $y;
   end { emitf @count, @x_sum, @y_sum}
 ' data/small
+
+
 count=5,x_sum=2.264761728567491,y_sum=2.585085709781158
 
Use **emit** to output an out-of-stream variable. If it's non-indexed you'll get a simple key-value pair: -
+
 cat data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -255,22 +263,28 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
-
+
 mlr put -q '@sum += $x; end { dump }' data/small
+
+
 {
   "sum": 2.264761728567491
 }
 
-
+
 mlr put -q '@sum += $x; end { emit @sum }' data/small
+
+
 sum=2.264761728567491
 
-If it's indexed then use as many names after ``emit`` as there are indices: +If it's indexed then use as many names after `emit` as there are indices: -
+
 mlr put -q '@sum[$a] += $x; end { dump }' data/small
+
+
 {
   "sum": {
     "pan": 0.3467901443380824,
@@ -280,15 +294,19 @@ If it's indexed then use as many names after ``emit`` as there are indices:
 }
 
-
+
 mlr put -q '@sum[$a] += $x; end { emit @sum, "a" }' data/small
+
+
 a=pan,sum=0.3467901443380824
 a=eks,sum=1.1400793586611044
 a=wye,sum=0.7778922255683036
 
-
+
 mlr put -q '@sum[$a][$b] += $x; end { dump }' data/small
+
+
 {
   "sum": {
     "pan": {
@@ -306,8 +324,10 @@ a=wye,sum=0.7778922255683036
 }
 
-
+
 mlr put -q '@sum[$a][$b] += $x; end { emit @sum, "a", "b" }' data/small
+
+
 a=pan,b=pan,sum=0.3467901443380824
 a=eks,b=pan,sum=0.7586799647899636
 a=eks,b=wye,sum=0.38139939387114097
@@ -315,8 +335,10 @@ a=wye,b=wye,sum=0.20460330576630303
 a=wye,b=pan,sum=0.5732889198020006
 
-
+
 mlr put -q '@sum[$a][$b][$i] += $x; end { dump }' data/small
+
+
 {
   "sum": {
     "pan": {
@@ -344,11 +366,13 @@ a=wye,b=pan,sum=0.5732889198020006
 }
 
-
+
 mlr put -q '
   @sum[$a][$b][$i] += $x;
   end { emit @sum, "a", "b", "i" }
 ' data/small
+
+
 a=pan,b=pan,i=1,sum=0.3467901443380824
 a=eks,b=pan,i=2,sum=0.7586799647899636
 a=eks,b=wye,i=4,sum=0.38139939387114097
@@ -356,10 +380,12 @@ a=wye,b=wye,i=3,sum=0.20460330576630303
 a=wye,b=pan,i=5,sum=0.5732889198020006
 
-Now for **emitp**: if you have as many names following ``emit`` as there are levels in the out-of-stream variable's hashmap, then ``emit`` and ``emitp`` do the same thing. Where they differ is when you don't specify as many names as there are hashmap levels. In this case, Miller needs to flatten multiple map indices down to output-record keys: ``emitp`` includes full prefixing (hence the ``p`` in ``emitp``) while ``emit`` takes the deepest hashmap key as the output-record key: +Now for **emitp**: if you have as many names following `emit` as there are levels in the out-of-stream variable's hashmap, then `emit` and `emitp` do the same thing. Where they differ is when you don't specify as many names as there are hashmap levels. In this case, Miller needs to flatten multiple map indices down to output-record keys: `emitp` includes full prefixing (hence the `p` in `emitp`) while `emit` takes the deepest hashmap key as the output-record key: -
+
 mlr put -q '@sum[$a][$b] += $x; end { dump }' data/small
+
+
 {
   "sum": {
     "pan": {
@@ -377,34 +403,44 @@ Now for **emitp**: if you have as many names following ``emit`` as there are lev
 }
 
-
+
 mlr put -q '@sum[$a][$b] += $x; end { emit @sum, "a" }' data/small
+
+
 a=pan,pan=0.3467901443380824
 a=eks,pan=0.7586799647899636,wye=0.38139939387114097
 a=wye,wye=0.20460330576630303,pan=0.5732889198020006
 
-
+
 mlr put -q '@sum[$a][$b] += $x; end { emit @sum }' data/small
+
+
 pan=0.3467901443380824
 pan=0.7586799647899636,wye=0.38139939387114097
 wye=0.20460330576630303,pan=0.5732889198020006
 
-
+
 mlr put -q '@sum[$a][$b] += $x; end { emitp @sum, "a" }' data/small
+
+
 a=pan,sum.pan=0.3467901443380824
 a=eks,sum.pan=0.7586799647899636,sum.wye=0.38139939387114097
 a=wye,sum.wye=0.20460330576630303,sum.pan=0.5732889198020006
 
-
+
 mlr put -q '@sum[$a][$b] += $x; end { emitp @sum }' data/small
+
+
 sum.pan.pan=0.3467901443380824,sum.eks.pan=0.7586799647899636,sum.eks.wye=0.38139939387114097,sum.wye.wye=0.20460330576630303,sum.wye.pan=0.5732889198020006
 
-
+
 mlr --oxtab put -q '@sum[$a][$b] += $x; end { emitp @sum }' data/small
+
+
 sum.pan.pan 0.3467901443380824
 sum.eks.pan 0.7586799647899636
 sum.eks.wye 0.38139939387114097
@@ -413,25 +449,31 @@ sum.wye.pan 0.5732889198020006
 
Use **--oflatsep** to specify the character which joins multilevel -keys for ``emitp`` (it defaults to a colon): +keys for `emitp` (it defaults to a colon): -
+
 mlr put -q --oflatsep / '@sum[$a][$b] += $x; end { emitp @sum, "a" }' data/small
+
+
 a=pan,sum.pan=0.3467901443380824
 a=eks,sum.pan=0.7586799647899636,sum.wye=0.38139939387114097
 a=wye,sum.wye=0.20460330576630303,sum.pan=0.5732889198020006
 
-
+
 mlr put -q --oflatsep / '@sum[$a][$b] += $x; end { emitp @sum }' data/small
+
+
 sum.pan.pan=0.3467901443380824,sum.eks.pan=0.7586799647899636,sum.eks.wye=0.38139939387114097,sum.wye.wye=0.20460330576630303,sum.wye.pan=0.5732889198020006
 
-
+
 mlr --oxtab put -q --oflatsep / '
   @sum[$a][$b] += $x;
   end { emitp @sum }
 ' data/small
+
+
 sum.pan.pan 0.3467901443380824
 sum.eks.pan 0.7586799647899636
 sum.eks.wye 0.38139939387114097
@@ -444,7 +486,7 @@ sum.wye.pan 0.5732889198020006
 You can emit **multiple map-valued expressions side-by-side** by
 including their names in parentheses:
 
-
+
 mlr --from data/medium --opprint put -q '
   @x_count[$a][$b] += 1;
   @x_sum[$a][$b] += $x;
@@ -455,6 +497,8 @@ including their names in parentheses:
       emit (@x_sum, @x_count, @x_mean), "a", "b"
   }
 '
+
+
 a   b   x_sum              x_count x_mean
 pan pan 219.1851288316854  427     0.5133141190437597
 pan wye 198.43293070748447 395     0.5023618498923658
@@ -483,18 +527,20 @@ hat hat 182.8535323148762  381     0.47993053101017374
 hat pan 168.5538067327806  363     0.4643355557376876
 
-What this does is walk through the first out-of-stream variable (``@x_sum`` in this example) as usual, then for each keylist found (e.g. ``pan,wye``), include the values for the remaining out-of-stream variables (here, ``@x_count`` and ``@x_mean``). You should use this when all out-of-stream variables in the emit statement have **the same shape and the same keylists**. +What this does is walk through the first out-of-stream variable (`@x_sum` in this example) as usual, then for each keylist found (e.g. `pan,wye`), include the values for the remaining out-of-stream variables (here, `@x_count` and `@x_mean`). You should use this when all out-of-stream variables in the emit statement have **the same shape and the same keylists**. ## Emit-all statements -Use **emit all** (or ``emit @*`` which is synonymous) to output all out-of-stream variables. You can use the following idiom to get various accumulators output side-by-side (reminiscent of ``mlr stats1``): +Use **emit all** (or `emit @*` which is synonymous) to output all out-of-stream variables. You can use the following idiom to get various accumulators output side-by-side (reminiscent of `mlr stats1`): -
+
 mlr --from data/small --opprint put -q '
   @v[$a][$b]["sum"] += $x;
   @v[$a][$b]["count"] += 1;
   end{emit @*,"a","b"}
 '
+
+
 a   b   v.sum               v.count
 pan pan 0.3467901443380824  1
 eks pan 0.7586799647899636  1
@@ -503,12 +549,14 @@ wye wye 0.20460330576630303 1
 wye pan 0.5732889198020006  1
 
-
+
 mlr --from data/small --opprint put -q '
   @sum[$a][$b] += $x;
   @count[$a][$b] += 1;
   end{emit @*,"a","b"}
 '
+
+
 a   b   sum
 pan pan 0.3467901443380824
 eks pan 0.7586799647899636
@@ -524,12 +572,14 @@ wye wye 1
 wye pan 1
 
-
+
 mlr --from data/small --opprint put -q '
   @sum[$a][$b] += $x;
   @count[$a][$b] += 1;
   end{emit (@sum, @count),"a","b"}
 '
+
+
 a   b   sum                 count
 pan pan 0.3467901443380824  1
 eks pan 0.7586799647899636  1
diff --git a/docs6b/docs/reference-dsl-output-statements.md.in b/docs6b/docs/reference-dsl-output-statements.md.in
index 578425b46..d7fbdc121 100644
--- a/docs6b/docs/reference-dsl-output-statements.md.in
+++ b/docs6b/docs/reference-dsl-output-statements.md.in
@@ -2,63 +2,57 @@
 
 You can **output** variable-values or expressions in **five ways**:
 
-* **Assign** them to stream-record fields. For example, ``$cumulative_sum = @sum``. For another example, ``$nr = NR`` adds a field named ``nr`` to each output record, containing the value of the built-in variable ``NR`` as of when that record was ingested.
+* **Assign** them to stream-record fields. For example, `$cumulative_sum = @sum`. For another example, `$nr = NR` adds a field named `nr` to each output record, containing the value of the built-in variable `NR` as of when that record was ingested.
 
-* Use the **print** or **eprint** keywords which immediately print an expression *directly to standard output or standard error*, respectively. Note that ``dump``, ``edump``, ``print``, and ``eprint`` don't output records which participate in ``then``-chaining; rather, they're just immediate prints to stdout/stderr. The ``printn`` and ``eprintn`` keywords are the same except that they don't print final newlines. Additionally, you can print to a specified file instead of stdout/stderr.
+* Use the **print** or **eprint** keywords which immediately print an expression *directly to standard output or standard error*, respectively. Note that `dump`, `edump`, `print`, and `eprint` don't output records which participate in `then`-chaining; rather, they're just immediate prints to stdout/stderr. The `printn` and `eprintn` keywords are the same except that they don't print final newlines. Additionally, you can print to a specified file instead of stdout/stderr.
 
 * Use the **dump** or **edump** keywords, which *immediately print all out-of-stream variables as a JSON data structure to the standard output or standard error* (respectively).
 
 * Use **tee** which formats the current stream record (not just an arbitrary string as with **print**) to a specific file.
 
-* Use **emit**/**emitp**/**emitf** to send out-of-stream variables' current values to the output record stream, e.g.  ``@sum += $x; emit @sum`` which produces an extra output record such as ``sum=3.1648382``.
+* Use **emit**/**emitp**/**emitf** to send out-of-stream variables' current values to the output record stream, e.g.  `@sum += $x; emit @sum` which produces an extra output record such as `sum=3.1648382`.
 
-For the first two options you are populating the output-records stream which feeds into the next verb in a ``then``-chain (if any), or which otherwise is formatted for output using ``--o...`` flags.
+For the first two options you are populating the output-records stream which feeds into the next verb in a `then`-chain (if any), or which otherwise is formatted for output using `--o...` flags.
 
 For the last three options you are sending output directly to standard output, standard error, or a file.
 
-.. _reference-dsl-print-statements:
-
 ## Print statements
 
-The ``print`` statement is perhaps self-explanatory, but with a few light caveats:
+The `print` statement is perhaps self-explanatory, but with a few light caveats:
 
-* There are four variants: ``print`` goes to stdout with final newline, ``printn`` goes to stdout without final newline (you can include one using "\n" in your output string), ``eprint`` goes to stderr with final newline, and ``eprintn`` goes to stderr without final newline.
+* There are four variants: `print` goes to stdout with final newline, `printn` goes to stdout without final newline (you can include one using "\n" in your output string), `eprint` goes to stderr with final newline, and `eprintn` goes to stderr without final newline.
 
-* Output goes directly to stdout/stderr, respectively: data produced this way do not go downstream to the next verb in a ``then``-chain. (Use ``emit`` for that.)
+* Output goes directly to stdout/stderr, respectively: data produced this way do not go downstream to the next verb in a `then`-chain. (Use `emit` for that.)
 
-* Print statements are for strings (``print "hello"``), or things which can be made into strings: numbers (``print 3``, ``print $a + $b``, or concatenations thereof (``print "a + b = " . ($a + $b)``). Maps (in ``$*``, map-valued out-of-stream or local variables, and map literals) aren't convertible into strings. If you print a map, you get ``{is-a-map}`` as output. Please use ``dump`` to print maps.
+* Print statements are for strings (`print "hello"`), or things which can be made into strings: numbers (`print 3`, `print $a + $b`, or concatenations thereof (`print "a + b = " . ($a + $b)`). Maps (in `$*`, map-valued out-of-stream or local variables, and map literals) aren't convertible into strings. If you print a map, you get `{is-a-map}` as output. Please use `dump` to print maps.
 
-* You can redirect print output to a file: ``mlr --from myfile.dat put 'print > "tap.txt", $x'`` ``mlr --from myfile.dat put 'o=$*; print > $a.".txt", $x'``.
+* You can redirect print output to a file: `mlr --from myfile.dat put 'print > "tap.txt", $x'` `mlr --from myfile.dat put 'o=$*; print > $a.".txt", $x'`.
 
-* See also :ref:`reference-dsl-redirected-output-statements` for examples.
-
-.. _reference-dsl-dump-statements:
+* See also [Redirected-output statements](reference-dsl-output-statements.md#redirected-output-statements) for examples.
 
 ## Dump statements
 
-The ``dump`` statement is for printing expressions, including maps, directly to stdout/stderr, respectively:
+The `dump` statement is for printing expressions, including maps, directly to stdout/stderr, respectively:
 
-* There are two variants: ``dump`` prints to stdout; ``edump`` prints to stderr.
+* There are two variants: `dump` prints to stdout; `edump` prints to stderr.
 
-* Output goes directly to stdout/stderr, respectively: data produced this way do not go downstream to the next verb in a ``then``-chain. (Use ``emit`` for that.)
+* Output goes directly to stdout/stderr, respectively: data produced this way do not go downstream to the next verb in a `then`-chain. (Use `emit` for that.)
 
-* You can use ``dump`` to output single strings, numbers, or expressions including map-valued data. Map-valued data are printed as JSON. Miller allows string and integer keys in its map literals while JSON allows only string keys, so use ``mlr put --jknquoteint`` if you want integer-valued map keys not double-quoted.
+* You can use `dump` to output single strings, numbers, or expressions including map-valued data. Map-valued data are printed as JSON. Miller allows string and integer keys in its map literals while JSON allows only string keys, so use `mlr put --jknquoteint` if you want integer-valued map keys not double-quoted.
 
-* If you use ``dump`` (or ``edump``) with no arguments, you get a JSON structure representing the current values of all out-of-stream variables.
+* If you use `dump` (or `edump`) with no arguments, you get a JSON structure representing the current values of all out-of-stream variables.
 
-* As with ``print``, you can redirect output to files.
+* As with `print`, you can redirect output to files.
 
-* See also :ref:`reference-dsl-redirected-output-statements` for examples.
+* See also [Redirected-output statements](reference-dsl-output-statements.md#redirected-output-statements) for examples.
 
 ## Tee statements
 
-Records produced by a ``mlr put`` go downstream to the next verb in your ``then``-chain, if any, or otherwise to standard output.  If you want to additionally copy out records to files, you can do that using ``tee``.
+Records produced by a `mlr put` go downstream to the next verb in your `then`-chain, if any, or otherwise to standard output.  If you want to additionally copy out records to files, you can do that using `tee`.
 
-The syntax is, by example, ``mlr --from myfile.dat put 'tee > "tap.dat", $*' then sort -n index``.  First is ``tee >``, then the filename expression (which can be an expression such as ``"tap.".$a.".dat"``), then a comma, then ``$*``. (Nothing else but ``$*`` is teeable.)
+The syntax is, by example, `mlr --from myfile.dat put 'tee > "tap.dat", $*' then sort -n index`.  First is `tee >`, then the filename expression (which can be an expression such as `"tap.".$a.".dat"`), then a comma, then `$*`. (Nothing else but `$*` is teeable.)
 
-See also :ref:`reference-dsl-redirected-output-statements` for examples.
-
-.. _reference-dsl-redirected-output-statements:
+See also [Redirected-output statements](reference-dsl-output-statements.md#redirected-output-statements) for examples.
 
 ## Redirected-output statements
 
@@ -66,7 +60,7 @@ The **print**, **dump** **tee**, **emitf**, **emit**, and **emitp** keywords all
 
 Details:
 
-* The ``print`` and ``dump`` keywords produce output immediately to standard output, or to specified file(s) or pipe-to command if present.
+* The `print` and `dump` keywords produce output immediately to standard output, or to specified file(s) or pipe-to command if present.
 
 GENMD_RUN_COMMAND
 mlr help keyword print
@@ -76,13 +70,13 @@ GENMD_RUN_COMMAND
 mlr help keyword dump
 GENMD_EOF
 
-* ``mlr put`` sends the current record (possibly modified by the ``put`` expression) to the output record stream. Records are then input to the following verb in a ``then``-chain (if any), else printed to standard output (unless ``put -q``). The **tee** keyword *additionally* writes the output record to specified file(s) or pipe-to command, or immediately to ``stdout``/``stderr``.
+* `mlr put` sends the current record (possibly modified by the `put` expression) to the output record stream. Records are then input to the following verb in a `then`-chain (if any), else printed to standard output (unless `put -q`). The **tee** keyword *additionally* writes the output record to specified file(s) or pipe-to command, or immediately to `stdout`/`stderr`.
 
 GENMD_RUN_COMMAND
 mlr help keyword tee
 GENMD_EOF
 
-* ``mlr put``'s ``emitf``, ``emitp``, and ``emit`` send out-of-stream variables to the output record stream. These are then input to the following verb in a ``then``-chain (if any), else printed to standard output. When redirected with ``>``, ``>>``, or ``|``, they *instead* write the out-of-stream variable(s) to specified file(s) or pipe-to command, or immediately to ``stdout``/``stderr``.
+* `mlr put`'s `emitf`, `emitp`, and `emit` send out-of-stream variables to the output record stream. These are then input to the following verb in a `then`-chain (if any), else printed to standard output. When redirected with `>`, `>>`, or `|`, they *instead* write the out-of-stream variable(s) to specified file(s) or pipe-to command, or immediately to `stdout`/`stderr`.
 
 GENMD_RUN_COMMAND
 mlr help keyword emitf
@@ -96,13 +90,11 @@ GENMD_RUN_COMMAND
 mlr help keyword emit
 GENMD_EOF
 
-.. _reference-dsl-emit-statements:
-
 ## Emit statements
 
-There are three variants: ``emitf``, ``emit``, and ``emitp``. Keep in mind that out-of-stream variables are a nested, multi-level hashmap (directly viewable as JSON using ``dump``), whereas Miller output records are lists of single-level key-value pairs. The three emit variants allow you to control how the multilevel hashmaps are flatten down to output records. You can emit any map-valued expression, including ``$*``, map-valued out-of-stream variables, the entire out-of-stream-variable collection ``@*``, map-valued local variables, map literals, or map-valued function return values.
+There are three variants: `emitf`, `emit`, and `emitp`. Keep in mind that out-of-stream variables are a nested, multi-level hashmap (directly viewable as JSON using `dump`), whereas Miller output records are lists of single-level key-value pairs. The three emit variants allow you to control how the multilevel hashmaps are flatten down to output records. You can emit any map-valued expression, including `$*`, map-valued out-of-stream variables, the entire out-of-stream-variable collection `@*`, map-valued local variables, map literals, or map-valued function return values.
 
-Use **emitf** to output several out-of-stream variables side-by-side in the same output record. For ``emitf`` these mustn't have indexing using ``@name[...]``. Example:
+Use **emitf** to output several out-of-stream variables side-by-side in the same output record. For `emitf` these mustn't have indexing using `@name[...]`. Example:
 
 GENMD_RUN_COMMAND
 mlr put -q '
@@ -127,7 +119,7 @@ GENMD_RUN_COMMAND
 mlr put -q '@sum += $x; end { emit @sum }' data/small
 GENMD_EOF
 
-If it's indexed then use as many names after ``emit`` as there are indices:
+If it's indexed then use as many names after `emit` as there are indices:
 
 GENMD_RUN_COMMAND
 mlr put -q '@sum[$a] += $x; end { dump }' data/small
@@ -156,7 +148,7 @@ mlr put -q '
 ' data/small
 GENMD_EOF
 
-Now for **emitp**: if you have as many names following ``emit`` as there are levels in the out-of-stream variable's hashmap, then ``emit`` and ``emitp`` do the same thing. Where they differ is when you don't specify as many names as there are hashmap levels. In this case, Miller needs to flatten multiple map indices down to output-record keys: ``emitp`` includes full prefixing (hence the ``p`` in ``emitp``) while ``emit`` takes the deepest hashmap key as the output-record key:
+Now for **emitp**: if you have as many names following `emit` as there are levels in the out-of-stream variable's hashmap, then `emit` and `emitp` do the same thing. Where they differ is when you don't specify as many names as there are hashmap levels. In this case, Miller needs to flatten multiple map indices down to output-record keys: `emitp` includes full prefixing (hence the `p` in `emitp`) while `emit` takes the deepest hashmap key as the output-record key:
 
 GENMD_RUN_COMMAND
 mlr put -q '@sum[$a][$b] += $x; end { dump }' data/small
@@ -183,7 +175,7 @@ mlr --oxtab put -q '@sum[$a][$b] += $x; end { emitp @sum }' data/small
 GENMD_EOF
 
 Use **--oflatsep** to specify the character which joins multilevel
-keys for ``emitp`` (it defaults to a colon):
+keys for `emitp` (it defaults to a colon):
 
 GENMD_RUN_COMMAND
 mlr put -q --oflatsep / '@sum[$a][$b] += $x; end { emitp @sum, "a" }' data/small
@@ -207,11 +199,11 @@ including their names in parentheses:
 
 GENMD_INCLUDE_AND_RUN_ESCAPED(data/emit-lashed.sh)
 
-What this does is walk through the first out-of-stream variable (``@x_sum`` in this example) as usual, then for each keylist found (e.g. ``pan,wye``), include the values for the remaining out-of-stream variables (here, ``@x_count`` and ``@x_mean``). You should use this when all out-of-stream variables in the emit statement have **the same shape and the same keylists**.
+What this does is walk through the first out-of-stream variable (`@x_sum` in this example) as usual, then for each keylist found (e.g. `pan,wye`), include the values for the remaining out-of-stream variables (here, `@x_count` and `@x_mean`). You should use this when all out-of-stream variables in the emit statement have **the same shape and the same keylists**.
 
 ## Emit-all statements
 
-Use **emit all** (or ``emit @*`` which is synonymous) to output all out-of-stream variables. You can use the following idiom to get various accumulators output side-by-side (reminiscent of ``mlr stats1``):
+Use **emit all** (or `emit @*` which is synonymous) to output all out-of-stream variables. You can use the following idiom to get various accumulators output side-by-side (reminiscent of `mlr stats1`):
 
 GENMD_RUN_COMMAND
 mlr --from data/small --opprint put -q '
diff --git a/docs6b/docs/reference-dsl-syntax.md b/docs6b/docs/reference-dsl-syntax.md
index f8fd85b2c..48e2493c9 100644
--- a/docs6b/docs/reference-dsl-syntax.md
+++ b/docs6b/docs/reference-dsl-syntax.md
@@ -5,8 +5,10 @@
 
 Multiple expressions may be given, separated by semicolons, and each may refer to the ones before:
 
-
+
 ruby -e '10.times{|i|puts "i=#{i}"}' | mlr --opprint put '$j = $i + 1; $k = $i +$j'
+
+
 i j  k
 0 1  1
 1 2  3
@@ -22,7 +24,7 @@ i j  k
 
 Newlines within the expression are ignored, which can help increase legibility of complex expressions:
 
-
+
 mlr --opprint put '
   $nf       = NF;
   $nr       = NR;
@@ -30,6 +32,8 @@ Newlines within the expression are ignored, which can help increase legibility o
   $filenum  = FILENUM;
   $filename = FILENAME
 ' data/small data/small2
+
+
 a   b   i     x                    y                    nf nr fnr filenum filename
 pan pan 1     0.3467901443380824   0.7268028627434533   5  1  1   1       data/small
 eks pan 2     0.7586799647899636   0.5221511083334797   5  2  2   1       data/small
@@ -43,22 +47,24 @@ hat wye 10002 0.321507044286237609 0.568893318795083758 5  9  4   2       data/s
 pan zee 10003 0.272054845593895200 0.425789896597056627 5  10 5   2       data/small2
 
-
+
 mlr --opprint filter '($x > 0.5 && $y < 0.5) || ($x < 0.5 && $y > 0.5)' \
   then stats2 -a corr -f x,y \
   data/medium
+
+
 x_y_corr
 -0.7479940285189345
 
-.. _reference-dsl-expressions-from-files: - ## Expressions from files -The simplest way to enter expressions for ``put`` and ``filter`` is between single quotes on the command line, e.g. +The simplest way to enter expressions for `put` and `filter` is between single quotes on the command line, e.g. -
+
 mlr --from data/small put '$xy = sqrt($x**2 + $y**2)'
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,xy=0.8052985815845617
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,xy=0.9209978658539777
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776,xy=0.3953756915115773
@@ -66,8 +72,10 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463,xy=0.404316851577441
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729,xy=1.036584492737304
 
-
+
 mlr --from data/small put 'func f(a, b) { return sqrt(a**2 + b**2) } $xy = f($x, $y)'
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,xy=0.8052985815845617
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,xy=0.9209978658539777
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776,xy=0.3953756915115773
@@ -78,16 +86,20 @@ a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729,xy=1.036584492737304
 You may, though, find it convenient to put expressions into files for reuse, and read them
 **using the -f option**. For example:
 
-
+
 cat data/fe-example-3.mlr
+
+
 func f(a, b) {
   return sqrt(a**2 + b**2)
 }
 $xy = f($x, $y)
 
-
+
 mlr --from data/small put -f data/fe-example-3.mlr
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,xy=0.8052985815845617
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,xy=0.9209978658539777
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776,xy=0.3953756915115773
@@ -97,15 +109,19 @@ a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729,xy=1.036584492737304
 
 If you have some of the logic in a file and you want to write the rest on the command line, you can **use the -f and -e options together**:
 
-
+
 cat data/fe-example-4.mlr
+
+
 func f(a, b) {
   return sqrt(a**2 + b**2)
 }
 
-
+
 mlr --from data/small put -f data/fe-example-4.mlr -e '$xy = f($x, $y)'
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,xy=0.8052985815845617
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,xy=0.9209978658539777
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776,xy=0.3953756915115773
@@ -115,15 +131,15 @@ a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729,xy=1.036584492737304
 
 A suggested use-case here is defining functions in files, and calling them from command-line expressions.
 
-Another suggested use-case is putting default parameter values in files, e.g. using ``begin{@count=is_present(@count)?@count:10}`` in the file, where you can precede that using ``begin{@count=40}`` using ``-e``.
+Another suggested use-case is putting default parameter values in files, e.g. using `begin{@count=is_present(@count)?@count:10}` in the file, where you can precede that using `begin{@count=40}` using `-e`.
 
-Moreover, you can have one or more ``-f`` expressions (maybe one function per file, for example) and one or more ``-e`` expressions on the command line.  If you mix ``-f`` and ``-e`` then the expressions are evaluated in the order encountered. (Since the expressions are all simply concatenated together in order, don't forget intervening semicolons: e.g. not ``mlr put -e '$x=1' -e '$y=2 ...'`` but rather ``mlr put -e '$x=1;' -e '$y=2' ...``.)
+Moreover, you can have one or more `-f` expressions (maybe one function per file, for example) and one or more `-e` expressions on the command line.  If you mix `-f` and `-e` then the expressions are evaluated in the order encountered. (Since the expressions are all simply concatenated together in order, don't forget intervening semicolons: e.g. not `mlr put -e '$x=1' -e '$y=2 ...'` but rather `mlr put -e '$x=1;' -e '$y=2' ...`.)
 
 ## Semicolons, commas, newlines, and curly braces
 
 Miller uses **semicolons as statement separators**, not statement terminators. This means you can write:
 
-
+
 mlr put 'x=1'
 mlr put 'x=1;$y=2'
 mlr put 'x=1;$y=2;'
@@ -132,19 +148,23 @@ mlr put 'x=1;;;;$y=2;'
 
 Semicolons are optional after closing curly braces (which close conditionals and loops as discussed below).
 
-
+
 echo x=1,y=2 | mlr put 'while (NF < 10) { $[NF+1] = ""}  $foo = "bar"'
+
+
 x=1,y=2,3=,4=,5=,6=,7=,8=,9=,10=,foo=bar
 
-
+
 echo x=1,y=2 | mlr put 'while (NF < 10) { $[NF+1] = ""}; $foo = "bar"'
+
+
 x=1,y=2,3=,4=,5=,6=,7=,8=,9=,10=,foo=bar
 
Semicolons are required between statements even if those statements are on separate lines. **Newlines** are for your convenience but have no syntactic meaning: line endings do not terminate statements. For example, adjacent assignment statements must be separated by semicolons even if those statements are on separate lines: -
+
 mlr put '
   $x = 1
   $y = 2 # Syntax error
@@ -158,7 +178,7 @@ mlr put '
 
 **Trailing commas** are allowed in function/subroutine definitions, function/subroutine callsites, and map literals. This is intended for (although not restricted to) the multi-line case:
 
-
+
 mlr --csvlite --from data/a.csv put '
   func f(
     num a,
@@ -176,6 +196,8 @@ mlr put '
     "v": NR,
   }
 '
+
+
 s,t,u,v
 3,-1,5,1
 9,-1,41,2
@@ -183,17 +205,17 @@ s,t,u,v
 
 Bodies for all compound statements must be enclosed in **curly braces**, even if the body is a single statement:
 
-
+
 mlr put 'if ($x == 1) $y = 2' # Syntax error
 
-
+
 mlr put 'if ($x == 1) { $y = 2 }' # This is OK
 
Bodies for compound statements may be empty: -
+
 mlr put 'if ($x == 1) { }' # This no-op is syntactically acceptable
 
diff --git a/docs6b/docs/reference-dsl-syntax.md.in b/docs6b/docs/reference-dsl-syntax.md.in index 6ef2c8b70..51d35dc83 100644 --- a/docs6b/docs/reference-dsl-syntax.md.in +++ b/docs6b/docs/reference-dsl-syntax.md.in @@ -18,11 +18,9 @@ mlr --opprint filter '($x > 0.5 && $y < 0.5) || ($x < 0.5 && $y > 0.5)' \ data/medium GENMD_EOF -.. _reference-dsl-expressions-from-files: - ## Expressions from files -The simplest way to enter expressions for ``put`` and ``filter`` is between single quotes on the command line, e.g. +The simplest way to enter expressions for `put` and `filter` is between single quotes on the command line, e.g. GENMD_INCLUDE_AND_RUN_ESCAPED(data/fe-example-1.sh) @@ -51,9 +49,9 @@ GENMD_EOF A suggested use-case here is defining functions in files, and calling them from command-line expressions. -Another suggested use-case is putting default parameter values in files, e.g. using ``begin{@count=is_present(@count)?@count:10}`` in the file, where you can precede that using ``begin{@count=40}`` using ``-e``. +Another suggested use-case is putting default parameter values in files, e.g. using `begin{@count=is_present(@count)?@count:10}` in the file, where you can precede that using `begin{@count=40}` using `-e`. -Moreover, you can have one or more ``-f`` expressions (maybe one function per file, for example) and one or more ``-e`` expressions on the command line. If you mix ``-f`` and ``-e`` then the expressions are evaluated in the order encountered. (Since the expressions are all simply concatenated together in order, don't forget intervening semicolons: e.g. not ``mlr put -e '$x=1' -e '$y=2 ...'`` but rather ``mlr put -e '$x=1;' -e '$y=2' ...``.) +Moreover, you can have one or more `-f` expressions (maybe one function per file, for example) and one or more `-e` expressions on the command line. If you mix `-f` and `-e` then the expressions are evaluated in the order encountered. (Since the expressions are all simply concatenated together in order, don't forget intervening semicolons: e.g. not `mlr put -e '$x=1' -e '$y=2 ...'` but rather `mlr put -e '$x=1;' -e '$y=2' ...`.) ## Semicolons, commas, newlines, and curly braces diff --git a/docs6b/docs/reference-dsl-unset-statements.md b/docs6b/docs/reference-dsl-unset-statements.md index c217f98b8..cc2fc539a 100644 --- a/docs6b/docs/reference-dsl-unset-statements.md +++ b/docs6b/docs/reference-dsl-unset-statements.md @@ -1,10 +1,12 @@ DSL reference: unset statements # -You can clear a map key by assigning the empty string as its value: ``$x=""`` or ``@x=""``. Using ``unset`` you can remove the key entirely. Examples: +You can clear a map key by assigning the empty string as its value: `$x=""` or `@x=""`. Using `unset` you can remove the key entirely. Examples: -
+
 cat data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -12,8 +14,10 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
-
+
 mlr put 'unset $x, $a' data/small
+
+
 b=pan,i=1,y=0.7268028627434533
 b=pan,i=2,y=0.5221511083334797
 b=wye,i=3,y=0.33831852551664776
@@ -21,10 +25,12 @@ b=wye,i=4,y=0.13418874328430463
 b=pan,i=5,y=0.8636244699032729
 
-This can also be done, of course, using ``mlr cut -x``. You can also clear out-of-stream or local variables, at the base name level, or at an indexed sublevel: +This can also be done, of course, using `mlr cut -x`. You can also clear out-of-stream or local variables, at the base name level, or at an indexed sublevel: -
+
 mlr put -q '@sum[$a][$b] += $x; end { dump; unset @sum; dump }' data/small
+
+
 {
   "sum": {
     "pan": {
@@ -43,8 +49,10 @@ This can also be done, of course, using ``mlr cut -x``. You can also clear out-o
 {}
 
-
+
 mlr put -q '@sum[$a][$b] += $x; end { dump; unset @sum["eks"]; dump }' data/small
+
+
 {
   "sum": {
     "pan": {
@@ -73,4 +81,4 @@ This can also be done, of course, using ``mlr cut -x``. You can also clear out-o
 }
 
-If you use ``unset all`` (or ``unset @*`` which is synonymous), that will unset all out-of-stream variables which have been defined up to that point. +If you use `unset all` (or `unset @*` which is synonymous), that will unset all out-of-stream variables which have been defined up to that point. diff --git a/docs6b/docs/reference-dsl-unset-statements.md.in b/docs6b/docs/reference-dsl-unset-statements.md.in index 721da78ec..c1994aa3a 100644 --- a/docs6b/docs/reference-dsl-unset-statements.md.in +++ b/docs6b/docs/reference-dsl-unset-statements.md.in @@ -1,6 +1,6 @@ DSL reference: unset statements # -You can clear a map key by assigning the empty string as its value: ``$x=""`` or ``@x=""``. Using ``unset`` you can remove the key entirely. Examples: +You can clear a map key by assigning the empty string as its value: `$x=""` or `@x=""`. Using `unset` you can remove the key entirely. Examples: GENMD_RUN_COMMAND cat data/small @@ -10,7 +10,7 @@ GENMD_RUN_COMMAND mlr put 'unset $x, $a' data/small GENMD_EOF -This can also be done, of course, using ``mlr cut -x``. You can also clear out-of-stream or local variables, at the base name level, or at an indexed sublevel: +This can also be done, of course, using `mlr cut -x`. You can also clear out-of-stream or local variables, at the base name level, or at an indexed sublevel: GENMD_RUN_COMMAND mlr put -q '@sum[$a][$b] += $x; end { dump; unset @sum; dump }' data/small @@ -20,4 +20,4 @@ GENMD_RUN_COMMAND mlr put -q '@sum[$a][$b] += $x; end { dump; unset @sum["eks"]; dump }' data/small GENMD_EOF -If you use ``unset all`` (or ``unset @*`` which is synonymous), that will unset all out-of-stream variables which have been defined up to that point. +If you use `unset all` (or `unset @*` which is synonymous), that will unset all out-of-stream variables which have been defined up to that point. diff --git a/docs6b/docs/reference-dsl-user-defined-functions.md b/docs6b/docs/reference-dsl-user-defined-functions.md index 823a8c8ba..a17a71da8 100644 --- a/docs6b/docs/reference-dsl-user-defined-functions.md +++ b/docs6b/docs/reference-dsl-user-defined-functions.md @@ -7,7 +7,7 @@ As of Miller 5.0.0 you can define your own functions, as well as subroutines. Here's the obligatory example of a recursive function to compute the factorial function: -
+
 mlr --opprint --from data/small put '
     func f(n) {
         if (is_numeric(n)) {
@@ -22,6 +22,8 @@ Here's the obligatory example of a recursive function to compute the factorial f
     $ox = f($x + NR);
     $oi = f($i);
 '
+
+
 a   b   i x                   y                   ox                  oi
 pan pan 1 0.3467901443380824  0.7268028627434533  0.46705354854811026 1
 eks pan 2 0.7586799647899636  0.5221511083334797  3.680838410072862   2
@@ -32,29 +34,29 @@ wye pan 5 0.5732889198020006  0.8636244699032729  211.38730958519247  120
 
 Properties of user-defined functions:
 
-* Function bodies start with ``func`` and a parameter list, defined outside of ``begin``, ``end``, or other ``func`` or ``subr`` blocks. (I.e. the Miller DSL has no nested functions.)
+* Function bodies start with `func` and a parameter list, defined outside of `begin`, `end`, or other `func` or `subr` blocks. (I.e. the Miller DSL has no nested functions.)
 
-* A function (uniqified by its name) may not be redefined: either by redefining a user-defined function, or by redefining a built-in function. However, functions and subroutines have separate namespaces: you can define a subroutine ``log`` which does not clash with the mathematical ``log`` function.
+* A function (uniqified by its name) may not be redefined: either by redefining a user-defined function, or by redefining a built-in function. However, functions and subroutines have separate namespaces: you can define a subroutine `log` which does not clash with the mathematical `log` function.
 
 * Functions may be defined either before or after use (there is an object-binding/linkage step at startup).  More specifically, functions may be either recursive or mutually recursive. Functions may not call subroutines.
 
-* Functions may be defined and called either within ``mlr put`` or ``mlr put``.
+* Functions may be defined and called either within `mlr put` or `mlr put`.
 
-* Functions have read access to ``$``-variables and ``@``-variables but may not modify them. See also :ref:`cookbook-memoization-with-oosvars` for an example.
+* Functions have read access to `$`-variables and `@`-variables but may not modify them. See also [Memoization with out-of-stream variables](misc-examples.md#memoization-with-out-of-stream-variables) for an example.
 
 * Argument values may be reassigned: they are not read-only.
 
-* When a return value is not implicitly returned, this results in a return value of absent-null. (In the example above, if there were records for which the argument to ``f`` is non-numeric, the assignments would be skipped.) See also the section on [xxxx](reference-main-null-data.md).
+* When a return value is not implicitly returned, this results in a return value of absent-null. (In the example above, if there were records for which the argument to `f` is non-numeric, the assignments would be skipped.) See also the section on [xxxx](reference-main-null-data.md).
 
-* See the section on :ref:`reference-dsl-local-variables` for information on scope and extent of arguments, as well as for information on the use of local variables within functions.
+* See the section on [Local variables](reference-dsl-variables.md#local-variables) for information on scope and extent of arguments, as well as for information on the use of local variables within functions.
 
-* See the section on :ref:`reference-dsl-expressions-from-files` for information on the use of ``-f`` and ``-e`` flags.
+* See the section on [Expressions from files](reference-dsl-syntax.md#expressions-from-files) for information on the use of `-f` and `-e` flags.
 
 ## User-defined subroutines
 
 Example:
 
-
+
 mlr --opprint --from data/small put -q '
   begin {
     @call_count = 0;
@@ -72,6 +74,8 @@ Example:
   print "NR=" . NR;
   call s(NR);
 '
+
+
 NR=1
 numcalls=1
 NR=2
@@ -86,18 +90,18 @@ numcalls=15
 
 Properties of user-defined subroutines:
 
-* Subroutine bodies start with ``subr`` and a parameter list, defined outside of ``begin``, ``end``, or other ``func`` or ``subr`` blocks. (I.e. the Miller DSL has no nested subroutines.)
+* Subroutine bodies start with `subr` and a parameter list, defined outside of `begin`, `end`, or other `func` or `subr` blocks. (I.e. the Miller DSL has no nested subroutines.)
 
-* A subroutine (uniqified by its name) may not be redefined. However, functions and subroutines have separate namespaces: you can define a subroutine ``log`` which does not clash with the mathematical ``log`` function.
+* A subroutine (uniqified by its name) may not be redefined. However, functions and subroutines have separate namespaces: you can define a subroutine `log` which does not clash with the mathematical `log` function.
 
 * Subroutines may be defined either before or after use (there is an object-binding/linkage step at startup).  More specifically, subroutines may be either recursive or mutually recursive. Subroutines may call functions.
 
-* Subroutines may be defined and called either within ``mlr put`` or ``mlr put``.
+* Subroutines may be defined and called either within `mlr put` or `mlr put`.
 
-* Subroutines have read/write access to ``$``-variables and ``@``-variables.
+* Subroutines have read/write access to `$`-variables and `@`-variables.
 
 * Argument values may be reassigned: they are not read-only.
 
-* See the section on :ref:`reference-dsl-local-variables` for information on scope and extent of arguments, as well as for information on the use of local variables within functions.
+* See the section on [local variables](reference-dsl-variables.md#local-variables) for information on scope and extent of arguments, as well as for information on the use of local variables within functions.
 
-* See the section on :ref:`reference-dsl-expressions-from-files` for information on the use of ``-f`` and ``-e`` flags.
+* See the section on [Expressions from files](reference-dsl-syntax.md#expressions-from-files) for information on the use of `-f` and `-e` flags.
diff --git a/docs6b/docs/reference-dsl-user-defined-functions.md.in b/docs6b/docs/reference-dsl-user-defined-functions.md.in
index 8855229fe..5b3764b7c 100644
--- a/docs6b/docs/reference-dsl-user-defined-functions.md.in
+++ b/docs6b/docs/reference-dsl-user-defined-functions.md.in
@@ -10,23 +10,23 @@ GENMD_INCLUDE_AND_RUN_ESCAPED(data/factorial-example.sh)
 
 Properties of user-defined functions:
 
-* Function bodies start with ``func`` and a parameter list, defined outside of ``begin``, ``end``, or other ``func`` or ``subr`` blocks. (I.e. the Miller DSL has no nested functions.)
+* Function bodies start with `func` and a parameter list, defined outside of `begin`, `end`, or other `func` or `subr` blocks. (I.e. the Miller DSL has no nested functions.)
 
-* A function (uniqified by its name) may not be redefined: either by redefining a user-defined function, or by redefining a built-in function. However, functions and subroutines have separate namespaces: you can define a subroutine ``log`` which does not clash with the mathematical ``log`` function.
+* A function (uniqified by its name) may not be redefined: either by redefining a user-defined function, or by redefining a built-in function. However, functions and subroutines have separate namespaces: you can define a subroutine `log` which does not clash with the mathematical `log` function.
 
 * Functions may be defined either before or after use (there is an object-binding/linkage step at startup).  More specifically, functions may be either recursive or mutually recursive. Functions may not call subroutines.
 
-* Functions may be defined and called either within ``mlr put`` or ``mlr put``.
+* Functions may be defined and called either within `mlr put` or `mlr put`.
 
-* Functions have read access to ``$``-variables and ``@``-variables but may not modify them. See also :ref:`cookbook-memoization-with-oosvars` for an example.
+* Functions have read access to `$`-variables and `@`-variables but may not modify them. See also [Memoization with out-of-stream variables](misc-examples.md#memoization-with-out-of-stream-variables) for an example.
 
 * Argument values may be reassigned: they are not read-only.
 
-* When a return value is not implicitly returned, this results in a return value of absent-null. (In the example above, if there were records for which the argument to ``f`` is non-numeric, the assignments would be skipped.) See also the section on [xxxx](reference-main-null-data.md).
+* When a return value is not implicitly returned, this results in a return value of absent-null. (In the example above, if there were records for which the argument to `f` is non-numeric, the assignments would be skipped.) See also the section on [xxxx](reference-main-null-data.md).
 
-* See the section on :ref:`reference-dsl-local-variables` for information on scope and extent of arguments, as well as for information on the use of local variables within functions.
+* See the section on [Local variables](reference-dsl-variables.md#local-variables) for information on scope and extent of arguments, as well as for information on the use of local variables within functions.
 
-* See the section on :ref:`reference-dsl-expressions-from-files` for information on the use of ``-f`` and ``-e`` flags.
+* See the section on [Expressions from files](reference-dsl-syntax.md#expressions-from-files) for information on the use of `-f` and `-e` flags.
 
 ## User-defined subroutines
 
@@ -36,18 +36,18 @@ GENMD_INCLUDE_AND_RUN_ESCAPED(data/subr-example.sh)
 
 Properties of user-defined subroutines:
 
-* Subroutine bodies start with ``subr`` and a parameter list, defined outside of ``begin``, ``end``, or other ``func`` or ``subr`` blocks. (I.e. the Miller DSL has no nested subroutines.)
+* Subroutine bodies start with `subr` and a parameter list, defined outside of `begin`, `end`, or other `func` or `subr` blocks. (I.e. the Miller DSL has no nested subroutines.)
 
-* A subroutine (uniqified by its name) may not be redefined. However, functions and subroutines have separate namespaces: you can define a subroutine ``log`` which does not clash with the mathematical ``log`` function.
+* A subroutine (uniqified by its name) may not be redefined. However, functions and subroutines have separate namespaces: you can define a subroutine `log` which does not clash with the mathematical `log` function.
 
 * Subroutines may be defined either before or after use (there is an object-binding/linkage step at startup).  More specifically, subroutines may be either recursive or mutually recursive. Subroutines may call functions.
 
-* Subroutines may be defined and called either within ``mlr put`` or ``mlr put``.
+* Subroutines may be defined and called either within `mlr put` or `mlr put`.
 
-* Subroutines have read/write access to ``$``-variables and ``@``-variables.
+* Subroutines have read/write access to `$`-variables and `@`-variables.
 
 * Argument values may be reassigned: they are not read-only.
 
-* See the section on :ref:`reference-dsl-local-variables` for information on scope and extent of arguments, as well as for information on the use of local variables within functions.
+* See the section on [local variables](reference-dsl-variables.md#local-variables) for information on scope and extent of arguments, as well as for information on the use of local variables within functions.
 
-* See the section on :ref:`reference-dsl-expressions-from-files` for information on the use of ``-f`` and ``-e`` flags.
+* See the section on [Expressions from files](reference-dsl-syntax.md#expressions-from-files) for information on the use of `-f` and `-e` flags.
diff --git a/docs6b/docs/reference-dsl-variables.md b/docs6b/docs/reference-dsl-variables.md
index b8046fae9..3081c71ea 100644
--- a/docs6b/docs/reference-dsl-variables.md
+++ b/docs6b/docs/reference-dsl-variables.md
@@ -3,11 +3,11 @@
 
 Miller has the following kinds of variables:
 
-**Built-in variables** such as ``NF``, ``NF``, ``FILENAME``, ``M_PI``, and ``M_E``.  These are all capital letters and are read-only (although some of them change value from one record to another).
+**Built-in variables** such as `NF`, `NF`, `FILENAME`, `M_PI`, and `M_E`.  These are all capital letters and are read-only (although some of them change value from one record to another).
 
-**Fields of stream records**, accessed using the ``$`` prefix. These refer to fields of the current data-stream record. For example, in ``echo x=1,y=2 | mlr put '$z = $x + $y'``, ``$x`` and ``$y`` refer to input fields, and ``$z`` refers to a new, computed output field. In a few contexts, presented below, you can refer to the entire record as ``$*``.
+**Fields of stream records**, accessed using the `$` prefix. These refer to fields of the current data-stream record. For example, in `echo x=1,y=2 | mlr put '$z = $x + $y'`, `$x` and `$y` refer to input fields, and `$z` refers to a new, computed output field. In a few contexts, presented below, you can refer to the entire record as `$*`.
 
-**Out-of-stream variables** accessed using the ``@`` prefix. These refer to data which persist from one record to the next, including in ``begin`` and ``end`` blocks (which execute before/after the record stream is consumed, respectively). You use them to remember values across records, such as sums, differences, counters, and so on.  In a few contexts, presented below, you can refer to the entire out-of-stream-variables collection as ``@*``.
+**Out-of-stream variables** accessed using the `@` prefix. These refer to data which persist from one record to the next, including in `begin` and `end` blocks (which execute before/after the record stream is consumed, respectively). You use them to remember values across records, such as sums, differences, counters, and so on.  In a few contexts, presented below, you can refer to the entire out-of-stream-variables collection as `@*`.
 
 **Local variables** are limited in scope and extent to the current statements being executed: these include function arguments, bound variables in for loops, and explicitly declared local variables.
 
@@ -15,19 +15,23 @@ Miller has the following kinds of variables:
 
 ## Built-in variables
 
-These are written all in capital letters, such as ``NR``, ``NF``, ``FILENAME``, and only a small, specific set of them is defined by Miller.
+These are written all in capital letters, such as `NR`, `NF`, `FILENAME`, and only a small, specific set of them is defined by Miller.
 
-Namely, Miller supports the following five built-in variables for :doc:`filter and put `, all ``awk``-inspired: ``NF``, ``NR``, ``FNR``, ``FILENUM``, and ``FILENAME``, as well as the mathematical constants ``M_PI`` and ``M_E``.  Lastly, the ``ENV`` hashmap allows read access to environment variables, e.g.  ``ENV["HOME"]`` or ``ENV["foo_".$hostname]``.
+Namely, Miller supports the following five built-in variables for [filter and put](reference-dsl.md), all `awk`-inspired: `NF`, `NR`, `FNR`, `FILENUM`, and `FILENAME`, as well as the mathematical constants `M_PI` and `M_E`.  Lastly, the `ENV` hashmap allows read access to environment variables, e.g.  `ENV["HOME"]` or `ENV["foo_".$hostname]`.
 
-
+
 mlr filter 'FNR == 2' data/small*
+
+
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 1=pan,2=pan,3=1,4=0.3467901443380824,5=0.7268028627434533
 a=wye,b=eks,i=10000,x=0.734806020620654365,y=0.884788571337605134
 
-
+
 mlr put '$fnr = FNR' data/small*
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,fnr=1
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,fnr=2
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776,fnr=3
@@ -46,19 +50,23 @@ a=hat,b=wye,i=10002,x=0.321507044286237609,y=0.568893318795083758,fnr=4
 a=pan,b=zee,i=10003,x=0.272054845593895200,y=0.425789896597056627,fnr=5
 
-Their values of ``NF``, ``NR``, ``FNR``, ``FILENUM``, and ``FILENAME`` change from one record to the next as Miller scans through your input data stream. The mathematical constants, of course, do not change; ``ENV`` is populated from the system environment variables at the time Miller starts and is read-only for the remainder of program execution. +Their values of `NF`, `NR`, `FNR`, `FILENUM`, and `FILENAME` change from one record to the next as Miller scans through your input data stream. The mathematical constants, of course, do not change; `ENV` is populated from the system environment variables at the time Miller starts and is read-only for the remainder of program execution. -Their **scope is global**: you can refer to them in any ``filter`` or ``put`` statement. Their values are assigned by the input-record reader: +Their **scope is global**: you can refer to them in any `filter` or `put` statement. Their values are assigned by the input-record reader: -
+
 mlr --csv put '$nr = NR' data/a.csv
+
+
 a,b,c,nr
 1,2,3,1
 4,5,6,2
 
-
+
 mlr --csv repeat -n 3 then put '$nr = NR' data/a.csv
+
+
 a,b,c,nr
 1,2,3,1
 1,2,3,1
@@ -68,24 +76,26 @@ a,b,c,nr
 4,5,6,2
 
-The **extent** is for the duration of the put/filter: in a ``begin`` statement (which executes before the fimd.input record is consumed) you will find ``NR=1`` and in an ``end`` statement (which is executed after the last input record is consumed) you will find ``NR`` to be the total number of records ingested. +The **extent** is for the duration of the put/filter: in a `begin` statement (which executes before the fimd.input record is consumed) you will find `NR=1` and in an `end` statement (which is executed after the last input record is consumed) you will find `NR` to be the total number of records ingested. -These are all **read-only** for the ``mlr put`` and ``mlr filter`` DSLs: they may be assigned from, e.g. ``$nr=NR``, but they may not be assigned to: ``NR=100`` is a syntax error. +These are all **read-only** for the `mlr put` and `mlr filter` DSLs: they may be assigned from, e.g. `$nr=NR`, but they may not be assigned to: `NR=100` is a syntax error. ## Field names -Names of fields within stream records must be specified using a ``$`` in :doc:`filter and put expressions `, even though the dollar signs don't appear in the data stream itself. For integer-indexed data, this looks like ``awk``'s ``$1,$2,$3``, except that Miller allows non-numeric names such as ``$quantity`` or ``$hostname``. Likewise, enclose string literals in double quotes in ``filter`` expressions even though they don't appear in file data. In particular, ``mlr filter '$x=="abc"'`` passes through the record ``x=abc``. +Names of fields within stream records must be specified using a `$` in [filter and put expressions](reference-dsl.md), even though the dollar signs don't appear in the data stream itself. For integer-indexed data, this looks like `awk`'s `$1,$2,$3`, except that Miller allows non-numeric names such as `$quantity` or `$hostname`. Likewise, enclose string literals in double quotes in `filter` expressions even though they don't appear in file data. In particular, `mlr filter '$x=="abc"'` passes through the record `x=abc`. -If field names have **special characters** such as ``.`` then you can use braces, e.g. ``'${field.name}'``. +If field names have **special characters** such as `.` then you can use braces, e.g. `'${field.name}'`. You may also use a **computed field name** in square brackets, e.g. -
+
 echo a=3,b=4 | mlr filter '$["x"] < 0.5'
 
-
+
 echo s=green,t=blue,a=3,b=4 | mlr put '$[$s."_".$t] = $a * $b'
+
+
 s=green,t=blue,a=3,b=4,green_blue=12
 
@@ -93,22 +103,24 @@ Notes: The names of record fields depend on the contents of your input data stream, and their values change from one record to the next as Miller scans through your input data stream. -Their **extent** is limited to the current record; their **scope** is the ``filter`` or ``put`` command in which they appear. +Their **extent** is limited to the current record; their **scope** is the `filter` or `put` command in which they appear. -These are **read-write**: you can do ``$y=2*$x``, ``$x=$x+1``, etc. +These are **read-write**: you can do `$y=2*$x`, `$x=$x+1`, etc. -Records are Miller's output: field names present in the input stream are passed through to output (written to standard output) unless fields are removed with ``cut``, or records are excluded with ``filter`` or ``put -q``, etc. Simply assign a value to a field and it will be output. +Records are Miller's output: field names present in the input stream are passed through to output (written to standard output) unless fields are removed with `cut`, or records are excluded with `filter` or `put -q`, etc. Simply assign a value to a field and it will be output. ## Positional field names Even though Miller's main selling point is name-indexing, sometimes you really want to refer to a field name by its positional index (starting from 1). -Use ``$[[3]]`` to access the name of field 3. More generally, any expression evaluating to an integer can go between ``$[[`` and ``]]``. +Use `$[[3]]` to access the name of field 3. More generally, any expression evaluating to an integer can go between `$[[` and `]]`. -Then using a computed field name, ``$[ $[[3]] ]`` is the value in the third field. This has the shorter equivalent notation ``$[[[3]]]``. +Then using a computed field name, `$[ $[[3]] ]` is the value in the third field. This has the shorter equivalent notation `$[[[3]]]`. -
+
 mlr cat data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -116,8 +128,10 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
-
+
 mlr put '$[[3]] = "NEW"' data/small
+
+
 a=pan,b=pan,NEW=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,NEW=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,NEW=3,x=0.20460330576630303,y=0.33831852551664776
@@ -125,8 +139,10 @@ a=eks,b=wye,NEW=4,x=0.38139939387114097,y=0.13418874328430463
 a=wye,b=pan,NEW=5,x=0.5732889198020006,y=0.8636244699032729
 
-
+
 mlr put '$[[[3]]] = "NEW"' data/small
+
+
 a=pan,b=pan,i=NEW,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=NEW,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=NEW,x=0.20460330576630303,y=0.33831852551664776
@@ -134,8 +150,10 @@ a=eks,b=wye,i=NEW,x=0.38139939387114097,y=0.13418874328430463
 a=wye,b=pan,i=NEW,x=0.5732889198020006,y=0.8636244699032729
 
-
+
 mlr put '$NEW = $[[NR]]' data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,NEW=a
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,NEW=b
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776,NEW=i
@@ -143,8 +161,10 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463,NEW=x
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729,NEW=y
 
-
+
 mlr put '$NEW = $[[[NR]]]' data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,NEW=pan
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,NEW=pan
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776,NEW=3
@@ -152,8 +172,10 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463,NEW=0.38139939387114
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729,NEW=0.8636244699032729
 
-
+
 mlr put '$[[[NR]]] = "NEW"' data/small
+
+
 a=NEW,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=NEW,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=NEW,x=0.20460330576630303,y=0.33831852551664776
@@ -161,10 +183,12 @@ a=eks,b=wye,i=4,x=NEW,y=0.13418874328430463
 a=wye,b=pan,i=5,x=0.5732889198020006,y=NEW
 
-Right-hand side accesses to non-existent fields -- i.e. with index less than 1 or greater than ``NF`` -- return an absent value. Likewise, left-hand side accesses only refer to fields which already exist. For example, if a field has 5 records then assigning the name or value of the 6th (or 600th) field results in a no-op. +Right-hand side accesses to non-existent fields -- i.e. with index less than 1 or greater than `NF` -- return an absent value. Likewise, left-hand side accesses only refer to fields which already exist. For example, if a field has 5 records then assigning the name or value of the 6th (or 600th) field results in a no-op. -
+
 mlr put '$[[6]] = "NEW"' data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -172,8 +196,10 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
-
+
 mlr put '$[[[6]]] = "NEW"' data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -183,46 +209,52 @@ a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
 ## Out-of-stream variables
 
-These are prefixed with an at-sign, e.g. ``@sum``.  Furthermore, unlike built-in variables and stream-record fields, they are maintained in an arbitrarily nested hashmap: you can do ``@sum += $quanity``, or ``@sum[$color] += $quanity``, or ``@sum[$color][$shape] += $quanity``. The keys for the multi-level hashmap can be any expression which evaluates to string or integer: e.g.  ``@sum[NR] = $a + $b``, ``@sum[$a."-".$b] = $x``, etc.
+These are prefixed with an at-sign, e.g. `@sum`.  Furthermore, unlike built-in variables and stream-record fields, they are maintained in an arbitrarily nested hashmap: you can do `@sum += $quanity`, or `@sum[$color] += $quanity`, or `@sum[$color][$shape] += $quanity`. The keys for the multi-level hashmap can be any expression which evaluates to string or integer: e.g.  `@sum[NR] = $a + $b`, `@sum[$a."-".$b] = $x`, etc.
 
 Their names and their values are entirely under your control; they change only when you assign to them.
 
-Just as for field names in stream records, if you want to define out-of-stream variables with **special characters** such as ``.`` then you can use braces, e.g. ``'@{variable.name}["index"]'``.
+Just as for field names in stream records, if you want to define out-of-stream variables with **special characters** such as `.` then you can use braces, e.g. `'@{variable.name}["index"]'`.
 
 You may use a **computed key** in square brackets, e.g.
 
-
+
 echo s=green,t=blue,a=3,b=4 | mlr put -q '@[$s."_".$t] = $a * $b; emit all'
+
+
 green_blue=12
 
-Out-of-stream variables are **scoped** to the ``put`` command in which they appear. In particular, if you have two or more ``put`` commands separated by ``then``, each put will have its own set of out-of-stream variables: +Out-of-stream variables are **scoped** to the `put` command in which they appear. In particular, if you have two or more `put` commands separated by `then`, each put will have its own set of out-of-stream variables: -
+
 cat data/a.dkvp
+
+
 a=1,b=2,c=3
 a=4,b=5,c=6
 
-
+
 mlr put '@sum += $a; end {emit @sum}' \
   then put 'is_present($a) {$a=10*$a; @sum += $a}; end {emit @sum}' \
   data/a.dkvp
+
+
 a=10,b=2,c=3
 a=40,b=5,c=6
 sum=5
 sum=50
 
-Out-of-stream variables' **extent** is from the start to the end of the record stream, i.e. every time the ``put`` or ``filter`` statement referring to them is executed. +Out-of-stream variables' **extent** is from the start to the end of the record stream, i.e. every time the `put` or `filter` statement referring to them is executed. -Out-of-stream variables are **read-write**: you can do ``$sum=@sum``, ``@sum=$sum``, etc. +Out-of-stream variables are **read-write**: you can do `$sum=@sum`, `@sum=$sum`, etc. ## Indexed out-of-stream variables -Using an index on the ``@count`` and ``@sum`` variables, we get the benefit of the ``-g`` (group-by) option which ``mlr stats1`` and various other Miller commands have: +Using an index on the `@count` and `@sum` variables, we get the benefit of the `-g` (group-by) option which `mlr stats1` and various other Miller commands have: -
+
 mlr put -q '
   @x_count[$a] += 1;
   @x_sum[$a] += $x;
@@ -231,6 +263,8 @@ Using an index on the ``@count`` and ``@sum`` variables, we get the benefit of t
     emit @x_sum, "a";
   }
 ' ./data/small
+
+
 a=pan,x_count=1
 a=eks,x_count=2
 a=wye,x_count=2
@@ -239,8 +273,10 @@ a=eks,x_sum=1.1400793586611044
 a=wye,x_sum=0.7778922255683036
 
-
+
 mlr stats1 -a count,sum -f x -g a ./data/small
+
+
 a=pan,x_count=1,x_sum=0.3467901443380824
 a=eks,x_count=2,x_sum=1.1400793586611044
 a=wye,x_count=2,x_sum=0.7778922255683036
@@ -248,7 +284,7 @@ a=wye,x_count=2,x_sum=0.7778922255683036
 
 Indices can be arbitrarily deep -- here there are two or more of them:
 
-
+
 mlr --from data/medium put -q '
   @x_count[$a][$b] += 1;
   @x_sum[$a][$b] += $x;
@@ -256,6 +292,8 @@ Indices can be arbitrarily deep -- here there are two or more of them:
     emit (@x_count, @x_sum), "a", "b";
   }
 '
+
+
 a=pan,b=pan,x_count=427,x_sum=219.1851288316854
 a=pan,b=wye,x_count=395,x_sum=198.43293070748447
 a=pan,b=eks,x_count=429,x_sum=216.07522773165525
@@ -283,11 +321,11 @@ a=hat,b=hat,x_count=381,x_sum=182.8535323148762
 a=hat,b=pan,x_count=363,x_sum=168.5538067327806
 
-The idea is that ``stats1``, and other Miller verbs, encapsulate frequently-used patterns with a minimum of keystroking (and run a little faster), whereas using out-of-stream variables you have more flexibility and control in what you do. +The idea is that `stats1`, and other Miller verbs, encapsulate frequently-used patterns with a minimum of keystroking (and run a little faster), whereas using out-of-stream variables you have more flexibility and control in what you do. Begin/end blocks can be mixed with pattern/action blocks. For example: -
+
 mlr put '
   begin {
     @num_total = 0;
@@ -302,6 +340,8 @@ Begin/end blocks can be mixed with pattern/action blocks. For example:
     emitf @num_total, @num_positive
   }
 ' data/put-gating-example-1.dkvp
+
+
 x=-1
 x=0
 x=1,y=0,z=0
@@ -310,15 +350,13 @@ x=3,y=0.4771212547196624,z=0.6907396432228734
 num_total=5,num_positive=3
 
-.. _reference-dsl-local-variables: - ## Local variables -Local variables are similar to out-of-stream variables, except that their extent is limited to the expressions in which they appear (and their basenames can't be computed using square brackets). There are three kinds of local variables: **arguments** to functions/subroutines, **variables bound within for-loops**, and **locals** defined within control blocks. They may be untyped using ``var``, or typed using ``num``, ``int``, ``float``, ``str``, ``bool``, and ``map``. +Local variables are similar to out-of-stream variables, except that their extent is limited to the expressions in which they appear (and their basenames can't be computed using square brackets). There are three kinds of local variables: **arguments** to functions/subroutines, **variables bound within for-loops**, and **locals** defined within control blocks. They may be untyped using `var`, or typed using `num`, `int`, `float`, `str`, `bool`, and `map`. For example: -
+
 # Here I'm using a specified random-number seed so this example always
 # produces the same output for this web document: in everyday practice we
 # would leave off the --seed 12345 part.
@@ -335,6 +373,8 @@ For example:
   num o = f(10, 20);                      # local to the top-level scope
   $o = o;
 '
+
+
 i=1,o=15.952526011537227
 i=2,o=12.782237754999116
 i=3,o=15.126606630220966
@@ -349,32 +389,34 @@ i=10,o=15.37686787628025
 
 Things which are completely unsurprising, resembling many other languages:
 
-* Parameter names are bound to their arguments but can be reassigned, e.g. if there is a parameter named ``a`` then you can reassign the value of ``a`` to be something else within the function if you like.
+* Parameter names are bound to their arguments but can be reassigned, e.g. if there is a parameter named `a` then you can reassign the value of `a` to be something else within the function if you like.
 
-* However, you cannot redeclare the *type* of an argument or a local: ``var a=1; var a=2`` is an error but ``var a=1;  a=2`` is OK.
+* However, you cannot redeclare the *type* of an argument or a local: `var a=1; var a=2` is an error but `var a=1;  a=2` is OK.
 
 * All argument-passing is positional rather than by name; arguments are passed by value, not by reference. (This is also true for map-valued variables: they are not, and cannot be, passed by reference)
 
-* You can define locals (using ``var``, ``num``, etc.) at any scope (if-statements, else-statements, while-loops, for-loops, or the top-level scope), and nested scopes will have access (more details on scope in the next section).  If you define a local variable with the same name inside an inner scope, then a new variable is created with the narrower scope.
+* You can define locals (using `var`, `num`, etc.) at any scope (if-statements, else-statements, while-loops, for-loops, or the top-level scope), and nested scopes will have access (more details on scope in the next section).  If you define a local variable with the same name inside an inner scope, then a new variable is created with the narrower scope.
 
-* If you assign to a local variable for the first time in a scope without declaring it as ``var``, ``num``, etc. then: if it exists in an outer scope, that outer-scope variable will be updated; if not, it will be defined in the current scope as if ``var`` had been used. (See also :ref:`reference-dsl-type-checking` for an example.) I recommend always declaring variables explicitly to make the intended scoping clear.
+* If you assign to a local variable for the first time in a scope without declaring it as `var`, `num`, etc. then: if it exists in an outer scope, that outer-scope variable will be updated; if not, it will be defined in the current scope as if `var` had been used. (See also [Type-checking](reference-dsl-variables.md#type-checking) for an example.) I recommend always declaring variables explicitly to make the intended scoping clear.
 
 * Functions and subroutines never have access to locals from their callee (unless passed by value as arguments).
 
 Things which are perhaps surprising compared to other languages:
 
-* Type declarations using ``var``, or typed using ``num``, ``int``, ``float``, ``str``, and ``bool`` are necessary to declare local variables.  Function arguments and variables bound in for-loops over stream records and out-of-stream variables are *implicitly* declared using ``var``. (Some examples are shown below.)
+* Type declarations using `var`, or typed using `num`, `int`, `float`, `str`, and `bool` are necessary to declare local variables.  Function arguments and variables bound in for-loops over stream records and out-of-stream variables are *implicitly* declared using `var`. (Some examples are shown below.)
 
-* Type-checking is done at assignment time. For example, ``float f = 0`` is an error (since ``0`` is an integer), as is ``float f = 0.0; f = 1``. For this reason I prefer to use ``num`` over ``float`` in most contexts since ``num`` encompasses integer and floating-point values. More information about type-checking is at :ref:`reference-dsl-type-checking`.
+* Type-checking is done at assignment time. For example, `float f = 0` is an error (since `0` is an integer), as is `float f = 0.0; f = 1`. For this reason I prefer to use `num` over `float` in most contexts since `num` encompasses integer and floating-point values. More information is at [Type-checking](reference-dsl-variables.md#type-checking).
 
-* Bound variables in for-loops over stream records and out-of-stream variables are implicitly local to that block. E.g. in ``for (k, v in $*) { ... }`` ``for ((k1, k2), v in @*) { ... }`` if there are ``k``, ``v``, etc. in the enclosing scope then those will be masked by the loop-local bound variables in the loop, and moreover the values of the loop-local bound variables are not available after the end of the loop.
+* Bound variables in for-loops over stream records and out-of-stream variables are implicitly local to that block. E.g. in `for (k, v in $*) { ... }` `for ((k1, k2), v in @*) { ... }` if there are `k`, `v`, etc. in the enclosing scope then those will be masked by the loop-local bound variables in the loop, and moreover the values of the loop-local bound variables are not available after the end of the loop.
 
-* For C-style triple-for loops, if a for-loop variable is defined using ``var``, ``int``, etc. then it is scoped to that for-loop. E.g. ``for (i = 0; i < 10; i += 1) { ... }`` and ``for (int i = 0; i < 10; i += 1) { ... }``. (This is unsurprising.). If there is no typedecl and an outer-scope variable of that name exists, then it is used. (This is also unsurprising.) But of there is no outer-scope variable of that name then the variable is scoped to the for-loop only.
+* For C-style triple-for loops, if a for-loop variable is defined using `var`, `int`, etc. then it is scoped to that for-loop. E.g. `for (i = 0; i < 10; i += 1) { ... }` and `for (int i = 0; i < 10; i += 1) { ... }`. (This is unsurprising.). If there is no typedecl and an outer-scope variable of that name exists, then it is used. (This is also unsurprising.) But of there is no outer-scope variable of that name then the variable is scoped to the for-loop only.
 
 The following example demonstrates the scope rules:
 
-
+
 cat data/scope-example.mlr
+
+
 func f(a) {      # argument is local to the function
   var b = 100;   # local to the function
   c = 100;       # local to the function; does not overwrite outer c
@@ -400,15 +442,19 @@ $outer_c = c;
 $outer_d = d;    # there is no outer d defined so no assignment happens
 
-
+
 cat data/scope-example.dat
+
+
 n=1,x=123
 n=2,x=456
 n=3,x=789
 
-
+
 mlr --oxtab --from data/scope-example.dat put -f data/scope-example.mlr
+
+
 n       1
 x       123
 outer_a 10
@@ -434,8 +480,10 @@ outer_c 60
 
 And this example demonstrates the type-declaration rules:
 
-
+
 cat data/type-decl-example.mlr
+
+
 subr s(a, str b, int c) {                         # a is implicitly var (untyped).
                                                   # b is explicitly str.
                                                   # c is explicitly int.
@@ -477,11 +525,11 @@ print "outer j =" . j;                            # j is undefined in this scope
 
 ## Map literals
 
-Miller's ``put``/``filter`` DSL has four kinds of hashmaps. **Stream records** are (single-level) maps from name to value. **Out-of-stream variables** and **local variables** can also be maps, although they can be multi-level hashmaps (e.g. ``@sum[$x][$y]``).  The fourth kind is **map literals**. These cannot be on the left-hand side of assignment expressions. Syntactically they look like JSON, although Miller allows string and integer keys in its map literals while JSON allows only string keys (e.g. ``"3"`` rather than ``3``).
+Miller's `put`/`filter` DSL has four kinds of hashmaps. **Stream records** are (single-level) maps from name to value. **Out-of-stream variables** and **local variables** can also be maps, although they can be multi-level hashmaps (e.g. `@sum[$x][$y]`).  The fourth kind is **map literals**. These cannot be on the left-hand side of assignment expressions. Syntactically they look like JSON, although Miller allows string and integer keys in its map literals while JSON allows only string keys (e.g. `"3"` rather than `3`).
 
-For example, the following swaps the input stream's ``a`` and ``i`` fields, modifies ``y``, and drops the rest:
+For example, the following swaps the input stream's `a` and `i` fields, modifies `y`, and drops the rest:
 
-
+
 mlr --opprint put '
   $* = {
     "a": $i,
@@ -489,6 +537,8 @@ For example, the following swaps the input stream's ``a`` and ``i`` fields, modi
     "y": $y * 10,
   }
 ' data/small
+
+
 a i   y
 1 pan 7.268028627434533
 2 eks 5.221511083334796
@@ -499,7 +549,7 @@ a i   y
 
 Likewise, you can assign map literals to out-of-stream variables or local variables; pass them as arguments to user-defined functions, return them from functions, and so on:
 
-
+
 mlr --from data/small put '
   func f(map m): map {
     m["x"] *= 200;
@@ -507,6 +557,8 @@ Likewise, you can assign map literals to out-of-stream variables or local variab
   }
   $* = f({"a": $a, "x": $x});
 '
+
+
 a=pan,x=69.35802886761648
 a=eks,x=151.73599295799272
 a=wye,x=40.92066115326061
@@ -516,7 +568,7 @@ a=wye,x=114.65778396040011
 
 Like out-of-stream and local variables, map literals can be multi-level:
 
-
+
 mlr --from data/small put -q '
   begin {
     @o = {
@@ -536,6 +588,8 @@ Like out-of-stream and local variables, map literals can be multi-level:
     dump @o;
   }
 '
+
+
 {
   "nrec": 5,
   "nkey": {
@@ -545,25 +599,22 @@ Like out-of-stream and local variables, map literals can be multi-level:
 }
 
-By default, map-valued expressions are dumped using JSON formatting. If you use ``dump`` to print a hashmap with integer keys and you don't want them double-quoted (JSON-style) then you can use ``mlr put --jknquoteint``. See also ``mlr put --help``. - -.. _reference-dsl-type-checking: +By default, map-valued expressions are dumped using JSON formatting. If you use `dump` to print a hashmap with integer keys and you don't want them double-quoted (JSON-style) then you can use `mlr put --jknquoteint`. See also `mlr put --help`. ## Type-checking -Miller's ``put``/``filter`` DSLs support two optional kinds of type-checking. One is inline **type-tests** and **type-assertions** within expressions. The other is **type declarations** for assignments to local variables, binding of arguments to user-defined functions, and return values from user-defined functions, These are discussed in the following subsections. +Miller's `put`/`filter` DSLs support two optional kinds of type-checking. One is inline **type-tests** and **type-assertions** within expressions. The other is **type declarations** for assignments to local variables, binding of arguments to user-defined functions, and return values from user-defined functions, These are discussed in the following subsections. Use of type-checking is entirely up to you: omit it if you want flexibility with heterogeneous data; use it if you want to help catch misspellings in your DSL code or unexpected irregularities in your input data. -.. _reference-dsl-type-tests-and-assertions: +### Type-test and type-assertion expressions -Type-test and type-assertion expressions -................................................................ +The following `is...` functions take a value and return a boolean indicating whether the argument is of the indicated type. The `assert_...` functions return their argument if it is of the specified type, and cause a fatal error otherwise: -The following ``is...`` functions take a value and return a boolean indicating whether the argument is of the indicated type. The ``assert_...`` functions return their argument if it is of the specified type, and cause a fatal error otherwise: - -
+
 mlr -f | grep ^is
+
+
 is_absent
 is_array
 is_bool
@@ -585,8 +636,10 @@ is_present
 is_string
 
-
+
 mlr -f | grep ^assert
+
+
 asserting_absent
 asserting_array
 asserting_bool
@@ -610,25 +663,24 @@ asserting_string
 
 See [Data-cleaning Examples](data-cleaning-examples.md) for examples of how to use these.
 
-Type-declarations for local variables, function parameter, and function return values
-...............................................................................................
+### Type-declarations for local variables, function parameter, and function return values
 
-Local variables can be defined either untyped as in ``x = 1``, or typed as in ``int x = 1``. Types include **var** (explicitly untyped), **int**, **float**, **num** (int or float), **str**, **bool**, and **map**. These optional type declarations are enforced at the time values are assigned to variables: whether at the initial value assignment as in ``int x = 1`` or in any subsequent assignments to the same variable farther down in the scope.
+Local variables can be defined either untyped as in `x = 1`, or typed as in `int x = 1`. Types include **var** (explicitly untyped), **int**, **float**, **num** (int or float), **str**, **bool**, and **map**. These optional type declarations are enforced at the time values are assigned to variables: whether at the initial value assignment as in `int x = 1` or in any subsequent assignments to the same variable farther down in the scope.
 
-The reason for ``num`` is that ``int`` and ``float`` typedecls are very precise:
+The reason for `num` is that `int` and `float` typedecls are very precise:
 
-
+
 float a = 0;   # Runtime error since 0 is int not float
 int   b = 1.0; # Runtime error since 1.0 is float not int
 num   c = 0;   # OK
 num   d = 1.0; # OK
 
-A suggestion is to use ``num`` for general use when you want numeric content, and use ``int`` when you genuinely want integer-only values, e.g. in loop indices or map keys (since Miller map keys can only be strings or ints). +A suggestion is to use `num` for general use when you want numeric content, and use `int` when you genuinely want integer-only values, e.g. in loop indices or map keys (since Miller map keys can only be strings or ints). -The ``var`` type declaration indicates no type restrictions, e.g. ``var x = 1`` has the same type restrictions on ``x`` as ``x = 1``. The difference is in intentional shadowing: if you have ``x = 1`` in outer scope and ``x = 2`` in inner scope (e.g. within a for-loop or an if-statement) then outer-scope ``x`` has value 2 after the second assignment. But if you have ``var x = 2`` in the inner scope, then you are declaring a variable scoped to the inner block.) For example: +The `var` type declaration indicates no type restrictions, e.g. `var x = 1` has the same type restrictions on `x` as `x = 1`. The difference is in intentional shadowing: if you have `x = 1` in outer scope and `x = 2` in inner scope (e.g. within a for-loop or an if-statement) then outer-scope `x` has value 2 after the second assignment. But if you have `var x = 2` in the inner scope, then you are declaring a variable scoped to the inner block.) For example: -
+
 x = 1;
 if (NR == 4) {
   x = 2; # Refers to outer-scope x: value changes from 1 to 2.
@@ -636,7 +688,7 @@ if (NR == 4) {
 print x; # Value of x is now two
 
-
+
 x = 1;
 if (NR == 4) {
   var x = 2; # Defines a new inner-scope x with value 2
@@ -646,7 +698,7 @@ print x;     # Value of this x is still 1
 
 Likewise function arguments can optionally be typed, with type enforced when the function is called:
 
-
+
 func f(map m, int i) {
   ...
 }
@@ -659,9 +711,9 @@ if (NR == 4) {
 print x;     # Value of this x is still 1
 
-Thirdly, function return values can be type-checked at the point of ``return`` using ``:`` and a typedecl after the parameter list: +Thirdly, function return values can be type-checked at the point of `return` using `:` and a typedecl after the parameter list: -
+
 func f(map m, int i): bool {
   ...
   ...
@@ -688,7 +740,7 @@ Please see [xxxx](reference-main-null-data.md).
 
 ## Aggregate variable assignments
 
-There are three remaining kinds of variable assignment using out-of-stream variables, the last two of which use the ``$*`` syntax:
+There are three remaining kinds of variable assignment using out-of-stream variables, the last two of which use the `$*` syntax:
 
 * Recursive copy of out-of-stream variables
 * Out-of-stream variable assigned to full stream record
@@ -696,8 +748,10 @@ There are three remaining kinds of variable assignment using out-of-stream varia
 
 Example recursive copy of out-of-stream variables:
 
-
+
 mlr --opprint put -q '@v["sum"] += $x; @v["count"] += 1; end{dump; @w = @v; dump}' data/small
+
+
 {
   "v": {
     "sum": 2.264761728567491,
@@ -718,8 +772,10 @@ Example recursive copy of out-of-stream variables:
 
 Example of out-of-stream variable assigned to full stream record, where the 2nd record is stashed, and the 4th record is overwritten with that:
 
-
+
 mlr put 'NR == 2 {@keep = $*}; NR == 4 {$* = @keep}' data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -727,10 +783,12 @@ a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
-Example of full stream record assigned to an out-of-stream variable, finding the record for which the ``x`` field has the largest value in the input stream: +Example of full stream record assigned to an out-of-stream variable, finding the record for which the `x` field has the largest value in the input stream: -
+
 cat data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -738,19 +796,23 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
-
+
 mlr --opprint put -q '
   is_null(@xmax) || $x > @xmax {@xmax=$x; @recmax=$*};
   end {emit @recmax}
 ' data/small
+
+
 a   b   i x                  y
 eks pan 2 0.7586799647899636 0.5221511083334797
 
## Keywords for filter and put -
+
 mlr help usage-keywords
+
+
 all: used in "emit", "emitp", and "unset" as a synonym for @*
 
 begin: defines a block of statements to be executed before input records
diff --git a/docs6b/docs/reference-dsl-variables.md.in b/docs6b/docs/reference-dsl-variables.md.in
index b16704d82..7ebebc225 100644
--- a/docs6b/docs/reference-dsl-variables.md.in
+++ b/docs6b/docs/reference-dsl-variables.md.in
@@ -2,11 +2,11 @@
 
 Miller has the following kinds of variables:
 
-**Built-in variables** such as ``NF``, ``NF``, ``FILENAME``, ``M_PI``, and ``M_E``.  These are all capital letters and are read-only (although some of them change value from one record to another).
+**Built-in variables** such as `NF`, `NF`, `FILENAME`, `M_PI`, and `M_E`.  These are all capital letters and are read-only (although some of them change value from one record to another).
 
-**Fields of stream records**, accessed using the ``$`` prefix. These refer to fields of the current data-stream record. For example, in ``echo x=1,y=2 | mlr put '$z = $x + $y'``, ``$x`` and ``$y`` refer to input fields, and ``$z`` refers to a new, computed output field. In a few contexts, presented below, you can refer to the entire record as ``$*``.
+**Fields of stream records**, accessed using the `$` prefix. These refer to fields of the current data-stream record. For example, in `echo x=1,y=2 | mlr put '$z = $x + $y'`, `$x` and `$y` refer to input fields, and `$z` refers to a new, computed output field. In a few contexts, presented below, you can refer to the entire record as `$*`.
 
-**Out-of-stream variables** accessed using the ``@`` prefix. These refer to data which persist from one record to the next, including in ``begin`` and ``end`` blocks (which execute before/after the record stream is consumed, respectively). You use them to remember values across records, such as sums, differences, counters, and so on.  In a few contexts, presented below, you can refer to the entire out-of-stream-variables collection as ``@*``.
+**Out-of-stream variables** accessed using the `@` prefix. These refer to data which persist from one record to the next, including in `begin` and `end` blocks (which execute before/after the record stream is consumed, respectively). You use them to remember values across records, such as sums, differences, counters, and so on.  In a few contexts, presented below, you can refer to the entire out-of-stream-variables collection as `@*`.
 
 **Local variables** are limited in scope and extent to the current statements being executed: these include function arguments, bound variables in for loops, and explicitly declared local variables.
 
@@ -14,9 +14,9 @@ Miller has the following kinds of variables:
 
 ## Built-in variables
 
-These are written all in capital letters, such as ``NR``, ``NF``, ``FILENAME``, and only a small, specific set of them is defined by Miller.
+These are written all in capital letters, such as `NR`, `NF`, `FILENAME`, and only a small, specific set of them is defined by Miller.
 
-Namely, Miller supports the following five built-in variables for :doc:`filter and put `, all ``awk``-inspired: ``NF``, ``NR``, ``FNR``, ``FILENUM``, and ``FILENAME``, as well as the mathematical constants ``M_PI`` and ``M_E``.  Lastly, the ``ENV`` hashmap allows read access to environment variables, e.g.  ``ENV["HOME"]`` or ``ENV["foo_".$hostname]``.
+Namely, Miller supports the following five built-in variables for [filter and put](reference-dsl.md), all `awk`-inspired: `NF`, `NR`, `FNR`, `FILENUM`, and `FILENAME`, as well as the mathematical constants `M_PI` and `M_E`.  Lastly, the `ENV` hashmap allows read access to environment variables, e.g.  `ENV["HOME"]` or `ENV["foo_".$hostname]`.
 
 GENMD_RUN_COMMAND
 mlr filter 'FNR == 2' data/small*
@@ -26,9 +26,9 @@ GENMD_RUN_COMMAND
 mlr put '$fnr = FNR' data/small*
 GENMD_EOF
 
-Their values of ``NF``, ``NR``, ``FNR``, ``FILENUM``, and ``FILENAME`` change from one record to the next as Miller scans through your input data stream. The mathematical constants, of course, do not change; ``ENV`` is populated from the system environment variables at the time Miller starts and is read-only for the remainder of program execution.
+Their values of `NF`, `NR`, `FNR`, `FILENUM`, and `FILENAME` change from one record to the next as Miller scans through your input data stream. The mathematical constants, of course, do not change; `ENV` is populated from the system environment variables at the time Miller starts and is read-only for the remainder of program execution.
 
-Their **scope is global**: you can refer to them in any ``filter`` or ``put`` statement. Their values are assigned by the input-record reader:
+Their **scope is global**: you can refer to them in any `filter` or `put` statement. Their values are assigned by the input-record reader:
 
 GENMD_RUN_COMMAND
 mlr --csv put '$nr = NR' data/a.csv
@@ -38,15 +38,15 @@ GENMD_RUN_COMMAND
 mlr --csv repeat -n 3 then put '$nr = NR' data/a.csv
 GENMD_EOF
 
-The **extent** is for the duration of the put/filter: in a ``begin`` statement (which executes before the fimd.input record is consumed) you will find ``NR=1`` and in an ``end`` statement (which is executed after the last input record is consumed) you will find ``NR`` to be the total number of records ingested.
+The **extent** is for the duration of the put/filter: in a `begin` statement (which executes before the fimd.input record is consumed) you will find `NR=1` and in an `end` statement (which is executed after the last input record is consumed) you will find `NR` to be the total number of records ingested.
 
-These are all **read-only** for the ``mlr put`` and ``mlr filter`` DSLs: they may be assigned from, e.g. ``$nr=NR``, but they may not be assigned to: ``NR=100`` is a syntax error.
+These are all **read-only** for the `mlr put` and `mlr filter` DSLs: they may be assigned from, e.g. `$nr=NR`, but they may not be assigned to: `NR=100` is a syntax error.
 
 ## Field names
 
-Names of fields within stream records must be specified using a ``$`` in :doc:`filter and put expressions `, even though the dollar signs don't appear in the data stream itself. For integer-indexed data, this looks like ``awk``'s ``$1,$2,$3``, except that Miller allows non-numeric names such as ``$quantity`` or ``$hostname``.  Likewise, enclose string literals in double quotes in ``filter`` expressions even though they don't appear in file data.  In particular, ``mlr filter '$x=="abc"'`` passes through the record ``x=abc``.
+Names of fields within stream records must be specified using a `$` in [filter and put expressions](reference-dsl.md), even though the dollar signs don't appear in the data stream itself. For integer-indexed data, this looks like `awk`'s `$1,$2,$3`, except that Miller allows non-numeric names such as `$quantity` or `$hostname`.  Likewise, enclose string literals in double quotes in `filter` expressions even though they don't appear in file data.  In particular, `mlr filter '$x=="abc"'` passes through the record `x=abc`.
 
-If field names have **special characters** such as ``.`` then you can use braces, e.g. ``'${field.name}'``.
+If field names have **special characters** such as `.` then you can use braces, e.g. `'${field.name}'`.
 
 You may also use a **computed field name** in square brackets, e.g.
 
@@ -62,19 +62,19 @@ Notes:
 
 The names of record fields depend on the contents of your input data stream, and their values change from one record to the next as Miller scans through your input data stream.
 
-Their **extent** is limited to the current record; their **scope** is the ``filter`` or ``put`` command in which they appear.
+Their **extent** is limited to the current record; their **scope** is the `filter` or `put` command in which they appear.
 
-These are **read-write**: you can do ``$y=2*$x``, ``$x=$x+1``, etc.
+These are **read-write**: you can do `$y=2*$x`, `$x=$x+1`, etc.
 
-Records are Miller's output: field names present in the input stream are passed through to output (written to standard output) unless fields are removed with ``cut``, or records are excluded with ``filter`` or ``put -q``, etc. Simply assign a value to a field and it will be output.
+Records are Miller's output: field names present in the input stream are passed through to output (written to standard output) unless fields are removed with `cut`, or records are excluded with `filter` or `put -q`, etc. Simply assign a value to a field and it will be output.
 
 ## Positional field names
 
 Even though Miller's main selling point is name-indexing, sometimes you really want to refer to a field name by its positional index (starting from 1).
 
-Use ``$[[3]]`` to access the name of field 3.  More generally, any expression evaluating to an integer can go between ``$[[`` and ``]]``.
+Use `$[[3]]` to access the name of field 3.  More generally, any expression evaluating to an integer can go between `$[[` and `]]`.
 
-Then using a computed field name, ``$[ $[[3]] ]`` is the value in the third field. This has the shorter equivalent notation ``$[[[3]]]``.
+Then using a computed field name, `$[ $[[3]] ]` is the value in the third field. This has the shorter equivalent notation `$[[[3]]]`.
 
 GENMD_RUN_COMMAND
 mlr cat data/small
@@ -100,7 +100,7 @@ GENMD_RUN_COMMAND
 mlr put '$[[[NR]]] = "NEW"' data/small
 GENMD_EOF
 
-Right-hand side accesses to non-existent fields -- i.e. with index less than 1 or greater than ``NF`` -- return an absent value. Likewise, left-hand side accesses only refer to fields which already exist. For example, if a field has 5 records then assigning the name or value of the 6th (or 600th) field results in a no-op.
+Right-hand side accesses to non-existent fields -- i.e. with index less than 1 or greater than `NF` -- return an absent value. Likewise, left-hand side accesses only refer to fields which already exist. For example, if a field has 5 records then assigning the name or value of the 6th (or 600th) field results in a no-op.
 
 GENMD_RUN_COMMAND
 mlr put '$[[6]] = "NEW"' data/small
@@ -112,11 +112,11 @@ GENMD_EOF
 
 ## Out-of-stream variables
 
-These are prefixed with an at-sign, e.g. ``@sum``.  Furthermore, unlike built-in variables and stream-record fields, they are maintained in an arbitrarily nested hashmap: you can do ``@sum += $quanity``, or ``@sum[$color] += $quanity``, or ``@sum[$color][$shape] += $quanity``. The keys for the multi-level hashmap can be any expression which evaluates to string or integer: e.g.  ``@sum[NR] = $a + $b``, ``@sum[$a."-".$b] = $x``, etc.
+These are prefixed with an at-sign, e.g. `@sum`.  Furthermore, unlike built-in variables and stream-record fields, they are maintained in an arbitrarily nested hashmap: you can do `@sum += $quanity`, or `@sum[$color] += $quanity`, or `@sum[$color][$shape] += $quanity`. The keys for the multi-level hashmap can be any expression which evaluates to string or integer: e.g.  `@sum[NR] = $a + $b`, `@sum[$a."-".$b] = $x`, etc.
 
 Their names and their values are entirely under your control; they change only when you assign to them.
 
-Just as for field names in stream records, if you want to define out-of-stream variables with **special characters** such as ``.`` then you can use braces, e.g. ``'@{variable.name}["index"]'``.
+Just as for field names in stream records, if you want to define out-of-stream variables with **special characters** such as `.` then you can use braces, e.g. `'@{variable.name}["index"]'`.
 
 You may use a **computed key** in square brackets, e.g.
 
@@ -124,7 +124,7 @@ GENMD_RUN_COMMAND
 echo s=green,t=blue,a=3,b=4 | mlr put -q '@[$s."_".$t] = $a * $b; emit all'
 GENMD_EOF
 
-Out-of-stream variables are **scoped** to the ``put`` command in which they appear.  In particular, if you have two or more ``put`` commands separated by ``then``, each put will have its own set of out-of-stream variables:
+Out-of-stream variables are **scoped** to the `put` command in which they appear.  In particular, if you have two or more `put` commands separated by `then`, each put will have its own set of out-of-stream variables:
 
 GENMD_RUN_COMMAND
 cat data/a.dkvp
@@ -136,13 +136,13 @@ mlr put '@sum += $a; end {emit @sum}' \
   data/a.dkvp
 GENMD_EOF
 
-Out-of-stream variables' **extent** is from the start to the end of the record stream, i.e. every time the ``put`` or ``filter`` statement referring to them is executed.
+Out-of-stream variables' **extent** is from the start to the end of the record stream, i.e. every time the `put` or `filter` statement referring to them is executed.
 
-Out-of-stream variables are **read-write**: you can do ``$sum=@sum``, ``@sum=$sum``, etc.
+Out-of-stream variables are **read-write**: you can do `$sum=@sum`, `@sum=$sum`, etc.
 
 ## Indexed out-of-stream variables
 
-Using an index on the ``@count`` and ``@sum`` variables, we get the benefit of the ``-g`` (group-by) option which ``mlr stats1`` and various other Miller commands have:
+Using an index on the `@count` and `@sum` variables, we get the benefit of the `-g` (group-by) option which `mlr stats1` and various other Miller commands have:
 
 GENMD_INCLUDE_AND_RUN_ESCAPED(data/begin-end-example-6.sh)
 
@@ -152,17 +152,15 @@ Indices can be arbitrarily deep -- here there are two or more of them:
 
 GENMD_INCLUDE_AND_RUN_ESCAPED(data/begin-end-example-6a.sh)
 
-The idea is that ``stats1``, and other Miller verbs, encapsulate frequently-used patterns with a minimum of keystroking (and run a little faster), whereas using out-of-stream variables you have more flexibility and control in what you do.
+The idea is that `stats1`, and other Miller verbs, encapsulate frequently-used patterns with a minimum of keystroking (and run a little faster), whereas using out-of-stream variables you have more flexibility and control in what you do.
 
 Begin/end blocks can be mixed with pattern/action blocks. For example:
 
 GENMD_INCLUDE_AND_RUN_ESCAPED(data/begin-end-example-8.sh)
 
-.. _reference-dsl-local-variables:
-
 ## Local variables
 
-Local variables are similar to out-of-stream variables, except that their extent is limited to the expressions in which they appear (and their basenames can't be computed using square brackets). There are three kinds of local variables: **arguments** to functions/subroutines, **variables bound within for-loops**, and **locals** defined within control blocks. They may be untyped using ``var``, or typed using ``num``, ``int``, ``float``, ``str``, ``bool``, and ``map``.
+Local variables are similar to out-of-stream variables, except that their extent is limited to the expressions in which they appear (and their basenames can't be computed using square brackets). There are three kinds of local variables: **arguments** to functions/subroutines, **variables bound within for-loops**, and **locals** defined within control blocks. They may be untyped using `var`, or typed using `num`, `int`, `float`, `str`, `bool`, and `map`.
 
 For example:
 
@@ -170,27 +168,27 @@ GENMD_INCLUDE_AND_RUN_ESCAPED(data/local-example-1.sh)
 
 Things which are completely unsurprising, resembling many other languages:
 
-* Parameter names are bound to their arguments but can be reassigned, e.g. if there is a parameter named ``a`` then you can reassign the value of ``a`` to be something else within the function if you like.
+* Parameter names are bound to their arguments but can be reassigned, e.g. if there is a parameter named `a` then you can reassign the value of `a` to be something else within the function if you like.
 
-* However, you cannot redeclare the *type* of an argument or a local: ``var a=1; var a=2`` is an error but ``var a=1;  a=2`` is OK.
+* However, you cannot redeclare the *type* of an argument or a local: `var a=1; var a=2` is an error but `var a=1;  a=2` is OK.
 
 * All argument-passing is positional rather than by name; arguments are passed by value, not by reference. (This is also true for map-valued variables: they are not, and cannot be, passed by reference)
 
-* You can define locals (using ``var``, ``num``, etc.) at any scope (if-statements, else-statements, while-loops, for-loops, or the top-level scope), and nested scopes will have access (more details on scope in the next section).  If you define a local variable with the same name inside an inner scope, then a new variable is created with the narrower scope.
+* You can define locals (using `var`, `num`, etc.) at any scope (if-statements, else-statements, while-loops, for-loops, or the top-level scope), and nested scopes will have access (more details on scope in the next section).  If you define a local variable with the same name inside an inner scope, then a new variable is created with the narrower scope.
 
-* If you assign to a local variable for the first time in a scope without declaring it as ``var``, ``num``, etc. then: if it exists in an outer scope, that outer-scope variable will be updated; if not, it will be defined in the current scope as if ``var`` had been used. (See also :ref:`reference-dsl-type-checking` for an example.) I recommend always declaring variables explicitly to make the intended scoping clear.
+* If you assign to a local variable for the first time in a scope without declaring it as `var`, `num`, etc. then: if it exists in an outer scope, that outer-scope variable will be updated; if not, it will be defined in the current scope as if `var` had been used. (See also [Type-checking](reference-dsl-variables.md#type-checking) for an example.) I recommend always declaring variables explicitly to make the intended scoping clear.
 
 * Functions and subroutines never have access to locals from their callee (unless passed by value as arguments).
 
 Things which are perhaps surprising compared to other languages:
 
-* Type declarations using ``var``, or typed using ``num``, ``int``, ``float``, ``str``, and ``bool`` are necessary to declare local variables.  Function arguments and variables bound in for-loops over stream records and out-of-stream variables are *implicitly* declared using ``var``. (Some examples are shown below.)
+* Type declarations using `var`, or typed using `num`, `int`, `float`, `str`, and `bool` are necessary to declare local variables.  Function arguments and variables bound in for-loops over stream records and out-of-stream variables are *implicitly* declared using `var`. (Some examples are shown below.)
 
-* Type-checking is done at assignment time. For example, ``float f = 0`` is an error (since ``0`` is an integer), as is ``float f = 0.0; f = 1``. For this reason I prefer to use ``num`` over ``float`` in most contexts since ``num`` encompasses integer and floating-point values. More information about type-checking is at :ref:`reference-dsl-type-checking`.
+* Type-checking is done at assignment time. For example, `float f = 0` is an error (since `0` is an integer), as is `float f = 0.0; f = 1`. For this reason I prefer to use `num` over `float` in most contexts since `num` encompasses integer and floating-point values. More information is at [Type-checking](reference-dsl-variables.md#type-checking).
 
-* Bound variables in for-loops over stream records and out-of-stream variables are implicitly local to that block. E.g. in ``for (k, v in $*) { ... }`` ``for ((k1, k2), v in @*) { ... }`` if there are ``k``, ``v``, etc. in the enclosing scope then those will be masked by the loop-local bound variables in the loop, and moreover the values of the loop-local bound variables are not available after the end of the loop.
+* Bound variables in for-loops over stream records and out-of-stream variables are implicitly local to that block. E.g. in `for (k, v in $*) { ... }` `for ((k1, k2), v in @*) { ... }` if there are `k`, `v`, etc. in the enclosing scope then those will be masked by the loop-local bound variables in the loop, and moreover the values of the loop-local bound variables are not available after the end of the loop.
 
-* For C-style triple-for loops, if a for-loop variable is defined using ``var``, ``int``, etc. then it is scoped to that for-loop. E.g. ``for (i = 0; i < 10; i += 1) { ... }`` and ``for (int i = 0; i < 10; i += 1) { ... }``. (This is unsurprising.). If there is no typedecl and an outer-scope variable of that name exists, then it is used. (This is also unsurprising.) But of there is no outer-scope variable of that name then the variable is scoped to the for-loop only.
+* For C-style triple-for loops, if a for-loop variable is defined using `var`, `int`, etc. then it is scoped to that for-loop. E.g. `for (i = 0; i < 10; i += 1) { ... }` and `for (int i = 0; i < 10; i += 1) { ... }`. (This is unsurprising.). If there is no typedecl and an outer-scope variable of that name exists, then it is used. (This is also unsurprising.) But of there is no outer-scope variable of that name then the variable is scoped to the for-loop only.
 
 The following example demonstrates the scope rules:
 
@@ -214,9 +212,9 @@ GENMD_EOF
 
 ## Map literals
 
-Miller's ``put``/``filter`` DSL has four kinds of hashmaps. **Stream records** are (single-level) maps from name to value. **Out-of-stream variables** and **local variables** can also be maps, although they can be multi-level hashmaps (e.g. ``@sum[$x][$y]``).  The fourth kind is **map literals**. These cannot be on the left-hand side of assignment expressions. Syntactically they look like JSON, although Miller allows string and integer keys in its map literals while JSON allows only string keys (e.g. ``"3"`` rather than ``3``).
+Miller's `put`/`filter` DSL has four kinds of hashmaps. **Stream records** are (single-level) maps from name to value. **Out-of-stream variables** and **local variables** can also be maps, although they can be multi-level hashmaps (e.g. `@sum[$x][$y]`).  The fourth kind is **map literals**. These cannot be on the left-hand side of assignment expressions. Syntactically they look like JSON, although Miller allows string and integer keys in its map literals while JSON allows only string keys (e.g. `"3"` rather than `3`).
 
-For example, the following swaps the input stream's ``a`` and ``i`` fields, modifies ``y``, and drops the rest:
+For example, the following swaps the input stream's `a` and `i` fields, modifies `y`, and drops the rest:
 
 GENMD_INCLUDE_AND_RUN_ESCAPED(data/map-literal-example-1.sh)
 
@@ -228,22 +226,17 @@ Like out-of-stream and local variables, map literals can be multi-level:
 
 GENMD_INCLUDE_AND_RUN_ESCAPED(data/map-literal-example-3.sh)
 
-By default, map-valued expressions are dumped using JSON formatting. If you use ``dump`` to print a hashmap with integer keys and you don't want them double-quoted (JSON-style) then you can use ``mlr put --jknquoteint``. See also ``mlr put --help``.
-
-.. _reference-dsl-type-checking:
+By default, map-valued expressions are dumped using JSON formatting. If you use `dump` to print a hashmap with integer keys and you don't want them double-quoted (JSON-style) then you can use `mlr put --jknquoteint`. See also `mlr put --help`.
 
 ## Type-checking
 
-Miller's ``put``/``filter`` DSLs support two optional kinds of type-checking.  One is inline **type-tests** and **type-assertions** within expressions.  The other is **type declarations** for assignments to local variables, binding of arguments to user-defined functions, and return values from user-defined functions, These are discussed in the following subsections.
+Miller's `put`/`filter` DSLs support two optional kinds of type-checking.  One is inline **type-tests** and **type-assertions** within expressions.  The other is **type declarations** for assignments to local variables, binding of arguments to user-defined functions, and return values from user-defined functions, These are discussed in the following subsections.
 
 Use of type-checking is entirely up to you: omit it if you want flexibility with heterogeneous data; use it if you want to help catch misspellings in your DSL code or unexpected irregularities in your input data.
 
-.. _reference-dsl-type-tests-and-assertions:
+### Type-test and type-assertion expressions
 
-Type-test and type-assertion expressions
-................................................................
-
-The following ``is...`` functions take a value and return a boolean indicating whether the argument is of the indicated type. The ``assert_...`` functions return their argument if it is of the specified type, and cause a fatal error otherwise:
+The following `is...` functions take a value and return a boolean indicating whether the argument is of the indicated type. The `assert_...` functions return their argument if it is of the specified type, and cause a fatal error otherwise:
 
 GENMD_RUN_COMMAND
 mlr -f | grep ^is
@@ -255,12 +248,11 @@ GENMD_EOF
 
 See [Data-cleaning Examples](data-cleaning-examples.md) for examples of how to use these.
 
-Type-declarations for local variables, function parameter, and function return values
-...............................................................................................
+### Type-declarations for local variables, function parameter, and function return values
 
-Local variables can be defined either untyped as in ``x = 1``, or typed as in ``int x = 1``. Types include **var** (explicitly untyped), **int**, **float**, **num** (int or float), **str**, **bool**, and **map**. These optional type declarations are enforced at the time values are assigned to variables: whether at the initial value assignment as in ``int x = 1`` or in any subsequent assignments to the same variable farther down in the scope.
+Local variables can be defined either untyped as in `x = 1`, or typed as in `int x = 1`. Types include **var** (explicitly untyped), **int**, **float**, **num** (int or float), **str**, **bool**, and **map**. These optional type declarations are enforced at the time values are assigned to variables: whether at the initial value assignment as in `int x = 1` or in any subsequent assignments to the same variable farther down in the scope.
 
-The reason for ``num`` is that ``int`` and ``float`` typedecls are very precise:
+The reason for `num` is that `int` and `float` typedecls are very precise:
 
 GENMD_CARDIFY
 float a = 0;   # Runtime error since 0 is int not float
@@ -269,9 +261,9 @@ num   c = 0;   # OK
 num   d = 1.0; # OK
 GENMD_EOF
 
-A suggestion is to use ``num`` for general use when you want numeric content, and use ``int`` when you genuinely want integer-only values, e.g. in loop indices or map keys (since Miller map keys can only be strings or ints).
+A suggestion is to use `num` for general use when you want numeric content, and use `int` when you genuinely want integer-only values, e.g. in loop indices or map keys (since Miller map keys can only be strings or ints).
 
-The ``var`` type declaration indicates no type restrictions, e.g. ``var x = 1`` has the same type restrictions on ``x`` as ``x = 1``. The difference is in intentional shadowing: if you have ``x = 1`` in outer scope and ``x = 2`` in inner scope (e.g. within a for-loop or an if-statement) then outer-scope ``x`` has value 2 after the second assignment.  But if you have ``var x = 2`` in the inner scope, then you are declaring a variable scoped to the inner block.) For example:
+The `var` type declaration indicates no type restrictions, e.g. `var x = 1` has the same type restrictions on `x` as `x = 1`. The difference is in intentional shadowing: if you have `x = 1` in outer scope and `x = 2` in inner scope (e.g. within a for-loop or an if-statement) then outer-scope `x` has value 2 after the second assignment.  But if you have `var x = 2` in the inner scope, then you are declaring a variable scoped to the inner block.) For example:
 
 GENMD_CARDIFY
 x = 1;
@@ -304,7 +296,7 @@ if (NR == 4) {
 print x;     # Value of this x is still 1
 GENMD_EOF
 
-Thirdly, function return values can be type-checked at the point of ``return`` using ``:`` and a typedecl after the parameter list:
+Thirdly, function return values can be type-checked at the point of `return` using `:` and a typedecl after the parameter list:
 
 GENMD_CARDIFY
 func f(map m, int i): bool {
@@ -333,7 +325,7 @@ Please see [xxxx](reference-main-null-data.md).
 
 ## Aggregate variable assignments
 
-There are three remaining kinds of variable assignment using out-of-stream variables, the last two of which use the ``$*`` syntax:
+There are three remaining kinds of variable assignment using out-of-stream variables, the last two of which use the `$*` syntax:
 
 * Recursive copy of out-of-stream variables
 * Out-of-stream variable assigned to full stream record
@@ -351,7 +343,7 @@ GENMD_RUN_COMMAND
 mlr put 'NR == 2 {@keep = $*}; NR == 4 {$* = @keep}' data/small
 GENMD_EOF
 
-Example of full stream record assigned to an out-of-stream variable, finding the record for which the ``x`` field has the largest value in the input stream:
+Example of full stream record assigned to an out-of-stream variable, finding the record for which the `x` field has the largest value in the input stream:
 
 GENMD_RUN_COMMAND
 cat data/small
diff --git a/docs6b/docs/reference-dsl.md b/docs6b/docs/reference-dsl.md
index a0416fe23..234127e57 100644
--- a/docs6b/docs/reference-dsl.md
+++ b/docs6b/docs/reference-dsl.md
@@ -3,12 +3,14 @@
 
 ## Overview
 
-Here's comparison of verbs and ``put``/``filter`` DSL expressions:
+Here's comparison of verbs and `put`/`filter` DSL expressions:
 
 Example:
 
-
+
 mlr stats1 -a sum -f x -g a data/small
+
+
 a=pan,x_sum=0.3467901443380824
 a=eks,x_sum=1.1400793586611044
 a=wye,x_sum=0.7778922255683036
@@ -22,8 +24,10 @@ a=wye,x_sum=0.7778922255683036
 
 Example:
 
-
+
 mlr  put -q '@x_sum[$a] += $x; end{emit @x_sum, "a"}' data/small
+
+
 a=pan,x_sum=0.3467901443380824
 a=eks,x_sum=1.1400793586611044
 a=wye,x_sum=0.7778922255683036
@@ -35,12 +39,14 @@ a=wye,x_sum=0.7778922255683036
 * There is more to learn
 * They are highly customizable
 
-Please see [Verbs Reference](reference-verbs.md) for information on verbs other than ``put`` and ``filter``.
+Please see [Verbs Reference](reference-verbs.md) for information on verbs other than `put` and `filter`.
 
-The essential usages of ``mlr filter`` and ``mlr put`` are for record-selection and record-updating expressions, respectively. For example, given the following input data:
+The essential usages of `mlr filter` and `mlr put` are for record-selection and record-updating expressions, respectively. For example, given the following input data:
 
-
+
 cat data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -48,18 +54,22 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
-you might retain only the records whose ``a`` field has value ``eks``: +you might retain only the records whose `a` field has value `eks`: -
+
 mlr filter '$a == "eks"' data/small
+
+
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
 
or you might add a new field which is a function of existing fields: -
+
 mlr put '$ab = $a . "_" . $b ' data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,ab=pan_pan
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,ab=eks_pan
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776,ab=wye_wye
@@ -67,13 +77,13 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463,ab=eks_wye
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729,ab=wye_pan
 
-The two verbs ``mlr filter`` and ``mlr put`` are essentially the same. The only differences are: +The two verbs `mlr filter` and `mlr put` are essentially the same. The only differences are: -* Expressions sent to ``mlr filter`` must end with a boolean expression, which is the filtering criterion; +* Expressions sent to `mlr filter` must end with a boolean expression, which is the filtering criterion; -* ``mlr filter`` expressions may not reference the ``filter`` keyword within them; and +* `mlr filter` expressions may not reference the `filter` keyword within them; and -* ``mlr filter`` expressions may not use ``tee``, ``emit``, ``emitp``, or ``emitf``. +* `mlr filter` expressions may not use `tee`, `emit`, `emitp`, or `emitf`. All the rest is the same: in particular, you can define and invoke functions and subroutines to help produce the final boolean statement, and record fields may be assigned to in the statements preceding the final boolean statement. diff --git a/docs6b/docs/reference-dsl.md.in b/docs6b/docs/reference-dsl.md.in index 86a8ef207..b2afa4e0b 100644 --- a/docs6b/docs/reference-dsl.md.in +++ b/docs6b/docs/reference-dsl.md.in @@ -2,7 +2,7 @@ ## Overview -Here's comparison of verbs and ``put``/``filter`` DSL expressions: +Here's comparison of verbs and `put`/`filter` DSL expressions: Example: @@ -28,15 +28,15 @@ GENMD_EOF * There is more to learn * They are highly customizable -Please see [Verbs Reference](reference-verbs.md) for information on verbs other than ``put`` and ``filter``. +Please see [Verbs Reference](reference-verbs.md) for information on verbs other than `put` and `filter`. -The essential usages of ``mlr filter`` and ``mlr put`` are for record-selection and record-updating expressions, respectively. For example, given the following input data: +The essential usages of `mlr filter` and `mlr put` are for record-selection and record-updating expressions, respectively. For example, given the following input data: GENMD_RUN_COMMAND cat data/small GENMD_EOF -you might retain only the records whose ``a`` field has value ``eks``: +you might retain only the records whose `a` field has value `eks`: GENMD_RUN_COMMAND mlr filter '$a == "eks"' data/small @@ -48,13 +48,13 @@ GENMD_RUN_COMMAND mlr put '$ab = $a . "_" . $b ' data/small GENMD_EOF -The two verbs ``mlr filter`` and ``mlr put`` are essentially the same. The only differences are: +The two verbs `mlr filter` and `mlr put` are essentially the same. The only differences are: -* Expressions sent to ``mlr filter`` must end with a boolean expression, which is the filtering criterion; +* Expressions sent to `mlr filter` must end with a boolean expression, which is the filtering criterion; -* ``mlr filter`` expressions may not reference the ``filter`` keyword within them; and +* `mlr filter` expressions may not reference the `filter` keyword within them; and -* ``mlr filter`` expressions may not use ``tee``, ``emit``, ``emitp``, or ``emitf``. +* `mlr filter` expressions may not use `tee`, `emit`, `emitp`, or `emitf`. All the rest is the same: in particular, you can define and invoke functions and subroutines to help produce the final boolean statement, and record fields may be assigned to in the statements preceding the final boolean statement. diff --git a/docs6b/docs/reference-main-arithmetic.md b/docs6b/docs/reference-main-arithmetic.md index b206243ab..cbe4a58f2 100644 --- a/docs6b/docs/reference-main-arithmetic.md +++ b/docs6b/docs/reference-main-arithmetic.md @@ -3,13 +3,13 @@ ## Input scanning -Numbers in Miller are double-precision float or 64-bit signed integers. Anything scannable as int, e.g ``123`` or ``0xabcd``, is treated as an integer; otherwise, input scannable as float (``4.56`` or ``8e9``) is treated as float; everything else is a string. +Numbers in Miller are double-precision float or 64-bit signed integers. Anything scannable as int, e.g `123` or `0xabcd`, is treated as an integer; otherwise, input scannable as float (`4.56` or `8e9`) is treated as float; everything else is a string. -If you want all numbers to be treated as floats, then you may use ``float()`` in your filter/put expressions (e.g. replacing ``$c = $a * $b`` with ``$c = float($a) * float($b)``) -- or, more simply, use ``mlr filter -F`` and ``mlr put -F`` which forces all numeric input, whether from expression literals or field values, to float. Likewise ``mlr stats1 -F`` and ``mlr step -F`` force integerable accumulators (such as ``count``) to be done in floating-point. +If you want all numbers to be treated as floats, then you may use `float()` in your filter/put expressions (e.g. replacing `$c = $a * $b` with `$c = float($a) * float($b)`) -- or, more simply, use `mlr filter -F` and `mlr put -F` which forces all numeric input, whether from expression literals or field values, to float. Likewise `mlr stats1 -F` and `mlr step -F` force integerable accumulators (such as `count`) to be done in floating-point. ## Conversion by math routines -For most math functions, integers are cast to float on input, and produce float output: e.g. ``exp(0) = 1.0`` rather than ``1``. The following, however, produce integer output if their inputs are integers: ``+`` ``-`` ``*`` ``/`` ``//`` ``%`` ``abs`` ``ceil`` ``floor`` ``max`` ``min`` ``round`` ``roundm`` ``sgn``. As well, ``stats1 -a min``, ``stats1 -a max``, ``stats1 -a sum``, ``step -a delta``, and ``step -a rsum`` produce integer output if their inputs are integers. +For most math functions, integers are cast to float on input, and produce float output: e.g. `exp(0) = 1.0` rather than `1`. The following, however, produce integer output if their inputs are integers: `+` `-` `*` `/` `//` `%` `abs` `ceil` `floor` `max` `min` `round` `roundm` `sgn`. As well, `stats1 -a min`, `stats1 -a max`, `stats1 -a sum`, `step -a delta`, and `step -a rsum` produce integer output if their inputs are integers. ## Conversion by arithmetic operators @@ -19,9 +19,9 @@ The short of it is that Miller does this transparently for you so you needn't th Implementation details of this, for the interested: integer adds and subtracts overflow by at most one bit so it suffices to check sign-changes. Thus, Miller allows you to add and subtract arbitrary 64-bit signed integers, converting only to float precisely when the result is less than -2\ :sup:`63` or greater than 2\ :sup:`63`\ -1. Multiplies, on the other hand, can overflow by a word size and a sign-change technique does not suffice to detect overflow. Instead Miller tests whether the floating-point product exceeds the representable integer range. Now, 64-bit integers have 64-bit precision while IEEE-doubles have only 52-bit mantissas -- so, there are 53 bits including implicit leading one. The following experiment explicitly demonstrates the resolution at this range: -
+
 64-bit integer     64-bit integer     Casted to double           Back to 64-bit
-in hex           in decimal                                    integer
+in hex             in decimal                                    integer
 0x7ffffffffffff9ff 9223372036854774271 9223372036854773760.000000 0x7ffffffffffff800
 0x7ffffffffffffa00 9223372036854774272 9223372036854773760.000000 0x7ffffffffffff800
 0x7ffffffffffffbff 9223372036854774783 9223372036854774784.000000 0x7ffffffffffffc00
@@ -38,6 +38,6 @@ That is, one cannot check an integer product to see if it is precisely greater t
 
 Division and remainder are [pythonic](http://python-history.blogspot.com/2010/08/why-pythons-integer-division-floors.html):
 
-* Quotient of integers is floating-point: ``7/2`` is ``3.5``.
-* Integer division is done with ``//``: ``7//2`` is ``3``.  This rounds toward the negative.
+* Quotient of integers is floating-point: `7/2` is `3.5`.
+* Integer division is done with `//`: `7//2` is `3`.  This rounds toward the negative.
 * Remainders are non-negative.
diff --git a/docs6b/docs/reference-main-arithmetic.md.in b/docs6b/docs/reference-main-arithmetic.md.in
index 67fc3db59..9f20490e6 100644
--- a/docs6b/docs/reference-main-arithmetic.md.in
+++ b/docs6b/docs/reference-main-arithmetic.md.in
@@ -2,13 +2,13 @@
 
 ## Input scanning
 
-Numbers in Miller are double-precision float or 64-bit signed integers. Anything scannable as int, e.g ``123`` or ``0xabcd``, is treated as an integer; otherwise, input scannable as float (``4.56`` or ``8e9``) is treated as float; everything else is a string.
+Numbers in Miller are double-precision float or 64-bit signed integers. Anything scannable as int, e.g `123` or `0xabcd`, is treated as an integer; otherwise, input scannable as float (`4.56` or `8e9`) is treated as float; everything else is a string.
 
-If you want all numbers to be treated as floats, then you may use ``float()`` in your filter/put expressions (e.g. replacing ``$c = $a * $b`` with ``$c = float($a) * float($b)``) -- or, more simply, use ``mlr filter -F`` and ``mlr put -F`` which forces all numeric input, whether from expression literals or field values, to float. Likewise ``mlr stats1 -F`` and ``mlr step -F`` force integerable accumulators (such as ``count``) to be done in floating-point.
+If you want all numbers to be treated as floats, then you may use `float()` in your filter/put expressions (e.g. replacing `$c = $a * $b` with `$c = float($a) * float($b)`) -- or, more simply, use `mlr filter -F` and `mlr put -F` which forces all numeric input, whether from expression literals or field values, to float. Likewise `mlr stats1 -F` and `mlr step -F` force integerable accumulators (such as `count`) to be done in floating-point.
 
 ## Conversion by math routines
 
-For most math functions, integers are cast to float on input, and produce float output: e.g. ``exp(0) = 1.0`` rather than ``1``.  The following, however, produce integer output if their inputs are integers: ``+`` ``-`` ``*`` ``/`` ``//`` ``%`` ``abs`` ``ceil`` ``floor`` ``max`` ``min`` ``round`` ``roundm`` ``sgn``. As well, ``stats1 -a min``, ``stats1 -a max``, ``stats1 -a sum``, ``step -a delta``, and ``step -a rsum`` produce integer output if their inputs are integers.
+For most math functions, integers are cast to float on input, and produce float output: e.g. `exp(0) = 1.0` rather than `1`.  The following, however, produce integer output if their inputs are integers: `+` `-` `*` `/` `//` `%` `abs` `ceil` `floor` `max` `min` `round` `roundm` `sgn`. As well, `stats1 -a min`, `stats1 -a max`, `stats1 -a sum`, `step -a delta`, and `step -a rsum` produce integer output if their inputs are integers.
 
 ## Conversion by arithmetic operators
 
@@ -20,7 +20,7 @@ Implementation details of this, for the interested: integer adds and subtracts o
 
 GENMD_CARDIFY
 64-bit integer     64-bit integer     Casted to double           Back to 64-bit
-in hex           in decimal                                    integer
+in hex             in decimal                                    integer
 0x7ffffffffffff9ff 9223372036854774271 9223372036854773760.000000 0x7ffffffffffff800
 0x7ffffffffffffa00 9223372036854774272 9223372036854773760.000000 0x7ffffffffffff800
 0x7ffffffffffffbff 9223372036854774783 9223372036854774784.000000 0x7ffffffffffffc00
@@ -37,6 +37,6 @@ That is, one cannot check an integer product to see if it is precisely greater t
 
 Division and remainder are [pythonic](http://python-history.blogspot.com/2010/08/why-pythons-integer-division-floors.html):
 
-* Quotient of integers is floating-point: ``7/2`` is ``3.5``.
-* Integer division is done with ``//``: ``7//2`` is ``3``.  This rounds toward the negative.
+* Quotient of integers is floating-point: `7/2` is `3.5`.
+* Integer division is done with `//`: `7//2` is `3`.  This rounds toward the negative.
 * Remainders are non-negative.
diff --git a/docs6b/docs/reference-main-auxiliary-commands.md b/docs6b/docs/reference-main-auxiliary-commands.md
index 446523c43..ea42583aa 100644
--- a/docs6b/docs/reference-main-auxiliary-commands.md
+++ b/docs6b/docs/reference-main-auxiliary-commands.md
@@ -3,8 +3,10 @@
 
 There are a few nearly-standalone programs which have nothing to do with the rest of Miller, do not participate in record streams, and do not deal with file formats. They might as well be little standalone executables but they're delivered within the main Miller executable for convenience.
 
-
+
 mlr aux-list
+
+
 Available subcommands:
   aux-list
   hex
@@ -17,8 +19,10 @@ Available subcommands:
 For more information, please invoke mlr {subcommand} --help.
 
-
+
 mlr lecat --help
+
+
 Usage: mlr lecat [options] {zero or more file names}
 Simply echoes input, but flags CR characters in red and LF characters in green.
 If zero file names are supplied, standard input is read.
@@ -27,8 +31,10 @@ Options:
 -h or --help: print this message
 
-
+
 mlr termcvt --help
+
+
 Usage: mlr termcvt [option] {zero or more file names}
 Option (exactly one is required):
 --cr2crlf
@@ -43,8 +49,10 @@ Zero file names means read from standard input.
 Output is always to standard output; files are not written in-place.
 
-
+
 mlr hex --help
+
+
 Usage: mlr hex [options] {zero or more file names}
 Simple hex-dump.
 If zero file names are supplied, standard input is read.
@@ -53,8 +61,10 @@ Options:
 -h or --help: print this message
 
-
+
 mlr unhex --help
+
+
 Usage: mlr unhex [option] {zero or more file names}
 Options:
 -h or --help: print this message
@@ -64,18 +74,24 @@ Output is always to standard output; files are not written in-place.
 
 Examples:
 
-
+
 echo 'Hello, world!' | mlr lecat --mono
+
+
 Hello, world![LF]
 
-
+
 echo 'Hello, world!' | mlr termcvt --lf2crlf | mlr lecat --mono
+
+
 Hello, world![CR][LF]
 
-
+
 mlr hex data/budget.csv
+
+
 00000000: 23 20 41 73  61 6e 61 20  2d 2d 20 68  65 72 65 20 |# Asana -- here |
 00000010: 61 72 65 20  74 68 65 20  62 75 64 67  65 74 20 66 |are the budget f|
 00000020: 69 67 75 72  65 73 20 79  6f 75 20 61  73 6b 65 64 |igures you asked|
@@ -85,8 +101,10 @@ Hello, world![CR][LF]
 00000060: 72 61 6e 67  65 2c 31 32  33 2e 34 35  0a          |range,123.45.|
 
-
+
 mlr hex -r data/budget.csv
+
+
 23 20 41 73  61 6e 61 20  2d 2d 20 68  65 72 65 20 
 61 72 65 20  74 68 65 20  62 75 64 67  65 74 20 66 
 69 67 75 72  65 73 20 79  6f 75 20 61  73 6b 65 64 
@@ -96,8 +114,10 @@ Hello, world![CR][LF]
 72 61 6e 67  65 2c 31 32  33 2e 34 35  0a          
 
-
+
 mlr hex -r data/budget.csv | sed 's/20/2a/g' | mlr unhex
+
+
 #*Asana*--*here*are*the*budget*figures*you*asked*for!
 type,quantity
 purple,456.78
diff --git a/docs6b/docs/reference-main-data-types.md b/docs6b/docs/reference-main-data-types.md
index 9385ba29f..df31be902 100644
--- a/docs6b/docs/reference-main-data-types.md
+++ b/docs6b/docs/reference-main-data-types.md
@@ -5,41 +5,41 @@ Miller's input and output are all string-oriented: there is (as of August 2015 a
 
 Field values are treated as numeric for the following:
 
-* Numeric sort: ``mlr sort -n``, ``mlr sort -nr``.
-* Statistics: ``mlr histogram``, ``mlr stats1``, ``mlr stats2``.
-* Cross-record arithmetic: ``mlr step``.
+* Numeric sort: `mlr sort -n`, `mlr sort -nr`.
+* Statistics: `mlr histogram`, `mlr stats1`, `mlr stats2`.
+* Cross-record arithmetic: `mlr step`.
 
-For ``mlr put`` and ``mlr filter``:
+For `mlr put` and `mlr filter`:
 
 * Miller's types for function processing are **empty-null** (empty string), **absent-null** (reads of unset right-hand sides, or fall-through non-explicit return values from user-defined functions), **error**, **string**, **float** (double-precision), **int** (64-bit signed), and **boolean**.
 
-* On input, string values representable as numbers, e.g. "3" or "3.1", are treated as int or float, respectively. If a record has ``x=1,y=2`` then ``mlr put '$z=$x+$y'`` will produce ``x=1,y=2,z=3``, and ``mlr put '$z=$x.$y'`` does not give an error simply because the dot operator has been generalized to stringify non-strings.  To coerce back to string for processing, use the ``string`` function: ``mlr put '$z=string($x).string($y)'`` will produce ``x=1,y=2,z=12``.
+* On input, string values representable as numbers, e.g. "3" or "3.1", are treated as int or float, respectively. If a record has `x=1,y=2` then `mlr put '$z=$x+$y'` will produce `x=1,y=2,z=3`, and `mlr put '$z=$x.$y'` does not give an error simply because the dot operator has been generalized to stringify non-strings.  To coerce back to string for processing, use the `string` function: `mlr put '$z=string($x).string($y)'` will produce `x=1,y=2,z=12`.
 
-* On input, string values representable as boolean  (e.g. ``"true"``, ``"false"``) are *not* automatically treated as boolean.  (This is because ``"true"`` and ``"false"`` are ordinary words, and auto string-to-boolean on a column consisting of words would result in some strings mixed with some booleans.) Use the ``boolean`` function to coerce: e.g. giving the record ``x=1,y=2,w=false`` to ``mlr put '$z=($x<$y) || boolean($w)'``.
+* On input, string values representable as boolean  (e.g. `"true"`, `"false"`) are *not* automatically treated as boolean.  (This is because `"true"` and `"false"` are ordinary words, and auto string-to-boolean on a column consisting of words would result in some strings mixed with some booleans.) Use the `boolean` function to coerce: e.g. giving the record `x=1,y=2,w=false` to `mlr put '$z=($x<$y) || boolean($w)'`.
 
-* Functions take types as described in ``mlr --help-all-functions``: for example, ``log10`` takes float input and produces float output, ``gmt2sec`` maps string to int, and ``sec2gmt`` maps int to string.
+* Functions take types as described in `mlr --help-all-functions`: for example, `log10` takes float input and produces float output, `gmt2sec` maps string to int, and `sec2gmt` maps int to string.
 
-* All math functions described in ``mlr --help-all-functions`` take integer as well as float input.
+* All math functions described in `mlr --help-all-functions` take integer as well as float input.
 
 ## String literals
 
-You can use the following backslash escapes for strings such as between the double quotes in contexts such as ``mlr filter '$name =~ "..."'``, ``mlr put '$name = $othername . "..."'``, ``mlr put '$name = sub($name, "...", "...")``, etc.:
+You can use the following backslash escapes for strings such as between the double quotes in contexts such as `mlr filter '$name =~ "..."'`, `mlr put '$name = $othername . "..."'`, `mlr put '$name = sub($name, "...", "...")`, etc.:
 
-* ``\a``: ASCII code 0x07 (alarm/bell)
-* ``\b``: ASCII code 0x08 (backspace)
-* ``\f``: ASCII code 0x0c (formfeed)
-* ``\n``: ASCII code 0x0a (LF/linefeed/newline)
-* ``\r``: ASCII code 0x0d (CR/carriage return)
-* ``\t``: ASCII code 0x09 (tab)
-* ``\v``: ASCII code 0x0b (vertical tab)
-* ``\\``: backslash
-* ``\"``: double quote
-* ``\123``: Octal 123, etc. for ``\000`` up to ``\377``
-* ``\x7f``: Hexadecimal 7f, etc. for ``\x00`` up to ``\xff``
+* `\a`: ASCII code 0x07 (alarm/bell)
+* `\b`: ASCII code 0x08 (backspace)
+* `\f`: ASCII code 0x0c (formfeed)
+* `\n`: ASCII code 0x0a (LF/linefeed/newline)
+* `\r`: ASCII code 0x0d (CR/carriage return)
+* `\t`: ASCII code 0x09 (tab)
+* `\v`: ASCII code 0x0b (vertical tab)
+* `\\`: backslash
+* `\"`: double quote
+* `\123`: Octal 123, etc. for `\000` up to `\377`
+* `\x7f`: Hexadecimal 7f, etc. for `\x00` up to `\xff`
 
-See also https://en.wikipedia.org/wiki/Escape_sequences_in_C.
+See also [https://en.wikipedia.org/wiki/Escape_sequences_in_C](https://en.wikipedia.org/wiki/Escape_sequences_in_C).
 
-These replacements apply only to strings you key in for the DSL expressions for ``filter`` and ``put``: that is, if you type ``\t`` in a string literal for a ``filter``/``put`` expression, it will be turned into a tab character. If you want a backslash followed by a ``t``, then please type ``\\t``.
+These replacements apply only to strings you key in for the DSL expressions for `filter` and `put`: that is, if you type `\t` in a string literal for a `filter`/`put` expression, it will be turned into a tab character. If you want a backslash followed by a `t`, then please type `\\t`.
 
-However, these replacements are not done automatically within your data stream. If you wish to make these replacements, you can do, for example, for a field named ``field``, ``mlr put '$field = gsub($field, "\\t", "\t")'``. If you need to make such a replacement for all fields in your data, you should probably simply use the system ``sed`` command.
+However, these replacements are not done automatically within your data stream. If you wish to make these replacements, you can do, for example, for a field named `field`, `mlr put '$field = gsub($field, "\\t", "\t")'`. If you need to make such a replacement for all fields in your data, you should probably simply use the system `sed` command.
 
diff --git a/docs6b/docs/reference-main-data-types.md.in b/docs6b/docs/reference-main-data-types.md.in
index ab5a68989..162baa3a5 100644
--- a/docs6b/docs/reference-main-data-types.md.in
+++ b/docs6b/docs/reference-main-data-types.md.in
@@ -4,41 +4,41 @@ Miller's input and output are all string-oriented: there is (as of August 2015 a
 
 Field values are treated as numeric for the following:
 
-* Numeric sort: ``mlr sort -n``, ``mlr sort -nr``.
-* Statistics: ``mlr histogram``, ``mlr stats1``, ``mlr stats2``.
-* Cross-record arithmetic: ``mlr step``.
+* Numeric sort: `mlr sort -n`, `mlr sort -nr`.
+* Statistics: `mlr histogram`, `mlr stats1`, `mlr stats2`.
+* Cross-record arithmetic: `mlr step`.
 
-For ``mlr put`` and ``mlr filter``:
+For `mlr put` and `mlr filter`:
 
 * Miller's types for function processing are **empty-null** (empty string), **absent-null** (reads of unset right-hand sides, or fall-through non-explicit return values from user-defined functions), **error**, **string**, **float** (double-precision), **int** (64-bit signed), and **boolean**.
 
-* On input, string values representable as numbers, e.g. "3" or "3.1", are treated as int or float, respectively. If a record has ``x=1,y=2`` then ``mlr put '$z=$x+$y'`` will produce ``x=1,y=2,z=3``, and ``mlr put '$z=$x.$y'`` does not give an error simply because the dot operator has been generalized to stringify non-strings.  To coerce back to string for processing, use the ``string`` function: ``mlr put '$z=string($x).string($y)'`` will produce ``x=1,y=2,z=12``.
+* On input, string values representable as numbers, e.g. "3" or "3.1", are treated as int or float, respectively. If a record has `x=1,y=2` then `mlr put '$z=$x+$y'` will produce `x=1,y=2,z=3`, and `mlr put '$z=$x.$y'` does not give an error simply because the dot operator has been generalized to stringify non-strings.  To coerce back to string for processing, use the `string` function: `mlr put '$z=string($x).string($y)'` will produce `x=1,y=2,z=12`.
 
-* On input, string values representable as boolean  (e.g. ``"true"``, ``"false"``) are *not* automatically treated as boolean.  (This is because ``"true"`` and ``"false"`` are ordinary words, and auto string-to-boolean on a column consisting of words would result in some strings mixed with some booleans.) Use the ``boolean`` function to coerce: e.g. giving the record ``x=1,y=2,w=false`` to ``mlr put '$z=($x<$y) || boolean($w)'``.
+* On input, string values representable as boolean  (e.g. `"true"`, `"false"`) are *not* automatically treated as boolean.  (This is because `"true"` and `"false"` are ordinary words, and auto string-to-boolean on a column consisting of words would result in some strings mixed with some booleans.) Use the `boolean` function to coerce: e.g. giving the record `x=1,y=2,w=false` to `mlr put '$z=($x<$y) || boolean($w)'`.
 
-* Functions take types as described in ``mlr --help-all-functions``: for example, ``log10`` takes float input and produces float output, ``gmt2sec`` maps string to int, and ``sec2gmt`` maps int to string.
+* Functions take types as described in `mlr --help-all-functions`: for example, `log10` takes float input and produces float output, `gmt2sec` maps string to int, and `sec2gmt` maps int to string.
 
-* All math functions described in ``mlr --help-all-functions`` take integer as well as float input.
+* All math functions described in `mlr --help-all-functions` take integer as well as float input.
 
 ## String literals
 
-You can use the following backslash escapes for strings such as between the double quotes in contexts such as ``mlr filter '$name =~ "..."'``, ``mlr put '$name = $othername . "..."'``, ``mlr put '$name = sub($name, "...", "...")``, etc.:
+You can use the following backslash escapes for strings such as between the double quotes in contexts such as `mlr filter '$name =~ "..."'`, `mlr put '$name = $othername . "..."'`, `mlr put '$name = sub($name, "...", "...")`, etc.:
 
-* ``\a``: ASCII code 0x07 (alarm/bell)
-* ``\b``: ASCII code 0x08 (backspace)
-* ``\f``: ASCII code 0x0c (formfeed)
-* ``\n``: ASCII code 0x0a (LF/linefeed/newline)
-* ``\r``: ASCII code 0x0d (CR/carriage return)
-* ``\t``: ASCII code 0x09 (tab)
-* ``\v``: ASCII code 0x0b (vertical tab)
-* ``\\``: backslash
-* ``\"``: double quote
-* ``\123``: Octal 123, etc. for ``\000`` up to ``\377``
-* ``\x7f``: Hexadecimal 7f, etc. for ``\x00`` up to ``\xff``
+* `\a`: ASCII code 0x07 (alarm/bell)
+* `\b`: ASCII code 0x08 (backspace)
+* `\f`: ASCII code 0x0c (formfeed)
+* `\n`: ASCII code 0x0a (LF/linefeed/newline)
+* `\r`: ASCII code 0x0d (CR/carriage return)
+* `\t`: ASCII code 0x09 (tab)
+* `\v`: ASCII code 0x0b (vertical tab)
+* `\\`: backslash
+* `\"`: double quote
+* `\123`: Octal 123, etc. for `\000` up to `\377`
+* `\x7f`: Hexadecimal 7f, etc. for `\x00` up to `\xff`
 
-See also https://en.wikipedia.org/wiki/Escape_sequences_in_C.
+See also [https://en.wikipedia.org/wiki/Escape_sequences_in_C](https://en.wikipedia.org/wiki/Escape_sequences_in_C).
 
-These replacements apply only to strings you key in for the DSL expressions for ``filter`` and ``put``: that is, if you type ``\t`` in a string literal for a ``filter``/``put`` expression, it will be turned into a tab character. If you want a backslash followed by a ``t``, then please type ``\\t``.
+These replacements apply only to strings you key in for the DSL expressions for `filter` and `put`: that is, if you type `\t` in a string literal for a `filter`/`put` expression, it will be turned into a tab character. If you want a backslash followed by a `t`, then please type `\\t`.
 
-However, these replacements are not done automatically within your data stream. If you wish to make these replacements, you can do, for example, for a field named ``field``, ``mlr put '$field = gsub($field, "\\t", "\t")'``. If you need to make such a replacement for all fields in your data, you should probably simply use the system ``sed`` command.
+However, these replacements are not done automatically within your data stream. If you wish to make these replacements, you can do, for example, for a field named `field`, `mlr put '$field = gsub($field, "\\t", "\t")'`. If you need to make such a replacement for all fields in your data, you should probably simply use the system `sed` command.
 
diff --git a/docs6b/docs/reference-main-env-vars.md b/docs6b/docs/reference-main-env-vars.md
index 5584a4a35..a8abc04bc 100644
--- a/docs6b/docs/reference-main-env-vars.md
+++ b/docs6b/docs/reference-main-env-vars.md
@@ -3,7 +3,7 @@
 
 The following environment variables affect how Miller works:
 
-* ``MLRRC``: see [Customization](customization.md)
-* ``MLR_NO_COLOR``, ``MLR_ALWAYS_COLOR``, ``MLR_KEY_COLOR``, ``MLR_VALUE_COLOR``, ``MLR_PASS_COLOR``, ``MLR_FAIL_COLOR``, ``MLR_REPL_PS1_COLOR``, ``MLR_REPL_PS2_COLOR``, ``MLR_HELP_COLOR``, ``TERM``, * ``MSYSTEM``: see [Output Colorization](output-colorization.md)
-* ``MLR_REPL_PS1``, ``MLR_REPL_PS2``: see [REPL](repl.md)
+* `MLRRC`: see [Customization](customization.md).
+* `MLR_NO_COLOR`, `MLR_ALWAYS_COLOR`, `MLR_KEY_COLOR`, `MLR_VALUE_COLOR`, `MLR_PASS_COLOR`, `MLR_FAIL_COLOR`, `MLR_REPL_PS1_COLOR`, `MLR_REPL_PS2_COLOR`, `MLR_HELP_COLOR`, `TERM`, * `MSYSTEM`: see [Output Colorization](output-colorization.md).
+* `MLR_REPL_PS1`, `MLR_REPL_PS2`: see [REPL](repl.md).
 
diff --git a/docs6b/docs/reference-main-env-vars.md.in b/docs6b/docs/reference-main-env-vars.md.in
index 6517d17c5..d3679d2c5 100644
--- a/docs6b/docs/reference-main-env-vars.md.in
+++ b/docs6b/docs/reference-main-env-vars.md.in
@@ -2,7 +2,7 @@
 
 The following environment variables affect how Miller works:
 
-* ``MLRRC``: see [Customization](customization.md)
-* ``MLR_NO_COLOR``, ``MLR_ALWAYS_COLOR``, ``MLR_KEY_COLOR``, ``MLR_VALUE_COLOR``, ``MLR_PASS_COLOR``, ``MLR_FAIL_COLOR``, ``MLR_REPL_PS1_COLOR``, ``MLR_REPL_PS2_COLOR``, ``MLR_HELP_COLOR``, ``TERM``, * ``MSYSTEM``: see [Output Colorization](output-colorization.md)
-* ``MLR_REPL_PS1``, ``MLR_REPL_PS2``: see [REPL](repl.md)
+* `MLRRC`: see [Customization](customization.md).
+* `MLR_NO_COLOR`, `MLR_ALWAYS_COLOR`, `MLR_KEY_COLOR`, `MLR_VALUE_COLOR`, `MLR_PASS_COLOR`, `MLR_FAIL_COLOR`, `MLR_REPL_PS1_COLOR`, `MLR_REPL_PS2_COLOR`, `MLR_HELP_COLOR`, `TERM`, * `MSYSTEM`: see [Output Colorization](output-colorization.md).
+* `MLR_REPL_PS1`, `MLR_REPL_PS2`: see [REPL](repl.md).
 
diff --git a/docs6b/docs/reference-main-io-options.md b/docs6b/docs/reference-main-io-options.md
index 644b76b5a..04ad3449c 100644
--- a/docs6b/docs/reference-main-io-options.md
+++ b/docs6b/docs/reference-main-io-options.md
@@ -5,7 +5,7 @@
 
 Options:
 
-
+
 --dkvp    --idkvp    --odkvp
 --nidx    --inidx    --onidx
 --csv     --icsv     --ocsv
@@ -15,10 +15,12 @@ Options:
 --json    --ijson    --ojson
 
-These are as discussed in [File Formats](file-formats.md), with the exception of ``--right`` which makes pretty-printed output right-aligned: +These are as discussed in [File Formats](file-formats.md), with the exception of `--right` which makes pretty-printed output right-aligned: -
+
 mlr --opprint cat data/small
+
+
 a   b   i x                   y
 pan pan 1 0.3467901443380824  0.7268028627434533
 eks pan 2 0.7586799647899636  0.5221511083334797
@@ -27,8 +29,10 @@ eks wye 4 0.38139939387114097 0.13418874328430463
 wye pan 5 0.5732889198020006  0.8636244699032729
 
-
+
 mlr --opprint --right cat data/small
+
+
   a   b i                   x                   y 
 pan pan 1  0.3467901443380824  0.7268028627434533 
 eks pan 2  0.7586799647899636  0.5221511083334797 
@@ -39,34 +43,34 @@ wye pan 5  0.5732889198020006  0.8636244699032729
 
 Additional notes:
 
-* Use ``--csv``, ``--pprint``, etc. when the input and output formats are the same.
+* Use `--csv`, `--pprint`, etc. when the input and output formats are the same.
 
-* Use ``--icsv --opprint``, etc. when you want format conversion as part of what Miller does to your data.
+* Use `--icsv --opprint`, etc. when you want format conversion as part of what Miller does to your data.
 
-* DKVP (key-value-pair) format is the default for input and output. So, ``--oxtab`` is the same as ``--idkvp --oxtab``.
+* DKVP (key-value-pair) format is the default for input and output. So, `--oxtab` is the same as `--idkvp --oxtab`.
 
-**Pro-tip:** Please use either **--format1**, or **--iformat1 --oformat2**.  If you use **--format1 --oformat2** then what happens is that flags are set up for input *and* output for format1, some of which are overwritten for output in format2. For technical reasons, having ``--oformat2`` clobber all the output-related effects of ``--format1`` also removes some flexibility from the command-line interface. See also https://github.com/johnkerl/miller/issues/180 and https://github.com/johnkerl/miller/issues/199.
+**Pro-tip:** Please use either **--format1**, or **--iformat1 --oformat2**.  If you use **--format1 --oformat2** then what happens is that flags are set up for input *and* output for format1, some of which are overwritten for output in format2. For technical reasons, having `--oformat2` clobber all the output-related effects of `--format1` also removes some flexibility from the command-line interface. See also [https://github.com/johnkerl/miller/issues/180](https://github.com/johnkerl/miller/issues/180) and [https://github.com/johnkerl/miller/issues/199](https://github.com/johnkerl/miller/issues/199).
 
 ## In-place mode
 
-Use the ``mlr -I`` flag to process files in-place. For example, ``mlr -I --csv cut -x -f unwanted_column_name mydata/*.csv`` will remove ``unwanted_column_name`` from all your ``*.csv`` files in your ``mydata/`` subdirectory.
+Use the `mlr -I` flag to process files in-place. For example, `mlr -I --csv cut -x -f unwanted_column_name mydata/*.csv` will remove `unwanted_column_name` from all your `*.csv` files in your `mydata/` subdirectory.
 
-By default, Miller output goes to the screen (or you can redirect a file using ``>`` or to another process using ``|``). With ``-I``, for each file name on the command line, output is written to a temporary file in the same directory. Miller writes its output into that temp file, which is then renamed over the original.  Then, processing continues on the next file. Each file is processed in isolation: if the output format is CSV, CSV headers will be present in each output file; statistics are only over each file's own records; and so on.
+By default, Miller output goes to the screen (or you can redirect a file using `>` or to another process using `|`). With `-I`, for each file name on the command line, output is written to a temporary file in the same directory. Miller writes its output into that temp file, which is then renamed over the original.  Then, processing continues on the next file. Each file is processed in isolation: if the output format is CSV, CSV headers will be present in each output file; statistics are only over each file's own records; and so on.
 
-Please see :ref:`10min-choices-for-printing-to-files` for examples.
+Please see [Choices for printing to files](10min.md#choices-for-printing-to-files) for examples.
 
 ## Compression
 
 Options:
 
-
+
 --prepipe {command}
 
-The prepipe command is anything which reads from standard input and produces data acceptable to Miller. Nominally this allows you to use whichever decompression utilities you have installed on your system, on a per-file basis. If the command has flags, quote them: e.g. ``mlr --prepipe 'zcat -cf'``. Examples: +The prepipe command is anything which reads from standard input and produces data acceptable to Miller. Nominally this allows you to use whichever decompression utilities you have installed on your system, on a per-file basis. If the command has flags, quote them: e.g. `mlr --prepipe 'zcat -cf'`. Examples: -
+
 # These two produce the same output:
 $ gunzip < myfile1.csv.gz | mlr cut -f hostname,uptime
 $ mlr --prepipe gunzip cut -f hostname,uptime myfile1.csv.gz
@@ -75,69 +79,73 @@ $ mlr --prepipe gunzip cut -f hostname,uptime myfile1.csv.gz myfile2.csv.gz
 $ mlr --prepipe gunzip --idkvp --oxtab cut -f hostname,uptime myfile1.dat.gz myfile2.dat.gz
 
-
+
 # Similar to the above, but with compressed output as well as input:
 $ gunzip < myfile1.csv.gz | mlr cut -f hostname,uptime | gzip > outfile.csv.gz
 $ mlr --prepipe gunzip cut -f hostname,uptime myfile1.csv.gz | gzip > outfile.csv.gz
 $ mlr --prepipe gunzip cut -f hostname,uptime myfile1.csv.gz myfile2.csv.gz | gzip > outfile.csv.gz
 
-
+
 # Similar to the above, but with different compression tools for input and output:
 $ gunzip < myfile1.csv.gz | mlr cut -f hostname,uptime | xz -z > outfile.csv.xz
 $ xz -cd < myfile1.csv.xz | mlr cut -f hostname,uptime | gzip > outfile.csv.xz
 $ mlr --prepipe 'xz -cd' cut -f hostname,uptime myfile1.csv.xz myfile2.csv.xz | xz -z > outfile.csv.xz
 
-.. _reference-separators: - ## Record/field/pair separators -Miller has record separators ``IRS`` and ``ORS``, field separators ``IFS`` and ``OFS``, and pair separators ``IPS`` and ``OPS``. For example, in the DKVP line ``a=1,b=2,c=3``, the record separator is newline, field separator is comma, and pair separator is the equals sign. These are the default values. +Miller has record separators `IRS` and `ORS`, field separators `IFS` and `OFS`, and pair separators `IPS` and `OPS`. For example, in the DKVP line `a=1,b=2,c=3`, the record separator is newline, field separator is comma, and pair separator is the equals sign. These are the default values. Options: -
+
 --rs --irs --ors
 --fs --ifs --ofs --repifs
 --ps --ips --ops
 
-* You can change a separator from input to output via e.g. ``--ifs = --ofs :``. Or, you can specify that the same separator is to be used for input and output via e.g. ``--fs :``. +* You can change a separator from input to output via e.g. `--ifs = --ofs :`. Or, you can specify that the same separator is to be used for input and output via e.g. `--fs :`. * The pair separator is only relevant to DKVP format. * Pretty-print and xtab formats ignore the separator arguments altogether. -* The ``--repifs`` means that multiple successive occurrences of the field separator count as one. For example, in CSV data we often signify nulls by empty strings, e.g. ``2,9,,,,,6,5,4``. On the other hand, if the field separator is a space, it might be more natural to parse ``2 4 5`` the same as ``2 4 5``: ``--repifs --ifs ' '`` lets this happen. In fact, the ``--ipprint`` option above is internally implemented in terms of ``--repifs``. +* The `--repifs` means that multiple successive occurrences of the field separator count as one. For example, in CSV data we often signify nulls by empty strings, e.g. `2,9,,,,,6,5,4`. On the other hand, if the field separator is a space, it might be more natural to parse `2 4 5` the same as `2 4 5`: `--repifs --ifs ' '` lets this happen. In fact, the `--ipprint` option above is internally implemented in terms of `--repifs`. -* Just write out the desired separator, e.g. ``--ofs '|'``. But you may use the symbolic names ``newline``, ``space``, ``tab``, ``pipe``, or ``semicolon`` if you like. +* Just write out the desired separator, e.g. `--ofs '|'`. But you may use the symbolic names `newline`, `space`, `tab`, `pipe`, or `semicolon` if you like. ## Number formatting -The command-line option ``--ofmt {format string}`` is the global number format for commands which generate numeric output, e.g. ``stats1``, ``stats2``, ``histogram``, and ``step``, as well as ``mlr put``. Examples: +The command-line option `--ofmt {format string}` is the global number format for commands which generate numeric output, e.g. `stats1`, `stats2`, `histogram`, and `step`, as well as `mlr put`. Examples: -
+
 --ofmt %.9le  --ofmt %.6lf  --ofmt %.0lf
 
-These are just familiar ``printf`` formats applied to double-precision numbers. Please don't use ``%s`` or ``%d``. Additionally, if you use leading width (e.g. ``%18.12lf``) then the output will contain embedded whitespace, which may not be what you want if you pipe the output to something else, particularly CSV. I use Miller's pretty-print format (``mlr --opprint``) to column-align numerical data. +These are just familiar `printf` formats applied to double-precision numbers. Please don't use `%s` or `%d`. Additionally, if you use leading width (e.g. `%18.12lf`) then the output will contain embedded whitespace, which may not be what you want if you pipe the output to something else, particularly CSV. I use Miller's pretty-print format (`mlr --opprint`) to column-align numerical data. -To apply formatting to a single field, overriding the global ``ofmt``, use ``fmtnum`` function within ``mlr put``. For example: +To apply formatting to a single field, overriding the global `ofmt`, use `fmtnum` function within `mlr put`. For example: -
+
 echo 'x=3.1,y=4.3' | mlr put '$z=fmtnum($x*$y,"%08lf")'
+
+
 x=3.1,y=4.3,z=%!l(float64=00013.33)f
 
-
+
 echo 'x=0xffff,y=0xff' | mlr put '$z=fmtnum(int($x*$y),"%08llx")'
+
+
 x=0xffff,y=0xff,z=%!l(int=16711425)lx
 
-Input conversion from hexadecimal is done automatically on fields handled by ``mlr put`` and ``mlr filter`` as long as the field value begins with "0x". To apply output conversion to hexadecimal on a single column, you may use ``fmtnum``, or the keystroke-saving ``hexfmt`` function. Example: +Input conversion from hexadecimal is done automatically on fields handled by `mlr put` and `mlr filter` as long as the field value begins with "0x". To apply output conversion to hexadecimal on a single column, you may use `fmtnum`, or the keystroke-saving `hexfmt` function. Example: -
+
 echo 'x=0xffff,y=0xff' | mlr put '$z=hexfmt($x*$y)'
+
+
 x=0xffff,y=0xff,z=0xfeff01
 
diff --git a/docs6b/docs/reference-main-io-options.md.in b/docs6b/docs/reference-main-io-options.md.in index b1118eba9..3ffe4ab8f 100644 --- a/docs6b/docs/reference-main-io-options.md.in +++ b/docs6b/docs/reference-main-io-options.md.in @@ -14,7 +14,7 @@ GENMD_CARDIFY --json --ijson --ojson GENMD_EOF -These are as discussed in [File Formats](file-formats.md), with the exception of ``--right`` which makes pretty-printed output right-aligned: +These are as discussed in [File Formats](file-formats.md), with the exception of `--right` which makes pretty-printed output right-aligned: GENMD_RUN_COMMAND mlr --opprint cat data/small @@ -26,21 +26,21 @@ GENMD_EOF Additional notes: -* Use ``--csv``, ``--pprint``, etc. when the input and output formats are the same. +* Use `--csv`, `--pprint`, etc. when the input and output formats are the same. -* Use ``--icsv --opprint``, etc. when you want format conversion as part of what Miller does to your data. +* Use `--icsv --opprint`, etc. when you want format conversion as part of what Miller does to your data. -* DKVP (key-value-pair) format is the default for input and output. So, ``--oxtab`` is the same as ``--idkvp --oxtab``. +* DKVP (key-value-pair) format is the default for input and output. So, `--oxtab` is the same as `--idkvp --oxtab`. -**Pro-tip:** Please use either **--format1**, or **--iformat1 --oformat2**. If you use **--format1 --oformat2** then what happens is that flags are set up for input *and* output for format1, some of which are overwritten for output in format2. For technical reasons, having ``--oformat2`` clobber all the output-related effects of ``--format1`` also removes some flexibility from the command-line interface. See also https://github.com/johnkerl/miller/issues/180 and https://github.com/johnkerl/miller/issues/199. +**Pro-tip:** Please use either **--format1**, or **--iformat1 --oformat2**. If you use **--format1 --oformat2** then what happens is that flags are set up for input *and* output for format1, some of which are overwritten for output in format2. For technical reasons, having `--oformat2` clobber all the output-related effects of `--format1` also removes some flexibility from the command-line interface. See also [https://github.com/johnkerl/miller/issues/180](https://github.com/johnkerl/miller/issues/180) and [https://github.com/johnkerl/miller/issues/199](https://github.com/johnkerl/miller/issues/199). ## In-place mode -Use the ``mlr -I`` flag to process files in-place. For example, ``mlr -I --csv cut -x -f unwanted_column_name mydata/*.csv`` will remove ``unwanted_column_name`` from all your ``*.csv`` files in your ``mydata/`` subdirectory. +Use the `mlr -I` flag to process files in-place. For example, `mlr -I --csv cut -x -f unwanted_column_name mydata/*.csv` will remove `unwanted_column_name` from all your `*.csv` files in your `mydata/` subdirectory. -By default, Miller output goes to the screen (or you can redirect a file using ``>`` or to another process using ``|``). With ``-I``, for each file name on the command line, output is written to a temporary file in the same directory. Miller writes its output into that temp file, which is then renamed over the original. Then, processing continues on the next file. Each file is processed in isolation: if the output format is CSV, CSV headers will be present in each output file; statistics are only over each file's own records; and so on. +By default, Miller output goes to the screen (or you can redirect a file using `>` or to another process using `|`). With `-I`, for each file name on the command line, output is written to a temporary file in the same directory. Miller writes its output into that temp file, which is then renamed over the original. Then, processing continues on the next file. Each file is processed in isolation: if the output format is CSV, CSV headers will be present in each output file; statistics are only over each file's own records; and so on. -Please see :ref:`10min-choices-for-printing-to-files` for examples. +Please see [Choices for printing to files](10min.md#choices-for-printing-to-files) for examples. ## Compression @@ -51,7 +51,7 @@ GENMD_CARDIFY GENMD_EOF -The prepipe command is anything which reads from standard input and produces data acceptable to Miller. Nominally this allows you to use whichever decompression utilities you have installed on your system, on a per-file basis. If the command has flags, quote them: e.g. ``mlr --prepipe 'zcat -cf'``. Examples: +The prepipe command is anything which reads from standard input and produces data acceptable to Miller. Nominally this allows you to use whichever decompression utilities you have installed on your system, on a per-file basis. If the command has flags, quote them: e.g. `mlr --prepipe 'zcat -cf'`. Examples: GENMD_CARDIFY # These two produce the same output: @@ -76,11 +76,9 @@ $ xz -cd < myfile1.csv.xz | mlr cut -f hostname,uptime | gzip > outfile.csv.xz $ mlr --prepipe 'xz -cd' cut -f hostname,uptime myfile1.csv.xz myfile2.csv.xz | xz -z > outfile.csv.xz GENMD_EOF -.. _reference-separators: - ## Record/field/pair separators -Miller has record separators ``IRS`` and ``ORS``, field separators ``IFS`` and ``OFS``, and pair separators ``IPS`` and ``OPS``. For example, in the DKVP line ``a=1,b=2,c=3``, the record separator is newline, field separator is comma, and pair separator is the equals sign. These are the default values. +Miller has record separators `IRS` and `ORS`, field separators `IFS` and `OFS`, and pair separators `IPS` and `OPS`. For example, in the DKVP line `a=1,b=2,c=3`, the record separator is newline, field separator is comma, and pair separator is the equals sign. These are the default values. Options: @@ -90,27 +88,27 @@ GENMD_CARDIFY --ps --ips --ops GENMD_EOF -* You can change a separator from input to output via e.g. ``--ifs = --ofs :``. Or, you can specify that the same separator is to be used for input and output via e.g. ``--fs :``. +* You can change a separator from input to output via e.g. `--ifs = --ofs :`. Or, you can specify that the same separator is to be used for input and output via e.g. `--fs :`. * The pair separator is only relevant to DKVP format. * Pretty-print and xtab formats ignore the separator arguments altogether. -* The ``--repifs`` means that multiple successive occurrences of the field separator count as one. For example, in CSV data we often signify nulls by empty strings, e.g. ``2,9,,,,,6,5,4``. On the other hand, if the field separator is a space, it might be more natural to parse ``2 4 5`` the same as ``2 4 5``: ``--repifs --ifs ' '`` lets this happen. In fact, the ``--ipprint`` option above is internally implemented in terms of ``--repifs``. +* The `--repifs` means that multiple successive occurrences of the field separator count as one. For example, in CSV data we often signify nulls by empty strings, e.g. `2,9,,,,,6,5,4`. On the other hand, if the field separator is a space, it might be more natural to parse `2 4 5` the same as `2 4 5`: `--repifs --ifs ' '` lets this happen. In fact, the `--ipprint` option above is internally implemented in terms of `--repifs`. -* Just write out the desired separator, e.g. ``--ofs '|'``. But you may use the symbolic names ``newline``, ``space``, ``tab``, ``pipe``, or ``semicolon`` if you like. +* Just write out the desired separator, e.g. `--ofs '|'`. But you may use the symbolic names `newline`, `space`, `tab`, `pipe`, or `semicolon` if you like. ## Number formatting -The command-line option ``--ofmt {format string}`` is the global number format for commands which generate numeric output, e.g. ``stats1``, ``stats2``, ``histogram``, and ``step``, as well as ``mlr put``. Examples: +The command-line option `--ofmt {format string}` is the global number format for commands which generate numeric output, e.g. `stats1`, `stats2`, `histogram`, and `step`, as well as `mlr put`. Examples: GENMD_CARDIFY --ofmt %.9le --ofmt %.6lf --ofmt %.0lf GENMD_EOF -These are just familiar ``printf`` formats applied to double-precision numbers. Please don't use ``%s`` or ``%d``. Additionally, if you use leading width (e.g. ``%18.12lf``) then the output will contain embedded whitespace, which may not be what you want if you pipe the output to something else, particularly CSV. I use Miller's pretty-print format (``mlr --opprint``) to column-align numerical data. +These are just familiar `printf` formats applied to double-precision numbers. Please don't use `%s` or `%d`. Additionally, if you use leading width (e.g. `%18.12lf`) then the output will contain embedded whitespace, which may not be what you want if you pipe the output to something else, particularly CSV. I use Miller's pretty-print format (`mlr --opprint`) to column-align numerical data. -To apply formatting to a single field, overriding the global ``ofmt``, use ``fmtnum`` function within ``mlr put``. For example: +To apply formatting to a single field, overriding the global `ofmt`, use `fmtnum` function within `mlr put`. For example: GENMD_RUN_COMMAND echo 'x=3.1,y=4.3' | mlr put '$z=fmtnum($x*$y,"%08lf")' @@ -120,7 +118,7 @@ GENMD_RUN_COMMAND echo 'x=0xffff,y=0xff' | mlr put '$z=fmtnum(int($x*$y),"%08llx")' GENMD_EOF -Input conversion from hexadecimal is done automatically on fields handled by ``mlr put`` and ``mlr filter`` as long as the field value begins with "0x". To apply output conversion to hexadecimal on a single column, you may use ``fmtnum``, or the keystroke-saving ``hexfmt`` function. Example: +Input conversion from hexadecimal is done automatically on fields handled by `mlr put` and `mlr filter` as long as the field value begins with "0x". To apply output conversion to hexadecimal on a single column, you may use `fmtnum`, or the keystroke-saving `hexfmt` function. Example: GENMD_RUN_COMMAND echo 'x=0xffff,y=0xff' | mlr put '$z=hexfmt($x*$y)' diff --git a/docs6b/docs/reference-main-null-data.md b/docs6b/docs/reference-main-null-data.md index 177df6433..1b53bf039 100644 --- a/docs6b/docs/reference-main-null-data.md +++ b/docs6b/docs/reference-main-null-data.md @@ -1,22 +1,24 @@ # Reference: null data -One of Miller's key features is its support for **heterogeneous** data. For example, take ``mlr sort``: if you try to sort on field ``hostname`` when not all records in the data stream *have* a field named ``hostname``, it is not an error (although you could pre-filter the data stream using ``mlr having-fields --at-least hostname then sort ...``). Rather, records lacking one or more sort keys are simply output contiguously by ``mlr sort``. +One of Miller's key features is its support for **heterogeneous** data. For example, take `mlr sort`: if you try to sort on field `hostname` when not all records in the data stream *have* a field named `hostname`, it is not an error (although you could pre-filter the data stream using `mlr having-fields --at-least hostname then sort ...`). Rather, records lacking one or more sort keys are simply output contiguously by `mlr sort`. Miller has two kinds of null data: -* **Empty (key present, value empty)**: a field name is present in a record (or in an out-of-stream variable) with empty value: e.g. ``x=,y=2`` in the data input stream, or assignment ``$x=""`` or ``@x=""`` in ``mlr put``. +* **Empty (key present, value empty)**: a field name is present in a record (or in an out-of-stream variable) with empty value: e.g. `x=,y=2` in the data input stream, or assignment `$x=""` or `@x=""` in `mlr put`. -* **Absent (key not present)**: a field name is not present, e.g. input record is ``x=1,y=2`` and a ``put`` or ``filter`` expression refers to ``$z``. Or, reading an out-of-stream variable which hasn't been assigned a value yet, e.g. ``mlr put -q '@sum += $x; end{emit @sum}'`` or ``mlr put -q '@sum[$a][$b] += $x; end{emit @sum, "a", "b"}'``. +* **Absent (key not present)**: a field name is not present, e.g. input record is `x=1,y=2` and a `put` or `filter` expression refers to `$z`. Or, reading an out-of-stream variable which hasn't been assigned a value yet, e.g. `mlr put -q '@sum += $x; end{emit @sum}'` or `mlr put -q '@sum[$a][$b] += $x; end{emit @sum, "a", "b"}'`. -You can test these programatically using the functions ``is_empty``/``is_not_empty``, ``is_absent``/``is_present``, and ``is_null``/``is_not_null``. For the last pair, note that null means either empty or absent. +You can test these programatically using the functions `is_empty`/`is_not_empty`, `is_absent`/`is_present`, and `is_null`/`is_not_null`. For the last pair, note that null means either empty or absent. Rules for null-handling: * Records with one or more empty sort-field values sort after records with all sort-field values present: -
+
 mlr cat data/sort-null.dat
+
+
 a=3,b=2
 a=1,b=8
 a=,b=4
@@ -24,8 +26,10 @@ x=9,b=10
 a=5,b=7
 
-
+
 mlr sort -n  a data/sort-null.dat
+
+
 a=1,b=8
 a=3,b=2
 a=5,b=7
@@ -33,8 +37,10 @@ a=,b=4
 x=9,b=10
 
-
+
 mlr sort -nr a data/sort-null.dat
+
+
 a=,b=4
 a=5,b=7
 a=3,b=2
@@ -44,37 +50,49 @@ x=9,b=10
 
 * Functions/operators which have one or more *empty* arguments produce empty output: e.g.
 
-
+
 echo 'x=2,y=3' | mlr put '$a=$x+$y'
+
+
 x=2,y=3,a=5
 
-
+
 echo 'x=,y=3' | mlr put '$a=$x+$y'
+
+
 x=,y=3,a=
 
-
+
 echo 'x=,y=3' | mlr put '$a=log($x);$b=log($y)'
+
+
 x=,y=3,a=,b=1.0986122886681096
 
-with the exception that the ``min`` and ``max`` functions are special: if one argument is non-null, it wins: +with the exception that the `min` and `max` functions are special: if one argument is non-null, it wins: -
+
 echo 'x=,y=3' | mlr put '$a=min($x,$y);$b=max($x,$y)'
+
+
 x=,y=3,a=3,b=
 
-* Functions of *absent* variables (e.g. ``mlr put '$y = log10($nonesuch)'``) evaluate to absent, and arithmetic/bitwise/boolean operators with both operands being absent evaluate to absent. Arithmetic operators with one absent operand return the other operand. More specifically, absent values act like zero for addition/subtraction, and one for multiplication: Furthermore, **any expression which evaluates to absent is not stored in the left-hand side of an assignment statement**: +* Functions of *absent* variables (e.g. `mlr put '$y = log10($nonesuch)'`) evaluate to absent, and arithmetic/bitwise/boolean operators with both operands being absent evaluate to absent. Arithmetic operators with one absent operand return the other operand. More specifically, absent values act like zero for addition/subtraction, and one for multiplication: Furthermore, **any expression which evaluates to absent is not stored in the left-hand side of an assignment statement**: -
+
 echo 'x=2,y=3' | mlr put '$a=$u+$v; $b=$u+$y; $c=$x+$y'
+
+
 x=2,y=3,b=3,c=5
 
-
+
 echo 'x=2,y=3' | mlr put '$a=min($x,$v);$b=max($u,$y);$c=min($u,$v)'
+
+
 x=2,y=3,a=2,b=3
 
@@ -82,18 +100,20 @@ x=2,y=3,a=2,b=3 The reasoning is as follows: -* Empty values are explicit in the data so they should explicitly affect accumulations: ``mlr put '@sum += $x'`` should accumulate numeric ``x`` values into the sum but an empty ``x``, when encountered in the input data stream, should make the sum non-numeric. To work around this you can use the ``is_not_null`` function as follows: ``mlr put 'is_not_null($x) { @sum += $x }'`` +* Empty values are explicit in the data so they should explicitly affect accumulations: `mlr put '@sum += $x'` should accumulate numeric `x` values into the sum but an empty `x`, when encountered in the input data stream, should make the sum non-numeric. To work around this you can use the `is_not_null` function as follows: `mlr put 'is_not_null($x) { @sum += $x }'` -* Absent stream-record values should not break accumulations, since Miller by design handles heterogenous data: the running ``@sum`` in ``mlr put '@sum += $x'`` should not be invalidated for records which have no ``x``. +* Absent stream-record values should not break accumulations, since Miller by design handles heterogenous data: the running `@sum` in `mlr put '@sum += $x'` should not be invalidated for records which have no `x`. -* Absent out-of-stream-variable values are precisely what allow you to write ``mlr put '@sum += $x'``. Otherwise you would have to write ``mlr put 'begin{@sum = 0}; @sum += $x'`` -- which is tolerable -- but for ``mlr put 'begin{...}; @sum[$a][$b] += $x'`` you'd have to pre-initialize ``@sum`` for all values of ``$a`` and ``$b`` in your input data stream, which is intolerable. +* Absent out-of-stream-variable values are precisely what allow you to write `mlr put '@sum += $x'`. Otherwise you would have to write `mlr put 'begin{@sum = 0}; @sum += $x'` -- which is tolerable -- but for `mlr put 'begin{...}; @sum[$a][$b] += $x'` you'd have to pre-initialize `@sum` for all values of `$a` and `$b` in your input data stream, which is intolerable. -* The penalty for the absent feature is that misspelled variables can be hard to find: e.g. in ``mlr put 'begin{@sumx = 10}; ...; update @sumx somehow per-record; ...; end {@something = @sum * 2}'`` the accumulator is spelt ``@sumx`` in the begin-block but ``@sum`` in the end-block, where since it is absent, ``@sum*2`` evaluates to 2. See also the section on :doc:`reference-dsl-errors`. +* The penalty for the absent feature is that misspelled variables can be hard to find: e.g. in `mlr put 'begin{@sumx = 10}; ...; update @sumx somehow per-record; ...; end {@something = @sum * 2}'` the accumulator is spelt `@sumx` in the begin-block but `@sum` in the end-block, where since it is absent, `@sum*2` evaluates to 2. See also the section on [DSL reference: errors and transparency](reference-dsl-errors.md). -Since absent plus absent is absent (and likewise for other operators), accumulations such as ``@sum += $x`` work correctly on heterogenous data, as do within-record formulas if both operands are absent. If one operand is present, you may get behavior you don't desire. To work around this -- namely, to set an output field only for records which have all the inputs present -- you can use a pattern-action block with ``is_present``: +Since absent plus absent is absent (and likewise for other operators), accumulations such as `@sum += $x` work correctly on heterogenous data, as do within-record formulas if both operands are absent. If one operand is present, you may get behavior you don't desire. To work around this -- namely, to set an output field only for records which have all the inputs present -- you can use a pattern-action block with `is_present`: -
+
 mlr cat data/het.dkvp
+
+
 resource=/path/to/file,loadsec=0.45,ok=true
 record_count=100,resource=/path/to/file
 resource=/path/to/second/file,loadsec=0.32,ok=true
@@ -101,8 +121,10 @@ record_count=150,resource=/path/to/second/file
 resource=/some/other/path,loadsec=0.97,ok=false
 
-
+
 mlr put 'is_present($loadsec) { $loadmillis = $loadsec * 1000 }' data/het.dkvp
+
+
 resource=/path/to/file,loadsec=0.45,ok=true,loadmillis=450
 record_count=100,resource=/path/to/file
 resource=/path/to/second/file,loadsec=0.32,ok=true,loadmillis=320
@@ -110,8 +132,10 @@ record_count=150,resource=/path/to/second/file
 resource=/some/other/path,loadsec=0.97,ok=false,loadmillis=970
 
-
+
 mlr put '$loadmillis = (is_present($loadsec) ? $loadsec : 0.0) * 1000' data/het.dkvp
+
+
 resource=/path/to/file,loadsec=0.45,ok=true,loadmillis=450
 record_count=100,resource=/path/to/file,loadmillis=0
 resource=/path/to/second/file,loadsec=0.32,ok=true,loadmillis=320
@@ -121,8 +145,10 @@ resource=/some/other/path,loadsec=0.97,ok=false,loadmillis=970
 
 If you're interested in a formal description of how empty and absent fields participate in arithmetic, here's a table for plus (other arithmetic/boolean/bitwise operators are similar):
 
-
+
 mlr help type-arithmetic-info
+
+
 (+)        | 1          2.5        (absent)   (error)   
 ------     + ------     ------     ------     ------    
 1          | 2          3.5        1          (error)   
diff --git a/docs6b/docs/reference-main-null-data.md.in b/docs6b/docs/reference-main-null-data.md.in
index d387d82ec..f3cabd519 100644
--- a/docs6b/docs/reference-main-null-data.md.in
+++ b/docs6b/docs/reference-main-null-data.md.in
@@ -1,14 +1,14 @@
 # Reference: null data
 
-One of Miller's key features is its support for **heterogeneous** data.  For example, take ``mlr sort``: if you try to sort on field ``hostname`` when not all records in the data stream *have* a field named ``hostname``, it is not an error (although you could pre-filter the data stream using ``mlr having-fields --at-least hostname then sort ...``).  Rather, records lacking one or more sort keys are simply output contiguously by ``mlr sort``.
+One of Miller's key features is its support for **heterogeneous** data.  For example, take `mlr sort`: if you try to sort on field `hostname` when not all records in the data stream *have* a field named `hostname`, it is not an error (although you could pre-filter the data stream using `mlr having-fields --at-least hostname then sort ...`).  Rather, records lacking one or more sort keys are simply output contiguously by `mlr sort`.
 
 Miller has two kinds of null data:
 
-* **Empty (key present, value empty)**: a field name is present in a record (or in an out-of-stream variable) with empty value: e.g. ``x=,y=2`` in the data input stream, or assignment ``$x=""`` or ``@x=""`` in ``mlr put``.
+* **Empty (key present, value empty)**: a field name is present in a record (or in an out-of-stream variable) with empty value: e.g. `x=,y=2` in the data input stream, or assignment `$x=""` or `@x=""` in `mlr put`.
 
-* **Absent (key not present)**: a field name is not present, e.g. input record is ``x=1,y=2`` and a ``put`` or ``filter`` expression refers to ``$z``. Or, reading an out-of-stream variable which hasn't been assigned a value yet, e.g.  ``mlr put -q '@sum += $x; end{emit @sum}'`` or ``mlr put -q '@sum[$a][$b] += $x; end{emit @sum, "a", "b"}'``.
+* **Absent (key not present)**: a field name is not present, e.g. input record is `x=1,y=2` and a `put` or `filter` expression refers to `$z`. Or, reading an out-of-stream variable which hasn't been assigned a value yet, e.g.  `mlr put -q '@sum += $x; end{emit @sum}'` or `mlr put -q '@sum[$a][$b] += $x; end{emit @sum, "a", "b"}'`.
 
-You can test these programatically using the functions ``is_empty``/``is_not_empty``, ``is_absent``/``is_present``, and ``is_null``/``is_not_null``. For the last pair, note that null means either empty or absent.
+You can test these programatically using the functions `is_empty`/`is_not_empty`, `is_absent`/`is_present`, and `is_null`/`is_not_null`. For the last pair, note that null means either empty or absent.
 
 Rules for null-handling:
 
@@ -40,13 +40,13 @@ GENMD_RUN_COMMAND
 echo 'x=,y=3' | mlr put '$a=log($x);$b=log($y)'
 GENMD_EOF
 
-with the exception that the ``min`` and ``max`` functions are special: if one argument is non-null, it wins:
+with the exception that the `min` and `max` functions are special: if one argument is non-null, it wins:
 
 GENMD_RUN_COMMAND
 echo 'x=,y=3' | mlr put '$a=min($x,$y);$b=max($x,$y)'
 GENMD_EOF
 
-* Functions of *absent* variables (e.g. ``mlr put '$y = log10($nonesuch)'``) evaluate to absent, and arithmetic/bitwise/boolean operators with both operands being absent evaluate to absent. Arithmetic operators with one absent operand return the other operand. More specifically, absent values act like zero for addition/subtraction, and one for multiplication: Furthermore, **any expression which evaluates to absent is not stored in the left-hand side of an assignment statement**:
+* Functions of *absent* variables (e.g. `mlr put '$y = log10($nonesuch)'`) evaluate to absent, and arithmetic/bitwise/boolean operators with both operands being absent evaluate to absent. Arithmetic operators with one absent operand return the other operand. More specifically, absent values act like zero for addition/subtraction, and one for multiplication: Furthermore, **any expression which evaluates to absent is not stored in the left-hand side of an assignment statement**:
 
 GENMD_RUN_COMMAND
 echo 'x=2,y=3' | mlr put '$a=$u+$v; $b=$u+$y; $c=$x+$y'
@@ -60,15 +60,15 @@ GENMD_EOF
 
 The reasoning is as follows:
 
-* Empty values are explicit in the data so they should explicitly affect accumulations: ``mlr put '@sum += $x'`` should accumulate numeric ``x`` values into the sum but an empty ``x``, when encountered in the input data stream, should make the sum non-numeric. To work around this you can use the ``is_not_null`` function as follows: ``mlr put 'is_not_null($x) { @sum += $x }'``
+* Empty values are explicit in the data so they should explicitly affect accumulations: `mlr put '@sum += $x'` should accumulate numeric `x` values into the sum but an empty `x`, when encountered in the input data stream, should make the sum non-numeric. To work around this you can use the `is_not_null` function as follows: `mlr put 'is_not_null($x) { @sum += $x }'`
 
-* Absent stream-record values should not break accumulations, since Miller by design handles heterogenous data: the running ``@sum`` in ``mlr put '@sum += $x'`` should not be invalidated for records which have no ``x``.
+* Absent stream-record values should not break accumulations, since Miller by design handles heterogenous data: the running `@sum` in `mlr put '@sum += $x'` should not be invalidated for records which have no `x`.
 
-* Absent out-of-stream-variable values are precisely what allow you to write ``mlr put '@sum += $x'``. Otherwise you would have to write ``mlr put 'begin{@sum = 0}; @sum += $x'`` -- which is tolerable -- but for ``mlr put 'begin{...}; @sum[$a][$b] += $x'`` you'd have to pre-initialize ``@sum`` for all values of ``$a`` and ``$b`` in your input data stream, which is intolerable.
+* Absent out-of-stream-variable values are precisely what allow you to write `mlr put '@sum += $x'`. Otherwise you would have to write `mlr put 'begin{@sum = 0}; @sum += $x'` -- which is tolerable -- but for `mlr put 'begin{...}; @sum[$a][$b] += $x'` you'd have to pre-initialize `@sum` for all values of `$a` and `$b` in your input data stream, which is intolerable.
 
-* The penalty for the absent feature is that misspelled variables can be hard to find: e.g. in ``mlr put 'begin{@sumx = 10}; ...; update @sumx somehow per-record; ...; end {@something = @sum * 2}'`` the accumulator is spelt ``@sumx`` in the begin-block but ``@sum`` in the end-block, where since it is absent, ``@sum*2`` evaluates to 2. See also the section on :doc:`reference-dsl-errors`.
+* The penalty for the absent feature is that misspelled variables can be hard to find: e.g. in `mlr put 'begin{@sumx = 10}; ...; update @sumx somehow per-record; ...; end {@something = @sum * 2}'` the accumulator is spelt `@sumx` in the begin-block but `@sum` in the end-block, where since it is absent, `@sum*2` evaluates to 2. See also the section on [DSL reference: errors and transparency](reference-dsl-errors.md).
 
-Since absent plus absent is absent (and likewise for other operators), accumulations such as ``@sum += $x`` work correctly on heterogenous data, as do within-record formulas if both operands are absent. If one operand is present, you may get behavior you don't desire.  To work around this -- namely, to set an output field only for records which have all the inputs present -- you can use a pattern-action block with ``is_present``:
+Since absent plus absent is absent (and likewise for other operators), accumulations such as `@sum += $x` work correctly on heterogenous data, as do within-record formulas if both operands are absent. If one operand is present, you may get behavior you don't desire.  To work around this -- namely, to set an output field only for records which have all the inputs present -- you can use a pattern-action block with `is_present`:
 
 GENMD_RUN_COMMAND
 mlr cat data/het.dkvp
diff --git a/docs6b/docs/reference-main-online-help.md b/docs6b/docs/reference-main-online-help.md
index b50307d4e..cb22bef40 100644
--- a/docs6b/docs/reference-main-online-help.md
+++ b/docs6b/docs/reference-main-online-help.md
@@ -5,8 +5,10 @@ TODO: expand this section
 
 Examples:
 
-
+
 mlr --help
+
+
 Usage: mlr [I/O options] {verb} [verb-dependent options ...] {zero or more file names}
 Output of one verb may be chained as input to another using "then", e.g.
   mlr stats1 -a min,mean,max -f flag,u,v -g color then sort -f color
@@ -14,8 +16,10 @@ Please see 'mlr help topics' for more information.
 Please also see https://johnkerl.org/miller6
 
-
+
 mlr sort --help
+
+
 Usage: mlr sort {flags}
 Sorts records primarily by the first specified field, secondarily by the second
 field, and so on.  (Any records not having all specified sort keys will appear
diff --git a/docs6b/docs/reference-main-overview.md b/docs6b/docs/reference-main-overview.md
index 4449bb25f..c65703d56 100644
--- a/docs6b/docs/reference-main-overview.md
+++ b/docs6b/docs/reference-main-overview.md
@@ -5,15 +5,17 @@
 
 The outline of an invocation of Miller is
 
-* ``mlr``
-* Options controlling input/output formatting, etc. (:doc:`reference-main-io-options`).
-* One or more verbs (such as ``cut``, ``sort``, etc.) ([Verbs Reference](reference-verbs.md)) -- chained together using ``then`` (:doc:`reference-main-then-chaining`). You use these to transform your data.
+* `mlr`
+* Options controlling input/output formatting, etc. ([Reference: I/O options](reference-main-io-options.md)).
+* One or more verbs (such as `cut`, `sort`, etc.) ([Verbs Reference](reference-verbs.md)) -- chained together using [then](reference-main-then-chaining.md)). You use these to transform your data.
 * Zero or more filenames, with input taken from standard input if there are no filenames present.
 
 For example, reading from a file:
 
-
+
 mlr --icsv --opprint head -n 2 then sort -f shape example.csv
+
+
 color  shape    flag index quantity rate
 red    square   true 15    79.2778  0.0130
 yellow triangle true 11    43.6498  9.8870
@@ -21,8 +23,10 @@ yellow triangle true 11    43.6498  9.8870
 
 Reading from standard input:
 
-
+
 cat example.csv | mlr --icsv --opprint head -n 2 then sort -f shape
+
+
 color  shape    flag index quantity rate
 red    square   true 15    79.2778  0.0130
 yellow triangle true 11    43.6498  9.8870
@@ -32,16 +36,18 @@ The rest of this reference section gives you full information on each of these p
 
 ## Verbs vs DSL
 
-When you type ``mlr {something} myfile.dat``, the ``{something}`` part is called a **verb**. It specifies how you want to transform your data. Most of the verbs are counterparts of built-in system tools like ``cut`` and ``sort`` -- but with file-format awareness, and giving you the ability to refer to fields by name.
+When you type `mlr {something} myfile.dat`, the `{something}` part is called a **verb**. It specifies how you want to transform your data. Most of the verbs are counterparts of built-in system tools like `cut` and `sort` -- but with file-format awareness, and giving you the ability to refer to fields by name.
 
-The verbs ``put`` and ``filter`` are special in that they have a rich expression language (domain-specific language, or "DSL"). More information about them can be found at [DSL reference](reference-dsl.md).
+The verbs `put` and `filter` are special in that they have a rich expression language (domain-specific language, or "DSL"). More information about them can be found at [DSL reference](reference-dsl.md).
 
-Here's a comparison of verbs and ``put``/``filter`` DSL expressions:
+Here's a comparison of verbs and `put`/`filter` DSL expressions:
 
 Example of using a verb for data processing:
 
-
+
 mlr stats1 -a sum -f x -g a data/small
+
+
 a=pan,x_sum=0.3467901443380824
 a=eks,x_sum=1.1400793586611044
 a=wye,x_sum=0.7778922255683036
@@ -55,8 +61,10 @@ a=wye,x_sum=0.7778922255683036
 
 Example of doing the same thing using a DSL expression:
 
-
+
 mlr  put -q '@x_sum[$a] += $x; end{emit @x_sum, "a"}' data/small
+
+
 a=pan,x_sum=0.3467901443380824
 a=eks,x_sum=1.1400793586611044
 a=wye,x_sum=0.7778922255683036
diff --git a/docs6b/docs/reference-main-overview.md.in b/docs6b/docs/reference-main-overview.md.in
index c0a34b3e4..d3031441f 100644
--- a/docs6b/docs/reference-main-overview.md.in
+++ b/docs6b/docs/reference-main-overview.md.in
@@ -4,9 +4,9 @@
 
 The outline of an invocation of Miller is
 
-* ``mlr``
-* Options controlling input/output formatting, etc. (:doc:`reference-main-io-options`).
-* One or more verbs (such as ``cut``, ``sort``, etc.) ([Verbs Reference](reference-verbs.md)) -- chained together using ``then`` (:doc:`reference-main-then-chaining`). You use these to transform your data.
+* `mlr`
+* Options controlling input/output formatting, etc. ([Reference: I/O options](reference-main-io-options.md)).
+* One or more verbs (such as `cut`, `sort`, etc.) ([Verbs Reference](reference-verbs.md)) -- chained together using [then](reference-main-then-chaining.md)). You use these to transform your data.
 * Zero or more filenames, with input taken from standard input if there are no filenames present.
 
 For example, reading from a file:
@@ -25,11 +25,11 @@ The rest of this reference section gives you full information on each of these p
 
 ## Verbs vs DSL
 
-When you type ``mlr {something} myfile.dat``, the ``{something}`` part is called a **verb**. It specifies how you want to transform your data. Most of the verbs are counterparts of built-in system tools like ``cut`` and ``sort`` -- but with file-format awareness, and giving you the ability to refer to fields by name.
+When you type `mlr {something} myfile.dat`, the `{something}` part is called a **verb**. It specifies how you want to transform your data. Most of the verbs are counterparts of built-in system tools like `cut` and `sort` -- but with file-format awareness, and giving you the ability to refer to fields by name.
 
-The verbs ``put`` and ``filter`` are special in that they have a rich expression language (domain-specific language, or "DSL"). More information about them can be found at [DSL reference](reference-dsl.md).
+The verbs `put` and `filter` are special in that they have a rich expression language (domain-specific language, or "DSL"). More information about them can be found at [DSL reference](reference-dsl.md).
 
-Here's a comparison of verbs and ``put``/``filter`` DSL expressions:
+Here's a comparison of verbs and `put`/`filter` DSL expressions:
 
 Example of using a verb for data processing:
 
diff --git a/docs6b/docs/reference-main-regular-expressions.md b/docs6b/docs/reference-main-regular-expressions.md
index 2a37f39bb..194110536 100644
--- a/docs6b/docs/reference-main-regular-expressions.md
+++ b/docs6b/docs/reference-main-regular-expressions.md
@@ -3,72 +3,76 @@
 
 Miller lets you use regular expressions (of type POSIX.2) in the following contexts:
 
-* In ``mlr filter`` with ``=~`` or ``!=~``, e.g. ``mlr filter '$url =~ "http.*com"'``
+* In `mlr filter` with `=~` or `!=~`, e.g. `mlr filter '$url =~ "http.*com"'`
 
-* In ``mlr put`` with ``sub`` or ``gsub``, e.g. ``mlr put '$url = sub($url, "http.*com", "")'``
+* In `mlr put` with `sub` or `gsub`, e.g. `mlr put '$url = sub($url, "http.*com", "")'`
 
-* In ``mlr having-fields``, e.g. ``mlr having-fields --any-matching '^sda[0-9]'``
+* In `mlr having-fields`, e.g. `mlr having-fields --any-matching '^sda[0-9]'`
 
-* In ``mlr cut``, e.g. ``mlr cut -r -f '^status$,^sda[0-9]'``
+* In `mlr cut`, e.g. `mlr cut -r -f '^status$,^sda[0-9]'`
 
-* In ``mlr rename``, e.g. ``mlr rename -r '^(sda[0-9]).*$,dev/\1'``
+* In `mlr rename`, e.g. `mlr rename -r '^(sda[0-9]).*$,dev/\1'`
 
-* In ``mlr grep``, e.g. ``mlr --csv grep 00188555487 myfiles*.csv``
+* In `mlr grep`, e.g. `mlr --csv grep 00188555487 myfiles*.csv`
 
 Points demonstrated by the above examples:
 
-* There are no implicit start-of-string or end-of-string anchors; please use ``^`` and/or ``$`` explicitly.
+* There are no implicit start-of-string or end-of-string anchors; please use `^` and/or `$` explicitly.
 
 * Miller regexes are wrapped with double quotes rather than slashes.
 
-* The ``i`` after the ending double quote indicates a case-insensitive regex.
+* The `i` after the ending double quote indicates a case-insensitive regex.
 
-* Capture groups are wrapped with ``(...)`` rather than ``\(...\)``; use ``\(`` and ``\)`` to match against parentheses.
+* Capture groups are wrapped with `(...)` rather than `\(...\)`; use `\(` and `\)` to match against parentheses.
 
-For ``filter`` and ``put``, if the regular expression is a string literal (the normal case), it is precompiled at process start and reused thereafter, which is efficient. If the regular expression is a more complex expression, including string concatenation using ``.``, or a column name (in which case you can take regular expressions from input data!), then regexes are compiled on each record which works but is less efficient. As well, in this case there is no way to specify case-insensitive matching.
+For `filter` and `put`, if the regular expression is a string literal (the normal case), it is precompiled at process start and reused thereafter, which is efficient. If the regular expression is a more complex expression, including string concatenation using `.`, or a column name (in which case you can take regular expressions from input data!), then regexes are compiled on each record which works but is less efficient. As well, in this case there is no way to specify case-insensitive matching.
 
 Example:
 
-
+
 cat data/regex-in-data.dat
+
+
 name=jane,regex=^j.*e$
 name=bill,regex=^b[ou]ll$
 name=bull,regex=^b[ou]ll$
 
-
+
 mlr filter '$name =~ $regex' data/regex-in-data.dat
+
+
 name=jane,regex=^j.*e$
 name=bull,regex=^b[ou]ll$
 
## Regex captures -Regex captures of the form ``\0`` through ``\9`` are supported as +Regex captures of the form `\0` through `\9` are supported as -* Captures have in-function context for ``sub`` and ``gsub``. For example, the first ``\1,\2`` pair belong to the first ``sub`` and the second ``\1,\2`` pair belong to the second ``sub``: +* Captures have in-function context for `sub` and `gsub`. For example, the first `\1,\2` pair belong to the first `sub` and the second `\1,\2` pair belong to the second `sub`: -
+
 mlr put '$b = sub($a, "(..)_(...)", "\2-\1"); $c = sub($a, "(..)_(.)(..)", ":\1:\2:\3")'
 
-* Captures endure for the entirety of a ``put`` for the ``=~`` and ``!=~`` operators. For example, here the ``\1,\2`` are set by the ``=~`` operator and are used by both subsequent assignment statements: +* Captures endure for the entirety of a `put` for the `=~` and `!=~` operators. For example, here the `\1,\2` are set by the `=~` operator and are used by both subsequent assignment statements: -
+
 mlr put '$a =~ "(..)_(....); $b = "left_\1"; $c = "right_\2"'
 
-* The captures are not retained across multiple puts. For example, here the ``\1,\2`` won't be expanded from the regex capture: +* The captures are not retained across multiple puts. For example, here the `\1,\2` won't be expanded from the regex capture: -
+
 mlr put '$a =~ "(..)_(....)' then {... something else ...} then put '$b = "left_\1"; $c = "right_\2"'
 
-* Captures are ignored in ``filter`` for the ``=~`` and ``!=~`` operators. For example, there is no mechanism provided to refer to the first ``(..)`` as ``\1`` or to the second ``(....)`` as ``\2`` in the following filter statement: +* Captures are ignored in `filter` for the `=~` and `!=~` operators. For example, there is no mechanism provided to refer to the first `(..)` as `\1` or to the second `(....)` as `\2` in the following filter statement: -
+
 mlr filter '$a =~ "(..)_(....)'
 
-* Up to nine matches are supported: ``\1`` through ``\9``, while ``\0`` is the entire match string; ``\15`` is treated as ``\1`` followed by an unrelated ``5``. +* Up to nine matches are supported: `\1` through `\9`, while `\0` is the entire match string; `\15` is treated as `\1` followed by an unrelated `5`. diff --git a/docs6b/docs/reference-main-regular-expressions.md.in b/docs6b/docs/reference-main-regular-expressions.md.in index 230c061cc..20442ae29 100644 --- a/docs6b/docs/reference-main-regular-expressions.md.in +++ b/docs6b/docs/reference-main-regular-expressions.md.in @@ -2,29 +2,29 @@ Miller lets you use regular expressions (of type POSIX.2) in the following contexts: -* In ``mlr filter`` with ``=~`` or ``!=~``, e.g. ``mlr filter '$url =~ "http.*com"'`` +* In `mlr filter` with `=~` or `!=~`, e.g. `mlr filter '$url =~ "http.*com"'` -* In ``mlr put`` with ``sub`` or ``gsub``, e.g. ``mlr put '$url = sub($url, "http.*com", "")'`` +* In `mlr put` with `sub` or `gsub`, e.g. `mlr put '$url = sub($url, "http.*com", "")'` -* In ``mlr having-fields``, e.g. ``mlr having-fields --any-matching '^sda[0-9]'`` +* In `mlr having-fields`, e.g. `mlr having-fields --any-matching '^sda[0-9]'` -* In ``mlr cut``, e.g. ``mlr cut -r -f '^status$,^sda[0-9]'`` +* In `mlr cut`, e.g. `mlr cut -r -f '^status$,^sda[0-9]'` -* In ``mlr rename``, e.g. ``mlr rename -r '^(sda[0-9]).*$,dev/\1'`` +* In `mlr rename`, e.g. `mlr rename -r '^(sda[0-9]).*$,dev/\1'` -* In ``mlr grep``, e.g. ``mlr --csv grep 00188555487 myfiles*.csv`` +* In `mlr grep`, e.g. `mlr --csv grep 00188555487 myfiles*.csv` Points demonstrated by the above examples: -* There are no implicit start-of-string or end-of-string anchors; please use ``^`` and/or ``$`` explicitly. +* There are no implicit start-of-string or end-of-string anchors; please use `^` and/or `$` explicitly. * Miller regexes are wrapped with double quotes rather than slashes. -* The ``i`` after the ending double quote indicates a case-insensitive regex. +* The `i` after the ending double quote indicates a case-insensitive regex. -* Capture groups are wrapped with ``(...)`` rather than ``\(...\)``; use ``\(`` and ``\)`` to match against parentheses. +* Capture groups are wrapped with `(...)` rather than `\(...\)`; use `\(` and `\)` to match against parentheses. -For ``filter`` and ``put``, if the regular expression is a string literal (the normal case), it is precompiled at process start and reused thereafter, which is efficient. If the regular expression is a more complex expression, including string concatenation using ``.``, or a column name (in which case you can take regular expressions from input data!), then regexes are compiled on each record which works but is less efficient. As well, in this case there is no way to specify case-insensitive matching. +For `filter` and `put`, if the regular expression is a string literal (the normal case), it is precompiled at process start and reused thereafter, which is efficient. If the regular expression is a more complex expression, including string concatenation using `.`, or a column name (in which case you can take regular expressions from input data!), then regexes are compiled on each record which works but is less efficient. As well, in this case there is no way to specify case-insensitive matching. Example: @@ -38,31 +38,31 @@ GENMD_EOF ## Regex captures -Regex captures of the form ``\0`` through ``\9`` are supported as +Regex captures of the form `\0` through `\9` are supported as -* Captures have in-function context for ``sub`` and ``gsub``. For example, the first ``\1,\2`` pair belong to the first ``sub`` and the second ``\1,\2`` pair belong to the second ``sub``: +* Captures have in-function context for `sub` and `gsub`. For example, the first `\1,\2` pair belong to the first `sub` and the second `\1,\2` pair belong to the second `sub`: GENMD_SHOW_COMMAND mlr put '$b = sub($a, "(..)_(...)", "\2-\1"); $c = sub($a, "(..)_(.)(..)", ":\1:\2:\3")' GENMD_EOF -* Captures endure for the entirety of a ``put`` for the ``=~`` and ``!=~`` operators. For example, here the ``\1,\2`` are set by the ``=~`` operator and are used by both subsequent assignment statements: +* Captures endure for the entirety of a `put` for the `=~` and `!=~` operators. For example, here the `\1,\2` are set by the `=~` operator and are used by both subsequent assignment statements: GENMD_SHOW_COMMAND mlr put '$a =~ "(..)_(....); $b = "left_\1"; $c = "right_\2"' GENMD_EOF -* The captures are not retained across multiple puts. For example, here the ``\1,\2`` won't be expanded from the regex capture: +* The captures are not retained across multiple puts. For example, here the `\1,\2` won't be expanded from the regex capture: GENMD_SHOW_COMMAND mlr put '$a =~ "(..)_(....)' then {... something else ...} then put '$b = "left_\1"; $c = "right_\2"' GENMD_EOF -* Captures are ignored in ``filter`` for the ``=~`` and ``!=~`` operators. For example, there is no mechanism provided to refer to the first ``(..)`` as ``\1`` or to the second ``(....)`` as ``\2`` in the following filter statement: +* Captures are ignored in `filter` for the `=~` and `!=~` operators. For example, there is no mechanism provided to refer to the first `(..)` as `\1` or to the second `(....)` as `\2` in the following filter statement: GENMD_CARDIFY mlr filter '$a =~ "(..)_(....)' GENMD_EOF -* Up to nine matches are supported: ``\1`` through ``\9``, while ``\0`` is the entire match string; ``\15`` is treated as ``\1`` followed by an unrelated ``5``. +* Up to nine matches are supported: `\1` through `\9`, while `\0` is the entire match string; `\15` is treated as `\1` followed by an unrelated `5`. diff --git a/docs6b/docs/reference-main-then-chaining.md b/docs6b/docs/reference-main-then-chaining.md index e5863b74b..ace997d6a 100644 --- a/docs6b/docs/reference-main-then-chaining.md +++ b/docs6b/docs/reference-main-then-chaining.md @@ -3,21 +3,21 @@ In accord with the [Unix philosophy](http://en.wikipedia.org/wiki/Unix_philosophy), you can pipe data into or out of Miller. For example: -
+
 mlr cut --complement -f os_version *.dat | mlr sort -f hostname,uptime
 
-You can, if you like, instead simply chain commands together using the ``then`` keyword: +You can, if you like, instead simply chain commands together using the `then` keyword: -
+
 mlr cut --complement -f os_version then sort -f hostname,uptime *.dat
 
-(You can precede the very first verb with ``then``, if you like, for symmetry.) +(You can precede the very first verb with `then`, if you like, for symmetry.) Here's a performance comparison: -
+
 % cat piped.sh
 mlr cut -x -f i,y data/big | mlr sort -n y > /dev/null
 
@@ -38,4 +38,4 @@ sys  0m0.137s
 
 There are two reasons to use then-chaining: one is for performance, although I don't expect this to be a win in all cases.  Using then-chaining avoids redundant string-parsing and string-formatting at each pipeline step: instead input records are parsed once, they are fed through each pipeline stage in memory, and then output records are formatted once. On the other hand, Miller is single-threaded, while modern systems are usually multi-processor, and when streaming-data programs operate through pipes, each one can use a CPU.  Rest assured you get the same results either way.
 
-The other reason to use then-chaining is for simplicity: you don't have re-type formatting flags (e.g. ``--csv --fs tab``) at every pipeline stage.
+The other reason to use then-chaining is for simplicity: you don't have re-type formatting flags (e.g. `--csv --fs tab`) at every pipeline stage.
diff --git a/docs6b/docs/reference-main-then-chaining.md.in b/docs6b/docs/reference-main-then-chaining.md.in
index de94f5fe6..296a435a6 100644
--- a/docs6b/docs/reference-main-then-chaining.md.in
+++ b/docs6b/docs/reference-main-then-chaining.md.in
@@ -6,13 +6,13 @@ GENMD_SHOW_COMMAND
 mlr cut --complement -f os_version *.dat | mlr sort -f hostname,uptime
 GENMD_EOF
 
-You can, if you like, instead simply chain commands together using the ``then`` keyword:
+You can, if you like, instead simply chain commands together using the `then` keyword:
 
 GENMD_SHOW_COMMAND
 mlr cut --complement -f os_version then sort -f hostname,uptime *.dat
 GENMD_EOF
 
-(You can precede the very first verb with ``then``, if you like, for symmetry.)
+(You can precede the very first verb with `then`, if you like, for symmetry.)
 
 Here's a performance comparison:
 
@@ -20,4 +20,4 @@ GENMD_INCLUDE_ESCAPED(data/then-chaining-performance.txt)
 
 There are two reasons to use then-chaining: one is for performance, although I don't expect this to be a win in all cases.  Using then-chaining avoids redundant string-parsing and string-formatting at each pipeline step: instead input records are parsed once, they are fed through each pipeline stage in memory, and then output records are formatted once. On the other hand, Miller is single-threaded, while modern systems are usually multi-processor, and when streaming-data programs operate through pipes, each one can use a CPU.  Rest assured you get the same results either way.
 
-The other reason to use then-chaining is for simplicity: you don't have re-type formatting flags (e.g. ``--csv --fs tab``) at every pipeline stage.
+The other reason to use then-chaining is for simplicity: you don't have re-type formatting flags (e.g. `--csv --fs tab`) at every pipeline stage.
diff --git a/docs6b/docs/reference-verbs.md b/docs6b/docs/reference-verbs.md
index be2ce43ef..594a57f66 100644
--- a/docs6b/docs/reference-verbs.md
+++ b/docs6b/docs/reference-verbs.md
@@ -3,10 +3,10 @@
 
 ## Overview
 
-Whereas the Unix toolkit is made of the separate executables ``cat``, ``tail``, ``cut``,
-``sort``, etc., Miller has subcommands, or **verbs**, invoked as follows:
+Whereas the Unix toolkit is made of the separate executables `cat`, `tail`, `cut`,
+`sort`, etc., Miller has subcommands, or **verbs**, invoked as follows:
 
-
+
 mlr tac *.dat
 mlr cut --complement -f os_version *.dat
 mlr sort -f hostname,uptime *.dat
@@ -14,48 +14,52 @@ mlr sort -f hostname,uptime *.dat
 
 These fall into categories as follows:
 
-* Analogs of their Unix-toolkit namesakes, discussed below as well as in [Unix-toolkit Context](feature-comparison.md): :ref:`reference-verbs-cat`, :ref:`reference-verbs-cut`, :ref:`reference-verbs-grep`, :ref:`reference-verbs-head`, :ref:`reference-verbs-join`, :ref:`reference-verbs-sort`, :ref:`reference-verbs-tac`, :ref:`reference-verbs-tail`, :ref:`reference-verbs-top`, :ref:`reference-verbs-uniq`.
+* Analogs of their Unix-toolkit namesakes, discussed below as well as in [Unix-toolkit Context](feature-comparison.md): [cat](reference-verbs.md#cat), [cut](reference-verbs.md#cut), [grep](reference-verbs.md#grep), [head](reference-verbs.md#head), [join](reference-verbs.md#join), [sort](reference-verbs.md#sort), [tac](reference-verbs.md#tac), [tail](reference-verbs.md#tail), [top](reference-verbs.md#top), [uniq](reference-verbs.md#uniq).
 
-* ``awk``-like functionality: :ref:`reference-verbs-filter`, :ref:`reference-verbs-put`, :ref:`reference-verbs-sec2gmt`, :ref:`reference-verbs-sec2gmtdate`, :ref:`reference-verbs-step`, :ref:`reference-verbs-tee`.
+* `awk`-like functionality: [filter](reference-verbs.md#filter), [put](reference-verbs.md#put), [sec2gmt](reference-verbs.md#sec2gmt), [sec2gmtdate](reference-verbs.md#sec2gmtdate), [step](reference-verbs.md#step), [tee](reference-verbs.md#tee).
 
-* Statistically oriented: :ref:`reference-verbs-bar`, :ref:`reference-verbs-bootstrap`, :ref:`reference-verbs-decimate`, :ref:`reference-verbs-histogram`, :ref:`reference-verbs-least-frequent`, :ref:`reference-verbs-most-frequent`, :ref:`reference-verbs-sample`, :ref:`reference-verbs-shuffle`, :ref:`reference-verbs-stats1`, :ref:`reference-verbs-stats2`.
+* Statistically oriented: [bar](reference-verbs.md#bar), [bootstrap](reference-verbs.md#bootstrap), [decimate](reference-verbs.md#decimate), [histogram](reference-verbs.md#histogram), [least-frequent](reference-verbs.md#least-frequent), [most-frequent](reference-verbs.md#most-frequent), [sample](reference-verbs.md#sample), [shuffle](reference-verbs.md#shuffle), [stats1](reference-verbs.md#stats1), [stats2](reference-verbs.md#stats2).
 
-* Particularly oriented toward [Record Heterogeneity](record-heterogeneity.md), although all Miller commands can handle heterogeneous records: :ref:`reference-verbs-group-by`, :ref:`reference-verbs-group-like`, :ref:`reference-verbs-having-fields`.
+* Particularly oriented toward [Record Heterogeneity](record-heterogeneity.md), although all Miller commands can handle heterogeneous records: [group-by](reference-verbs.md#group-by), [group-like](reference-verbs.md#group-like), [having-fields](reference-verbs.md#having-fields).
 
-* These draw from other sources (see also [How Original Is Miller?](originality.md)): :ref:`reference-verbs-count-distinct` is SQL-ish, and :ref:`reference-verbs-rename` can be done by ``sed`` (which does it faster: see [Performance](performance.md). Verbs: :ref:`reference-verbs-check`, :ref:`reference-verbs-count-distinct`, :ref:`reference-verbs-label`, :ref:`reference-verbs-merge-fields`, :ref:`reference-verbs-nest`, :ref:`reference-verbs-nothing`, :ref:`reference-verbs-regularize`, :ref:`reference-verbs-rename`, :ref:`reference-verbs-reorder`, :ref:`reference-verbs-reshape`, :ref:`reference-verbs-seqgen`.
-
-.. _reference-verbs-altkv:
+* These draw from other sources (see also [How Original Is Miller?](originality.md)): [count-distinct](reference-verbs.md#count-distinct) is SQL-ish, and [rename](reference-verbs.md#rename) can be done by `sed` (which does it faster: see [Performance](performance.md). Verbs: [check](reference-verbs.md#check), [count-distinct](reference-verbs.md#count-distinct), [label](reference-verbs.md#label), [merge-fields](reference-verbs.md#merge-fields), [nest](reference-verbs.md#nest), [nothing](reference-verbs.md#nothing), [regularize](reference-verbs.md#regularize), [rename](reference-verbs.md#rename), [reorder](reference-verbs.md#reorder), [reshape](reference-verbs.md#reshape), [seqgen](reference-verbs.md#seqgen).
 
 ## altkv
 
 Map list of values to alternating key/value pairs.
 
-
+
 mlr altkv -h
+
+
 Usage: mlr altkv [options]
 Given fields with values of the form a,b,c,d,e,f emits a=b,c=d,e=f pairs.
 Options:
 -h|--help Show this message.
 
-
+
 echo 'a,b,c,d,e,f' | mlr altkv
+
+
 a=b,c=d,e=f
 
-
+
 echo 'a,b,c,d,e,f,g' | mlr altkv
+
+
 a=b,c=d,e=f,4=g
 
-.. _reference-verbs-bar: - ## bar Cheesy bar-charting. -
+
 mlr bar -h
+
+
 Usage: mlr bar [options]
 Replaces a numeric field with a number of asterisks, allowing for cheesy
 bar plots. These align best with --opprint or --oxtab output format.
@@ -74,8 +78,10 @@ However you can make them all longer if you so desire.
 -h|--help Show this message.
 
-
+
 mlr --opprint cat data/small
+
+
 a   b   i x                   y
 pan pan 1 0.3467901443380824  0.7268028627434533
 eks pan 2 0.7586799647899636  0.5221511083334797
@@ -84,8 +90,10 @@ eks wye 4 0.38139939387114097 0.13418874328430463
 wye pan 5 0.5732889198020006  0.8636244699032729
 
-
+
 mlr --opprint bar --lo 0 --hi 1 -f x,y data/small
+
+
 a   b   i x                                        y
 pan pan 1 *************........................... *****************************...........
 eks pan 2 ******************************.......... ********************....................
@@ -94,8 +102,10 @@ eks wye 4 ***************......................... *****........................
 wye pan 5 **********************.................. **********************************......
 
-
+
 mlr --opprint bar --lo 0.4 --hi 0.6 -f x,y data/small
+
+
 a   b   i x                                        y
 pan pan 1 #....................................... ***************************************#
 eks pan 2 ***************************************# ************************................
@@ -104,8 +114,10 @@ eks wye 4 #....................................... #............................
 wye pan 5 **********************************...... ***************************************#
 
-
+
 mlr --opprint bar --auto -f x,y data/small
+
+
 a   b   i x                                                                                 y
 pan pan 1 [0.20460330576630303]**********..............................[0.7586799647899636] [0.13418874328430463]********************************........[0.8636244699032729]
 eks pan 2 [0.20460330576630303]***************************************#[0.7586799647899636] [0.13418874328430463]*********************...................[0.8636244699032729]
@@ -114,12 +126,12 @@ eks wye 4 [0.20460330576630303]************............................[0.758679
 wye pan 5 [0.20460330576630303]**************************..............[0.7586799647899636] [0.13418874328430463]***************************************#[0.8636244699032729]
 
-.. _reference-verbs-bootstrap: - ## bootstrap -
+
 mlr bootstrap --help
+
+
 Usage: mlr bootstrap [options]
 Emits an n-sample, with replacement, of the input records.
 See also mlr sample and mlr shuffle.
@@ -135,8 +147,10 @@ The canonical use for bootstrap sampling is to put error bars on statistical qua
     hard-coded, not live-code, since random sampling would generate different data on each doc run
     which would needlessly complicate git diff
 
-
+
 mlr --opprint stats1 -a mean,count -f u -g color data/colored-shapes.dkvp
+
+
 color  u_mean   u_count
 yellow 0.497129 1413
 red    0.492560 4641
@@ -146,8 +160,10 @@ blue   0.517717 1470
 orange 0.490532 303
 
-
+
 mlr --opprint bootstrap then stats1 -a mean,count -f u -g color data/colored-shapes.dkvp
+
+
 color  u_mean   u_count
 yellow 0.500651 1380
 purple 0.501556 1111
@@ -157,8 +173,10 @@ blue   0.512529 1496
 orange 0.521030 321
 
-
+
 mlr --opprint bootstrap then stats1 -a mean,count -f u -g color data/colored-shapes.dkvp
+
+
 color  u_mean   u_count
 yellow 0.498046 1485
 blue   0.513576 1417
@@ -168,8 +186,10 @@ green  0.496803 1075
 purple 0.486337 1199
 
-
+
 mlr --opprint bootstrap then stats1 -a mean,count -f u -g color data/colored-shapes.dkvp
+
+
 color  u_mean   u_count
 blue   0.522921 1447
 red    0.490717 4617
@@ -179,14 +199,14 @@ green  0.507569 1111
 orange 0.468014 292
 
-.. _reference-verbs-cat: - ## cat Most useful for format conversions (see [File Formats](file-formats.md), and concatenating multiple same-schema CSV files to have the same header: -
+
 mlr cat -h
+
+
 Usage: mlr cat [options]
 Passes input records directly to output. Most useful for format conversion.
 Options:
@@ -196,29 +216,37 @@ Options:
 -h|--help Show this message.
 
-
+
 cat data/a.csv
+
+
 a,b,c
 1,2,3
 4,5,6
 
-
+
 cat data/b.csv
+
+
 a,b,c
 7,8,9
 
-
+
 mlr --csv cat data/a.csv data/b.csv
+
+
 a,b,c
 1,2,3
 4,5,6
 7,8,9
 
-
+
 mlr --icsv --oxtab cat data/a.csv data/b.csv
+
+
 a 1
 b 2
 c 3
@@ -232,16 +260,20 @@ b 8
 c 9
 
-
+
 mlr --csv cat -n data/a.csv data/b.csv
+
+
 n,a,b,c
 1,1,2,3
 2,4,5,6
 3,7,8,9
 
-
+
 mlr --opprint cat data/small
+
+
 a   b   i x                   y
 pan pan 1 0.3467901443380824  0.7268028627434533
 eks pan 2 0.7586799647899636  0.5221511083334797
@@ -250,8 +282,10 @@ eks wye 4 0.38139939387114097 0.13418874328430463
 wye pan 5 0.5732889198020006  0.8636244699032729
 
-
+
 mlr --opprint cat -n -g a data/small
+
+
 n a   b   i x                   y
 1 pan pan 1 0.3467901443380824  0.7268028627434533
 1 eks pan 2 0.7586799647899636  0.5221511083334797
@@ -260,12 +294,12 @@ n a   b   i x                   y
 2 wye pan 5 0.5732889198020006  0.8636244699032729
 
-.. _reference-verbs-check: - ## check -
+
 mlr check --help
+
+
 Usage: mlr check [options]
 Consumes records without printing any output.
 Useful for doing a well-formatted check on input data.
@@ -273,12 +307,12 @@ Options:
 -h|--help Show this message.
 
-.. _reference-verbs-clean-whitespace: - ## clean-whitespace -
+
 mlr clean-whitespace --help
+
+
 Usage: mlr clean-whitespace [options]
 For each record, for each field in the record, whitespace-cleans the keys and/or
 values. Whitespace-cleaning entails stripping leading and trailing whitespace,
@@ -294,8 +328,10 @@ leave off -k as well as -v.
 -h|--help Show this message.
 
-
+
 mlr --icsv --ojson cat data/clean-whitespace.csv
+
+
 {
   "  Name  ": "  Ann  Simons",
   " Preference  ": "  blue  "
@@ -310,8 +346,10 @@ leave off -k as well as -v.
 }
 
-
+
 mlr --icsv --ojson clean-whitespace -k data/clean-whitespace.csv
+
+
 {
   "Name": "  Ann  Simons",
   "Preference": "  blue  "
@@ -326,8 +364,10 @@ leave off -k as well as -v.
 }
 
-
+
 mlr --icsv --ojson clean-whitespace -v data/clean-whitespace.csv
+
+
 {
   "  Name  ": "Ann Simons",
   " Preference  ": "blue"
@@ -342,8 +382,10 @@ leave off -k as well as -v.
 }
 
-
+
 mlr --icsv --ojson clean-whitespace data/clean-whitespace.csv
+
+
 {
   "Name": "Ann Simons",
   "Preference": "blue"
@@ -360,18 +402,18 @@ leave off -k as well as -v.
 
 Function links:
 
-* :ref:`reference-dsl-lstrip`
-* :ref:`reference-dsl-rstrip`
-* :ref:`reference-dsl-strip`
-* :ref:`reference-dsl-collapse_whitespace`
-* :ref:`reference-dsl-clean_whitespace`
-
-.. _reference-verbs-count:
+* [lstrip](reference-dsl-builtin-functions.md#lstrip)
+* [rstrip](reference-dsl-builtin-functions.md#rstrip)
+* [strip](reference-dsl-builtin-functions.md#strip)
+* [collapse_whitespace](reference-dsl-builtin-functions.md#collapse_whitespace)
+* [clean_whitespace](reference-dsl-builtin-functions.md#clean_whitespace)
 
 ## count
 
-
+
 mlr count --help
+
+
 Usage: mlr count [options]
 Prints number of records, optionally grouped by distinct values for specified field names.
 Options:
@@ -381,13 +423,17 @@ Options:
 -h|--help Show this message.
 
-
+
 mlr count data/medium
+
+
 count=10000
 
-
+
 mlr count -g a data/medium
+
+
 a=pan,count=2081
 a=eks,count=1965
 a=wye,count=1966
@@ -395,13 +441,17 @@ a=zee,count=2047
 a=hat,count=1941
 
-
+
 mlr count -n -g a data/medium
+
+
 count=5
 
-
+
 mlr count -g b data/medium
+
+
 b=pan,count=1942
 b=wye,count=2057
 b=zee,count=1943
@@ -409,13 +459,17 @@ b=eks,count=2008
 b=hat,count=2050
 
-
+
 mlr count -n -g b data/medium
+
+
 count=5
 
-
+
 mlr count -g a,b data/medium
+
+
 a=pan,b=pan,count=427
 a=eks,b=pan,count=371
 a=wye,b=wye,count=377
@@ -443,12 +497,12 @@ a=eks,b=hat,count=417
 a=wye,b=eks,count=386
 
-.. _reference-verbs-count-distinct: - ## count-distinct -
+
 mlr count-distinct --help
+
+
 Usage: mlr count-distinct [options]
 Prints number of records having distinct values for specified field names.
 Same as uniq -c.
@@ -465,8 +519,10 @@ Options:
               values separately.
 
-
+
 mlr count-distinct -f a,b then sort -nr count data/medium
+
+
 a=zee,b=wye,count=455
 a=pan,b=eks,count=429
 a=pan,b=pan,count=427
@@ -494,8 +550,10 @@ a=hat,b=pan,count=363
 a=eks,b=zee,count=357
 
-
+
 mlr count-distinct -u -f a,b data/medium
+
+
 field=a,value=pan,count=2081
 field=a,value=eks,count=1965
 field=a,value=wye,count=1966
@@ -508,8 +566,10 @@ field=b,value=eks,count=2008
 field=b,value=hat,count=2050
 
-
+
 mlr count-distinct -f a,b -o someothername then sort -nr someothername data/medium
+
+
 a=zee,b=wye,someothername=455
 a=pan,b=eks,someothername=429
 a=pan,b=pan,someothername=427
@@ -537,17 +597,19 @@ a=hat,b=pan,someothername=363
 a=eks,b=zee,someothername=357
 
-
+
 mlr count-distinct -n -f a,b data/medium
+
+
 count=25
 
-.. _reference-verbs-count-similar: - ## count-similar -
+
 mlr count-similar --help
+
+
 Usage: mlr count-similar [options]
 Ingests all records, then emits each record augmented by a count of
 the number of other records having the same group-by field values.
@@ -557,8 +619,10 @@ Options:
 -h|--help Show this message.
 
-
+
 mlr --opprint head -n 20 data/medium
+
+
 a   b   i  x                   y
 pan pan 1  0.3467901443380824  0.7268028627434533
 eks pan 2  0.7586799647899636  0.5221511083334797
@@ -582,8 +646,10 @@ zee pan 19 0.43144132839222604 0.8442204830496998
 eks wye 20 0.38245149780530685 0.4730652428100751
 
-
+
 mlr --opprint head -n 20 then count-similar -g a data/medium
+
+
 a   b   i  x                   y                    count
 pan pan 1  0.3467901443380824  0.7268028627434533   4
 pan wye 10 0.5026260055412137  0.9526183602969864   4
@@ -607,8 +673,10 @@ hat wye 9  0.03144187646093577 0.7495507603507059   2
 hat zee 18 0.05727869223575699 0.13343527626645157  2
 
-
+
 mlr --opprint head -n 20 then count-similar -g a then sort -f a data/medium
+
+
 a   b   i  x                   y                    count
 eks pan 2  0.7586799647899636  0.5221511083334797   7
 eks wye 4  0.38139939387114097 0.13418874328430463  7
@@ -632,12 +700,12 @@ zee eks 17 0.29081949506712723 0.054478717073354166 5
 zee pan 19 0.43144132839222604 0.8442204830496998   5
 
-.. _reference-verbs-cut: - ## cut -
+
 mlr cut --help
+
+
 Usage: mlr cut [options]
 Passes through input records with specified fields included/excluded.
 Options:
@@ -658,8 +726,10 @@ Examples:
   mlr cut -r -f '^status$,"sda[0-9]"i' (this is case-insensitive)
 
-
+
 mlr --opprint cat data/small
+
+
 a   b   i x                   y
 pan pan 1 0.3467901443380824  0.7268028627434533
 eks pan 2 0.7586799647899636  0.5221511083334797
@@ -668,8 +738,10 @@ eks wye 4 0.38139939387114097 0.13418874328430463
 wye pan 5 0.5732889198020006  0.8636244699032729
 
-
+
 mlr --opprint cut -f y,x,i data/small
+
+
 i x                   y
 1 0.3467901443380824  0.7268028627434533
 2 0.7586799647899636  0.5221511083334797
@@ -678,22 +750,26 @@ i x                   y
 5 0.5732889198020006  0.8636244699032729
 
-
+
 echo 'a=1,b=2,c=3' | mlr cut -f b,c,a
+
+
 a=1,b=2,c=3
 
-
+
 echo 'a=1,b=2,c=3' | mlr cut -o -f b,c,a
+
+
 b=2,c=3,a=1
 
-.. _reference-verbs-decimate: - ## decimate -
+
 mlr decimate --help
+
+
 Usage: mlr decimate [options]
 Passes through one of every n records, optionally by category.
 Options:
@@ -704,12 +780,12 @@ Options:
 -h|--help Show this message.
 
-.. _reference-verbs-fill-down: - ## fill-down -
+
 mlr fill-down --help
+
+
 Usage: mlr fill-down [options]
 If a given record has a missing value for a given field, fill that from
 the corresponding value from a previous record, if any.
@@ -726,36 +802,42 @@ Options:
  -h|--help Show this message.
 
-
+
 cat data/fill-down.csv
+
+
 a,b,c
 1,,3
 4,5,6
 7,,9
 
-
+
 mlr --csv fill-down -f b data/fill-down.csv
+
+
 a,b,c
 1,,3
 4,5,6
 7,5,9
 
-
+
 mlr --csv fill-down -a -f b data/fill-down.csv
+
+
 a,b,c
 1,,3
 4,5,6
 7,,9
 
-.. _reference-verbs-filter: - ## filter -
+
 mlr filter --help
+
+
 Usage: mlr put [options] {DSL expression}
 Options:
 -f {file name} File containing a DSL expression. If the filename is a directory,
@@ -812,14 +894,14 @@ Parser-info options:
 
 ### Features which filter shares with put
 
-Please see [DSL reference](reference-dsl.md) for more information about the expression language for ``mlr filter``.
-
-.. _reference-verbs-format-values:
+Please see [DSL reference](reference-dsl.md) for more information about the expression language for `mlr filter`.
 
 ## format-values
 
-
+
 mlr format-values --help
+
+
 Usage: mlr format-values [options]
 Applies format strings to all field values, depending on autodetected type.
 * If a field value is detected to be integer, applies integer format.
@@ -851,8 +933,10 @@ Options:
                     apply the float format.
 
-
+
 mlr --opprint format-values data/small
+
+
 a   b   i x        y
 pan pan 1 0.346790 0.726803
 eks pan 2 0.758680 0.522151
@@ -861,8 +945,10 @@ eks wye 4 0.381399 0.134189
 wye pan 5 0.573289 0.863624
 
-
+
 mlr --opprint format-values -n data/small
+
+
 a   b   i        x        y
 pan pan 1.000000 0.346790 0.726803
 eks pan 2.000000 0.758680 0.522151
@@ -871,8 +957,10 @@ eks wye 4.000000 0.381399 0.134189
 wye pan 5.000000 0.573289 0.863624
 
-
+
 mlr --opprint format-values -i %08llx -f %.6le -s X%sX data/small
+
+
 a     b     i                   x                      y
 XpanX XpanX %!l(int=00000001)lx %!l(float64=0.34679)e  %!l(float64=0.726803)e
 XeksX XpanX %!l(int=00000002)lx %!l(float64=0.75868)e  %!l(float64=0.522151)e
@@ -881,8 +969,10 @@ XeksX XwyeX %!l(int=00000004)lx %!l(float64=0.381399)e %!l(float64=0.134189)e
 XwyeX XpanX %!l(int=00000005)lx %!l(float64=0.573289)e %!l(float64=0.863624)e
 
-
+
 mlr --opprint format-values -i %08llx -f %.6le -s X%sX -n data/small
+
+
 a     b     i               x                      y
 XpanX XpanX %!l(float64=1)e %!l(float64=0.34679)e  %!l(float64=0.726803)e
 XeksX XpanX %!l(float64=2)e %!l(float64=0.75868)e  %!l(float64=0.522151)e
@@ -891,12 +981,12 @@ XeksX XwyeX %!l(float64=4)e %!l(float64=0.381399)e %!l(float64=0.134189)e
 XwyeX XpanX %!l(float64=5)e %!l(float64=0.573289)e %!l(float64=0.863624)e
 
-.. _reference-verbs-fraction: - ## fraction -
+
 mlr fraction --help
+
+
 Usage: mlr fraction [options]
 For each record's value in specified fields, computes the ratio of that
 value to the sum of values in that field over all input records.
@@ -921,7 +1011,7 @@ Options:
 
 For example, suppose you have the following CSV file:
 
-
+
 u=female,v=red,n=2458
 u=female,v=green,n=192
 u=female,v=blue,n=337
@@ -936,10 +1026,12 @@ u=male,v=yellow,n=1192
 u=male,v=orange,n=448
 
-Then we can see what each record's ``n`` contributes to the total ``n``: +Then we can see what each record's `n` contributes to the total `n`: -
+
 mlr --opprint fraction -f n data/fraction-example.csv
+
+
 u      v      n    n_fraction
 female red    2458 0.32638427831629263
 female green  192  0.025494622228123754
@@ -955,10 +1047,12 @@ male   yellow 1192 0.15827911299960165
 male   orange 448  0.0594874518656221
 
-Using ``-g`` we can split those out by gender, or by color: +Using `-g` we can split those out by gender, or by color: -
+
 mlr --opprint fraction -f n -g u data/fraction-example.csv
+
+
 u      v      n    n_fraction
 female red    2458 0.7073381294964028
 female green  192  0.05525179856115108
@@ -974,8 +1068,10 @@ male   yellow 1192 0.2938856015779093
 male   orange 448  0.11045364891518737
 
-
+
 mlr --opprint fraction -f n -g v data/fraction-example.csv
+
+
 u      v      n    n_fraction
 female red    2458 0.9450211457131872
 female green  192  0.45823389021479716
@@ -993,10 +1089,12 @@ male   orange 448  0.9634408602150538
 
 We can see, for example, that 70.9% of females have red (on the left) while 94.5% of reds are for females.
 
-To convert fractions to percents, you may use ``-p``:
+To convert fractions to percents, you may use `-p`:
 
-
+
 mlr --opprint fraction -f n -p data/fraction-example.csv
+
+
 u      v      n    n_percent
 female red    2458 32.638427831629265
 female green  192  2.5494622228123753
@@ -1012,10 +1110,12 @@ male   yellow 1192 15.827911299960165
 male   orange 448  5.94874518656221
 
-Another often-used idiom is to convert from a point distribution to a cumulative distribution, also known as "running sums". Here, you can use ``-c``: +Another often-used idiom is to convert from a point distribution to a cumulative distribution, also known as "running sums". Here, you can use `-c`: -
+
 mlr --opprint fraction -f n -p -c data/fraction-example.csv
+
+
 u      v      n    n_cumulative_percent
 female red    2458 32.638427831629265
 female green  192  35.18789005444164
@@ -1031,8 +1131,10 @@ male   yellow 1192 94.0512548134378
 male   orange 448  100
 
-
+
 mlr --opprint fraction -f n -g u -p -c data/fraction-example.csv
+
+
 u      v      n    n_cumulative_percent
 female red    2458 70.73381294964028
 female green  192  76.2589928057554
@@ -1048,12 +1150,12 @@ male   yellow 1192 88.95463510848126
 male   orange 448  100
 
-.. _reference-verbs-grep: - ## grep -
+
 mlr grep -h
+
+
 Usage: mlr grep [options] {regular expression}
 Passes through records which match the regular expression.
 Options:
@@ -1073,21 +1175,23 @@ features of system grep, you can do
   "mlr --odkvp ... | grep ... | mlr --idkvp ..."
 
-.. _reference-verbs-group-by: - ## group-by -
+
 mlr group-by --help
+
+
 Usage: mlr group-by [options] {comma-separated field names}
 Outputs records in batches having identical values at specified field names.Options:
 -h|--help Show this message.
 
-This is similar to ``sort`` but with less work. Namely, Miller's sort has three steps: read through the data and append linked lists of records, one for each unique combination of the key-field values; after all records are read, sort the key-field values; then print each record-list. The group-by operation simply omits the middle sort. An example should make this more clear. +This is similar to `sort` but with less work. Namely, Miller's sort has three steps: read through the data and append linked lists of records, one for each unique combination of the key-field values; after all records are read, sort the key-field values; then print each record-list. The group-by operation simply omits the middle sort. An example should make this more clear. -
+
 mlr --opprint group-by a data/small
+
+
 a   b   i x                   y
 pan pan 1 0.3467901443380824  0.7268028627434533
 eks pan 2 0.7586799647899636  0.5221511083334797
@@ -1096,8 +1200,10 @@ wye wye 3 0.20460330576630303 0.33831852551664776
 wye pan 5 0.5732889198020006  0.8636244699032729
 
-
+
 mlr --opprint sort -f a data/small
+
+
 a   b   i x                   y
 eks pan 2 0.7586799647899636  0.5221511083334797
 eks wye 4 0.38139939387114097 0.13418874328430463
@@ -1106,14 +1212,14 @@ wye wye 3 0.20460330576630303 0.33831852551664776
 wye pan 5 0.5732889198020006  0.8636244699032729
 
-In this example, since the sort is on field ``a``, the first step is to group together all records having the same value for field ``a``; the second step is to sort the distinct ``a``-field values ``pan``, ``eks``, and ``wye`` into ``eks``, ``pan``, and ``wye``; the third step is to print out the record-list for ``a=eks``, then the record-list for ``a=pan``, then the record-list for ``a=wye``. The group-by operation omits the middle sort and just puts like records together, for those times when a sort isn't desired. In particular, the ordering of group-by fields for group-by is the order in which they were encountered in the data stream, which in some cases may be more interesting to you. - -.. _reference-verbs-group-like: +In this example, since the sort is on field `a`, the first step is to group together all records having the same value for field `a`; the second step is to sort the distinct `a`-field values `pan`, `eks`, and `wye` into `eks`, `pan`, and `wye`; the third step is to print out the record-list for `a=eks`, then the record-list for `a=pan`, then the record-list for `a=wye`. The group-by operation omits the middle sort and just puts like records together, for those times when a sort isn't desired. In particular, the ordering of group-by fields for group-by is the order in which they were encountered in the data stream, which in some cases may be more interesting to you. ## group-like -
+
 mlr group-like --help
+
+
 Usage: mlr group-like [options]
 Outputs records in batches having identical field names.Options:
 -h|--help Show this message.
@@ -1121,8 +1227,10 @@ Outputs records in batches having identical field names.Options:
 
 This groups together records having the same schema (i.e. same ordered list of field names) which is useful for making sense of time-ordered output as described in [Record Heterogeneity](record-heterogeneity.md) -- in particular, in preparation for CSV or pretty-print output.
 
-
+
 mlr cat data/het.dkvp
+
+
 resource=/path/to/file,loadsec=0.45,ok=true
 record_count=100,resource=/path/to/file
 resource=/path/to/second/file,loadsec=0.32,ok=true
@@ -1130,8 +1238,10 @@ record_count=150,resource=/path/to/second/file
 resource=/some/other/path,loadsec=0.97,ok=false
 
-
+
 mlr --opprint group-like data/het.dkvp
+
+
 resource             loadsec ok
 /path/to/file        0.45    true
 /path/to/second/file 0.32    true
@@ -1142,12 +1252,12 @@ record_count resource
 150          /path/to/second/file
 
-.. _reference-verbs-having-fields: - ## having-fields -
+
 mlr having-fields --help
+
+
 Usage: mlr having-fields [options]
 Conditionally passes through records depending on each record's field names.
 Options:
@@ -1164,10 +1274,12 @@ Examples:
   mlr having-fields --any-matching '"sda[0-9]"i' (this is case-insensitive)
 
-Similar to :ref:`reference-verbs-group-like`, this retains records with specified schema. +Similar to [group-like](reference-verbs.md#group-like), this retains records with specified schema. -
+
 mlr cat data/het.dkvp
+
+
 resource=/path/to/file,loadsec=0.45,ok=true
 record_count=100,resource=/path/to/file
 resource=/path/to/second/file,loadsec=0.32,ok=true
@@ -1175,8 +1287,10 @@ record_count=150,resource=/path/to/second/file
 resource=/some/other/path,loadsec=0.97,ok=false
 
-
+
 mlr having-fields --at-least resource data/het.dkvp
+
+
 resource=/path/to/file,loadsec=0.45,ok=true
 record_count=100,resource=/path/to/file
 resource=/path/to/second/file,loadsec=0.32,ok=true
@@ -1184,19 +1298,21 @@ record_count=150,resource=/path/to/second/file
 resource=/some/other/path,loadsec=0.97,ok=false
 
-
+
 mlr having-fields --which-are resource,ok,loadsec data/het.dkvp
+
+
 resource=/path/to/file,loadsec=0.45,ok=true
 resource=/path/to/second/file,loadsec=0.32,ok=true
 resource=/some/other/path,loadsec=0.97,ok=false
 
-.. _reference-verbs-head: - ## head -
+
 mlr head --help
+
+
 Usage: mlr head [options]
 Passes through the first n records, optionally by category.
 Options:
@@ -1205,10 +1321,12 @@ Options:
 -h|--help Show this message.
 
-Note that ``head`` is distinct from :ref:`reference-verbs-top` -- ``head`` shows fields which appear fimd.in the data stream; ``top`` shows fields which are numerically largest (or smallest). +Note that `head` is distinct from [top](reference-verbs.md#top) -- `head` shows fields which appear fimd.in the data stream; `top` shows fields which are numerically largest (or smallest). -
+
 mlr --opprint head -n 4 data/medium
+
+
 a   b   i x                   y
 pan pan 1 0.3467901443380824  0.7268028627434533
 eks pan 2 0.7586799647899636  0.5221511083334797
@@ -1216,8 +1334,10 @@ wye wye 3 0.20460330576630303 0.33831852551664776
 eks wye 4 0.38139939387114097 0.13418874328430463
 
-
+
 mlr --opprint head -n 1 -g b data/medium
+
+
 a   b   i  x                   y
 pan pan 1  0.3467901443380824  0.7268028627434533
 wye wye 3  0.20460330576630303 0.33831852551664776
@@ -1226,12 +1346,12 @@ zee eks 17 0.29081949506712723 0.054478717073354166
 wye hat 24 0.7286126830627567  0.19441962592638418
 
-.. _reference-verbs-histogram: - ## histogram -
+
 mlr histogram --help
+
+
 Just a histogram. Input values < lo or > hi are not counted.
 Usage: mlr histogram [options]
 -f {a,b,c}    Value-field names for histogram counts
@@ -1244,12 +1364,14 @@ Usage: mlr histogram [options]
 -h|--help Show this message.
 
-This is just a histogram; there's not too much to say here. A note about binning, by example: Suppose you use ``--lo 0.0 --hi 1.0 --nbins 10 -f x``. The input numbers less than 0 or greater than 1 aren't counted in any bin. Input numbers equal to 1 are counted in the last bin. That is, bin 0 has ``0.0 ≤ x < 0.1``, bin 1 has ``0.1 ≤ x < 0.2``, etc., but bin 9 has ``0.9 ≤ x ≤ 1.0``. +This is just a histogram; there's not too much to say here. A note about binning, by example: Suppose you use `--lo 0.0 --hi 1.0 --nbins 10 -f x`. The input numbers less than 0 or greater than 1 aren't counted in any bin. Input numbers equal to 1 are counted in the last bin. That is, bin 0 has `0.0 ≤ x < 0.1`, bin 1 has `0.1 ≤ x < 0.2`, etc., but bin 9 has `0.9 ≤ x ≤ 1.0`. -
+
 mlr --opprint put '$x2=$x**2;$x3=$x2*$x' \
   then histogram -f x,x2,x3 --lo 0 --hi 1 --nbins 10 \
   data/medium
+
+
 bin_lo bin_hi x_count x2_count x3_count
 0      0.1    1072    3231     4661
 0.1    0.2    938     1254     1184
@@ -1263,10 +1385,12 @@ bin_lo bin_hi x_count x2_count x3_count
 0.9    1      1013    507      341
 
-
+
 mlr --opprint put '$x2=$x**2;$x3=$x2*$x' \
   then histogram -f x,x2,x3 --lo 0 --hi 1 --nbins 10 -o my_ \
   data/medium
+
+
 my_bin_lo my_bin_hi my_x_count my_x2_count my_x3_count
 0         0.1       1072       3231        4661
 0.1       0.2       938        1254        1184
@@ -1280,12 +1404,12 @@ my_bin_lo my_bin_hi my_x_count my_x2_count my_x3_count
 0.9       1         1013       507         341
 
-.. _reference-verbs-join: - ## join -
+
 mlr join --help
+
+
 Usage: mlr sort {flags}
 Sorts records primarily by the first specified field, secondarily by the second
 field, and so on.  (Any records not having all specified sort keys will appear
@@ -1311,8 +1435,10 @@ Examples:
 
 Join larger table with IDs with smaller ID-to-name lookup table, showing only paired records:
 
-
+
 mlr --icsvlite --opprint cat data/join-left-example.csv
+
+
 id  name
 100 alice
 200 bob
@@ -1321,8 +1447,10 @@ id  name
 500 edgar
 
-
+
 mlr --icsvlite --opprint cat data/join-right-example.csv
+
+
 status  idcode
 present 400
 present 100
@@ -1346,10 +1474,12 @@ present 400
 present 300
 
-
+
 mlr --icsvlite --opprint \
   join -u -j id -r idcode -f data/join-left-example.csv \
   data/join-right-example.csv
+
+
 id  name  status
 400 david present
 100 alice present
@@ -1374,10 +1504,12 @@ id  name  status
 
 Same, but with sorting the input first:
 
-
+
 mlr --icsvlite --opprint sort -f idcode \
   then join -j id -r idcode -f data/join-left-example.csv \
   data/join-right-example.csv
+
+
 id  name  status
 100 alice present
 100 alice present
@@ -1402,10 +1534,12 @@ id  name  status
 
 Same, but showing only unpaired records:
 
-
+
 mlr --icsvlite --opprint \
   join --np --ul --ur -u -j id -r idcode -f data/join-left-example.csv \
   data/join-right-example.csv
+
+
 status  idcode
 missing 600
 
@@ -1415,8 +1549,10 @@ id  name
 
 Use prefixing options to disambiguate between otherwise identical non-join field names:
 
-
+
 mlr --csvlite --opprint cat data/self-join.csv data/self-join.csv
+
+
 a b c
 1 2 3
 1 4 5
@@ -1424,8 +1560,10 @@ a b c
 1 4 5
 
-
+
 mlr --csvlite --opprint join -j a --lp left_ --rp right_ -f data/self-join.csv data/self-join.csv
+
+
 a left_b left_c right_b right_c
 1 2      3      2       3
 1 4      5      2       3
@@ -1435,8 +1573,10 @@ a left_b left_c right_b right_c
 
 Use zero join columns:
 
-
+
 mlr --csvlite --opprint join -j "" --lp left_ --rp right_ -f data/self-join.csv data/self-join.csv
+
+
 left_a left_b left_c right_a right_b right_c
 1      2      3      1       2       3
 1      4      5      1       2       3
@@ -1444,12 +1584,12 @@ left_a left_b left_c right_a right_b right_c
 1      4      5      1       4       5
 
-.. _reference-verbs-label: - ## label -
+
 mlr label --help
+
+
 Usage: mlr label [options] {new1,new2,new3,...}
 Given n comma-separated names, renames the first n fields of each record to
 have the respective name. (Fields past the nth are left with their original
@@ -1460,11 +1600,11 @@ Options:
 -h|--help Show this message.
 
-See also :ref:`reference-verbs-rename`. +See also [rename](reference-verbs.md#rename). -Example: Files such as ``/etc/passwd``, ``/etc/group``, and so on have implicit field names which are found in section-5 manpages. These field names may be made explicit as follows: +Example: Files such as `/etc/passwd`, `/etc/group`, and so on have implicit field names which are found in section-5 manpages. These field names may be made explicit as follows: -
+
 % grep -v '^#' /etc/passwd | mlr --nidx --fs : --opprint label name,password,uid,gid,gecos,home_dir,shell | head
 name                  password uid gid gecos                         home_dir           shell
 nobody                *        -2  -2  Unprivileged User             /var/empty         /usr/bin/false
@@ -1478,18 +1618,22 @@ _lp                   *        26  26  Printing Services             /var/spool/
 _postfix              *        27  27  Postfix Mail Server           /var/spool/postfix /usr/bin/false
 
-Likewise, if you have CSV/CSV-lite input data which has somehow been bereft of its header line, you can re-add a header line using ``--implicit-csv-header`` and ``label``: +Likewise, if you have CSV/CSV-lite input data which has somehow been bereft of its header line, you can re-add a header line using `--implicit-csv-header` and `label`: -
+
 cat data/headerless.csv
+
+
 John,23,present
 Fred,34,present
 Alice,56,missing
 Carol,45,present
 
-
+
 mlr  --csv --implicit-csv-header cat data/headerless.csv
+
+
 1,2,3
 John,23,present
 Fred,34,present
@@ -1497,8 +1641,10 @@ Alice,56,missing
 Carol,45,present
 
-
+
 mlr  --csv --implicit-csv-header label name,age,status data/headerless.csv
+
+
 name,age,status
 John,23,present
 Fred,34,present
@@ -1506,8 +1652,10 @@ Alice,56,missing
 Carol,45,present
 
-
+
 mlr --icsv --implicit-csv-header --opprint label name,age,status data/headerless.csv
+
+
 name  age status
 John  23  present
 Fred  34  present
@@ -1515,12 +1663,12 @@ Alice 56  missing
 Carol 45  present
 
-.. _reference-verbs-least-frequent: - ## least-frequent -
+
 mlr least-frequent -h
+
+
 Usage: mlr least-frequent [options]
 Shows the least frequently occurring distinct values for specified field names.
 The first entry is the statistical anti-mode; the remaining are runners-up.
@@ -1532,16 +1680,20 @@ Options:
 See also "mlr most-frequent".
 
-
+
 mlr --opprint --from data/colored-shapes.dkvp least-frequent -f shape -n 5
+
+
 shape    count
 circle   2591
 triangle 3372
 square   4115
 
-
+
 mlr --opprint --from data/colored-shapes.dkvp least-frequent -f shape,color -n 5
+
+
 shape    color  count
 circle   orange 68
 triangle orange 107
@@ -1550,8 +1702,10 @@ circle   green  287
 circle   purple 289
 
-
+
 mlr --opprint --from data/colored-shapes.dkvp least-frequent -f shape,color -n 5 -o someothername
+
+
 shape    color  someothername
 circle   orange 68
 triangle orange 107
@@ -1560,8 +1714,10 @@ circle   green  287
 circle   purple 289
 
-
+
 mlr --opprint --from data/colored-shapes.dkvp least-frequent -f shape,color -n 5 -b
+
+
 shape    color
 circle   orange
 triangle orange
@@ -1570,14 +1726,14 @@ circle   green
 circle   purple
 
-See also :ref:`reference-verbs-most-frequent`. - -.. _reference-verbs-merge-fields: +See also [most-frequent](reference-verbs.md#most-frequent). ## merge-fields -
+
 mlr merge-fields --help
+
+
 Usage: mlr merge-fields [options]
 Computes univariate statistics for each input record, accumulated across
 specified fields.
@@ -1623,40 +1779,46 @@ Example: mlr merge-fields -a sum,count -c in_,out_
   "b_y", and "b_out_x" collapses to "b_x".
 
-This is like ``mlr stats1`` but all accumulation is done across fields within each given record: horizontal rather than vertical statistics, if you will. +This is like `mlr stats1` but all accumulation is done across fields within each given record: horizontal rather than vertical statistics, if you will. Examples: -
+
 mlr --csvlite --opprint cat data/inout.csv
+
+
 a_in a_out b_in b_out
 436  490   446  195
 526  320   963  780
 220  888   705  831
 
-
+
 mlr --csvlite --opprint merge-fields -a min,max,sum -c _in,_out data/inout.csv
+
+
 a_min a_max a_sum b_min b_max b_sum
 436   490   926   195   446   641
 320   526   846   780   963   1743
 220   888   1108  705   831   1536
 
-
+
 mlr --csvlite --opprint merge-fields -k -a sum -c _in,_out data/inout.csv
+
+
 a_in a_out b_in b_out a_sum b_sum
 436  490   446  195   926   641
 526  320   963  780   846   1743
 220  888   705  831   1108  1536
 
-.. _reference-verbs-most-frequent: - ## most-frequent -
+
 mlr most-frequent -h
+
+
 Usage: mlr most-frequent [options]
 Shows the most frequently occurring distinct values for specified field names.
 The first entry is the statistical mode; the remaining are runners-up.
@@ -1668,16 +1830,20 @@ Options:
 See also "mlr least-frequent".
 
-
+
 mlr --opprint --from data/colored-shapes.dkvp most-frequent -f shape -n 5
+
+
 shape    count
 square   4115
 triangle 3372
 circle   2591
 
-
+
 mlr --opprint --from data/colored-shapes.dkvp most-frequent -f shape,color -n 5
+
+
 shape    color  count
 square   red    1874
 triangle red    1560
@@ -1686,8 +1852,10 @@ square   yellow 589
 square   blue   589
 
-
+
 mlr --opprint --from data/colored-shapes.dkvp most-frequent -f shape,color -n 5 -o someothername
+
+
 shape    color  someothername
 square   red    1874
 triangle red    1560
@@ -1696,8 +1864,10 @@ square   yellow 589
 square   blue   589
 
-
+
 mlr --opprint --from data/colored-shapes.dkvp most-frequent -f shape,color -n 5 -b
+
+
 shape    color
 square   red
 triangle red
@@ -1706,14 +1876,14 @@ square   yellow
 square   blue
 
-See also :ref:`reference-verbs-least-frequent`. - -.. _reference-verbs-nest: +See also [least-frequent](reference-verbs.md#least-frequent). ## nest -
+
 mlr nest -h
+
+
 Usage: mlr nest [options]
 Explodes specified field values into separate fields/records, or reverses this.
 Options:
@@ -1763,12 +1933,12 @@ Notes:
 See also mlr reshape.
 
-.. _reference-verbs-nothing: - ## nothing -
+
 mlr nothing -h
+
+
 Usage: mlr nothing [options]
 Drops all input records. Useful for testing, or after tee/print/etc. have
 produced other output.
@@ -1776,12 +1946,12 @@ Options:
 -h|--help Show this message.
 
-.. _reference-verbs-put: - ## put -
+
 mlr put --help
+
+
 Usage: mlr put [options] {DSL expression}
 Options:
 -f {file name} File containing a DSL expression. If the filename is a directory,
@@ -1838,45 +2008,49 @@ Parser-info options:
 
 ### Features which put shares with filter
 
-Please see the [DSL reference](reference-dsl.md) for more information about the expression language for ``mlr put``.
-
-.. _reference-verbs-regularize:
+Please see the [DSL reference](reference-dsl.md) for more information about the expression language for `mlr put`.
 
 ## regularize
 
-
+
 mlr regularize --help
+
+
 Usage: mlr regularize [options]
 Outputs records sorted lexically ascending by keys.Options:
 -h|--help Show this message.
 
-This exists since hash-map software in various languages and tools encountered in the wild does not always print similar rows with fields in the same order: ``mlr regularize`` helps clean that up. +This exists since hash-map software in various languages and tools encountered in the wild does not always print similar rows with fields in the same order: `mlr regularize` helps clean that up. -See also :ref:`reference-verbs-reorder`. - -.. _reference-verbs-remove-empty-columns: +See also [reorder](reference-verbs.md#reorder). ## remove-empty-columns -
+
 mlr remove-empty-columns --help
+
+
 Usage: mlr remove-empty-columns [options]
 Omits fields which are empty on every input row. Non-streaming.
 Options:
 -h|--help Show this message.
 
-
+
 cat data/remove-empty-columns.csv
+
+
 a,b,c,d,e
 1,,3,,5
 2,,4,,5
 3,,5,,7
 
-
+
 mlr --csv remove-empty-columns data/remove-empty-columns.csv
+
+
 a,c,e
 1,3,5
 2,4,5
@@ -1885,12 +2059,12 @@ a,c,e
 
 Since this verb needs to read all records to see if any of them has a non-empty value for a given field name, it is non-streaming: it will ingest all records before writing any.
 
-.. _reference-verbs-rename:
-
 ## rename
 
-
+
 mlr rename --help
+
+
 Usage: mlr rename [options] {old1,new1,old2,new2,...}
 Renames specified fields.
 Options:
@@ -1914,8 +2088,10 @@ mlr rename -r 'Date_([0-9]+).*,\1' Rename all such fields to be of the form 2015
 mlr rename -r '"name"i,Name'       Rename "name", "Name", "NAME", etc. to "Name"
 
-
+
 mlr --opprint cat data/small
+
+
 a   b   i x                   y
 pan pan 1 0.3467901443380824  0.7268028627434533
 eks pan 2 0.7586799647899636  0.5221511083334797
@@ -1924,8 +2100,10 @@ eks wye 4 0.38139939387114097 0.13418874328430463
 wye pan 5 0.5732889198020006  0.8636244699032729
 
-
+
 mlr --opprint rename i,INDEX,b,COLUMN2 data/small
+
+
 a   COLUMN2 INDEX x                   y
 pan pan     1     0.3467901443380824  0.7268028627434533
 eks pan     2     0.7586799647899636  0.5221511083334797
@@ -1934,10 +2112,12 @@ eks wye     4     0.38139939387114097 0.13418874328430463
 wye pan     5     0.5732889198020006  0.8636244699032729
 
-As discussed in [Performance](performance.md), ``sed`` is significantly faster than Miller at doing this. However, Miller is format-aware, so it knows to do renames only within specified field keys and not any others, nor in field values which may happen to contain the same pattern. Example: +As discussed in [Performance](performance.md), `sed` is significantly faster than Miller at doing this. However, Miller is format-aware, so it knows to do renames only within specified field keys and not any others, nor in field values which may happen to contain the same pattern. Example: -
+
 sed 's/y/COLUMN5/g' data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,COLUMN5=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,COLUMN5=0.5221511083334797
 a=wCOLUMN5e,b=wCOLUMN5e,i=3,x=0.20460330576630303,COLUMN5=0.33831852551664776
@@ -1945,8 +2125,10 @@ a=eks,b=wCOLUMN5e,i=4,x=0.38139939387114097,COLUMN5=0.13418874328430463
 a=wCOLUMN5e,b=pan,i=5,x=0.5732889198020006,COLUMN5=0.8636244699032729
 
-
+
 mlr rename y,COLUMN5 data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,COLUMN5=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,COLUMN5=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,COLUMN5=0.33831852551664776
@@ -1954,14 +2136,14 @@ a=eks,b=wye,i=4,x=0.38139939387114097,COLUMN5=0.13418874328430463
 a=wye,b=pan,i=5,x=0.5732889198020006,COLUMN5=0.8636244699032729
 
-See also :ref:`reference-verbs-label`. - -.. _reference-verbs-reorder: +See also [label](reference-verbs.md#label). ## reorder -
+
 mlr reorder --help
+
+
 Usage: mlr reorder [options]
 Moves specified names to start of record, or end of record.
 Options:
@@ -1984,8 +2166,10 @@ This pivots specified field names to the start or end of the record -- for
 example when you have highly multi-column data and you want to bring a field or
 two to the front of line where you can give a quick visual scan.
 
-
+
 mlr --opprint cat data/small
+
+
 a   b   i x                   y
 pan pan 1 0.3467901443380824  0.7268028627434533
 eks pan 2 0.7586799647899636  0.5221511083334797
@@ -1994,8 +2178,10 @@ eks wye 4 0.38139939387114097 0.13418874328430463
 wye pan 5 0.5732889198020006  0.8636244699032729
 
-
+
 mlr --opprint reorder -f i,b data/small
+
+
 i b   a   x                   y
 1 pan pan 0.3467901443380824  0.7268028627434533
 2 pan eks 0.7586799647899636  0.5221511083334797
@@ -2004,8 +2190,10 @@ i b   a   x                   y
 5 pan wye 0.5732889198020006  0.8636244699032729
 
-
+
 mlr --opprint reorder -e -f i,b data/small
+
+
 a   x                   y                   i b
 pan 0.3467901443380824  0.7268028627434533  1 pan
 eks 0.7586799647899636  0.5221511083334797  2 pan
@@ -2014,12 +2202,12 @@ eks 0.38139939387114097 0.13418874328430463 4 wye
 wye 0.5732889198020006  0.8636244699032729  5 pan
 
-.. _reference-verbs-repeat: - ## repeat -
+
 mlr repeat --help
+
+
 Usage: mlr repeat [options]
 Copies input records to output records multiple times.
 Options must be exactly one of the following:
@@ -2048,18 +2236,22 @@ produces:
 
This is useful in at least two ways: one, as a data-generator as in the -above example using ``urand()``; two, for reconstructing individual +above example using `urand()`; two, for reconstructing individual samples from data which has been count-aggregated: -
+
 cat data/repeat-example.dat
+
+
 color=blue,count=5
 color=red,count=4
 color=green,count=3
 
-
+
 mlr repeat -f count then cut -x -f count data/repeat-example.dat
+
+
 color=blue
 color=blue
 color=blue
@@ -2074,16 +2266,16 @@ color=green
 color=green
 
-After expansion with ``repeat``, such data can then be sent on to -``stats1 -a mode``, or (if the data are numeric) to ``stats1 -a -p10,p50,p90``, etc. - -.. _reference-verbs-reshape: +After expansion with `repeat`, such data can then be sent on to +`stats1 -a mode`, or (if the data are numeric) to `stats1 -a +p10,p50,p90`, etc. ## reshape -
+
 mlr reshape --help
+
+
 Usage: mlr reshape [options]
 Wide-to-long options:
   -i {input field names}   -o {key-field name,value-field name}
@@ -2141,12 +2333,12 @@ Examples:
 See also mlr nest.
 
-.. _reference-verbs-sample: - ## sample -
+
 mlr sample --help
+
+
 Usage: mlr sample [options]
 Reservoir sampling (subsampling without replacement), optionally by category.
 See also mlr bootstrap and mlr shuffle.
@@ -2158,11 +2350,11 @@ Options:
 
 This is reservoir-sampling: select *k* items from *n* with
 uniform probability and no repeats in the sample. (If *n* is less than
-*k*, then of course only *n* samples are produced.) With ``-g
-{field names}``, produce a *k*-sample for each distinct value of the
+*k*, then of course only *n* samples are produced.) With `-g
+{field names}`, produce a *k*-sample for each distinct value of the
 specified field names.
 
-
+
 $ mlr --opprint sample -k 4 data/colored-shapes.dkvp 
 color  shape    flag i     u                   v                    w                   x
 purple triangle 0    90122 0.9986871176198068  0.3037738877233719   0.5154934457238382  5.365962021016529
@@ -2210,15 +2402,15 @@ yellow square   1    158   0.41527900739142165 0.7118027080775757   0.4200799665
 
Note that no output is produced until all inputs are in. Another way to do -sampling, which works in the streaming case, is ``mlr filter 'urand() & -0.001'`` where you tune the 0.001 to meet your needs. - -.. _reference-verbs-sec2gmt: +sampling, which works in the streaming case, is `mlr filter 'urand() & +0.001'` where you tune the 0.001 to meet your needs. ## sec2gmt -
+
 mlr sec2gmt -h
+
+
 Usage: mlr sec2gmt [options] {comma-separated list of field names}
 Replaces a numeric field representing seconds since the epoch with the
 corresponding GMT timestamp; leaves non-numbers as-is. This is nothing
@@ -2234,12 +2426,12 @@ Options:
 -h|--help Show this message.
 
-.. _reference-verbs-sec2gmtdate: - ## sec2gmtdate -
+
 mlr sec2gmtdate -h
+
+
 Usage: ../c/mlr sec2gmtdate {comma-separated list of field names}
 Replaces a numeric field representing seconds since the epoch with the
 corresponding GMT year-month-day timestamp; leaves non-numbers as-is.
@@ -2249,12 +2441,12 @@ is the same as
   ../c/mlr put '$time1=sec2gmtdate($time1);$time2=sec2gmtdate($time2)'
 
-.. _reference-verbs-seqgen: - ## seqgen -
+
 mlr seqgen -h
+
+
 Usage: mlr seqgen [options]
 Passes input records directly to output. Most useful for format conversion.
 Produces a sequence of counters.  Discards the input record stream. Produces
@@ -2271,8 +2463,10 @@ stop, and step are all integers. Step may be negative. It may not be zero
 unless start == stop.
 
-
+
 mlr seqgen --stop 10
+
+
 i=1
 i=2
 i=3
@@ -2285,8 +2479,10 @@ i=9
 i=10
 
-
+
 mlr seqgen --start 20 --stop 40 --step 4
+
+
 i=20
 i=24
 i=28
@@ -2295,8 +2491,10 @@ i=36
 i=40
 
-
+
 mlr seqgen --start 40 --stop 20 --step -4
+
+
 i=40
 i=36
 i=32
@@ -2305,12 +2503,12 @@ i=24
 i=20
 
-.. _reference-verbs-shuffle: - ## shuffle -
+
 mlr shuffle -h
+
+
 Usage: mlr shuffle [options]
 Outputs records randomly permuted. No output records are produced until
 all input records are read. See also mlr bootstrap and mlr sample.
@@ -2318,12 +2516,12 @@ Options:
 -h|--help Show this message.
 
-.. _reference-verbs-skip-trivial-records: - ## skip-trivial-records -
+
 mlr skip-trivial-records -h
+
+
 Usage: mlr skip-trivial-records [options]
 Passes through all records except those with zero fields,
 or those for which all fields have empty value.
@@ -2331,8 +2529,10 @@ Options:
 -h|--help Show this message.
 
-
+
 cat data/trivial-records.csv
+
+
 a,b,c
 1,2,3
 4,,6
@@ -2340,20 +2540,22 @@ a,b,c
 ,8,9
 
-
+
 mlr --csv skip-trivial-records data/trivial-records.csv
+
+
 a,b,c
 1,2,3
 4,,6
 ,8,9
 
-.. _reference-verbs-sort: - ## sort -
+
 mlr sort --help
+
+
 Usage: mlr sort {flags}
 Sorts records primarily by the first specified field, secondarily by the second
 field, and so on.  (Any records not having all specified sort keys will appear
@@ -2377,8 +2579,10 @@ which is the same as:
 
 Example:
 
-
+
 mlr --opprint sort -f a -nr x data/small
+
+
 a   b   i x                   y
 eks pan 2 0.7586799647899636  0.5221511083334797
 eks wye 4 0.38139939387114097 0.13418874328430463
@@ -2389,8 +2593,10 @@ wye wye 3 0.20460330576630303 0.33831852551664776
 
 Here's an example filtering log data: suppose multiple threads (labeled here by color) are all logging progress counts to a single log file. The log file is (by nature) chronological, so the progress of various threads is interleaved:
 
-
+
 head -n 10 data/multicountdown.dat
+
+
 upsec=0.002,color=green,count=1203
 upsec=0.083,color=red,count=3817
 upsec=0.188,color=red,count=3801
@@ -2404,11 +2610,13 @@ upsec=1.327,color=purple,count=917
 
We can group these by thread by sorting on the thread ID (here, -``color``). Since Miller's sort is stable, this means that +`color`). Since Miller's sort is stable, this means that timestamps within each thread's log data are still chronological: -
+
 head -n 20 data/multicountdown.dat | mlr --opprint sort -f color
+
+
 upsec              color  count
 0.395              blue   2697
 0.671              blue   2684
@@ -2435,28 +2643,32 @@ upsec              color  count
 Any records not having all specified sort keys will appear at the end of the output, in the order they
 were encountered, regardless of the specified sort order:
 
-
+
 mlr sort -n  x data/sort-missing.dkvp
+
+
 x=1
 x=2
 x=4
 a=3
 
-
+
 mlr sort -nr x data/sort-missing.dkvp
+
+
 x=4
 x=2
 x=1
 a=3
 
-.. _reference-verbs-sort-within-records: - ## sort-within-records -
+
 mlr sort-within-records -h
+
+
 Usage: mlr sort-within-records [options]
 Outputs records sorted lexically ascending by keys.
 Options:
@@ -2464,8 +2676,10 @@ Options:
 -h|--help Show this message.
 
-
+
 cat data/sort-within-records.json
+
+
 {
   "a": 1,
   "b": 2,
@@ -2483,8 +2697,10 @@ Options:
 }
 
-
+
 mlr --ijson --opprint cat data/sort-within-records.json
+
+
 a b c
 1 2 3
 
@@ -2495,8 +2711,10 @@ c b a
 7 8 9
 
-
+
 mlr --json sort-within-records data/sort-within-records.json
+
+
 {
   "a": 1,
   "b": 2,
@@ -2514,20 +2732,22 @@ c b a
 }
 
-
+
 mlr --ijson --opprint sort-within-records data/sort-within-records.json
+
+
 a b c
 1 2 3
 5 4 6
 9 8 7
 
-.. _reference-verbs-stats1: - ## stats1 -
+
 mlr stats1 --help
+
+
 Usage: mlr stats1 [options]
 Computes univariate statistics for one or more given fields, accumulated across
 the input record stream.
@@ -2583,11 +2803,13 @@ Notes:
 
These are simple univariate statistics on one or more number-valued fields -(``count`` and ``mode`` apply to non-numeric fields as well), +(`count` and `mode` apply to non-numeric fields as well), optionally categorized by one or more other fields. -
+
 mlr --oxtab stats1 -a count,sum,min,p10,p50,mean,p90,max -f x,y data/medium
+
+
 x_count 10000
 x_sum   4986.019681679581
 x_min   0.00004509679127584487
@@ -2606,8 +2828,10 @@ y_p90   0.9053657573378745
 y_max   0.9999648102177897
 
-
+
 mlr --opprint stats1 -a mean -f x,y -g b then sort -f b data/medium
+
+
 b   x_mean             y_mean
 eks 0.5063609846272304 0.510292657158104
 hat 0.4878988625336502 0.5131176341556505
@@ -2616,10 +2840,12 @@ wye 0.4975928392133964 0.5045964890907357
 zee 0.5042419022900586 0.5029967546798116
 
-
+
 mlr --opprint stats1 -a p50,p99 -f u,v -g color \
   then put '$ur=$u_p99/$u_p50;$vr=$v_p99/$v_p50' \
   data/colored-shapes.dkvp
+
+
 color  u_p50               u_p99              v_p50               v_p99              ur                 vr
 yellow 0.5010187906650703  0.9890464545334569 0.5206303554834582  0.9870337429747029 1.9740705797093183 1.8958436298977264
 red    0.48503770531462564 0.9900536015797581 0.49258608624814926 0.9944442307252868 2.0411889441410493 2.0188232239761583
@@ -2629,28 +2855,32 @@ blue   0.525225660059      0.9926547550299167 0.48516993577967726 0.993872833141
 orange 0.4835478569328253  0.9936350141409035 0.48091255603363914 0.9891023960550895 2.0548845370623567 2.0567198415711636
 
-
+
 mlr --opprint count-distinct -f shape then sort -nr count data/colored-shapes.dkvp
+
+
 shape    count
 square   4115
 triangle 3372
 circle   2591
 
-
+
 mlr --opprint stats1 -a mode -f color -g shape data/colored-shapes.dkvp
+
+
 shape    color_mode
 triangle red
 square   red
 circle   red
 
-.. _reference-verbs-stats2: - ## stats2 -
+
 mlr stats2 --help
+
+
 Usage: mlr stats2 [options]
 Computes bivariate statistics for one or more given field-name pairs,
 accumulated across the input record stream.
@@ -2682,10 +2912,12 @@ Example: mlr stats2 -a corr -f x,y
 These are simple bivariate statistics on one or more pairs of number-valued
 fields, optionally categorized by one or more fields.
 
-
+
 mlr --oxtab put '$x2=$x*$x; $xy=$x*$y; $y2=$y**2' \
   then stats2 -a cov,corr -f x,y,y,y,x2,xy,x2,y2 \
   data/medium
+
+
 x_y_cov    0.000042574820827444476
 x_y_corr   0.0005042001844467462
 y_y_cov    0.08461122467974003
@@ -2696,10 +2928,12 @@ x2_y2_cov  -0.00030953725962542085
 x2_y2_corr -0.0034249088761121966
 
-
+
 mlr --opprint put '$x2=$x*$x; $xy=$x*$y; $y2=$y**2' \
   then stats2 -a linreg-ols,r2 -f x,y,y,y,xy,y2 -g a \
   data/medium
+
+
 a   x_y_ols_m             x_y_ols_b           x_y_ols_n x_y_r2                  y_y_ols_m y_y_ols_b y_y_ols_n y_y_r2 xy_y2_ols_m        xy_y2_ols_b         xy_y2_ols_n xy_y2_r2
 pan 0.01702551273681908   0.5004028922897639  2081      0.00028691820445814767  1         0         2081      1      0.8781320866715662 0.11908230147563566 2081        0.41749827377311266
 eks 0.0407804923685586    0.48140207967651016 1965      0.0016461239223448587   1         0         1965      1      0.8978728611690183 0.10734054433612333 1965        0.45563223864254526
@@ -2708,11 +2942,11 @@ zee 0.0027812364960399147 0.5043070448033061  2047      0.000007751652858786137
 hat -0.018620577041095078 0.5179005397264935  1941      0.0003520036646055585   1         0         1941      1      0.8412305086345014 0.13557328318623216 1941        0.3687944261732265
 
-Here's an example simple line-fit. The ``x`` and ``y`` -fields of the ``data/medium`` dataset are just independent uniformly +Here's an example simple line-fit. The `x` and `y` +fields of the `data/medium` dataset are just independent uniformly distributed on the unit interval. Here we remove half the data and fit a line to it. -
+
 
 # Prepare input data:
 mlr filter '($x<.5 && $y<.5) || ($x>.5 && $y>.5)' data/medium > data/medium-squares
@@ -2743,10 +2977,12 @@ I use [pgr](https://github.com/johnkerl/pgr) for plotting; here's a screenshot.
 
 (Thanks Drew Kunas for a good conversation about PCA!)
 
-Here's an example estimating time-to-completion for a set of jobs. Input data comes from a log file, with number of work units left to do in the ``count`` field and accumulated seconds in the ``upsec`` field, labeled by the ``color`` field:
+Here's an example estimating time-to-completion for a set of jobs. Input data comes from a log file, with number of work units left to do in the `count` field and accumulated seconds in the `upsec` field, labeled by the `color` field:
 
-
+
 head -n 10 data/multicountdown.dat
+
+
 upsec=0.002,color=green,count=1203
 upsec=0.083,color=red,count=3817
 upsec=0.188,color=red,count=3801
@@ -2759,12 +2995,14 @@ upsec=1.093,color=blue,count=2662
 upsec=1.327,color=purple,count=917
 
-We can do a linear regression on count remaining as a function of time: with ``c = m*u+b`` we want to find the time when the count goes to zero, i.e. ``u=-b/m``. +We can do a linear regression on count remaining as a function of time: with `c = m*u+b` we want to find the time when the count goes to zero, i.e. `u=-b/m`. -
+
 mlr --oxtab stats2 -a linreg-pca -f upsec,count -g color \
   then put '$donesec = -$upsec_count_pca_b/$upsec_count_pca_m' \
   data/multicountdown.dat
+
+
 color                   green
 upsec_count_pca_m       -32.75691673397728
 upsec_count_pca_b       1213.7227296044375
@@ -2794,12 +3032,12 @@ upsec_count_pca_quality 0.9999908956206317
 donesec                 25.10852919630297
 
-.. _reference-verbs-step: - ## step -
+
 mlr step --help
+
+
 Usage: mlr step [options]
 Computes values dependent on the previous record, optionally grouped by category.
 Options:
@@ -2839,10 +3077,12 @@ https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
 for more information on EWMA.
 
-Most Miller commands are record-at-a-time, with the exception of ``stats1``, ``stats2``, and ``histogram`` which compute aggregate output. The ``step`` command is intermediate: it allows the option of adding fields which are functions of fields from previous records. Rsum is short for *running sum*. +Most Miller commands are record-at-a-time, with the exception of `stats1`, `stats2`, and `histogram` which compute aggregate output. The `step` command is intermediate: it allows the option of adding fields which are functions of fields from previous records. Rsum is short for *running sum*. -
+
 mlr --opprint step -a shift,delta,rsum,counter -f x data/medium | head -15
+
+
 a   b   i     x                      y                      x_shift                x_delta                 x_rsum             x_counter
 pan pan 1     0.3467901443380824     0.7268028627434533     -                      0                       0.3467901443380824 1
 eks pan 2     0.7586799647899636     0.5221511083334797     0.3467901443380824     0.41188982045188116     1.105470109128046  2
@@ -2860,8 +3100,10 @@ eks pan 13    0.4915175580479536     0.7709126592971468     0.3676141320555616
 eks zee 14    0.5207382318405251     0.34141681118811673    0.4915175580479536     0.02922067379257154     6.709212604625001  14
 
-
+
 mlr --opprint step -a shift,delta,rsum,counter -f x -g a data/medium | head -15
+
+
 a   b   i     x                      y                      x_shift                x_delta                 x_rsum              x_counter
 pan pan 1     0.3467901443380824     0.7268028627434533     -                      0                       0.3467901443380824  1
 eks pan 2     0.7586799647899636     0.5221511083334797     -                      0                       0.7586799647899636  1
@@ -2879,8 +3121,10 @@ eks pan 13    0.4915175580479536     0.7709126592971468     0.6117840605678454
 eks zee 14    0.5207382318405251     0.34141681118811673    0.4915175580479536     0.02922067379257154     2.7641192091174287  5
 
-
+
 mlr --opprint step -a ewma -f x -d 0.1,0.9 data/medium | head -15
+
+
 a   b   i     x                      y                      x_ewma_0.1          x_ewma_0.9
 pan pan 1     0.3467901443380824     0.7268028627434533     0.3467901443380824  0.3467901443380824
 eks pan 2     0.7586799647899636     0.5221511083334797     0.3879791263832706  0.7174909827447755
@@ -2898,8 +3142,10 @@ eks pan 13    0.4915175580479536     0.7709126592971468     0.4465877702644416
 eks zee 14    0.5207382318405251     0.34141681118811673    0.4540028164220499  0.51696937840094
 
-
+
 mlr --opprint step -a ewma -f x -d 0.1,0.9 -o smooth,rough data/medium | head -15
+
+
 a   b   i     x                      y                      x_ewma_smooth       x_ewma_rough
 pan pan 1     0.3467901443380824     0.7268028627434533     0.3467901443380824  0.3467901443380824
 eks pan 2     0.7586799647899636     0.5221511083334797     0.3879791263832706  0.7174909827447755
@@ -2920,7 +3166,7 @@ eks zee 14    0.5207382318405251     0.34141681118811673    0.4540028164220499
 
 Example deriving uptime-delta from system uptime:
 
-
+
 $ each 10 uptime | mlr -p step -a delta -f 11 
 ...
 20:08 up 36 days, 10:38, 5 users, load averages: 1.42 1.62 1.73 0.000000
@@ -2934,12 +3180,12 @@ $ each 10 uptime | mlr -p step -a delta -f 11
 
 
-.. _reference-verbs-tac: - ## tac -
+
 mlr tac --help
+
+
 Usage: mlr tac [options]
 Prints records in reverse order from the order in which they were encountered.
 Options:
@@ -2948,41 +3194,49 @@ Options:
 
 Prints the records in the input stream in reverse order. Note: this requires Miller to retain all input records in memory before any output records are produced.
 
-
+
 mlr --icsv --opprint cat data/a.csv
+
+
 a b c
 1 2 3
 4 5 6
 
-
+
 mlr --icsv --opprint cat data/b.csv
+
+
 a b c
 7 8 9
 
-
+
 mlr --icsv --opprint tac data/a.csv data/b.csv
+
+
 a b c
 7 8 9
 4 5 6
 1 2 3
 
-
+
 mlr --icsv --opprint put '$filename=FILENAME' then tac data/a.csv data/b.csv
+
+
 a b c filename
 7 8 9 data/b.csv
 4 5 6 data/a.csv
 1 2 3 data/a.csv
 
-.. _reference-verbs-tail: - ## tail -
+
 mlr tail --help
+
+
 Usage: mlr tail [options]
 Passes through the last n records, optionally by category.
 Options:
@@ -2993,8 +3247,10 @@ Options:
 
 Prints the last *n* records in the input stream, optionally by category.
 
-
+
 mlr --opprint tail -n 4 data/colored-shapes.dkvp
+
+
 color  shape    flag i     u                    v                   w                   x
 blue   square   1    99974 0.6189062525431605   0.2637962404841453  0.5311465405784674  6.210738209085753
 blue   triangle 0    99976 0.008110504040268474 0.8267274952432482  0.4732962944898885  6.146956761817328
@@ -3002,20 +3258,22 @@ yellow triangle 0    99990 0.3839424618160777   0.55952913620132    0.5113763011
 yellow circle   1    99994 0.764950884927175    0.25284227383991364 0.49969878539567425 5.013809741826425
 
-
+
 mlr --opprint tail -n 1 -g shape data/colored-shapes.dkvp
+
+
 color  shape    flag i     u                  v                   w                   x
 yellow triangle 0    99990 0.3839424618160777 0.55952913620132    0.5113763011485609  4.307973891915119
 blue   square   1    99974 0.6189062525431605 0.2637962404841453  0.5311465405784674  6.210738209085753
 yellow circle   1    99994 0.764950884927175  0.25284227383991364 0.49969878539567425 5.013809741826425
 
-.. _reference-verbs-tee: - ## tee -
+
 mlr tee --help
+
+
 Usage: mlr tee [options] {filename}
 Options:
 -a    Append to existing file, if any, rather than overwriting.
@@ -3028,12 +3286,12 @@ is written in JSON format.
 -h|--help Show this message.
 
-.. _reference-verbs-template: - ## template -
+
 mlr template --help
+
+
 Usage: mlr template [options]
 Places input-record fields in the order specified by list of column names.
 If the input record is missing a specified field, it will be filled with the fill-with.
@@ -3049,12 +3307,12 @@ Example:
 * Output record is a=1,b=,c=3.
 
-.. _reference-verbs-top: - ## top -
+
 mlr top --help
+
+
 Usage: mlr top [options]
 -f {a,b,c}    Value-field names for top counts.
 -g {d,e,f}    Optional group-by-field names for top counts.
@@ -3069,10 +3327,12 @@ Prints the n records with smallest/largest values at specified fields,
 optionally by category.
 
-Note that ``top`` is distinct from :ref:`reference-verbs-head` -- ``head`` shows fields which appear fimd.in the data stream; ``top`` shows fields which are numerically largest (or smallest). +Note that `top` is distinct from [head](reference-verbs.md#head) -- `head` shows fields which appear fimd.in the data stream; `top` shows fields which are numerically largest (or smallest). -
+
 mlr --opprint top -n 4 -f x data/medium
+
+
 top_idx x_top
 1       0.999952670371898
 2       0.9998228522652893
@@ -3080,8 +3340,10 @@ top_idx x_top
 4       0.9995625801977208
 
-
+
 mlr --opprint top -n 4 -f x -o someothername data/medium
+
+
 someothername x_top
 1             0.999952670371898
 2             0.9998228522652893
@@ -3089,8 +3351,10 @@ someothername x_top
 4             0.9995625801977208
 
-
+
 mlr --opprint top -n 2 -f x -g a then sort -f a data/medium
+
+
 a   top_idx x_top
 eks 1       0.9988110946859143
 eks 2       0.9985342548358704
@@ -3104,12 +3368,12 @@ zee 1       0.9994904324789629
 zee 2       0.9994378171787394
 
-.. _reference-verbs-uniq: - ## uniq -
+
 mlr uniq --help
+
+
 Usage: mlr uniq [options]
 Prints distinct values for specified field names. With -c, same as
 count-distinct. For uniq, -f is a synonym for -g.
@@ -3125,15 +3389,19 @@ Options:
               With neither -c nor -n, produces unique records.
 
-There are two main ways to use ``mlr uniq``: the first way is with ``-g`` to specify group-by columns. +There are two main ways to use `mlr uniq`: the first way is with `-g` to specify group-by columns. -
+
 wc -l data/colored-shapes.dkvp
+
+
    10078 data/colored-shapes.dkvp
 
-
+
 mlr uniq -g color,shape data/colored-shapes.dkvp
+
+
 color=yellow,shape=triangle
 color=red,shape=square
 color=red,shape=circle
@@ -3154,8 +3422,10 @@ color=orange,shape=square
 color=orange,shape=circle
 
-
+
 mlr --opprint uniq -g color,shape -c then sort -f color,shape data/colored-shapes.dkvp
+
+
 color  shape    count
 blue   circle   384
 blue   square   589
@@ -3177,10 +3447,12 @@ yellow square   589
 yellow triangle 468
 
-
+
 mlr --opprint uniq -g color,shape -c -o someothername \
   then sort -nr someothername \
   data/colored-shapes.dkvp
+
+
 color  shape    someothername
 red    square   1874
 red    triangle 1560
@@ -3202,16 +3474,20 @@ orange triangle 107
 orange circle   68
 
-
+
 mlr --opprint uniq -n -g color,shape data/colored-shapes.dkvp
+
+
 count
 18
 
-The second main way to use ``mlr uniq`` is without group-by columns, using ``-a`` instead: +The second main way to use `mlr uniq` is without group-by columns, using `-a` instead: -
+
 cat data/repeats.dkvp
+
+
 color=red,shape=square,flag=0
 color=purple,shape=triangle,flag=0
 color=yellow,shape=circle,flag=1
@@ -3271,13 +3547,17 @@ color=yellow,shape=circle,flag=1
 color=purple,shape=square,flag=0
 
-
+
 wc -l data/repeats.dkvp
+
+
       57 data/repeats.dkvp
 
-
+
 mlr --opprint uniq -a data/repeats.dkvp
+
+
 color  shape    flag
 red    square   0
 purple triangle 0
@@ -3288,14 +3568,18 @@ red    square   1
 yellow triangle 1
 
-
+
 mlr --opprint uniq -a -n data/repeats.dkvp
+
+
 count
 7
 
-
+
 mlr --opprint uniq -a -c data/repeats.dkvp
+
+
 count color  shape    flag
 17    red    square   0
 11    purple triangle 0
@@ -3306,12 +3590,12 @@ count color  shape    flag
 2     yellow triangle 1
 
-.. _reference-verbs-unsparsify: - ## unsparsify -
+
 mlr unsparsify --help
+
+
 Usage: mlr unsparsify [options]
 Prints records with the union of field names over all input records.
 For field names absent in a given record but present in others, fills in
@@ -3329,16 +3613,20 @@ being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and
 
 Examples:
 
-
+
 cat data/sparse.json
+
+
 {"a":1,"b":2,"v":3}
 {"u":1,"b":2}
 {"a":1,"v":2,"x":3}
 {"v":1,"w":2}
 
-
+
 mlr --json unsparsify data/sparse.json
+
+
 {
   "a": 1,
   "b": 2,
@@ -3373,8 +3661,10 @@ Examples:
 }
 
-
+
 mlr --ijson --opprint unsparsify data/sparse.json
+
+
 a b v u x w
 1 2 3 - - -
 - 2 - 1 - -
@@ -3382,8 +3672,10 @@ a b v u x w
 - - 1 - - 2
 
-
+
 mlr --ijson --opprint unsparsify --fill-with missing data/sparse.json
+
+
 a       b       v       u       x       w
 1       2       3       missing missing missing
 missing 2       missing 1       missing missing
@@ -3391,8 +3683,10 @@ missing 2       missing 1       missing missing
 missing missing 1       missing missing 2
 
-
+
 mlr --ijson --opprint unsparsify -f a,b,u data/sparse.json
+
+
 a b v u
 1 2 3 -
 
@@ -3406,8 +3700,10 @@ v w a b u
 1 2 - - -
 
-
+
 mlr --ijson --opprint unsparsify -f a,b,u,v,w,x then regularize data/sparse.json
+
+
 a b v u w x
 1 2 3 - - -
 - 2 - 1 - -
diff --git a/docs6b/docs/reference-verbs.md.in b/docs6b/docs/reference-verbs.md.in
index a5e93e431..a1c8cc210 100644
--- a/docs6b/docs/reference-verbs.md.in
+++ b/docs6b/docs/reference-verbs.md.in
@@ -2,24 +2,22 @@
 
 ## Overview
 
-Whereas the Unix toolkit is made of the separate executables ``cat``, ``tail``, ``cut``,
-``sort``, etc., Miller has subcommands, or **verbs**, invoked as follows:
+Whereas the Unix toolkit is made of the separate executables `cat`, `tail`, `cut`,
+`sort`, etc., Miller has subcommands, or **verbs**, invoked as follows:
 
 GENMD_INCLUDE_ESCAPED(data/subcommand-example.txt)
 
 These fall into categories as follows:
 
-* Analogs of their Unix-toolkit namesakes, discussed below as well as in [Unix-toolkit Context](feature-comparison.md): :ref:`reference-verbs-cat`, :ref:`reference-verbs-cut`, :ref:`reference-verbs-grep`, :ref:`reference-verbs-head`, :ref:`reference-verbs-join`, :ref:`reference-verbs-sort`, :ref:`reference-verbs-tac`, :ref:`reference-verbs-tail`, :ref:`reference-verbs-top`, :ref:`reference-verbs-uniq`.
+* Analogs of their Unix-toolkit namesakes, discussed below as well as in [Unix-toolkit Context](feature-comparison.md): [cat](reference-verbs.md#cat), [cut](reference-verbs.md#cut), [grep](reference-verbs.md#grep), [head](reference-verbs.md#head), [join](reference-verbs.md#join), [sort](reference-verbs.md#sort), [tac](reference-verbs.md#tac), [tail](reference-verbs.md#tail), [top](reference-verbs.md#top), [uniq](reference-verbs.md#uniq).
 
-* ``awk``-like functionality: :ref:`reference-verbs-filter`, :ref:`reference-verbs-put`, :ref:`reference-verbs-sec2gmt`, :ref:`reference-verbs-sec2gmtdate`, :ref:`reference-verbs-step`, :ref:`reference-verbs-tee`.
+* `awk`-like functionality: [filter](reference-verbs.md#filter), [put](reference-verbs.md#put), [sec2gmt](reference-verbs.md#sec2gmt), [sec2gmtdate](reference-verbs.md#sec2gmtdate), [step](reference-verbs.md#step), [tee](reference-verbs.md#tee).
 
-* Statistically oriented: :ref:`reference-verbs-bar`, :ref:`reference-verbs-bootstrap`, :ref:`reference-verbs-decimate`, :ref:`reference-verbs-histogram`, :ref:`reference-verbs-least-frequent`, :ref:`reference-verbs-most-frequent`, :ref:`reference-verbs-sample`, :ref:`reference-verbs-shuffle`, :ref:`reference-verbs-stats1`, :ref:`reference-verbs-stats2`.
+* Statistically oriented: [bar](reference-verbs.md#bar), [bootstrap](reference-verbs.md#bootstrap), [decimate](reference-verbs.md#decimate), [histogram](reference-verbs.md#histogram), [least-frequent](reference-verbs.md#least-frequent), [most-frequent](reference-verbs.md#most-frequent), [sample](reference-verbs.md#sample), [shuffle](reference-verbs.md#shuffle), [stats1](reference-verbs.md#stats1), [stats2](reference-verbs.md#stats2).
 
-* Particularly oriented toward [Record Heterogeneity](record-heterogeneity.md), although all Miller commands can handle heterogeneous records: :ref:`reference-verbs-group-by`, :ref:`reference-verbs-group-like`, :ref:`reference-verbs-having-fields`.
+* Particularly oriented toward [Record Heterogeneity](record-heterogeneity.md), although all Miller commands can handle heterogeneous records: [group-by](reference-verbs.md#group-by), [group-like](reference-verbs.md#group-like), [having-fields](reference-verbs.md#having-fields).
 
-* These draw from other sources (see also [How Original Is Miller?](originality.md)): :ref:`reference-verbs-count-distinct` is SQL-ish, and :ref:`reference-verbs-rename` can be done by ``sed`` (which does it faster: see [Performance](performance.md). Verbs: :ref:`reference-verbs-check`, :ref:`reference-verbs-count-distinct`, :ref:`reference-verbs-label`, :ref:`reference-verbs-merge-fields`, :ref:`reference-verbs-nest`, :ref:`reference-verbs-nothing`, :ref:`reference-verbs-regularize`, :ref:`reference-verbs-rename`, :ref:`reference-verbs-reorder`, :ref:`reference-verbs-reshape`, :ref:`reference-verbs-seqgen`.
-
-.. _reference-verbs-altkv:
+* These draw from other sources (see also [How Original Is Miller?](originality.md)): [count-distinct](reference-verbs.md#count-distinct) is SQL-ish, and [rename](reference-verbs.md#rename) can be done by `sed` (which does it faster: see [Performance](performance.md). Verbs: [check](reference-verbs.md#check), [count-distinct](reference-verbs.md#count-distinct), [label](reference-verbs.md#label), [merge-fields](reference-verbs.md#merge-fields), [nest](reference-verbs.md#nest), [nothing](reference-verbs.md#nothing), [regularize](reference-verbs.md#regularize), [rename](reference-verbs.md#rename), [reorder](reference-verbs.md#reorder), [reshape](reference-verbs.md#reshape), [seqgen](reference-verbs.md#seqgen).
 
 ## altkv
 
@@ -37,8 +35,6 @@ GENMD_RUN_COMMAND
 echo 'a,b,c,d,e,f,g' | mlr altkv
 GENMD_EOF
 
-.. _reference-verbs-bar:
-
 ## bar
 
 Cheesy bar-charting.
@@ -63,8 +59,6 @@ GENMD_RUN_COMMAND
 mlr --opprint bar --auto -f x,y data/small
 GENMD_EOF
 
-.. _reference-verbs-bootstrap:
-
 ## bootstrap
 
 GENMD_RUN_COMMAND
@@ -121,8 +115,6 @@ green  0.507569 1111
 orange 0.468014 292
 GENMD_EOF
 
-.. _reference-verbs-cat:
-
 ## cat
 
 Most useful for format conversions (see [File Formats](file-formats.md), and concatenating multiple same-schema CSV files to have the same header:
@@ -159,16 +151,12 @@ GENMD_RUN_COMMAND
 mlr --opprint cat -n -g a data/small
 GENMD_EOF
 
-.. _reference-verbs-check:
-
 ## check
 
 GENMD_RUN_COMMAND
 mlr check --help
 GENMD_EOF
 
-.. _reference-verbs-clean-whitespace:
-
 ## clean-whitespace
 
 GENMD_RUN_COMMAND
@@ -193,13 +181,11 @@ GENMD_EOF
 
 Function links:
 
-* :ref:`reference-dsl-lstrip`
-* :ref:`reference-dsl-rstrip`
-* :ref:`reference-dsl-strip`
-* :ref:`reference-dsl-collapse_whitespace`
-* :ref:`reference-dsl-clean_whitespace`
-
-.. _reference-verbs-count:
+* [lstrip](reference-dsl-builtin-functions.md#lstrip)
+* [rstrip](reference-dsl-builtin-functions.md#rstrip)
+* [strip](reference-dsl-builtin-functions.md#strip)
+* [collapse_whitespace](reference-dsl-builtin-functions.md#collapse_whitespace)
+* [clean_whitespace](reference-dsl-builtin-functions.md#clean_whitespace)
 
 ## count
 
@@ -231,8 +217,6 @@ GENMD_RUN_COMMAND
 mlr count -g a,b data/medium
 GENMD_EOF
 
-.. _reference-verbs-count-distinct:
-
 ## count-distinct
 
 GENMD_RUN_COMMAND
@@ -255,8 +239,6 @@ GENMD_RUN_COMMAND
 mlr count-distinct -n -f a,b data/medium
 GENMD_EOF
 
-.. _reference-verbs-count-similar:
-
 ## count-similar
 
 GENMD_RUN_COMMAND
@@ -275,8 +257,6 @@ GENMD_RUN_COMMAND
 mlr --opprint head -n 20 then count-similar -g a then sort -f a data/medium
 GENMD_EOF
 
-.. _reference-verbs-cut:
-
 ## cut
 
 GENMD_RUN_COMMAND
@@ -299,16 +279,12 @@ GENMD_RUN_COMMAND
 echo 'a=1,b=2,c=3' | mlr cut -o -f b,c,a
 GENMD_EOF
 
-.. _reference-verbs-decimate:
-
 ## decimate
 
 GENMD_RUN_COMMAND
 mlr decimate --help
 GENMD_EOF
 
-.. _reference-verbs-fill-down:
-
 ## fill-down
 
 GENMD_RUN_COMMAND
@@ -327,8 +303,6 @@ GENMD_RUN_COMMAND
 mlr --csv fill-down -a -f b data/fill-down.csv
 GENMD_EOF
 
-.. _reference-verbs-filter:
-
 ## filter
 
 GENMD_RUN_COMMAND
@@ -337,9 +311,7 @@ GENMD_EOF
 
 ### Features which filter shares with put
 
-Please see [DSL reference](reference-dsl.md) for more information about the expression language for ``mlr filter``.
-
-.. _reference-verbs-format-values:
+Please see [DSL reference](reference-dsl.md) for more information about the expression language for `mlr filter`.
 
 ## format-values
 
@@ -363,8 +335,6 @@ GENMD_RUN_COMMAND
 mlr --opprint format-values -i %08llx -f %.6le -s X%sX -n data/small
 GENMD_EOF
 
-.. _reference-verbs-fraction:
-
 ## fraction
 
 GENMD_RUN_COMMAND
@@ -375,13 +345,13 @@ For example, suppose you have the following CSV file:
 
 GENMD_INCLUDE_ESCAPED(data/fraction-example.csv)
 
-Then we can see what each record's ``n`` contributes to the total ``n``:
+Then we can see what each record's `n` contributes to the total `n`:
 
 GENMD_RUN_COMMAND
 mlr --opprint fraction -f n data/fraction-example.csv
 GENMD_EOF
 
-Using ``-g`` we can split those out by gender, or by color:
+Using `-g` we can split those out by gender, or by color:
 
 GENMD_RUN_COMMAND
 mlr --opprint fraction -f n -g u data/fraction-example.csv
@@ -393,13 +363,13 @@ GENMD_EOF
 
 We can see, for example, that 70.9% of females have red (on the left) while 94.5% of reds are for females.
 
-To convert fractions to percents, you may use ``-p``:
+To convert fractions to percents, you may use `-p`:
 
 GENMD_RUN_COMMAND
 mlr --opprint fraction -f n -p data/fraction-example.csv
 GENMD_EOF
 
-Another often-used idiom is to convert from a point distribution to a cumulative distribution, also known as "running sums". Here, you can use ``-c``:
+Another often-used idiom is to convert from a point distribution to a cumulative distribution, also known as "running sums". Here, you can use `-c`:
 
 GENMD_RUN_COMMAND
 mlr --opprint fraction -f n -p -c data/fraction-example.csv
@@ -409,23 +379,19 @@ GENMD_RUN_COMMAND
 mlr --opprint fraction -f n -g u -p -c data/fraction-example.csv
 GENMD_EOF
 
-.. _reference-verbs-grep:
-
 ## grep
 
 GENMD_RUN_COMMAND
 mlr grep -h
 GENMD_EOF
 
-.. _reference-verbs-group-by:
-
 ## group-by
 
 GENMD_RUN_COMMAND
 mlr group-by --help
 GENMD_EOF
 
-This is similar to ``sort`` but with less work. Namely, Miller's sort has three steps: read through the data and append linked lists of records, one for each unique combination of the key-field values; after all records are read, sort the key-field values; then print each record-list. The group-by operation simply omits the middle sort.  An example should make this more clear.
+This is similar to `sort` but with less work. Namely, Miller's sort has three steps: read through the data and append linked lists of records, one for each unique combination of the key-field values; after all records are read, sort the key-field values; then print each record-list. The group-by operation simply omits the middle sort.  An example should make this more clear.
 
 GENMD_RUN_COMMAND
 mlr --opprint group-by a data/small
@@ -435,9 +401,7 @@ GENMD_RUN_COMMAND
 mlr --opprint sort -f a data/small
 GENMD_EOF
 
-In this example, since the sort is on field ``a``, the first step is to group together all records having the same value for field ``a``; the second step is to sort the distinct ``a``-field values ``pan``, ``eks``, and ``wye`` into ``eks``, ``pan``, and ``wye``; the third step is to print out the record-list for ``a=eks``, then the record-list for ``a=pan``, then the record-list for ``a=wye``.  The group-by operation omits the middle sort and just puts like records together, for those times when a sort isn't desired. In particular, the ordering of group-by fields for group-by is the order in which they were encountered in the data stream, which in some cases may be more interesting to you.
-
-.. _reference-verbs-group-like:
+In this example, since the sort is on field `a`, the first step is to group together all records having the same value for field `a`; the second step is to sort the distinct `a`-field values `pan`, `eks`, and `wye` into `eks`, `pan`, and `wye`; the third step is to print out the record-list for `a=eks`, then the record-list for `a=pan`, then the record-list for `a=wye`.  The group-by operation omits the middle sort and just puts like records together, for those times when a sort isn't desired. In particular, the ordering of group-by fields for group-by is the order in which they were encountered in the data stream, which in some cases may be more interesting to you.
 
 ## group-like
 
@@ -455,15 +419,13 @@ GENMD_RUN_COMMAND
 mlr --opprint group-like data/het.dkvp
 GENMD_EOF
 
-.. _reference-verbs-having-fields:
-
 ## having-fields
 
 GENMD_RUN_COMMAND
 mlr having-fields --help
 GENMD_EOF
 
-Similar to :ref:`reference-verbs-group-like`, this retains records with specified schema.
+Similar to [group-like](reference-verbs.md#group-like), this retains records with specified schema.
 
 GENMD_RUN_COMMAND
 mlr cat data/het.dkvp
@@ -477,15 +439,13 @@ GENMD_RUN_COMMAND
 mlr having-fields --which-are resource,ok,loadsec data/het.dkvp
 GENMD_EOF
 
-.. _reference-verbs-head:
-
 ## head
 
 GENMD_RUN_COMMAND
 mlr head --help
 GENMD_EOF
 
-Note that ``head`` is distinct from :ref:`reference-verbs-top` -- ``head`` shows fields which appear fimd.in the data stream; ``top`` shows fields which are numerically largest (or smallest).
+Note that `head` is distinct from [top](reference-verbs.md#top) -- `head` shows fields which appear fimd.in the data stream; `top` shows fields which are numerically largest (or smallest).
 
 GENMD_RUN_COMMAND
 mlr --opprint head -n 4 data/medium
@@ -495,15 +455,13 @@ GENMD_RUN_COMMAND
 mlr --opprint head -n 1 -g b data/medium
 GENMD_EOF
 
-.. _reference-verbs-histogram:
-
 ## histogram
 
 GENMD_RUN_COMMAND
 mlr histogram --help
 GENMD_EOF
 
-This is just a histogram; there's not too much to say here. A note about binning, by example: Suppose you use ``--lo 0.0 --hi 1.0 --nbins 10 -f x``.  The input numbers less than 0 or greater than 1 aren't counted in any bin.  Input numbers equal to 1 are counted in the last bin. That is, bin 0 has ``0.0 ≤ x < 0.1``, bin 1 has ``0.1 ≤ x < 0.2``, etc., but bin 9 has ``0.9 ≤ x ≤ 1.0``.
+This is just a histogram; there's not too much to say here. A note about binning, by example: Suppose you use `--lo 0.0 --hi 1.0 --nbins 10 -f x`.  The input numbers less than 0 or greater than 1 aren't counted in any bin.  Input numbers equal to 1 are counted in the last bin. That is, bin 0 has `0.0 ≤ x < 0.1`, bin 1 has `0.1 ≤ x < 0.2`, etc., but bin 9 has `0.9 ≤ x ≤ 1.0`.
 
 GENMD_RUN_COMMAND
 mlr --opprint put '$x2=$x**2;$x3=$x2*$x' \
@@ -517,8 +475,6 @@ mlr --opprint put '$x2=$x**2;$x3=$x2*$x' \
   data/medium
 GENMD_EOF
 
-.. _reference-verbs-join:
-
 ## join
 
 GENMD_RUN_COMMAND
@@ -575,21 +531,19 @@ GENMD_RUN_COMMAND
 mlr --csvlite --opprint join -j "" --lp left_ --rp right_ -f data/self-join.csv data/self-join.csv
 GENMD_EOF
 
-.. _reference-verbs-label:
-
 ## label
 
 GENMD_RUN_COMMAND
 mlr label --help
 GENMD_EOF
 
-See also :ref:`reference-verbs-rename`.
+See also [rename](reference-verbs.md#rename).
 
-Example: Files such as ``/etc/passwd``, ``/etc/group``, and so on have implicit field names which are found in section-5 manpages. These field names may be made explicit as follows:
+Example: Files such as `/etc/passwd`, `/etc/group`, and so on have implicit field names which are found in section-5 manpages. These field names may be made explicit as follows:
 
 GENMD_INCLUDE_ESCAPED(data/label-example.txt)
 
-Likewise, if you have CSV/CSV-lite input data which has somehow been bereft of its header line, you can re-add a header line using ``--implicit-csv-header`` and ``label``:
+Likewise, if you have CSV/CSV-lite input data which has somehow been bereft of its header line, you can re-add a header line using `--implicit-csv-header` and `label`:
 
 GENMD_RUN_COMMAND
 cat data/headerless.csv
@@ -607,8 +561,6 @@ GENMD_RUN_COMMAND
 mlr --icsv --implicit-csv-header --opprint label name,age,status data/headerless.csv
 GENMD_EOF
 
-.. _reference-verbs-least-frequent:
-
 ## least-frequent
 
 GENMD_RUN_COMMAND
@@ -631,9 +583,7 @@ GENMD_RUN_COMMAND
 mlr --opprint --from data/colored-shapes.dkvp least-frequent -f shape,color -n 5 -b
 GENMD_EOF
 
-See also :ref:`reference-verbs-most-frequent`.
-
-.. _reference-verbs-merge-fields:
+See also [most-frequent](reference-verbs.md#most-frequent).
 
 ## merge-fields
 
@@ -641,7 +591,7 @@ GENMD_RUN_COMMAND
 mlr merge-fields --help
 GENMD_EOF
 
-This is like ``mlr stats1`` but all accumulation is done across fields within each given record: horizontal rather than vertical statistics, if you will.
+This is like `mlr stats1` but all accumulation is done across fields within each given record: horizontal rather than vertical statistics, if you will.
 
 Examples:
 
@@ -657,8 +607,6 @@ GENMD_RUN_COMMAND
 mlr --csvlite --opprint merge-fields -k -a sum -c _in,_out data/inout.csv
 GENMD_EOF
 
-.. _reference-verbs-most-frequent:
-
 ## most-frequent
 
 GENMD_RUN_COMMAND
@@ -681,9 +629,7 @@ GENMD_RUN_COMMAND
 mlr --opprint --from data/colored-shapes.dkvp most-frequent -f shape,color -n 5 -b
 GENMD_EOF
 
-See also :ref:`reference-verbs-least-frequent`.
-
-.. _reference-verbs-nest:
+See also [least-frequent](reference-verbs.md#least-frequent).
 
 ## nest
 
@@ -691,16 +637,12 @@ GENMD_RUN_COMMAND
 mlr nest -h
 GENMD_EOF
 
-.. _reference-verbs-nothing:
-
 ## nothing
 
 GENMD_RUN_COMMAND
 mlr nothing -h
 GENMD_EOF
 
-.. _reference-verbs-put:
-
 ## put
 
 GENMD_RUN_COMMAND
@@ -709,9 +651,7 @@ GENMD_EOF
 
 ### Features which put shares with filter
 
-Please see the [DSL reference](reference-dsl.md) for more information about the expression language for ``mlr put``.
-
-.. _reference-verbs-regularize:
+Please see the [DSL reference](reference-dsl.md) for more information about the expression language for `mlr put`.
 
 ## regularize
 
@@ -719,11 +659,9 @@ GENMD_RUN_COMMAND
 mlr regularize --help
 GENMD_EOF
 
-This exists since hash-map software in various languages and tools encountered in the wild does not always print similar rows with fields in the same order: ``mlr regularize`` helps clean that up.
+This exists since hash-map software in various languages and tools encountered in the wild does not always print similar rows with fields in the same order: `mlr regularize` helps clean that up.
 
-See also :ref:`reference-verbs-reorder`.
-
-.. _reference-verbs-remove-empty-columns:
+See also [reorder](reference-verbs.md#reorder).
 
 ## remove-empty-columns
 
@@ -741,8 +679,6 @@ GENMD_EOF
 
 Since this verb needs to read all records to see if any of them has a non-empty value for a given field name, it is non-streaming: it will ingest all records before writing any.
 
-.. _reference-verbs-rename:
-
 ## rename
 
 GENMD_RUN_COMMAND
@@ -757,7 +693,7 @@ GENMD_RUN_COMMAND
 mlr --opprint rename i,INDEX,b,COLUMN2 data/small
 GENMD_EOF
 
-As discussed in [Performance](performance.md), ``sed`` is significantly faster than Miller at doing this. However, Miller is format-aware, so it knows to do renames only within specified field keys and not any others, nor in field values which may happen to contain the same pattern. Example:
+As discussed in [Performance](performance.md), `sed` is significantly faster than Miller at doing this. However, Miller is format-aware, so it knows to do renames only within specified field keys and not any others, nor in field values which may happen to contain the same pattern. Example:
 
 GENMD_RUN_COMMAND
 sed 's/y/COLUMN5/g' data/small
@@ -767,9 +703,7 @@ GENMD_RUN_COMMAND
 mlr rename y,COLUMN5 data/small
 GENMD_EOF
 
-See also :ref:`reference-verbs-label`.
-
-.. _reference-verbs-reorder:
+See also [label](reference-verbs.md#label).
 
 ## reorder
 
@@ -793,8 +727,6 @@ GENMD_RUN_COMMAND
 mlr --opprint reorder -e -f i,b data/small
 GENMD_EOF
 
-.. _reference-verbs-repeat:
-
 ## repeat
 
 GENMD_RUN_COMMAND
@@ -802,7 +734,7 @@ mlr repeat --help
 GENMD_EOF
 
 This is useful in at least two ways: one, as a data-generator as in the
-above example using ``urand()``; two, for reconstructing individual
+above example using `urand()`; two, for reconstructing individual
 samples from data which has been count-aggregated:
 
 GENMD_RUN_COMMAND
@@ -813,11 +745,9 @@ GENMD_RUN_COMMAND
 mlr repeat -f count then cut -x -f count data/repeat-example.dat
 GENMD_EOF
 
-After expansion with ``repeat``, such data can then be sent on to
-``stats1 -a mode``, or (if the data are numeric) to ``stats1 -a
-p10,p50,p90``, etc.
-
-.. _reference-verbs-reshape:
+After expansion with `repeat`, such data can then be sent on to
+`stats1 -a mode`, or (if the data are numeric) to `stats1 -a
+p10,p50,p90`, etc.
 
 ## reshape
 
@@ -825,8 +755,6 @@ GENMD_RUN_COMMAND
 mlr reshape --help
 GENMD_EOF
 
-.. _reference-verbs-sample:
-
 ## sample
 
 GENMD_RUN_COMMAND
@@ -835,17 +763,15 @@ GENMD_EOF
 
 This is reservoir-sampling: select *k* items from *n* with
 uniform probability and no repeats in the sample. (If *n* is less than
-*k*, then of course only *n* samples are produced.) With ``-g
-{field names}``, produce a *k*-sample for each distinct value of the
+*k*, then of course only *n* samples are produced.) With `-g
+{field names}`, produce a *k*-sample for each distinct value of the
 specified field names.
 
 GENMD_INCLUDE_ESCAPED(data/sample-example.txt)
 
 Note that no output is produced until all inputs are in. Another way to do
-sampling, which works in the streaming case, is ``mlr filter 'urand() &
-0.001'`` where you tune the 0.001 to meet your needs.
-
-.. _reference-verbs-sec2gmt:
+sampling, which works in the streaming case, is `mlr filter 'urand() &
+0.001'` where you tune the 0.001 to meet your needs.
 
 ## sec2gmt
 
@@ -853,16 +779,12 @@ GENMD_RUN_COMMAND
 mlr sec2gmt -h
 GENMD_EOF
 
-.. _reference-verbs-sec2gmtdate:
-
 ## sec2gmtdate
 
 GENMD_RUN_COMMAND
 mlr sec2gmtdate -h
 GENMD_EOF
 
-.. _reference-verbs-seqgen:
-
 ## seqgen
 
 GENMD_RUN_COMMAND
@@ -881,16 +803,12 @@ GENMD_RUN_COMMAND
 mlr seqgen --start 40 --stop 20 --step -4
 GENMD_EOF
 
-.. _reference-verbs-shuffle:
-
 ## shuffle
 
 GENMD_RUN_COMMAND
 mlr shuffle -h
 GENMD_EOF
 
-.. _reference-verbs-skip-trivial-records:
-
 ## skip-trivial-records
 
 GENMD_RUN_COMMAND
@@ -905,8 +823,6 @@ GENMD_RUN_COMMAND
 mlr --csv skip-trivial-records data/trivial-records.csv
 GENMD_EOF
 
-.. _reference-verbs-sort:
-
 ## sort
 
 GENMD_RUN_COMMAND
@@ -926,7 +842,7 @@ head -n 10 data/multicountdown.dat
 GENMD_EOF
 
 We can group these by thread by sorting on the thread ID (here,
-``color``). Since Miller's sort is stable, this means that
+`color`). Since Miller's sort is stable, this means that
 timestamps within each thread's log data are still chronological:
 
 GENMD_RUN_COMMAND
@@ -944,8 +860,6 @@ GENMD_RUN_COMMAND
 mlr sort -nr x data/sort-missing.dkvp
 GENMD_EOF
 
-.. _reference-verbs-sort-within-records:
-
 ## sort-within-records
 
 GENMD_RUN_COMMAND
@@ -968,8 +882,6 @@ GENMD_RUN_COMMAND
 mlr --ijson --opprint sort-within-records data/sort-within-records.json
 GENMD_EOF
 
-.. _reference-verbs-stats1:
-
 ## stats1
 
 GENMD_RUN_COMMAND
@@ -977,7 +889,7 @@ mlr stats1 --help
 GENMD_EOF
 
 These are simple univariate statistics on one or more number-valued fields
-(``count`` and ``mode`` apply to non-numeric fields as well),
+(`count` and `mode` apply to non-numeric fields as well),
 optionally categorized by one or more other fields.
 
 GENMD_RUN_COMMAND
@@ -1002,8 +914,6 @@ GENMD_RUN_COMMAND
 mlr --opprint stats1 -a mode -f color -g shape data/colored-shapes.dkvp
 GENMD_EOF
 
-.. _reference-verbs-stats2:
-
 ## stats2
 
 GENMD_RUN_COMMAND
@@ -1025,8 +935,8 @@ mlr --opprint put '$x2=$x*$x; $xy=$x*$y; $y2=$y**2' \
   data/medium
 GENMD_EOF
 
-Here's an example simple line-fit. The ``x`` and ``y``
-fields of the ``data/medium`` dataset are just independent uniformly
+Here's an example simple line-fit. The `x` and `y`
+fields of the `data/medium` dataset are just independent uniformly
 distributed on the unit interval. Here we remove half the data and fit a line to it.
 
 GENMD_INCLUDE_ESCAPED(data/linreg-example.txt)
@@ -1037,13 +947,13 @@ I use [pgr](https://github.com/johnkerl/pgr) for plotting; here's a screenshot.
 
 (Thanks Drew Kunas for a good conversation about PCA!)
 
-Here's an example estimating time-to-completion for a set of jobs. Input data comes from a log file, with number of work units left to do in the ``count`` field and accumulated seconds in the ``upsec`` field, labeled by the ``color`` field:
+Here's an example estimating time-to-completion for a set of jobs. Input data comes from a log file, with number of work units left to do in the `count` field and accumulated seconds in the `upsec` field, labeled by the `color` field:
 
 GENMD_RUN_COMMAND
 head -n 10 data/multicountdown.dat
 GENMD_EOF
 
-We can do a linear regression on count remaining as a function of time: with ``c = m*u+b`` we want to find the time when the count goes to zero, i.e. ``u=-b/m``.
+We can do a linear regression on count remaining as a function of time: with `c = m*u+b` we want to find the time when the count goes to zero, i.e. `u=-b/m`.
 
 GENMD_RUN_COMMAND
 mlr --oxtab stats2 -a linreg-pca -f upsec,count -g color \
@@ -1051,15 +961,13 @@ mlr --oxtab stats2 -a linreg-pca -f upsec,count -g color \
   data/multicountdown.dat
 GENMD_EOF
 
-.. _reference-verbs-step:
-
 ## step
 
 GENMD_RUN_COMMAND
 mlr step --help
 GENMD_EOF
 
-Most Miller commands are record-at-a-time, with the exception of ``stats1``, ``stats2``, and ``histogram`` which compute aggregate output. The ``step`` command is intermediate: it allows the option of adding fields which are functions of fields from previous records. Rsum is short for *running sum*.
+Most Miller commands are record-at-a-time, with the exception of `stats1`, `stats2`, and `histogram` which compute aggregate output. The `step` command is intermediate: it allows the option of adding fields which are functions of fields from previous records. Rsum is short for *running sum*.
 
 GENMD_RUN_COMMAND
 mlr --opprint step -a shift,delta,rsum,counter -f x data/medium | head -15
@@ -1082,8 +990,6 @@ Example deriving uptime-delta from system uptime:
 
 GENMD_INCLUDE_ESCAPED(data/ping-delta-example.txt)
 
-.. _reference-verbs-tac:
-
 ## tac
 
 GENMD_RUN_COMMAND
@@ -1108,8 +1014,6 @@ GENMD_RUN_COMMAND
 mlr --icsv --opprint put '$filename=FILENAME' then tac data/a.csv data/b.csv
 GENMD_EOF
 
-.. _reference-verbs-tail:
-
 ## tail
 
 GENMD_RUN_COMMAND
@@ -1126,31 +1030,25 @@ GENMD_RUN_COMMAND
 mlr --opprint tail -n 1 -g shape data/colored-shapes.dkvp
 GENMD_EOF
 
-.. _reference-verbs-tee:
-
 ## tee
 
 GENMD_RUN_COMMAND
 mlr tee --help
 GENMD_EOF
 
-.. _reference-verbs-template:
-
 ## template
 
 GENMD_RUN_COMMAND
 mlr template --help
 GENMD_EOF
 
-.. _reference-verbs-top:
-
 ## top
 
 GENMD_RUN_COMMAND
 mlr top --help
 GENMD_EOF
 
-Note that ``top`` is distinct from :ref:`reference-verbs-head` -- ``head`` shows fields which appear fimd.in the data stream; ``top`` shows fields which are numerically largest (or smallest).
+Note that `top` is distinct from [head](reference-verbs.md#head) -- `head` shows fields which appear fimd.in the data stream; `top` shows fields which are numerically largest (or smallest).
 
 GENMD_RUN_COMMAND
 mlr --opprint top -n 4 -f x data/medium
@@ -1164,15 +1062,13 @@ GENMD_RUN_COMMAND
 mlr --opprint top -n 2 -f x -g a then sort -f a data/medium
 GENMD_EOF
 
-.. _reference-verbs-uniq:
-
 ## uniq
 
 GENMD_RUN_COMMAND
 mlr uniq --help
 GENMD_EOF
 
-There are two main ways to use ``mlr uniq``: the first way is with ``-g`` to specify group-by columns.
+There are two main ways to use `mlr uniq`: the first way is with `-g` to specify group-by columns.
 
 GENMD_RUN_COMMAND
 wc -l data/colored-shapes.dkvp
@@ -1196,7 +1092,7 @@ GENMD_RUN_COMMAND
 mlr --opprint uniq -n -g color,shape data/colored-shapes.dkvp
 GENMD_EOF
 
-The second main way to use ``mlr uniq`` is without group-by columns, using ``-a`` instead:
+The second main way to use `mlr uniq` is without group-by columns, using `-a` instead:
 
 GENMD_RUN_COMMAND
 cat data/repeats.dkvp
@@ -1218,8 +1114,6 @@ GENMD_RUN_COMMAND
 mlr --opprint uniq -a -c data/repeats.dkvp
 GENMD_EOF
 
-.. _reference-verbs-unsparsify:
-
 ## unsparsify
 
 GENMD_RUN_COMMAND
diff --git a/docs6b/docs/release-docs.md b/docs6b/docs/release-docs.md
index d8b64472e..682ff63f5 100644
--- a/docs6b/docs/release-docs.md
+++ b/docs6b/docs/release-docs.md
@@ -1,7 +1,7 @@
 
 # Documents by release
 
-As of September 2020, for 5.9.1 onward, release-specific docs will be handled automatically by https://miller.readthedocs.io whenever a new release is tagged at https://github.com/johnkerl/miller/releases.
+As of September 2020, for 5.9.1 onward, release-specific docs will be handled automatically by [https://miller.readthedocs.io](https://miller.readthedocs.io) whenever a new release is tagged at [https://github.com/johnkerl/miller/releases](https://github.com/johnkerl/miller/releases).
 
 Information here is for documents from before the readthedocs port:
 
diff --git a/docs6b/docs/release-docs.md.in b/docs6b/docs/release-docs.md.in
index dd91a1f6a..19d86db44 100644
--- a/docs6b/docs/release-docs.md.in
+++ b/docs6b/docs/release-docs.md.in
@@ -1,6 +1,6 @@
 # Documents by release
 
-As of September 2020, for 5.9.1 onward, release-specific docs will be handled automatically by https://miller.readthedocs.io whenever a new release is tagged at https://github.com/johnkerl/miller/releases.
+As of September 2020, for 5.9.1 onward, release-specific docs will be handled automatically by [https://miller.readthedocs.io](https://miller.readthedocs.io) whenever a new release is tagged at [https://github.com/johnkerl/miller/releases](https://github.com/johnkerl/miller/releases).
 
 Information here is for documents from before the readthedocs port:
 
diff --git a/docs6b/docs/repl.md b/docs6b/docs/repl.md
index 2b6f76ca0..61d4ef86f 100644
--- a/docs6b/docs/repl.md
+++ b/docs6b/docs/repl.md
@@ -1,12 +1,14 @@
 
 # The REPL
 
-The Miller REPL (read-evaluate-print loop) is an interactive counterpart to record-processing using the ``put``/``filter`` language. (A REPL is anything that evaluates what you type into it -- like ``python`` with no arguments, or Ruby's ``irb``, or ``node`` with no arguments, etc.)
+The Miller REPL (read-evaluate-print loop) is an interactive counterpart to record-processing using the `put`/`filter` language. (A REPL is anything that evaluates what you type into it -- like `python` with no arguments, or Ruby's `irb`, or `node` with no arguments, etc.)
 
 Miller's REPL isn't a source-level debugger which lets you execute one source-code *statement* at a time -- however, it does let you operate on one *record* at a time. Further, it lets you use "immediate expressions", namely, you can interact with the language without having to provide data from an input file.
 
-
+
 mlr repl
+
+
 
 [mlr] 1 + 2
 3
@@ -14,19 +16,21 @@ Miller's REPL isn't a source-level debugger which lets you execute one source-co
 
 ## Using Miller without the REPL
 
-Using ``put`` and ``filter``, you can do the following as we've seen above:
+Using `put` and `filter`, you can do the following as we've seen above:
 
-* Specify input format (e.g. ``--icsv``), output format (e.g. ``--ojson``), etc. using command-line flags.
+* Specify input format (e.g. `--icsv`), output format (e.g. `--ojson`), etc. using command-line flags.
 * Specify filenames on the command line.
-* Define ``begin {...}`` blocks which are executed before the first record is read.
-* Define ``end {...}`` blocks which are executed after the last record is read.
-* Define user-defined functions/subroutines using ``func`` and ``subr``.
-* Specify statements to be executed on each record -- which are anything outside of ``begin``/``end``/``func``/``subr``.
+* Define `begin {...}` blocks which are executed before the first record is read.
+* Define `end {...}` blocks which are executed after the last record is read.
+* Define user-defined functions/subroutines using `func` and `subr`.
+* Specify statements to be executed on each record -- which are anything outside of `begin`/`end`/`func`/`subr`.
 * Example:
 
-
+
 mlr --icsv --ojson --from example.csv head -n 2 \
   then put 'begin {print "HELLO"} $z = $x + $y; end {print "GOODBYE"}'
+
+
 HELLO
 {
   "color": "yellow",
@@ -51,50 +55,52 @@ GOODBYE
 
 Using the REPL, by contrast, you get interactive control over those same steps:
 
-* Specify input format (e.g. ``--icsv``), output format (e.g. ``--ojson``), etc. using command-line flags.
-* REPL-only statements (non-DSL statements) start with ``:``, such as ``:help`` or ``:quit``
-  or ``:open``.
-* Specify filenames either on the command line or via ``:open`` at the Miller REPL.
-* Read records one at a time using ``:read``.
-* Write the current record (maybe after you've modified it with things like ``$z = $x + $y``)
-  using ``:write``. This goes to the terminal; you can use ``:> {filename}`` to make writes
-  go to a file, or ``:>> {filename}`` to append.
-* You can type ``:reopen`` to go back to the start of the same file(s) you specified
-  with ``:open``.
-* Skip ahead using statements ``:skip 10`` or ``:skip until NR == 100`` or
-  ``:skip until $status_code != 200``.
+* Specify input format (e.g. `--icsv`), output format (e.g. `--ojson`), etc. using command-line flags.
+* REPL-only statements (non-DSL statements) start with `:`, such as `:help` or `:quit`
+  or `:open`.
+* Specify filenames either on the command line or via `:open` at the Miller REPL.
+* Read records one at a time using `:read`.
+* Write the current record (maybe after you've modified it with things like `$z = $x + $y`)
+  using `:write`. This goes to the terminal; you can use `:> {filename}` to make writes
+  go to a file, or `:>> {filename}` to append.
+* You can type `:reopen` to go back to the start of the same file(s) you specified
+  with `:open`.
+* Skip ahead using statements `:skip 10` or `:skip until NR == 100` or
+  `:skip until $status_code != 200`.
 * Similarly, but processing records rather than skipping past them, using
-  ``:process`` rather than ``:skip``. Like ``:write``, these go to the screen;
-  use ``:> {filename}`` or ``:>> {filename}`` to log to a file instead.
-* Define ``begin {...}`` blocks; invoke them at will using ``:begin``.
-* Define ``end {...}`` blocks; invoke them at will using ``:end``.
-* Define user-defined functions/subroutines using ``func``/``subr``; call them from other statements.
+  `:process` rather than `:skip`. Like `:write`, these go to the screen;
+  use `:> {filename}` or `:>> {filename}` to log to a file instead.
+* Define `begin {...}` blocks; invoke them at will using `:begin`.
+* Define `end {...}` blocks; invoke them at will using `:end`.
+* Define user-defined functions/subroutines using `func`/`subr`; call them from other statements.
 * Interactively specify statements to be executed immediately on the current record.
-* Load any of the above from Miller-script files using ``:load``.
+* Load any of the above from Miller-script files using `:load`.
 
 The input "record" by default is the empty map but you can do things like
-``$x=3``, or ``unset $y``, or ``$* = {"x": 3, "y": 4}`` to populate it. Or, ``:open
-foo.dat`` followed by ``:read`` to populate it from a data file.
+`$x=3`, or `unset $y`, or `$* = {"x": 3, "y": 4}` to populate it. Or, `:open
+foo.dat` followed by `:read` to populate it from a data file.
 
-Non-assignment expressions, such as ``7`` or ``true``, operate as filter conditions
-in the ``put`` DSL: they can be used to specify whether a record will or won't be
+Non-assignment expressions, such as `7` or `true`, operate as filter conditions
+in the `put` DSL: they can be used to specify whether a record will or won't be
 included in the output-record stream.  But here in the REPL, they are simply
-printed to the terminal, e.g. if you type ``1+2``, you will see ``3``.
+printed to the terminal, e.g. if you type `1+2`, you will see `3`.
 
 ## Entering multi-line statements
 
-* To enter multi-line statements, enter ``<`` on a line by itself, then the code (taking care
+* To enter multi-line statements, enter `<` on a line by itself, then the code (taking care
   for semicolons), then ">" on a line by itself. These will be executed immediately.
-* If you enter ``<<`` on a line by itself, then the code, then ``>>`` on a line by
+* If you enter `<<` on a line by itself, then the code, then `>>` on a line by
   itself, the statements will be remembered for executing on records with
-  ``:main``, as if you had done ``:load`` to load statements from a file.
+  `:main`, as if you had done `:load` to load statements from a file.
 
 ## Examples
 
 Use the REPL to look at arithmetic:
 
-
+
 mlr repl
+
+
 
 [mlr] 6/3
 2
@@ -111,8 +117,10 @@ float
 
 Read the first record from a small file:
 
-
+
 mlr repl
+
+
 
 [mlr] :open foo.dat
 
@@ -138,8 +146,10 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463,z=4.381399393871141
 
 Skip until deep into a larger file, then inspect a record:
 
-
+
 mlr repl --csv
+
+
 
 [mlr] :open data/colored-shapes.csv
 [mlr] :skip until NR == 10000
@@ -160,11 +170,11 @@ Skip until deep into a larger file, then inspect a record:
 ## History-editing
 
 No command-line-history-editing feature is built in but **rlwrap mlr repl** is a
-delight. You may need ``brew install rlwrap``, ``sudo apt-get install rlwrap``,
+delight. You may need `brew install rlwrap`, `sudo apt-get install rlwrap`,
 etc. depending on your platform.
 
-Suggestion: ``alias mrpl='rlwrap mlr repl'`` in your shell's startup file.
+Suggestion: `alias mrpl='rlwrap mlr repl'` in your shell's startup file.
 
 ## On-line help
 
-After ``mlr repl``, type ``:help`` to see more about your options. In particular, ``:help examples``.
+After `mlr repl`, type `:help` to see more about your options. In particular, `:help examples`.
diff --git a/docs6b/docs/repl.md.in b/docs6b/docs/repl.md.in
index 5d0ed19fc..b86d21648 100644
--- a/docs6b/docs/repl.md.in
+++ b/docs6b/docs/repl.md.in
@@ -1,6 +1,6 @@
 # The REPL
 
-The Miller REPL (read-evaluate-print loop) is an interactive counterpart to record-processing using the ``put``/``filter`` language. (A REPL is anything that evaluates what you type into it -- like ``python`` with no arguments, or Ruby's ``irb``, or ``node`` with no arguments, etc.)
+The Miller REPL (read-evaluate-print loop) is an interactive counterpart to record-processing using the `put`/`filter` language. (A REPL is anything that evaluates what you type into it -- like `python` with no arguments, or Ruby's `irb`, or `node` with no arguments, etc.)
 
 Miller's REPL isn't a source-level debugger which lets you execute one source-code *statement* at a time -- however, it does let you operate on one *record* at a time. Further, it lets you use "immediate expressions", namely, you can interact with the language without having to provide data from an input file.
 
@@ -13,14 +13,14 @@ GENMD_EOF
 
 ## Using Miller without the REPL
 
-Using ``put`` and ``filter``, you can do the following as we've seen above:
+Using `put` and `filter`, you can do the following as we've seen above:
 
-* Specify input format (e.g. ``--icsv``), output format (e.g. ``--ojson``), etc. using command-line flags.
+* Specify input format (e.g. `--icsv`), output format (e.g. `--ojson`), etc. using command-line flags.
 * Specify filenames on the command line.
-* Define ``begin {...}`` blocks which are executed before the first record is read.
-* Define ``end {...}`` blocks which are executed after the last record is read.
-* Define user-defined functions/subroutines using ``func`` and ``subr``.
-* Specify statements to be executed on each record -- which are anything outside of ``begin``/``end``/``func``/``subr``.
+* Define `begin {...}` blocks which are executed before the first record is read.
+* Define `end {...}` blocks which are executed after the last record is read.
+* Define user-defined functions/subroutines using `func` and `subr`.
+* Specify statements to be executed on each record -- which are anything outside of `begin`/`end`/`func`/`subr`.
 * Example:
 
 GENMD_RUN_COMMAND
@@ -32,43 +32,43 @@ GENMD_EOF
 
 Using the REPL, by contrast, you get interactive control over those same steps:
 
-* Specify input format (e.g. ``--icsv``), output format (e.g. ``--ojson``), etc. using command-line flags.
-* REPL-only statements (non-DSL statements) start with ``:``, such as ``:help`` or ``:quit``
-  or ``:open``.
-* Specify filenames either on the command line or via ``:open`` at the Miller REPL.
-* Read records one at a time using ``:read``.
-* Write the current record (maybe after you've modified it with things like ``$z = $x + $y``)
-  using ``:write``. This goes to the terminal; you can use ``:> {filename}`` to make writes
-  go to a file, or ``:>> {filename}`` to append.
-* You can type ``:reopen`` to go back to the start of the same file(s) you specified
-  with ``:open``.
-* Skip ahead using statements ``:skip 10`` or ``:skip until NR == 100`` or
-  ``:skip until $status_code != 200``.
+* Specify input format (e.g. `--icsv`), output format (e.g. `--ojson`), etc. using command-line flags.
+* REPL-only statements (non-DSL statements) start with `:`, such as `:help` or `:quit`
+  or `:open`.
+* Specify filenames either on the command line or via `:open` at the Miller REPL.
+* Read records one at a time using `:read`.
+* Write the current record (maybe after you've modified it with things like `$z = $x + $y`)
+  using `:write`. This goes to the terminal; you can use `:> {filename}` to make writes
+  go to a file, or `:>> {filename}` to append.
+* You can type `:reopen` to go back to the start of the same file(s) you specified
+  with `:open`.
+* Skip ahead using statements `:skip 10` or `:skip until NR == 100` or
+  `:skip until $status_code != 200`.
 * Similarly, but processing records rather than skipping past them, using
-  ``:process`` rather than ``:skip``. Like ``:write``, these go to the screen;
-  use ``:> {filename}`` or ``:>> {filename}`` to log to a file instead.
-* Define ``begin {...}`` blocks; invoke them at will using ``:begin``.
-* Define ``end {...}`` blocks; invoke them at will using ``:end``.
-* Define user-defined functions/subroutines using ``func``/``subr``; call them from other statements.
+  `:process` rather than `:skip`. Like `:write`, these go to the screen;
+  use `:> {filename}` or `:>> {filename}` to log to a file instead.
+* Define `begin {...}` blocks; invoke them at will using `:begin`.
+* Define `end {...}` blocks; invoke them at will using `:end`.
+* Define user-defined functions/subroutines using `func`/`subr`; call them from other statements.
 * Interactively specify statements to be executed immediately on the current record.
-* Load any of the above from Miller-script files using ``:load``.
+* Load any of the above from Miller-script files using `:load`.
 
 The input "record" by default is the empty map but you can do things like
-``$x=3``, or ``unset $y``, or ``$* = {"x": 3, "y": 4}`` to populate it. Or, ``:open
-foo.dat`` followed by ``:read`` to populate it from a data file.
+`$x=3`, or `unset $y`, or `$* = {"x": 3, "y": 4}` to populate it. Or, `:open
+foo.dat` followed by `:read` to populate it from a data file.
 
-Non-assignment expressions, such as ``7`` or ``true``, operate as filter conditions
-in the ``put`` DSL: they can be used to specify whether a record will or won't be
+Non-assignment expressions, such as `7` or `true`, operate as filter conditions
+in the `put` DSL: they can be used to specify whether a record will or won't be
 included in the output-record stream.  But here in the REPL, they are simply
-printed to the terminal, e.g. if you type ``1+2``, you will see ``3``.
+printed to the terminal, e.g. if you type `1+2`, you will see `3`.
 
 ## Entering multi-line statements
 
-* To enter multi-line statements, enter ``<`` on a line by itself, then the code (taking care
+* To enter multi-line statements, enter `<` on a line by itself, then the code (taking care
   for semicolons), then ">" on a line by itself. These will be executed immediately.
-* If you enter ``<<`` on a line by itself, then the code, then ``>>`` on a line by
+* If you enter `<<` on a line by itself, then the code, then `>>` on a line by
   itself, the statements will be remembered for executing on records with
-  ``:main``, as if you had done ``:load`` to load statements from a file.
+  `:main`, as if you had done `:load` to load statements from a file.
 
 ## Examples
 
@@ -141,11 +141,11 @@ GENMD_EOF
 ## History-editing
 
 No command-line-history-editing feature is built in but **rlwrap mlr repl** is a
-delight. You may need ``brew install rlwrap``, ``sudo apt-get install rlwrap``,
+delight. You may need `brew install rlwrap`, `sudo apt-get install rlwrap`,
 etc. depending on your platform.
 
-Suggestion: ``alias mrpl='rlwrap mlr repl'`` in your shell's startup file.
+Suggestion: `alias mrpl='rlwrap mlr repl'` in your shell's startup file.
 
 ## On-line help
 
-After ``mlr repl``, type ``:help`` to see more about your options. In particular, ``:help examples``.
+After `mlr repl`, type `:help` to see more about your options. In particular, `:help examples`.
diff --git a/docs6b/docs/shapes-of-data.md b/docs6b/docs/shapes-of-data.md
index cd1873dd6..137cbb38e 100644
--- a/docs6b/docs/shapes-of-data.md
+++ b/docs6b/docs/shapes-of-data.md
@@ -3,29 +3,33 @@
 
 ## No output at all
 
-Try ``od -xcv`` and/or ``cat -e`` on your file to check for non-printable characters.
+Try `od -xcv` and/or `cat -e` on your file to check for non-printable characters.
 
-If you're using Miller version less than 5.0.0 (try ``mlr --version`` on your system to find out), when the line-ending-autodetect feature was introduced, please see http://johnkerl.org/miller-releases/miller-4.5.0/doc/index.html.
+If you're using Miller version less than 5.0.0 (try `mlr --version` on your system to find out), when the line-ending-autodetect feature was introduced, please see [http://johnkerl.org/miller-releases/miller-4.5.0/doc/index.html](http://johnkerl.org/miller-releases/miller-4.5.0/doc/index.html).
 
 ## Fields not selected
 
-Check the field-separators of the data, e.g. with the command-line ``head`` program. Example: for CSV, Miller's default record separator is comma; if your data is tab-delimited, e.g. ``aTABbTABc``, then Miller won't find three fields named ``a``, ``b``, and ``c`` but rather just one named ``aTABbTABc``.  Solution in this case: ``mlr --fs tab {remaining arguments ...}``.
+Check the field-separators of the data, e.g. with the command-line `head` program. Example: for CSV, Miller's default record separator is comma; if your data is tab-delimited, e.g. `aTABbTABc`, then Miller won't find three fields named `a`, `b`, and `c` but rather just one named `aTABbTABc`.  Solution in this case: `mlr --fs tab {remaining arguments ...}`.
 
-Also try ``od -xcv`` and/or ``cat -e`` on your file to check for non-printable characters.
+Also try `od -xcv` and/or `cat -e` on your file to check for non-printable characters.
 
 ## Diagnosing delimiter specifications
 
-Use the ``file`` command to see if there are CR/LF terminators (in this case, # there are not):
+Use the `file` command to see if there are CR/LF terminators (in this case, # there are not):
 
-
+
 file data/colours.csv 
+
+
 data/colours.csv: UTF-8 Unicode text
 
Look at the file to find names of fields -
+
 cat data/colours.csv 
+
+
 KEY;DE;EN;ES;FI;FR;IT;NL;PL;RO;TR
 masterdata_colourcode_1;Weiß;White;Blanco;Valkoinen;Blanc;Bianco;Wit;Biały;Alb;Beyaz
 masterdata_colourcode_2;Schwarz;Black;Negro;Musta;Noir;Nero;Zwart;Czarny;Negru;Siyah
@@ -33,24 +37,30 @@ masterdata_colourcode_2;Schwarz;Black;Negro;Musta;Noir;Nero;Zwart;Czarny;Negru;S
 
 Extract a few fields:
 
-
+
 mlr --csv cut -f KEY,PL,RO data/colours.csv 
+
+
 (only blank lines appear)
 
Use XTAB output format to get a sharper picture of where records/fields are being split: -
+
 mlr --icsv --oxtab cat data/colours.csv 
+
+
 KEY;DE;EN;ES;FI;FR;IT;NL;PL;RO;TR masterdata_colourcode_1;Weiß;White;Blanco;Valkoinen;Blanc;Bianco;Wit;Biały;Alb;Beyaz
 
 KEY;DE;EN;ES;FI;FR;IT;NL;PL;RO;TR masterdata_colourcode_2;Schwarz;Black;Negro;Musta;Noir;Nero;Zwart;Czarny;Negru;Siyah
 
-Using XTAB output format makes it clearer that ``KEY;DE;...;RO;TR`` is being treated as a single field name in the CSV header, and likewise each subsequent line is being treated as a single field value. This is because the default field separator is a comma but we have semicolons here. Use XTAB again with different field separator (``--fs semicolon``): +Using XTAB output format makes it clearer that `KEY;DE;...;RO;TR` is being treated as a single field name in the CSV header, and likewise each subsequent line is being treated as a single field value. This is because the default field separator is a comma but we have semicolons here. Use XTAB again with different field separator (`--fs semicolon`): -
+
 mlr --icsv --ifs semicolon --oxtab cat data/colours.csv 
+
+
 KEY masterdata_colourcode_1
 DE  Weiß
 EN  White
@@ -78,8 +88,10 @@ TR  Siyah
 
 Using the new field-separator, retry the cut:
 
-
+
 mlr --csv --fs semicolon cut -f KEY,PL,RO data/colours.csv 
+
+
 KEY;PL;RO
 masterdata_colourcode_1;Biały;Alb
 masterdata_colourcode_2;Czarny;Negru
@@ -87,51 +99,67 @@ masterdata_colourcode_2;Czarny;Negru
 
 ## I assigned $9 and it's not 9th
 
-Miller records are ordered lists of key-value pairs. For NIDX format, DKVP format when keys are missing, or CSV/CSV-lite format with ``--implicit-csv-header``, Miller will sequentially assign keys of the form ``1``, ``2``, etc. But these are not integer array indices: they're just field names taken from the initial field ordering in the input data, when it is originally read from the input file(s).
+Miller records are ordered lists of key-value pairs. For NIDX format, DKVP format when keys are missing, or CSV/CSV-lite format with `--implicit-csv-header`, Miller will sequentially assign keys of the form `1`, `2`, etc. But these are not integer array indices: they're just field names taken from the initial field ordering in the input data, when it is originally read from the input file(s).
 
-
+
 echo x,y,z | mlr --dkvp cat
+
+
 1=x,2=y,3=z
 
-
+
 echo x,y,z | mlr --dkvp put '$6="a";$4="b";$55="cde"'
+
+
 1=x,2=y,3=z,6=a,4=b,55=cde
 
-
+
 echo x,y,z | mlr --nidx cat
+
+
 x,y,z
 
-
+
 echo x,y,z | mlr --csv --implicit-csv-header cat
+
+
 1,2,3
 x,y,z
 
-
+
 echo x,y,z | mlr --dkvp rename 2,999
+
+
 1=x,999=y,3=z
 
-
+
 echo x,y,z | mlr --dkvp rename 2,newname
+
+
 1=x,newname=y,3=z
 
-
+
 echo x,y,z | mlr --csv --implicit-csv-header reorder -f 3,1,2
+
+
 3,1,2
 z,x,y
 
## Why doesn't mlr cut put fields in the order I want? -Example: columns ``x,i,a`` were requested but they appear here in the order ``a,i,x``: +Example: columns `x,i,a` were requested but they appear here in the order `a,i,x`: -
+
 cat data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -139,8 +167,10 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
-
+
 mlr cut -f x,i,a data/small
+
+
 a=pan,i=1,x=0.3467901443380824
 a=eks,i=2,x=0.7586799647899636
 a=wye,i=3,x=0.20460330576630303
@@ -148,12 +178,14 @@ a=eks,i=4,x=0.38139939387114097
 a=wye,i=5,x=0.5732889198020006
 
-The issue is that Miller's ``cut``, by default, outputs cut fields in the order they appear in the input data. This design decision was made intentionally to parallel the Unix/Linux system ``cut`` command, which has the same semantics. +The issue is that Miller's `cut`, by default, outputs cut fields in the order they appear in the input data. This design decision was made intentionally to parallel the Unix/Linux system `cut` command, which has the same semantics. -The solution is to use the ``-o`` option: +The solution is to use the `-o` option: -
+
 mlr cut -o -f x,i,a data/small
+
+
 x=0.3467901443380824,i=1,a=pan
 x=0.7586799647899636,i=2,a=eks
 x=0.20460330576630303,i=3,a=wye
@@ -163,10 +195,12 @@ x=0.5732889198020006,i=5,a=wye
 
 ## Numbering and renumbering records
 
-The ``awk``-like built-in variable ``NR`` is incremented for each input record:
+The `awk`-like built-in variable `NR` is incremented for each input record:
 
-
+
 cat data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -174,8 +208,10 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
-
+
 mlr put '$nr = NR' data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,nr=1
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,nr=2
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776,nr=3
@@ -185,44 +221,52 @@ a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729,nr=5
 
 However, this is the record number within the original input stream -- not after any filtering you may have done:
 
-
+
 mlr filter '$a == "wye"' then put '$nr = NR' data/small
+
+
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776,nr=3
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729,nr=5
 
-There are two good options here. One is to use the ``cat`` verb with ``-n``: +There are two good options here. One is to use the `cat` verb with `-n`: -
+
 mlr filter '$a == "wye"' then cat -n data/small
+
+
 n=1,a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
 n=2,a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
-The other is to keep your own counter within the ``put`` DSL: +The other is to keep your own counter within the `put` DSL: -
+
 mlr filter '$a == "wye"' then put 'begin {@n = 1} $n = @n; @n += 1' data/small
+
+
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776,n=1
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729,n=2
 
-The difference is a matter of taste (although ``mlr cat -n`` puts the counter first). +The difference is a matter of taste (although `mlr cat -n` puts the counter first). ## Splitting nested fields Suppose you have a TSV file like this: -
+
 a	b
 x	z
 s	u:v:w
 
-The simplest option is to use :ref:`mlr nest `: +The simplest option is to use [nest](reference-verbs.md#nest): -
+
 mlr --tsv nest --explode --values --across-records -f b --nested-fs : data/nested.tsv
+
+
 a	b
 x	z
 s	u
@@ -230,8 +274,10 @@ s	v
 s	w
 
-
+
 mlr --tsv nest --explode --values --across-fields  -f b --nested-fs : data/nested.tsv
+
+
 a	b_1
 x	z
 
@@ -239,17 +285,19 @@ a	b_1	b_2	b_3
 s	u	v	w
 
-While ``mlr nest`` is simplest, let's also take a look at a few ways to do this using the ``put`` DSL. +While `mlr nest` is simplest, let's also take a look at a few ways to do this using the `put` DSL. -One option to split out the colon-delimited values in the ``b`` column is to use ``splitnv`` to create an integer-indexed map and loop over it, adding new fields to the current record: +One option to split out the colon-delimited values in the `b` column is to use `splitnv` to create an integer-indexed map and loop over it, adding new fields to the current record: -
+
 mlr --from data/nested.tsv --itsv --oxtab put '
   o = splitnv($b, ":");
   for (k,v in o) {
     $["p".k]=v
   }
 '
+
+
 a  x
 b  z
 p1 z
@@ -261,9 +309,9 @@ p2 v
 p3 w
 
-while another is to loop over the same map from ``splitnv`` and use it (with ``put -q`` to suppress printing the original record) to produce multiple records: +while another is to loop over the same map from `splitnv` and use it (with `put -q` to suppress printing the original record) to produce multiple records: -
+
 mlr --from data/nested.tsv --itsv --oxtab put -q '
   o = splitnv($b, ":");
   for (k,v in o) {
@@ -271,6 +319,8 @@ while another is to loop over the same map from ``splitnv`` and use it (with ``p
     emit x
   }
 '
+
+
 a x
 b z
 
@@ -284,13 +334,15 @@ a s
 b w
 
-
+
 mlr --from data/nested.tsv --tsv put -q '
   o = splitnv($b, ":");
   for (k,v in o) {
     x = mapsum($*, {"b":v}); emit x
   }
 '
+
+
 a	b
 x	z
 s	u
@@ -300,15 +352,15 @@ s	w
 
 ## Options for dealing with duplicate rows
 
-If your data has records appearing multiple times, you can use :ref:`mlr uniq ` to show and/or count the unique records.
+If your data has records appearing multiple times, you can use [uniq](reference-verbs.md#uniq) to show and/or count the unique records.
 
-If you want to look at partial uniqueness -- for example, show only the first record for each unique combination of the ``account_id`` and ``account_status`` fields -- you might use ``mlr head -n 1 -g account_id,account_status``. Please also see :ref:`mlr head `.
+If you want to look at partial uniqueness -- for example, show only the first record for each unique combination of the `account_id` and `account_status` fields -- you might use `mlr head -n 1 -g account_id,account_status`. Please also see [head](reference-verbs.md#head).
 
 ## Rectangularizing data
 
 Suppose you have a method (in whatever language) which is printing things of the form
 
-
+
 outer=1
 outer=2
 outer=3
@@ -316,7 +368,7 @@ outer=3
 
 and then calls another method which prints things of the form
 
-
+
 middle=10
 middle=11
 middle=12
@@ -328,7 +380,7 @@ middle=31
 
 and then, perhaps, that second method calls a third method which prints things of the form
 
-
+
 inner1=100,inner2=101
 inner1=120,inner2=121
 inner1=200,inner2=201
@@ -340,7 +392,7 @@ inner1=313,inner2=314
 
 with the result that your program's output is
 
-
+
 outer=1
 middle=10
 inner1=100,inner2=101
@@ -362,7 +414,7 @@ inner1=313,inner2=314
 
 The idea here is that middles starting with a 1 belong to the outer value of 1, and so on.  (For example, the outer values might be account IDs, the middle values might be invoice IDs, and the inner values might be invoice line-items.) If you want all the middle and inner lines to have the context of which outers they belong to, you can modify your software to pass all those through your methods. Alternatively, don't refactor your code just to handle some ad-hoc log-data formatting -- instead, use the following to rectangularize the data.  The idea is to use an out-of-stream variable to accumulate fields across records. Clear that variable when you see an outer ID; accumulate fields; emit output when you see the inner IDs.
 
-
+
 mlr --from data/rect.txt put -q '
   is_present($outer) {
     unset @r
@@ -373,6 +425,8 @@ The idea here is that middles starting with a 1 belong to the outer value of 1,
   is_present($inner1) {
     emit @r
   }'
+
+
 outer=1,middle=10,inner1=100,inner2=101
 outer=1,middle=12,inner1=120,inner2=121
 outer=2,middle=20,inner1=200,inner2=201
diff --git a/docs6b/docs/shapes-of-data.md.in b/docs6b/docs/shapes-of-data.md.in
index f42cadf13..b152c5db8 100644
--- a/docs6b/docs/shapes-of-data.md.in
+++ b/docs6b/docs/shapes-of-data.md.in
@@ -2,19 +2,19 @@
 
 ## No output at all
 
-Try ``od -xcv`` and/or ``cat -e`` on your file to check for non-printable characters.
+Try `od -xcv` and/or `cat -e` on your file to check for non-printable characters.
 
-If you're using Miller version less than 5.0.0 (try ``mlr --version`` on your system to find out), when the line-ending-autodetect feature was introduced, please see http://johnkerl.org/miller-releases/miller-4.5.0/doc/index.html.
+If you're using Miller version less than 5.0.0 (try `mlr --version` on your system to find out), when the line-ending-autodetect feature was introduced, please see [http://johnkerl.org/miller-releases/miller-4.5.0/doc/index.html](http://johnkerl.org/miller-releases/miller-4.5.0/doc/index.html).
 
 ## Fields not selected
 
-Check the field-separators of the data, e.g. with the command-line ``head`` program. Example: for CSV, Miller's default record separator is comma; if your data is tab-delimited, e.g. ``aTABbTABc``, then Miller won't find three fields named ``a``, ``b``, and ``c`` but rather just one named ``aTABbTABc``.  Solution in this case: ``mlr --fs tab {remaining arguments ...}``.
+Check the field-separators of the data, e.g. with the command-line `head` program. Example: for CSV, Miller's default record separator is comma; if your data is tab-delimited, e.g. `aTABbTABc`, then Miller won't find three fields named `a`, `b`, and `c` but rather just one named `aTABbTABc`.  Solution in this case: `mlr --fs tab {remaining arguments ...}`.
 
-Also try ``od -xcv`` and/or ``cat -e`` on your file to check for non-printable characters.
+Also try `od -xcv` and/or `cat -e` on your file to check for non-printable characters.
 
 ## Diagnosing delimiter specifications
 
-Use the ``file`` command to see if there are CR/LF terminators (in this case, # there are not):
+Use the `file` command to see if there are CR/LF terminators (in this case, # there are not):
 
 GENMD_CARDIFY_HIGHLIGHT_ONE
 file data/colours.csv 
@@ -46,7 +46,7 @@ KEY;DE;EN;ES;FI;FR;IT;NL;PL;RO;TR masterdata_colourcode_1;Weiß;White;Blanco;Val
 KEY;DE;EN;ES;FI;FR;IT;NL;PL;RO;TR masterdata_colourcode_2;Schwarz;Black;Negro;Musta;Noir;Nero;Zwart;Czarny;Negru;Siyah
 GENMD_EOF
 
-Using XTAB output format makes it clearer that ``KEY;DE;...;RO;TR`` is being treated as a single field name in the CSV header, and likewise each subsequent line is being treated as a single field value. This is because the default field separator is a comma but we have semicolons here.  Use XTAB again with different field separator (``--fs semicolon``):
+Using XTAB output format makes it clearer that `KEY;DE;...;RO;TR` is being treated as a single field name in the CSV header, and likewise each subsequent line is being treated as a single field value. This is because the default field separator is a comma but we have semicolons here.  Use XTAB again with different field separator (`--fs semicolon`):
 
 GENMD_CARDIFY_HIGHLIGHT_ONE
 mlr --icsv --ifs semicolon --oxtab cat data/colours.csv 
@@ -86,7 +86,7 @@ GENMD_EOF
 
 ## I assigned $9 and it's not 9th
 
-Miller records are ordered lists of key-value pairs. For NIDX format, DKVP format when keys are missing, or CSV/CSV-lite format with ``--implicit-csv-header``, Miller will sequentially assign keys of the form ``1``, ``2``, etc. But these are not integer array indices: they're just field names taken from the initial field ordering in the input data, when it is originally read from the input file(s).
+Miller records are ordered lists of key-value pairs. For NIDX format, DKVP format when keys are missing, or CSV/CSV-lite format with `--implicit-csv-header`, Miller will sequentially assign keys of the form `1`, `2`, etc. But these are not integer array indices: they're just field names taken from the initial field ordering in the input data, when it is originally read from the input file(s).
 
 GENMD_RUN_COMMAND
 echo x,y,z | mlr --dkvp cat
@@ -118,7 +118,7 @@ GENMD_EOF
 
 ## Why doesn't mlr cut put fields in the order I want?
 
-Example: columns ``x,i,a`` were requested but they appear here in the order ``a,i,x``:
+Example: columns `x,i,a` were requested but they appear here in the order `a,i,x`:
 
 GENMD_RUN_COMMAND
 cat data/small
@@ -128,9 +128,9 @@ GENMD_RUN_COMMAND
 mlr cut -f x,i,a data/small
 GENMD_EOF
 
-The issue is that Miller's ``cut``, by default, outputs cut fields in the order they appear in the input data. This design decision was made intentionally to parallel the Unix/Linux system ``cut`` command, which has the same semantics.
+The issue is that Miller's `cut`, by default, outputs cut fields in the order they appear in the input data. This design decision was made intentionally to parallel the Unix/Linux system `cut` command, which has the same semantics.
 
-The solution is to use the ``-o`` option:
+The solution is to use the `-o` option:
 
 GENMD_RUN_COMMAND
 mlr cut -o -f x,i,a data/small
@@ -138,7 +138,7 @@ GENMD_EOF
 
 ## Numbering and renumbering records
 
-The ``awk``-like built-in variable ``NR`` is incremented for each input record:
+The `awk`-like built-in variable `NR` is incremented for each input record:
 
 GENMD_RUN_COMMAND
 cat data/small
@@ -154,19 +154,19 @@ GENMD_RUN_COMMAND
 mlr filter '$a == "wye"' then put '$nr = NR' data/small
 GENMD_EOF
 
-There are two good options here. One is to use the ``cat`` verb with ``-n``:
+There are two good options here. One is to use the `cat` verb with `-n`:
 
 GENMD_RUN_COMMAND
 mlr filter '$a == "wye"' then cat -n data/small
 GENMD_EOF
 
-The other is to keep your own counter within the ``put`` DSL:
+The other is to keep your own counter within the `put` DSL:
 
 GENMD_RUN_COMMAND
 mlr filter '$a == "wye"' then put 'begin {@n = 1} $n = @n; @n += 1' data/small
 GENMD_EOF
 
-The difference is a matter of taste (although ``mlr cat -n`` puts the counter first).
+The difference is a matter of taste (although `mlr cat -n` puts the counter first).
 
 ## Splitting nested fields
 
@@ -174,7 +174,7 @@ Suppose you have a TSV file like this:
 
 GENMD_INCLUDE_ESCAPED(data/nested.tsv)
 
-The simplest option is to use :ref:`mlr nest `:
+The simplest option is to use [nest](reference-verbs.md#nest):
 
 GENMD_RUN_COMMAND
 mlr --tsv nest --explode --values --across-records -f b --nested-fs : data/nested.tsv
@@ -184,9 +184,9 @@ GENMD_RUN_COMMAND
 mlr --tsv nest --explode --values --across-fields  -f b --nested-fs : data/nested.tsv
 GENMD_EOF
 
-While ``mlr nest`` is simplest, let's also take a look at a few ways to do this using the ``put`` DSL.
+While `mlr nest` is simplest, let's also take a look at a few ways to do this using the `put` DSL.
 
-One option to split out the colon-delimited values in the ``b`` column is to use ``splitnv`` to create an integer-indexed map and loop over it, adding new fields to the current record:
+One option to split out the colon-delimited values in the `b` column is to use `splitnv` to create an integer-indexed map and loop over it, adding new fields to the current record:
 
 GENMD_RUN_COMMAND
 mlr --from data/nested.tsv --itsv --oxtab put '
@@ -197,7 +197,7 @@ mlr --from data/nested.tsv --itsv --oxtab put '
 '
 GENMD_EOF
 
-while another is to loop over the same map from ``splitnv`` and use it (with ``put -q`` to suppress printing the original record) to produce multiple records:
+while another is to loop over the same map from `splitnv` and use it (with `put -q` to suppress printing the original record) to produce multiple records:
 
 GENMD_RUN_COMMAND
 mlr --from data/nested.tsv --itsv --oxtab put -q '
@@ -220,9 +220,9 @@ GENMD_EOF
 
 ## Options for dealing with duplicate rows
 
-If your data has records appearing multiple times, you can use :ref:`mlr uniq ` to show and/or count the unique records.
+If your data has records appearing multiple times, you can use [uniq](reference-verbs.md#uniq) to show and/or count the unique records.
 
-If you want to look at partial uniqueness -- for example, show only the first record for each unique combination of the ``account_id`` and ``account_status`` fields -- you might use ``mlr head -n 1 -g account_id,account_status``. Please also see :ref:`mlr head `.
+If you want to look at partial uniqueness -- for example, show only the first record for each unique combination of the `account_id` and `account_status` fields -- you might use `mlr head -n 1 -g account_id,account_status`. Please also see [head](reference-verbs.md#head).
 
 ## Rectangularizing data
 
diff --git a/docs6b/docs/shell-commands.md b/docs6b/docs/shell-commands.md
index dfdfe7187..c0c8eb59a 100644
--- a/docs6b/docs/shell-commands.md
+++ b/docs6b/docs/shell-commands.md
@@ -3,10 +3,12 @@
 
 TODO: while-read example from issues
 
-The :ref:`reference-dsl-system` DSL function allows you to run a specific shell command and put its output -- minus the final newline -- into a record field. The command itself is any string, either a literal string, or a concatenation of strings, perhaps including other field values or what have you.
+The [system](reference-dsl.md#system) DSL function allows you to run a specific shell command and put its output -- minus the final newline -- into a record field. The command itself is any string, either a literal string, or a concatenation of strings, perhaps including other field values or what have you.
 
-
+
 mlr --opprint put '$o = system("echo hello world")' data/small
+
+
 a   b   i x                   y                   o
 pan pan 1 0.3467901443380824  0.7268028627434533  hello world
 eks pan 2 0.7586799647899636  0.5221511083334797  hello world
@@ -15,8 +17,10 @@ eks wye 4 0.38139939387114097 0.13418874328430463 hello world
 wye pan 5 0.5732889198020006  0.8636244699032729  hello world
 
-
+
 mlr --opprint put '$o = system("echo {" . NR . "}")' data/small
+
+
 a   b   i x                   y                   o
 pan pan 1 0.3467901443380824  0.7268028627434533  {1}
 eks pan 2 0.7586799647899636  0.5221511083334797  {2}
@@ -25,8 +29,10 @@ eks wye 4 0.38139939387114097 0.13418874328430463 {4}
 wye pan 5 0.5732889198020006  0.8636244699032729  {5}
 
-
+
 mlr --opprint put '$o = system("echo -n ".$a."| sha1sum")' data/small
+
+
 a   b   i x                   y                   o
 pan pan 1 0.3467901443380824  0.7268028627434533  f29c748220331c273ef16d5115f6ecd799947f13  -
 eks pan 2 0.7586799647899636  0.5221511083334797  456d988ecb3bf1b75f057fc6e9fe70db464e9388  -
@@ -35,13 +41,15 @@ eks wye 4 0.38139939387114097 0.13418874328430463 456d988ecb3bf1b75f057fc6e9fe70
 wye pan 5 0.5732889198020006  0.8636244699032729  eab0de043d67f441c7fd1e335f0ca38708e6ebf7  -
 
-Note that running a subprocess on every record takes a non-trivial amount of time. Comparing asking the system ``date`` command for the current time in nanoseconds versus computing it in process: +Note that running a subprocess on every record takes a non-trivial amount of time. Comparing asking the system `date` command for the current time in nanoseconds versus computing it in process: .. hard-coded, not live-code, since %N doesn't exist on all platforms -
+
 mlr --opprint put '$t=system("date +%s.%N")' then step -a delta -f t data/small
+
+
 a   b   i x                   y                   t                    t_delta
 pan pan 1 0.3467901443380824  0.7268028627434533  1568774318.513903817 0
 eks pan 2 0.7586799647899636  0.5221511083334797  1568774318.514722876 0.000819
@@ -50,8 +58,10 @@ eks wye 4 0.38139939387114097 0.13418874328430463 1568774318.516547441 0.000929
 wye pan 5 0.5732889198020006  0.8636244699032729  1568774318.517518828 0.000971
 
-
+
 mlr --opprint put '$t=systime()' then step -a delta -f t data/small
+
+
 a   b   i x                   y                   t                 t_delta
 pan pan 1 0.3467901443380824  0.7268028627434533  1568774318.518699 0
 eks pan 2 0.7586799647899636  0.5221511083334797  1568774318.518717 0.000018
diff --git a/docs6b/docs/shell-commands.md.in b/docs6b/docs/shell-commands.md.in
index afce3ddf3..39efbfb48 100644
--- a/docs6b/docs/shell-commands.md.in
+++ b/docs6b/docs/shell-commands.md.in
@@ -2,7 +2,7 @@
 
 TODO: while-read example from issues
 
-The :ref:`reference-dsl-system` DSL function allows you to run a specific shell command and put its output -- minus the final newline -- into a record field. The command itself is any string, either a literal string, or a concatenation of strings, perhaps including other field values or what have you.
+The [system](reference-dsl.md#system) DSL function allows you to run a specific shell command and put its output -- minus the final newline -- into a record field. The command itself is any string, either a literal string, or a concatenation of strings, perhaps including other field values or what have you.
 
 GENMD_RUN_COMMAND
 mlr --opprint put '$o = system("echo hello world")' data/small
@@ -16,7 +16,7 @@ GENMD_RUN_COMMAND
 mlr --opprint put '$o = system("echo -n ".$a."| sha1sum")' data/small
 GENMD_EOF
 
-Note that running a subprocess on every record takes a non-trivial amount of time. Comparing asking the system ``date`` command for the current time in nanoseconds versus computing it in process:
+Note that running a subprocess on every record takes a non-trivial amount of time. Comparing asking the system `date` command for the current time in nanoseconds versus computing it in process:
 
 ..
     hard-coded, not live-code, since %N doesn't exist on all platforms
diff --git a/docs6b/docs/special-symbols-and-formatting.md b/docs6b/docs/special-symbols-and-formatting.md
index b245f9ee0..634cd8404 100644
--- a/docs6b/docs/special-symbols-and-formatting.md
+++ b/docs6b/docs/special-symbols-and-formatting.md
@@ -3,19 +3,23 @@
 
 ## How can I handle commas-as-data in various formats?
 
-:doc:`CSV ` handles this well and by design:
+[CSV](file-formats.md) handles this well and by design:
 
-
+
 cat commas.csv
+
+
 Name,Role
 "Xiao, Lin",administrator
 "Khavari, Darius",tester
 
-Likewise :ref:`file-formats-json`: +Likewise [JSON](file-formats.md#json): -
+
 mlr --icsv --ojson cat commas.csv
+
+
 {
   "Name": "Xiao, Lin",
   "Role": "administrator"
@@ -26,10 +30,12 @@ Likewise :ref:`file-formats-json`:
 }
 
-For Miller's :ref:`vertical-tabular format ` there is no escaping for carriage returns, but commas work fine: +For Miller's [XTAB](file-formats.md#xtab-vertical-tabular) there is no escaping for carriage returns, but commas work fine: -
+
 mlr --icsv --oxtab cat commas.csv
+
+
 Name Xiao, Lin
 Role administrator
 
@@ -37,18 +43,22 @@ Name Khavari, Darius
 Role tester
 
-But for :ref:`Key-value_pairs ` and :ref:`index-numbered `, commas are the default field separator. And -- as of Miller 5.4.0 anyway -- there is no CSV-style double-quote-handling like there is for CSV. So commas within the data look like delimiters: +But for [key-value-pairs](file-formats.md#dkvp-key-value-pairs) and [index-numbered](file-formats.md#nidx-index-numbered-toolkit-style) formats, commas are the default field separator. And -- as of Miller 5.4.0 anyway -- there is no CSV-style double-quote-handling like there is for CSV. So commas within the data look like delimiters: -
+
 mlr --icsv --odkvp cat commas.csv
+
+
 Name=Xiao, Lin,Role=administrator
 Name=Khavari, Darius,Role=tester
 
One solution is to use a different delimiter, such as a pipe character: -
+
 mlr --icsv --odkvp --ofs pipe cat commas.csv
+
+
 Name=Xiao, Lin|Role=administrator
 Name=Khavari, Darius|Role=tester
 
@@ -56,8 +66,10 @@ Name=Khavari, Darius|Role=tester To be extra-sure to avoid data/delimiter clashes, you can also use control characters as delimiters -- here, control-A: -
+
 mlr --icsv --odkvp --ofs '\001'  cat commas.csv | cat -v
+
+
 Name=Xiao, Lin\001Role=administrator
 Name=Khavari, Darius\001Role=tester
 
@@ -66,8 +78,10 @@ Name=Khavari, Darius\001Role=tester Simply surround the field names with curly braces: -
+
 echo 'x.a=3,y:b=4,z/c=5' | mlr put '${product.all} = ${x.a} * ${y:b} * ${z/c}'
+
+
 x.a=3,y:b=4,z/c=5,product.all=60
 
@@ -75,12 +89,14 @@ x.a=3,y:b=4,z/c=5,product.all=60 This is a little tricky due to the shell's handling of quotes. For simplicity, let's first put an update script into a file: -
+
 $a = "It's OK, I said, then 'for now'."
 
-
+
 echo a=bcd | mlr put -f data/single-quote-example.mlr
+
+
 a=It's OK, I said, then 'for now'.
 
@@ -88,40 +104,48 @@ So, it's simple: Miller's DSL uses double quotes for strings, and you can put si Without putting the update expression in a file, it's messier: -
+
 echo a=bcd | mlr put '$a="It'\''s OK, I said, '\''for now'\''."'
+
+
 a=It's OK, I said, 'for now'.
 
-The idea is that the outermost single-quotes are to protect the ``put`` expression from the shell, and the double quotes within them are for Miller. To get a single quote in the middle there, you need to actually put it *outside* the single-quoting for the shell. The pieces are the following, all concatenated together: +The idea is that the outermost single-quotes are to protect the `put` expression from the shell, and the double quotes within them are for Miller. To get a single quote in the middle there, you need to actually put it *outside* the single-quoting for the shell. The pieces are the following, all concatenated together: -* ``$a="It`` -* ``\'`` -* ``s OK, I said,`` -* ``\'`` -* ``for now`` -* ``\'`` -* ``.`` +* `$a="It` +* `\'` +* `s OK, I said,` +* `\'` +* `for now` +* `\'` +* `.` ## How to escape '?' in regexes? One way is to use square brackets; an alternative is to use simple string-substitution rather than a regular expression. -
+
 cat data/question.dat
+
+
 a=is it?,b=it is!
 
-
+
 mlr --oxtab put '$c = gsub($a, "[?]"," ...")' data/question.dat
+
+
 a is it?
 b it is!
 c is it ...
 
-
+
 mlr --oxtab put '$c = ssub($a, "?"," ...")' data/question.dat
+
+
 a is it?
 b it is!
 c is it ...
 
-The ``ssub`` function exists precisely for this reason: so you don't have to escape anything. +The `ssub` function exists precisely for this reason: so you don't have to escape anything. diff --git a/docs6b/docs/special-symbols-and-formatting.md.in b/docs6b/docs/special-symbols-and-formatting.md.in index 3ad97ce39..97fd841a1 100644 --- a/docs6b/docs/special-symbols-and-formatting.md.in +++ b/docs6b/docs/special-symbols-and-formatting.md.in @@ -2,25 +2,25 @@ ## How can I handle commas-as-data in various formats? -:doc:`CSV ` handles this well and by design: +[CSV](file-formats.md) handles this well and by design: GENMD_RUN_COMMAND cat commas.csv GENMD_EOF -Likewise :ref:`file-formats-json`: +Likewise [JSON](file-formats.md#json): GENMD_RUN_COMMAND mlr --icsv --ojson cat commas.csv GENMD_EOF -For Miller's :ref:`vertical-tabular format ` there is no escaping for carriage returns, but commas work fine: +For Miller's [XTAB](file-formats.md#xtab-vertical-tabular) there is no escaping for carriage returns, but commas work fine: GENMD_RUN_COMMAND mlr --icsv --oxtab cat commas.csv GENMD_EOF -But for :ref:`Key-value_pairs ` and :ref:`index-numbered `, commas are the default field separator. And -- as of Miller 5.4.0 anyway -- there is no CSV-style double-quote-handling like there is for CSV. So commas within the data look like delimiters: +But for [key-value-pairs](file-formats.md#dkvp-key-value-pairs) and [index-numbered](file-formats.md#nidx-index-numbered-toolkit-style) formats, commas are the default field separator. And -- as of Miller 5.4.0 anyway -- there is no CSV-style double-quote-handling like there is for CSV. So commas within the data look like delimiters: GENMD_RUN_COMMAND mlr --icsv --odkvp cat commas.csv @@ -65,15 +65,15 @@ GENMD_RUN_COMMAND echo a=bcd | mlr put '$a="It'\''s OK, I said, '\''for now'\''."' GENMD_EOF -The idea is that the outermost single-quotes are to protect the ``put`` expression from the shell, and the double quotes within them are for Miller. To get a single quote in the middle there, you need to actually put it *outside* the single-quoting for the shell. The pieces are the following, all concatenated together: +The idea is that the outermost single-quotes are to protect the `put` expression from the shell, and the double quotes within them are for Miller. To get a single quote in the middle there, you need to actually put it *outside* the single-quoting for the shell. The pieces are the following, all concatenated together: -* ``$a="It`` -* ``\'`` -* ``s OK, I said,`` -* ``\'`` -* ``for now`` -* ``\'`` -* ``.`` +* `$a="It` +* `\'` +* `s OK, I said,` +* `\'` +* `for now` +* `\'` +* `.` ## How to escape '?' in regexes? @@ -89,4 +89,4 @@ GENMD_RUN_COMMAND mlr --oxtab put '$c = ssub($a, "?"," ...")' data/question.dat GENMD_EOF -The ``ssub`` function exists precisely for this reason: so you don't have to escape anything. +The `ssub` function exists precisely for this reason: so you don't have to escape anything. diff --git a/docs6b/docs/sql-examples.md b/docs6b/docs/sql-examples.md index 71e68eca6..cfa7cea22 100644 --- a/docs6b/docs/sql-examples.md +++ b/docs6b/docs/sql-examples.md @@ -1,16 +1,16 @@ # SQL examples -.. _sql-output-examples: - ## SQL-output examples -I like to produce SQL-query output with header-column and tab delimiter: this is CSV but with a tab instead of a comma, also known as TSV. Then I post-process with ``mlr --tsv`` or ``mlr --tsvlite``. This means I can do some (or all, or none) of my data processing within SQL queries, and some (or none, or all) of my data processing using Miller -- whichever is most convenient for my needs at the moment. +I like to produce SQL-query output with header-column and tab delimiter: this is CSV but with a tab instead of a comma, also known as TSV. Then I post-process with `mlr --tsv` or `mlr --tsvlite`. This means I can do some (or all, or none) of my data processing within SQL queries, and some (or none, or all) of my data processing using Miller -- whichever is most convenient for my needs at the moment. -For example, using default output formatting in ``mysql`` we get formatting like Miller's ``--opprint --barred``: +For example, using default output formatting in `mysql` we get formatting like Miller's `--opprint --barred`: -
+
 mysql --database=mydb -e 'show columns in mytable'
+
+
 +------------------+--------------+------+-----+---------+-------+
 | Field            | Type         | Null | Key | Default | Extra |
 +------------------+--------------+------+-----+---------+-------+
@@ -22,10 +22,12 @@ For example, using default output formatting in ``mysql`` we get formatting like
 +------------------+--------------+------+-----+---------+-------+
 
-Using ``mysql``'s ``-B`` we get TSV output: +Using `mysql`'s `-B` we get TSV output: -
+
 mysql --database=mydb -B -e 'show columns in mytable' | mlr --itsvlite --opprint cat
+
+
 Field            Type         Null Key Default Extra
 id               bigint(20)   NO  MUL NULL -
 category         varchar(256) NO  -   NULL -
@@ -36,8 +38,10 @@ last_update_time int(11)      YES -   NULL -
 
 Since Miller handles TSV output, we can do as much or as little processing as we want in the SQL query, then send the rest on to Miller. This includes outputting as JSON, doing further selects/joins in Miller, doing stats, etc.  etc.:
 
-
+
 mysql --database=mydb -B -e 'show columns in mytable' | mlr --itsvlite --ojson --jlistwrap --jvstack cat
+
+
 [
   {
     "Field": "id",
@@ -82,12 +86,14 @@ Since Miller handles TSV output, we can do as much or as little processing as we
 ]
 
-
+
 mysql --database=mydb -B -e 'select * from mytable' > query.tsv
 
-
+
 mlr --from query.tsv --t2p stats1 -a count -f id -g category,assigned_to
+
+
 category assigned_to id_count
 special  10000978    207
 special  10003924    385
@@ -100,15 +106,13 @@ standard 10009872    108
 
 Again, all the examples in the CSV section apply here -- just change the input-format flags.
 
-.. _sql-input-examples:
-
 ## SQL-input examples
 
 One use of NIDX (value-only, no keys) format is for loading up SQL tables.
 
 Create and load SQL table:
 
-
+
 mysql> CREATE TABLE abixy(
   a VARCHAR(32),
   b VARCHAR(32),
@@ -151,7 +155,7 @@ mysql> SELECT * FROM abixy LIMIT 10;
 
 Aggregate counts within SQL:
 
-
+
 mysql> SELECT a, b, COUNT(*) AS count FROM abixy GROUP BY a, b ORDER BY COUNT DESC;
 +------+------+-------+
 | a    | b    | count |
@@ -187,8 +191,10 @@ mysql> SELECT a, b, COUNT(*) AS count FROM abixy GROUP BY a, b ORDER BY COUNT DE
 
 Aggregate counts within Miller:
 
-
+
 mlr --opprint uniq -c -g a,b then sort -nr count data/medium
+
+
 a   b   count
 zee wye 455
 pan eks 429
@@ -209,8 +215,10 @@ eks zee 357
 
 Pipe SQL output to aggregate counts within Miller:
 
-
+
 mysql -D miller -B -e 'select * from abixy' | mlr --itsv --opprint uniq -c -g a,b then sort -nr count
+
+
 a   b   count
 zee wye 455
 pan eks 429
diff --git a/docs6b/docs/sql-examples.md.in b/docs6b/docs/sql-examples.md.in
index a1547b2e6..7559893c7 100644
--- a/docs6b/docs/sql-examples.md.in
+++ b/docs6b/docs/sql-examples.md.in
@@ -1,12 +1,10 @@
 # SQL examples
 
-.. _sql-output-examples:
-
 ## SQL-output examples
 
-I like to produce SQL-query output with header-column and tab delimiter: this is CSV but with a tab instead of a comma, also known as TSV. Then I post-process with ``mlr --tsv`` or ``mlr --tsvlite``.  This means I can do some (or all, or none) of my data processing within SQL queries, and some (or none, or all) of my data processing using Miller -- whichever is most convenient for my needs at the moment.
+I like to produce SQL-query output with header-column and tab delimiter: this is CSV but with a tab instead of a comma, also known as TSV. Then I post-process with `mlr --tsv` or `mlr --tsvlite`.  This means I can do some (or all, or none) of my data processing within SQL queries, and some (or none, or all) of my data processing using Miller -- whichever is most convenient for my needs at the moment.
 
-For example, using default output formatting in ``mysql`` we get formatting like Miller's ``--opprint --barred``:
+For example, using default output formatting in `mysql` we get formatting like Miller's `--opprint --barred`:
 
 GENMD_CARDIFY_HIGHLIGHT_ONE
 mysql --database=mydb -e 'show columns in mytable'
@@ -21,7 +19,7 @@ mysql --database=mydb -e 'show columns in mytable'
 +------------------+--------------+------+-----+---------+-------+
 GENMD_EOF
 
-Using ``mysql``'s ``-B`` we get TSV output:
+Using `mysql`'s `-B` we get TSV output:
 
 GENMD_CARDIFY_HIGHLIGHT_ONE
 mysql --database=mydb -B -e 'show columns in mytable' | mlr --itsvlite --opprint cat
@@ -99,8 +97,6 @@ GENMD_EOF
 
 Again, all the examples in the CSV section apply here -- just change the input-format flags.
 
-.. _sql-input-examples:
-
 ## SQL-input examples
 
 One use of NIDX (value-only, no keys) format is for loading up SQL tables.
diff --git a/docs6b/docs/statistics-examples.md b/docs6b/docs/statistics-examples.md
index 3c6f166b7..c201aa3e9 100644
--- a/docs6b/docs/statistics-examples.md
+++ b/docs6b/docs/statistics-examples.md
@@ -5,18 +5,20 @@
 
 For one or more specified field names, simply compute p25 and p75, then write the IQR as the difference of p75 and p25:
 
-
+
 mlr --oxtab stats1 -f x -a p25,p75 \
     then put '$x_iqr = $x_p75 - $x_p25' \
     data/medium 
+
+
 x_p25 0.24667037823231752
 x_p75 0.7481860062358446
 x_iqr 0.5015156280035271
 
-For wildcarded field names, first compute p25 and p75, then loop over field names with ``p25`` in them: +For wildcarded field names, first compute p25 and p75, then loop over field names with `p25` in them: -
+
 mlr --oxtab stats1 --fr '[i-z]' -a p25,p75 \
     then put 'for (k,v in $*) {
       if (k =~ "(.*)_p25") {
@@ -28,9 +30,9 @@ For wildcarded field names, first compute p25 and p75, then loop over field name
 
 ## Computing weighted means
 
-This might be more elegantly implemented as an option within the ``stats1`` verb. Meanwhile, it's expressible within the DSL:
+This might be more elegantly implemented as an option within the `stats1` verb. Meanwhile, it's expressible within the DSL:
 
-
+
 mlr --from data/medium put -q '
   # Using the y field for weighting in this example
   weight = $y;
@@ -55,6 +57,8 @@ This might be more elegantly implemented as an option within the ``stats1`` verb
     #emit mean, "a";
     emit (wmean, mean), "a";
   }'
+
+
 a=pan,wmean=4979.563722208067,mean=5028.259010091302
 a=eks,wmean=4890.3815931472145,mean=4956.2900763358775
 a=wye,wmean=4946.987746229947,mean=4920.001017293998
diff --git a/docs6b/docs/statistics-examples.md.in b/docs6b/docs/statistics-examples.md.in
index 41872240d..382f75775 100644
--- a/docs6b/docs/statistics-examples.md.in
+++ b/docs6b/docs/statistics-examples.md.in
@@ -6,12 +6,12 @@ For one or more specified field names, simply compute p25 and p75, then write th
 
 GENMD_INCLUDE_AND_RUN_ESCAPED(data/iqr1.sh)
 
-For wildcarded field names, first compute p25 and p75, then loop over field names with ``p25`` in them:
+For wildcarded field names, first compute p25 and p75, then loop over field names with `p25` in them:
 
 GENMD_INCLUDE_AND_RUN_ESCAPED(data/iqrn.sh)
 
 ## Computing weighted means
 
-This might be more elegantly implemented as an option within the ``stats1`` verb. Meanwhile, it's expressible within the DSL:
+This might be more elegantly implemented as an option within the `stats1` verb. Meanwhile, it's expressible within the DSL:
 
 GENMD_INCLUDE_AND_RUN_ESCAPED(data/weighted-mean.sh)
diff --git a/docs6b/docs/then-chaining.md b/docs6b/docs/then-chaining.md
index c9470479e..38a971d8f 100644
--- a/docs6b/docs/then-chaining.md
+++ b/docs6b/docs/then-chaining.md
@@ -7,8 +7,10 @@ Then-chaining found in Miller is intended to function the same as Unix pipes, bu
 
 First, look at the input data:
 
-
+
 cat data/then-example.csv
+
+
 Status,Payment_Type,Amount
 paid,cash,10.00
 pending,debit,20.00
@@ -17,10 +19,12 @@ pending,credit,40.00
 paid,debit,30.00
 
-Next, run the first step of your command, omitting anything from the first ``then`` onward: +Next, run the first step of your command, omitting anything from the first `then` onward: -
+
 mlr --icsv --opprint count-distinct -f Status,Payment_Type data/then-example.csv
+
+
 Status  Payment_Type count
 paid    cash         2
 pending debit        1
@@ -28,12 +32,14 @@ pending credit       1
 paid    debit        1
 
-After that, run it with the next ``then`` step included: +After that, run it with the next `then` step included: -
+
 mlr --icsv --opprint count-distinct -f Status,Payment_Type \
   then sort -nr count \
   data/then-example.csv
+
+
 Status  Payment_Type count
 paid    cash         2
 pending debit        1
@@ -41,13 +47,15 @@ pending credit       1
 paid    debit        1
 
-Now if you use ``then`` to include another verb after that, the columns ``Status``, ``Payment_Type``, and ``count`` will be the input to that verb. +Now if you use `then` to include another verb after that, the columns `Status`, `Payment_Type`, and `count` will be the input to that verb. Note, by the way, that you'll get the same results using pipes: -
+
 mlr --csv count-distinct -f Status,Payment_Type data/then-example.csv \
 | mlr --icsv --opprint sort -nr count
+
+
 Status  Payment_Type count
 paid    cash         2
 pending debit        1
@@ -59,8 +67,10 @@ paid    debit        1
 
 Given this input data:
 
-
+
 cat data/small
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -68,24 +78,28 @@ a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
-why don't I see ``NR=1`` and ``NR=2`` here?? +why don't I see `NR=1` and `NR=2` here?? -
+
 mlr filter '$x > 0.5' then put '$NR = NR' data/small
+
+
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,NR=2
 a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729,NR=5
 
-The reason is that ``NR`` is computed for the original input records and isn't dynamically updated. By contrast, ``NF`` is dynamically updated: it's the number of fields in the current record, and if you add/remove a field, the value of ``NF`` will change: +The reason is that `NR` is computed for the original input records and isn't dynamically updated. By contrast, `NF` is dynamically updated: it's the number of fields in the current record, and if you add/remove a field, the value of `NF` will change: -
+
 echo x=1,y=2,z=3 | mlr put '$nf1 = NF; $u = 4; $nf2 = NF; unset $x,$y,$z; $nf3 = NF'
+
+
 nf1=3,u=4,nf2=5,nf3=3
 
-``NR``, by contrast (and ``FNR`` as well), retains the value from the original input stream, and records may be dropped by a ``filter`` within a ``then``-chain. To recover consecutive record numbers, you can use out-of-stream variables as follows: +`NR`, by contrast (and `FNR` as well), retains the value from the original input stream, and records may be dropped by a `filter` within a `then`-chain. To recover consecutive record numbers, you can use out-of-stream variables as follows: -
+
 mlr --opprint --from data/small put '
   begin{ @nr1 = 0 }
   @nr1 += 1;
@@ -97,15 +111,19 @@ nf1=3,u=4,nf2=5,nf3=3
   @nr2 += 1;
   $nr2 = @nr2
 '
+
+
 a   b   i x                  y                  nr1 nr2
 eks pan 2 0.7586799647899636 0.5221511083334797 2   1
 wye pan 5 0.5732889198020006 0.8636244699032729 5   2
 
-Or, simply use ``mlr cat -n``: +Or, simply use `mlr cat -n`: -
+
 mlr filter '$x > 0.5' then cat -n data/small
+
+
 n=1,a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 n=2,a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
diff --git a/docs6b/docs/then-chaining.md.in b/docs6b/docs/then-chaining.md.in index cfa643ce1..e5d00399d 100644 --- a/docs6b/docs/then-chaining.md.in +++ b/docs6b/docs/then-chaining.md.in @@ -10,13 +10,13 @@ GENMD_RUN_COMMAND cat data/then-example.csv GENMD_EOF -Next, run the first step of your command, omitting anything from the first ``then`` onward: +Next, run the first step of your command, omitting anything from the first `then` onward: GENMD_RUN_COMMAND mlr --icsv --opprint count-distinct -f Status,Payment_Type data/then-example.csv GENMD_EOF -After that, run it with the next ``then`` step included: +After that, run it with the next `then` step included: GENMD_RUN_COMMAND mlr --icsv --opprint count-distinct -f Status,Payment_Type \ @@ -24,7 +24,7 @@ mlr --icsv --opprint count-distinct -f Status,Payment_Type \ data/then-example.csv GENMD_EOF -Now if you use ``then`` to include another verb after that, the columns ``Status``, ``Payment_Type``, and ``count`` will be the input to that verb. +Now if you use `then` to include another verb after that, the columns `Status`, `Payment_Type`, and `count` will be the input to that verb. Note, by the way, that you'll get the same results using pipes: @@ -41,23 +41,23 @@ GENMD_RUN_COMMAND cat data/small GENMD_EOF -why don't I see ``NR=1`` and ``NR=2`` here?? +why don't I see `NR=1` and `NR=2` here?? GENMD_RUN_COMMAND mlr filter '$x > 0.5' then put '$NR = NR' data/small GENMD_EOF -The reason is that ``NR`` is computed for the original input records and isn't dynamically updated. By contrast, ``NF`` is dynamically updated: it's the number of fields in the current record, and if you add/remove a field, the value of ``NF`` will change: +The reason is that `NR` is computed for the original input records and isn't dynamically updated. By contrast, `NF` is dynamically updated: it's the number of fields in the current record, and if you add/remove a field, the value of `NF` will change: GENMD_RUN_COMMAND echo x=1,y=2,z=3 | mlr put '$nf1 = NF; $u = 4; $nf2 = NF; unset $x,$y,$z; $nf3 = NF' GENMD_EOF -``NR``, by contrast (and ``FNR`` as well), retains the value from the original input stream, and records may be dropped by a ``filter`` within a ``then``-chain. To recover consecutive record numbers, you can use out-of-stream variables as follows: +`NR`, by contrast (and `FNR` as well), retains the value from the original input stream, and records may be dropped by a `filter` within a `then`-chain. To recover consecutive record numbers, you can use out-of-stream variables as follows: GENMD_INCLUDE_AND_RUN_ESCAPED(data/dynamic-nr.sh) -Or, simply use ``mlr cat -n``: +Or, simply use `mlr cat -n`: GENMD_RUN_COMMAND mlr filter '$x > 0.5' then cat -n data/small diff --git a/docs6b/docs/two-pass-algorithms.md b/docs6b/docs/two-pass-algorithms.md index 207fe8c41..e59a1842b 100644 --- a/docs6b/docs/two-pass-algorithms.md +++ b/docs6b/docs/two-pass-algorithms.md @@ -3,12 +3,14 @@ ## Overview -Miller is a streaming record processor; commands are performed once per record. This makes Miller particularly suitable for single-pass algorithms, allowing many of its verbs to process files that are (much) larger than the amount of RAM present in your system. (Of course, Miller verbs such as ``sort``, ``tac``, etc. all must ingest and retain all input records before emitting any output records.) You can also use out-of-stream variables to perform multi-pass computations, at the price of retaining all input records in memory. +Miller is a streaming record processor; commands are performed once per record. This makes Miller particularly suitable for single-pass algorithms, allowing many of its verbs to process files that are (much) larger than the amount of RAM present in your system. (Of course, Miller verbs such as `sort`, `tac`, etc. all must ingest and retain all input records before emitting any output records.) You can also use out-of-stream variables to perform multi-pass computations, at the price of retaining all input records in memory. One of Miller's strengths is its compact notation: for example, given input of the form -
+
 head -n 5 ./data/medium
+
+
 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
@@ -18,15 +20,19 @@ a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
 
 you can simply do
 
-
+
 mlr --oxtab stats1 -a sum -f x ./data/medium
+
+
 x_sum 4986.019681679581
 
or -
+
 mlr --opprint stats1 -a sum -f x -g b ./data/medium
+
+
 b   x_sum
 pan 965.7636699425815
 wye 1023.5484702619565
@@ -37,25 +43,29 @@ hat 1000.192668193983
 
 rather than the more tedious
 
-
+
 mlr --oxtab put -q '
   @x_sum += $x;
   end {
     emit @x_sum
   }
 ' data/medium
+
+
 x_sum 4986.019681679581
 
or -
+
 mlr --opprint put -q '
   @x_sum[$b] += $x;
   end {
     emit @x_sum, "b"
   }
 ' data/medium
+
+
 b   x_sum
 pan 965.7636699425815
 wye 1023.5484702619565
@@ -64,9 +74,9 @@ eks 1016.7728571314786
 hat 1000.192668193983
 
-The former (``mlr stats1`` et al.) has the advantages of being easier to type, being less error-prone to type, and running faster. +The former (`mlr stats1` et al.) has the advantages of being easier to type, being less error-prone to type, and running faster. -Nonetheless, out-of-stream variables (which I whimsically call *oosvars*), begin/end blocks, and emit statements give you the ability to implement logic -- if you wish to do so -- which isn't present in other Miller verbs. (If you find yourself often using the same out-of-stream-variable logic over and over, please file a request at https://github.com/johnkerl/miller/issues to get it implemented directly in Go as a Miller verb of its own.) +Nonetheless, out-of-stream variables (which I whimsically call *oosvars*), begin/end blocks, and emit statements give you the ability to implement logic -- if you wish to do so -- which isn't present in other Miller verbs. (If you find yourself often using the same out-of-stream-variable logic over and over, please file a request at [https://github.com/johnkerl/miller/issues](https://github.com/johnkerl/miller/issues) to get it implemented directly in Go as a Miller verb of its own.) The following examples compute some things using oosvars which are already computable using Miller verbs, by way of providing food for thought. @@ -74,7 +84,7 @@ The following examples compute some things using oosvars which are already compu For example, mapping numeric values down a column to the percentage between their min and max values is two-pass: on the first pass you find the min and max values, then on the second, map each record's value to a percentage. -
+
 mlr --from data/small --opprint put -q '
   # These are executed once per record, which is the first pass.
   # The key is to use NR to index an out-of-stream variable to
@@ -91,6 +101,8 @@ For example, mapping numeric values down a column to the percentage between thei
     emit (@x, @x_pct), "NR"
   }
 '
+
+
 NR x                   x_pct
 1  0.3467901443380824  25.66194338926441
 2  0.7586799647899636  100
@@ -103,7 +115,7 @@ NR x                   x_pct
 
 Similarly, finding the total record count requires first reading through all the data:
 
-
+
 mlr --opprint --from data/small put -q '
   @records[NR] = $*;
   end {
@@ -115,6 +127,8 @@ Similarly, finding the total record count requires first reading through all the
     emit @records,"I"
   }
 ' then reorder -f I,N,PCT
+
+
 I N PCT     a   b   i x                   y
 1 5 (error) pan pan 1 0.3467901443380824  0.7268028627434533
 2 5 (error) eks pan 2 0.7586799647899636  0.5221511083334797
@@ -125,10 +139,12 @@ I N PCT     a   b   i x                   y
 
 ## Records having max value
 
-The idea is to retain records having the largest value of ``n`` in the following data:
+The idea is to retain records having the largest value of `n` in the following data:
 
-
+
 mlr --itsv --opprint cat data/maxrows.tsv
+
+
 a      b      n score
 purple red    5 0.743231
 blue   purple 2 0.093710
@@ -163,10 +179,12 @@ red    purple 4 0.477187
 blue   red    4 0.007487
 
-Of course, the largest value of ``n`` isn't known until after all data have been read. Using an out-of-stream variable we can retain all records as they are read, then filter them at the end: +Of course, the largest value of `n` isn't known until after all data have been read. Using an out-of-stream variable we can retain all records as they are read, then filter them at the end: -
+
 cat data/maxrows.mlr
+
+
 # Retain all records
 @records[NR] = $*;
 # Track max value of n
@@ -184,8 +202,10 @@ end {
 }
 
-
+
 mlr --itsv --opprint put -q -f data/maxrows.mlr data/maxrows.tsv
+
+
 a      b      n score
 purple red    5 0.743231
 purple red    5 0.389055
@@ -202,7 +222,7 @@ green  purple 5 0.203577
 
 Suppose you have some heterogeneous data like this:
 
-
+
 { "qoh": 29874, "rate": 1.68, "latency": 0.02 }
 { "name": "alice", "uid": 572 }
 { "qoh": 1227, "rate": 1.01, "latency": 0.07 }
@@ -219,7 +239,7 @@ Suppose you have some heterogeneous data like this:
 
 A reasonable question to ask is, how many occurrences of each field are there? And, what percentage of total row count has each of them? Since the denominator of the percentage is not known until the end, this is a two-pass algorithm:
 
-
+
 for (key in $*) {
   @key_counts[key] += 1;
 }
@@ -237,8 +257,10 @@ end {
 
 Then
 
-
+
 mlr --json put -q -f data/feature-count.mlr data/features.json
+
+
 {
   "record_count": 12
 }
@@ -292,8 +314,10 @@ Then
 }
 
-
+
 mlr --ijson --opprint put -q -f data/feature-count.mlr data/features.json
+
+
 record_count
 12
 
@@ -320,18 +344,22 @@ The previous section discussed how to fill out missing data fields within CSV wi
 
 For example, suppose you have JSON input like this:
 
-
+
 cat data/sparse.json
+
+
 {"a":1,"b":2,"v":3}
 {"u":1,"b":2}
 {"a":1,"v":2,"x":3}
 {"v":1,"w":2}
 
-There are field names ``a``, ``b``, ``v``, ``u``, ``x``, ``w`` in the data -- but not all in every record. Since we don't know the names of all the keys until we've read them all, this needs to be a two-pass algorithm. On the first pass, remember all the unique key names and all the records; on the second pass, loop through the records filling in absent values, then producing output. Use ``put -q`` since we don't want to produce per-record output, only emitting output in the ``end`` block: +There are field names `a`, `b`, `v`, `u`, `x`, `w` in the data -- but not all in every record. Since we don't know the names of all the keys until we've read them all, this needs to be a two-pass algorithm. On the first pass, remember all the unique key names and all the records; on the second pass, loop through the records filling in absent values, then producing output. Use `put -q` since we don't want to produce per-record output, only emitting output in the `end` block: -
+
 cat data/unsparsify.mlr
+
+
 # First pass:
 # Remember all unique key names:
 for (k in $*) {
@@ -360,8 +388,10 @@ end {
 }
 
-
+
 mlr --json put -q -f data/unsparsify.mlr data/sparse.json
+
+
 {
   "a": 1,
   "b": 2,
@@ -396,8 +426,10 @@ end {
 }
 
-
+
 mlr --ijson --ocsv put -q -f data/unsparsify.mlr data/sparse.json
+
+
 a,b,v,u,x,w
 1,2,3,,,
 ,2,,1,,
@@ -405,8 +437,10 @@ a,b,v,u,x,w
 ,,1,,,2
 
-
+
 mlr --ijson --opprint put -q -f data/unsparsify.mlr data/sparse.json
+
+
 a b v u x w
 1 2 3 - - -
 - 2 - 1 - -
@@ -414,17 +448,19 @@ a b v u x w
 - - 1 - - 2
 
-There is a keystroke-saving verb for this: :ref:`mlr unsparsify `. +There is a keystroke-saving verb for this: [unsparsify](reference-verbs.md#unsparsify). ## Mean without/with oosvars -
+
 mlr --opprint stats1 -a mean -f x data/medium
+
+
 x_mean
 0.49860196816795804
 
-
+
 mlr --opprint put -q '
   @x_sum += $x;
   @x_count += 1;
@@ -433,14 +469,18 @@ x_mean
     emit @x_mean
   }
 ' data/medium
+
+
 x_mean
 0.49860196816795804
 
## Keyed mean without/with oosvars -
+
 mlr --opprint stats1 -a mean -f x -g a,b data/medium
+
+
 a   b   x_mean
 pan pan 0.5133141190437597
 eks pan 0.48507555383425127
@@ -469,7 +509,7 @@ eks hat 0.5006790659966355
 wye eks 0.5306035254809106
 
-
+
 mlr --opprint put -q '
   @x_sum[$a][$b] += $x;
   @x_count[$a][$b] += 1;
@@ -480,6 +520,8 @@ wye eks 0.5306035254809106
     emit @x_mean, "a", "b"
   }
 ' data/medium
+
+
 a   b   x_mean
 pan pan 0.5133141190437597
 pan wye 0.5023618498923658
@@ -510,8 +552,10 @@ hat pan 0.4643355557376876
 
 ## Variance and standard deviation without/with oosvars
 
-
+
 mlr --oxtab stats1 -a count,sum,mean,var,stddev -f x data/medium
+
+
 x_count  10000
 x_sum    4986.019681679581
 x_mean   0.49860196816795804
@@ -519,8 +563,10 @@ x_var    0.08426974433144456
 x_stddev 0.2902925151144007
 
-
+
 cat variance.mlr
+
+
 @n += 1;
 @sumx += $x;
 @sumx2 += $x**2;
@@ -532,8 +578,10 @@ end {
 }
 
-
+
 mlr --oxtab put -q -f variance.mlr data/medium
+
+
 n      10000
 sumx   4986.019681679581
 sumx2  3328.652400179729
@@ -546,26 +594,32 @@ You can also do this keyed, of course, imitating the keyed-mean example above.
 
 ## Min/max without/with oosvars
 
-
+
 mlr --oxtab stats1 -a min,max -f x data/medium
+
+
 x_min 0.00004509679127584487
 x_max 0.999952670371898
 
-
+
 mlr --oxtab put -q '
   @x_min = min(@x_min, $x);
   @x_max = max(@x_max, $x);
   end{emitf @x_min, @x_max}
 ' data/medium
+
+
 x_min 0.00004509679127584487
 x_max 0.999952670371898
 
## Keyed min/max without/with oosvars -
+
 mlr --opprint stats1 -a min,max -f x -g a data/medium
+
+
 a   x_min                  x_max
 pan 0.00020390740306253097 0.9994029107062516
 eks 0.0006917972627396018  0.9988110946859143
@@ -574,7 +628,7 @@ zee 0.0005486114815762555  0.9994904324789629
 hat 0.00004509679127584487 0.999952670371898
 
-
+
 mlr --opprint --from data/medium put -q '
   @min[$a] = min(@min[$a], $x);
   @max[$a] = max(@max[$a], $x);
@@ -582,6 +636,8 @@ hat 0.00004509679127584487 0.999952670371898
     emit (@min, @max), "a";
   }
 '
+
+
 a   min                    max
 pan 0.00020390740306253097 0.9994029107062516
 eks 0.0006917972627396018  0.9988110946859143
@@ -592,8 +648,10 @@ hat 0.00004509679127584487 0.999952670371898
 
 ## Delta without/with oosvars
 
-
+
 mlr --opprint step -a delta -f x data/small
+
+
 a   b   i x                   y                   x_delta
 pan pan 1 0.3467901443380824  0.7268028627434533  0
 eks pan 2 0.7586799647899636  0.5221511083334797  0.41188982045188116
@@ -602,11 +660,13 @@ eks wye 4 0.38139939387114097 0.13418874328430463 0.17679608810483793
 wye pan 5 0.5732889198020006  0.8636244699032729  0.19188952593085962
 
-
+
 mlr --opprint put '
   $x_delta = is_present(@last) ? $x - @last : 0;
   @last = $x
 ' data/small
+
+
 a   b   i x                   y                   x_delta
 pan pan 1 0.3467901443380824  0.7268028627434533  0
 eks pan 2 0.7586799647899636  0.5221511083334797  0.41188982045188116
@@ -617,8 +677,10 @@ wye pan 5 0.5732889198020006  0.8636244699032729  0.19188952593085962
 
 ## Keyed delta without/with oosvars
 
-
+
 mlr --opprint step -a delta -f x -g a data/small
+
+
 a   b   i x                   y                   x_delta
 pan pan 1 0.3467901443380824  0.7268028627434533  0
 eks pan 2 0.7586799647899636  0.5221511083334797  0
@@ -627,11 +689,13 @@ eks wye 4 0.38139939387114097 0.13418874328430463 -0.3772805709188226
 wye pan 5 0.5732889198020006  0.8636244699032729  0.36868561403569755
 
-
+
 mlr --opprint put '
   $x_delta = is_present(@last[$a]) ? $x - @last[$a] : 0;
   @last[$a]=$x
 ' data/small
+
+
 a   b   i x                   y                   x_delta
 pan pan 1 0.3467901443380824  0.7268028627434533  0
 eks pan 2 0.7586799647899636  0.5221511083334797  0
@@ -642,8 +706,10 @@ wye pan 5 0.5732889198020006  0.8636244699032729  0.36868561403569755
 
 ## Exponentially weighted moving averages without/with oosvars
 
-
+
 mlr --opprint step -a ewma -d 0.1 -f x data/small
+
+
 a   b   i x                   y                   x_ewma_0.1
 pan pan 1 0.3467901443380824  0.7268028627434533  0.3467901443380824
 eks pan 2 0.7586799647899636  0.5221511083334797  0.3879791263832706
@@ -652,12 +718,14 @@ eks wye 4 0.38139939387114097 0.13418874328430463 0.37081732927653055
 wye pan 5 0.5732889198020006  0.8636244699032729  0.3910644883290776
 
-
+
 mlr --opprint put '
   begin{ @a=0.1 };
   $e = NR==1 ? $x : @a * $x + (1 - @a) * @e;
   @e=$e
 ' data/small
+
+
 a   b   i x                   y                   e
 pan pan 1 0.3467901443380824  0.7268028627434533  0.3467901443380824
 eks pan 2 0.7586799647899636  0.5221511083334797  0.3879791263832706
diff --git a/docs6b/docs/two-pass-algorithms.md.in b/docs6b/docs/two-pass-algorithms.md.in
index b7efc296f..889d3f44b 100644
--- a/docs6b/docs/two-pass-algorithms.md.in
+++ b/docs6b/docs/two-pass-algorithms.md.in
@@ -2,7 +2,7 @@
 
 ## Overview
 
-Miller is a streaming record processor; commands are performed once per record. This makes Miller particularly suitable for single-pass algorithms, allowing many of its verbs to process files that are (much) larger than the amount of RAM present in your system. (Of course, Miller verbs such as ``sort``, ``tac``, etc. all must ingest and retain all input records before emitting any output records.) You can also use out-of-stream variables to perform multi-pass computations, at the price of retaining all input records in memory.
+Miller is a streaming record processor; commands are performed once per record. This makes Miller particularly suitable for single-pass algorithms, allowing many of its verbs to process files that are (much) larger than the amount of RAM present in your system. (Of course, Miller verbs such as `sort`, `tac`, etc. all must ingest and retain all input records before emitting any output records.) You can also use out-of-stream variables to perform multi-pass computations, at the price of retaining all input records in memory.
 
 One of Miller's strengths is its compact notation: for example, given input of the form
 
@@ -30,9 +30,9 @@ or
 
 GENMD_INCLUDE_AND_RUN_ESCAPED(oosvar-example-sum-grouped.sh)
 
-The former (``mlr stats1`` et al.) has the advantages of being easier to type, being less error-prone to type, and running faster.
+The former (`mlr stats1` et al.) has the advantages of being easier to type, being less error-prone to type, and running faster.
 
-Nonetheless, out-of-stream variables (which I whimsically call *oosvars*), begin/end blocks, and emit statements give you the ability to implement logic -- if you wish to do so -- which isn't present in other Miller verbs.  (If you find yourself often using the same out-of-stream-variable logic over and over, please file a request at https://github.com/johnkerl/miller/issues to get it implemented directly in Go as a Miller verb of its own.)
+Nonetheless, out-of-stream variables (which I whimsically call *oosvars*), begin/end blocks, and emit statements give you the ability to implement logic -- if you wish to do so -- which isn't present in other Miller verbs.  (If you find yourself often using the same out-of-stream-variable logic over and over, please file a request at [https://github.com/johnkerl/miller/issues](https://github.com/johnkerl/miller/issues) to get it implemented directly in Go as a Miller verb of its own.)
 
 The following examples compute some things using oosvars which are already computable using Miller verbs, by way of providing food for thought.
 
@@ -50,13 +50,13 @@ GENMD_INCLUDE_AND_RUN_ESCAPED(data/two-pass-record-numbers.sh)
 
 ## Records having max value
 
-The idea is to retain records having the largest value of ``n`` in the following data:
+The idea is to retain records having the largest value of `n` in the following data:
 
 GENMD_RUN_COMMAND
 mlr --itsv --opprint cat data/maxrows.tsv
 GENMD_EOF
 
-Of course, the largest value of ``n`` isn't known until after all data have been read. Using an out-of-stream variable we can retain all records as they are read, then filter them at the end:
+Of course, the largest value of `n` isn't known until after all data have been read. Using an out-of-stream variable we can retain all records as they are read, then filter them at the end:
 
 GENMD_RUN_COMMAND
 cat data/maxrows.mlr
@@ -96,7 +96,7 @@ GENMD_RUN_COMMAND
 cat data/sparse.json
 GENMD_EOF
 
-There are field names ``a``, ``b``, ``v``, ``u``, ``x``, ``w`` in the data -- but not all in every record.  Since we don't know the names of all the keys until we've read them all, this needs to be a two-pass algorithm. On the first pass, remember all the unique key names and all the records; on the second pass, loop through the records filling in absent values, then producing output. Use ``put -q`` since we don't want to produce per-record output, only emitting output in the ``end`` block:
+There are field names `a`, `b`, `v`, `u`, `x`, `w` in the data -- but not all in every record.  Since we don't know the names of all the keys until we've read them all, this needs to be a two-pass algorithm. On the first pass, remember all the unique key names and all the records; on the second pass, loop through the records filling in absent values, then producing output. Use `put -q` since we don't want to produce per-record output, only emitting output in the `end` block:
 
 GENMD_RUN_COMMAND
 cat data/unsparsify.mlr
@@ -114,7 +114,7 @@ GENMD_RUN_COMMAND
 mlr --ijson --opprint put -q -f data/unsparsify.mlr data/sparse.json
 GENMD_EOF
 
-There is a keystroke-saving verb for this: :ref:`mlr unsparsify `.
+There is a keystroke-saving verb for this: [unsparsify](reference-verbs.md#unsparsify).
 
 ## Mean without/with oosvars
 
diff --git a/docs6b/docs/why.md b/docs6b/docs/why.md
index 3f6154880..6d79a2ea4 100644
--- a/docs6b/docs/why.md
+++ b/docs6b/docs/why.md
@@ -9,11 +9,11 @@ For background, I'm a software engineer, with a heavy devops bent and a non-triv
 
 But now there's this neat little tool **which seems to be useful for people in various disciplines**. I don't even know entirely *who*. I can click through Github starrers and read a bit about what they seem to do, but not everyone that uses Miller is even *on* Github (or stars things). I've gotten a lot of feature requests through Github -- but only from people who are Github users.  Not everyone's a coder (it seems like a lot of Miller's Github starrers are devops folks like myself, or data-science-ish people, or biology/genomics folks.) A lot of people care 100% about CSV. And so on.
 
-So I wonder (please drop a note at https://github.com/johnkerl/miller/issues) does Miller do what you need? Do you use it for all sorts of things, or just one or two nice things? Are there things you wish it did but it doesn't? Is it almost there, or just nowhere near what you want? Are there not enough features or way too many? Are the docs too complicated; do you have a hard time finding out how to do what you want? Should I think differently about what this tool even *is* in the first place? Should I think differently about who it's for?
+So I wonder (please drop a note at [https://github.com/johnkerl/miller/issues](https://github.com/johnkerl/miller/issues)) does Miller do what you need? Do you use it for all sorts of things, or just one or two nice things? Are there things you wish it did but it doesn't? Is it almost there, or just nowhere near what you want? Are there not enough features or way too many? Are the docs too complicated; do you have a hard time finding out how to do what you want? Should I think differently about what this tool even *is* in the first place? Should I think differently about who it's for?
 
 ## What was Miller created to do?
 
-First: there are tools like ``xsv`` which handles CSV marvelously and ``jq`` which handles JSON marvelously, and so on -- but I over the years of my career in the software industry I've found myself, and others, doing a lot of ad-hoc things which really were fundamentally the same *except* for format. So the number one thing about Miller is doing common things while supporting **multiple formats**: (a) ingest a list of records where a record is a list of key-value pairs (however represented in the input files); (b) transform that stream of records; (c) emit the transformed stream -- either in the same format as input, or in a different format.
+First: there are tools like `xsv` which handles CSV marvelously and `jq` which handles JSON marvelously, and so on -- but I over the years of my career in the software industry I've found myself, and others, doing a lot of ad-hoc things which really were fundamentally the same *except* for format. So the number one thing about Miller is doing common things while supporting **multiple formats**: (a) ingest a list of records where a record is a list of key-value pairs (however represented in the input files); (b) transform that stream of records; (c) emit the transformed stream -- either in the same format as input, or in a different format.
 
 Second thing, a lot like the first: just as I didn't want to build something only for a single file format, I didn't want to build something only for one problem domain. In my work doing software engineering, devops, data engineering, etc. I saw a lot of commonalities and I wanted to **solve as many problems simultaneously as possible**.
 
@@ -23,9 +23,9 @@ Fourth: it had to be **fast**. This precludes all sorts of very nice things writ
 
 Fifth thing: I wanted Miller to be **pipe-friendly and interoperate with other command-line tools**.  Since the basic paradigm is ingest records, transform records, emit records -- where the input and output formats can be the same or different, and the transform can be complex, or just pass-through -- this means you can use it to transform data, or re-format it, or both. So if you just want to do data-cleaning/prep/formatting and do all the "real" work in R, you can. If you just want a little glue script between other tools you can get that. And if you want to do non-trivial data-reduction in Miller you can.
 
-Sixth thing: Must have **comprehensive documentation and unit-test**. Since Miller handles a lot of formats and solves a lot of problems, there's a lot to test and a lot to keep working correctly as I add features or optimize. And I wanted it to be able to explain itself -- not only through web docs like the one you're reading but also through ``man mlr`` and ``mlr --help``, ``mlr sort --help``, etc.
+Sixth thing: Must have **comprehensive documentation and unit-test**. Since Miller handles a lot of formats and solves a lot of problems, there's a lot to test and a lot to keep working correctly as I add features or optimize. And I wanted it to be able to explain itself -- not only through web docs like the one you're reading but also through `man mlr` and `mlr --help`, `mlr sort --help`, etc.
 
-Seventh thing: **Must have a domain-specific language** (DSL) **but also must let you do common things without it**. All those little verbs Miller has to help you *avoid* having to write for-loops are great. I use them for keystroke-saving: ``mlr stats1 -a mean,stddev,min,max -f quantity``, for example, without you having to write for-loops or define accumulator variables. But you also have to be able to break out of that and write arbitrary code when you want to: ``mlr put '$distance = $rate * $time'`` or anything else you can think up. In Perl/AWK/etc.  it's all DSL. In xsv et al.  it's all verbs. In Miller I like having the combination.
+Seventh thing: **Must have a domain-specific language** (DSL) **but also must let you do common things without it**. All those little verbs Miller has to help you *avoid* having to write for-loops are great. I use them for keystroke-saving: `mlr stats1 -a mean,stddev,min,max -f quantity`, for example, without you having to write for-loops or define accumulator variables. But you also have to be able to break out of that and write arbitrary code when you want to: `mlr put '$distance = $rate * $time'` or anything else you can think up. In Perl/AWK/etc.  it's all DSL. In xsv et al.  it's all verbs. In Miller I like having the combination.
 
 Eighth thing: It's an **awful lot of fun to write**. In my experience I didn't find any tools which do multi-format, streaming, efficient, multi-purpose, with DSL and non-DSL, so I wrote one. But I don't guarantee it's unique in the world. It fills a niche in the world (people use it) but it also fills a niche in my life.
 
@@ -35,14 +35,14 @@ Miller is command-line-only by design. People who want a graphical user interfac
 
 Another tradeoff: supporting lists of records -- each with only one depth -- keeps me supporting only what can be expressed in *all* of those formats.  E.g. in JSON you can have lists of lists of lists which Miller just doesn't handle. So Miller can't (and won't) handle arbitrary JSON because it only handles tabular data which can be expressed in a variety of formats.
 
-A third tradeoff is doing build-from-scratch in a low-level language. It'd be quicker to write (but slower to run) if written in a high-level language. If Miller were written in Python, it would be implemented in significantly fewer lines of code than its current Go implementation. The DSL would just be an ``eval`` of Python code. And it would run slower, but maybe not enough slower to be a problem for most folks. Later I found out about the [rows](https://github.com/turicas/rows) tool -- if you find Miller useful, you should check out ``rows`` as well.
+A third tradeoff is doing build-from-scratch in a low-level language. It'd be quicker to write (but slower to run) if written in a high-level language. If Miller were written in Python, it would be implemented in significantly fewer lines of code than its current Go implementation. The DSL would just be an `eval` of Python code. And it would run slower, but maybe not enough slower to be a problem for most folks. Later I found out about the [rows](https://github.com/turicas/rows) tool -- if you find Miller useful, you should check out `rows` as well.
 
-A fourth tradeoff is in the DSL (more visibly so in 5.0.0 but already in pre-5.0.0): how much to make it dynamically typed -- so you can just say y=x+1 with a minimum number of keystrokes -- vs. having it do a good job of telling you when you've made a typo. This is a common paradigm across *all* languages.  Some like Ruby you don't declare anything and they're quick to code little stuff in but programs of even a few thousand lines (which isn't large in the software world) become insanely unmanageable.  Then Java at the other extreme which is very typesafe but you have to type in a lot of punctuation, angle brackets, datatypes, repetition, etc. just to be able to get anything done. And some in the middle like Go which are typesafe but with type inference which aim to do the best of both. In the Miller (5.0.0) DSL you get ``y=x+1`` by default but you can have things like ``int y = x+1`` etc. so the typesafety is opt-in. See also :ref:`reference-dsl-type-checking` for more information on type-checking.
+A fourth tradeoff is in the DSL (more visibly so in 5.0.0 but already in pre-5.0.0): how much to make it dynamically typed -- so you can just say y=x+1 with a minimum number of keystrokes -- vs. having it do a good job of telling you when you've made a typo. This is a common paradigm across *all* languages.  Some like Ruby you don't declare anything and they're quick to code little stuff in but programs of even a few thousand lines (which isn't large in the software world) become insanely unmanageable.  Then Java at the other extreme which is very typesafe but you have to type in a lot of punctuation, angle brackets, datatypes, repetition, etc. just to be able to get anything done. And some in the middle like Go which are typesafe but with type inference which aim to do the best of both. In the Miller (5.0.0) DSL you get `y=x+1` by default but you can have things like `int y = x+1` etc. so the typesafety is opt-in. See also [Type-checking](reference-dsl-variables.md#type-checking) for more information on type-checking.
 
 ## Related tools
 
-Here's a comprehensive list: https://github.com/dbohdan/structured-text-tools. It doesn't mention [rows](https://github.com/turicas/rows) so here's a plug for that as well.
+Here's a comprehensive list: [https://github.com/dbohdan/structured-text-tools](https://github.com/dbohdan/structured-text-tools). It doesn't mention [rows](https://github.com/turicas/rows) so here's a plug for that as well.
 
 ## Moving forward
 
-I originally aimed Miller at people who already know what ``sed``/``awk``/``cut``/``sort``/``join`` are and wanted some options. But as time goes by I realize that tools like this can be useful to folks who *don't* know what those things are; people who aren't primarily coders; people who are scientists, or data scientists. These days some journalists do data analysis.  So moving forward in terms of docs, I am working on having more cookbook, follow-by-example stuff in addition to the existing language-reference kinds of stuff.  And continuing to seek out input from people who use Miller on where to go next.
+I originally aimed Miller at people who already know what `sed`/`awk`/`cut`/`sort`/`join` are and wanted some options. But as time goes by I realize that tools like this can be useful to folks who *don't* know what those things are; people who aren't primarily coders; people who are scientists, or data scientists. These days some journalists do data analysis.  So moving forward in terms of docs, I am working on having more cookbook, follow-by-example stuff in addition to the existing language-reference kinds of stuff.  And continuing to seek out input from people who use Miller on where to go next.
diff --git a/docs6b/docs/why.md.in b/docs6b/docs/why.md.in
index 42277b9e8..12581f4d8 100644
--- a/docs6b/docs/why.md.in
+++ b/docs6b/docs/why.md.in
@@ -8,11 +8,11 @@ For background, I'm a software engineer, with a heavy devops bent and a non-triv
 
 But now there's this neat little tool **which seems to be useful for people in various disciplines**. I don't even know entirely *who*. I can click through Github starrers and read a bit about what they seem to do, but not everyone that uses Miller is even *on* Github (or stars things). I've gotten a lot of feature requests through Github -- but only from people who are Github users.  Not everyone's a coder (it seems like a lot of Miller's Github starrers are devops folks like myself, or data-science-ish people, or biology/genomics folks.) A lot of people care 100% about CSV. And so on.
 
-So I wonder (please drop a note at https://github.com/johnkerl/miller/issues) does Miller do what you need? Do you use it for all sorts of things, or just one or two nice things? Are there things you wish it did but it doesn't? Is it almost there, or just nowhere near what you want? Are there not enough features or way too many? Are the docs too complicated; do you have a hard time finding out how to do what you want? Should I think differently about what this tool even *is* in the first place? Should I think differently about who it's for?
+So I wonder (please drop a note at [https://github.com/johnkerl/miller/issues](https://github.com/johnkerl/miller/issues)) does Miller do what you need? Do you use it for all sorts of things, or just one or two nice things? Are there things you wish it did but it doesn't? Is it almost there, or just nowhere near what you want? Are there not enough features or way too many? Are the docs too complicated; do you have a hard time finding out how to do what you want? Should I think differently about what this tool even *is* in the first place? Should I think differently about who it's for?
 
 ## What was Miller created to do?
 
-First: there are tools like ``xsv`` which handles CSV marvelously and ``jq`` which handles JSON marvelously, and so on -- but I over the years of my career in the software industry I've found myself, and others, doing a lot of ad-hoc things which really were fundamentally the same *except* for format. So the number one thing about Miller is doing common things while supporting **multiple formats**: (a) ingest a list of records where a record is a list of key-value pairs (however represented in the input files); (b) transform that stream of records; (c) emit the transformed stream -- either in the same format as input, or in a different format.
+First: there are tools like `xsv` which handles CSV marvelously and `jq` which handles JSON marvelously, and so on -- but I over the years of my career in the software industry I've found myself, and others, doing a lot of ad-hoc things which really were fundamentally the same *except* for format. So the number one thing about Miller is doing common things while supporting **multiple formats**: (a) ingest a list of records where a record is a list of key-value pairs (however represented in the input files); (b) transform that stream of records; (c) emit the transformed stream -- either in the same format as input, or in a different format.
 
 Second thing, a lot like the first: just as I didn't want to build something only for a single file format, I didn't want to build something only for one problem domain. In my work doing software engineering, devops, data engineering, etc. I saw a lot of commonalities and I wanted to **solve as many problems simultaneously as possible**.
 
@@ -22,9 +22,9 @@ Fourth: it had to be **fast**. This precludes all sorts of very nice things writ
 
 Fifth thing: I wanted Miller to be **pipe-friendly and interoperate with other command-line tools**.  Since the basic paradigm is ingest records, transform records, emit records -- where the input and output formats can be the same or different, and the transform can be complex, or just pass-through -- this means you can use it to transform data, or re-format it, or both. So if you just want to do data-cleaning/prep/formatting and do all the "real" work in R, you can. If you just want a little glue script between other tools you can get that. And if you want to do non-trivial data-reduction in Miller you can.
 
-Sixth thing: Must have **comprehensive documentation and unit-test**. Since Miller handles a lot of formats and solves a lot of problems, there's a lot to test and a lot to keep working correctly as I add features or optimize. And I wanted it to be able to explain itself -- not only through web docs like the one you're reading but also through ``man mlr`` and ``mlr --help``, ``mlr sort --help``, etc.
+Sixth thing: Must have **comprehensive documentation and unit-test**. Since Miller handles a lot of formats and solves a lot of problems, there's a lot to test and a lot to keep working correctly as I add features or optimize. And I wanted it to be able to explain itself -- not only through web docs like the one you're reading but also through `man mlr` and `mlr --help`, `mlr sort --help`, etc.
 
-Seventh thing: **Must have a domain-specific language** (DSL) **but also must let you do common things without it**. All those little verbs Miller has to help you *avoid* having to write for-loops are great. I use them for keystroke-saving: ``mlr stats1 -a mean,stddev,min,max -f quantity``, for example, without you having to write for-loops or define accumulator variables. But you also have to be able to break out of that and write arbitrary code when you want to: ``mlr put '$distance = $rate * $time'`` or anything else you can think up. In Perl/AWK/etc.  it's all DSL. In xsv et al.  it's all verbs. In Miller I like having the combination.
+Seventh thing: **Must have a domain-specific language** (DSL) **but also must let you do common things without it**. All those little verbs Miller has to help you *avoid* having to write for-loops are great. I use them for keystroke-saving: `mlr stats1 -a mean,stddev,min,max -f quantity`, for example, without you having to write for-loops or define accumulator variables. But you also have to be able to break out of that and write arbitrary code when you want to: `mlr put '$distance = $rate * $time'` or anything else you can think up. In Perl/AWK/etc.  it's all DSL. In xsv et al.  it's all verbs. In Miller I like having the combination.
 
 Eighth thing: It's an **awful lot of fun to write**. In my experience I didn't find any tools which do multi-format, streaming, efficient, multi-purpose, with DSL and non-DSL, so I wrote one. But I don't guarantee it's unique in the world. It fills a niche in the world (people use it) but it also fills a niche in my life.
 
@@ -34,14 +34,14 @@ Miller is command-line-only by design. People who want a graphical user interfac
 
 Another tradeoff: supporting lists of records -- each with only one depth -- keeps me supporting only what can be expressed in *all* of those formats.  E.g. in JSON you can have lists of lists of lists which Miller just doesn't handle. So Miller can't (and won't) handle arbitrary JSON because it only handles tabular data which can be expressed in a variety of formats.
 
-A third tradeoff is doing build-from-scratch in a low-level language. It'd be quicker to write (but slower to run) if written in a high-level language. If Miller were written in Python, it would be implemented in significantly fewer lines of code than its current Go implementation. The DSL would just be an ``eval`` of Python code. And it would run slower, but maybe not enough slower to be a problem for most folks. Later I found out about the [rows](https://github.com/turicas/rows) tool -- if you find Miller useful, you should check out ``rows`` as well.
+A third tradeoff is doing build-from-scratch in a low-level language. It'd be quicker to write (but slower to run) if written in a high-level language. If Miller were written in Python, it would be implemented in significantly fewer lines of code than its current Go implementation. The DSL would just be an `eval` of Python code. And it would run slower, but maybe not enough slower to be a problem for most folks. Later I found out about the [rows](https://github.com/turicas/rows) tool -- if you find Miller useful, you should check out `rows` as well.
 
-A fourth tradeoff is in the DSL (more visibly so in 5.0.0 but already in pre-5.0.0): how much to make it dynamically typed -- so you can just say y=x+1 with a minimum number of keystrokes -- vs. having it do a good job of telling you when you've made a typo. This is a common paradigm across *all* languages.  Some like Ruby you don't declare anything and they're quick to code little stuff in but programs of even a few thousand lines (which isn't large in the software world) become insanely unmanageable.  Then Java at the other extreme which is very typesafe but you have to type in a lot of punctuation, angle brackets, datatypes, repetition, etc. just to be able to get anything done. And some in the middle like Go which are typesafe but with type inference which aim to do the best of both. In the Miller (5.0.0) DSL you get ``y=x+1`` by default but you can have things like ``int y = x+1`` etc. so the typesafety is opt-in. See also :ref:`reference-dsl-type-checking` for more information on type-checking.
+A fourth tradeoff is in the DSL (more visibly so in 5.0.0 but already in pre-5.0.0): how much to make it dynamically typed -- so you can just say y=x+1 with a minimum number of keystrokes -- vs. having it do a good job of telling you when you've made a typo. This is a common paradigm across *all* languages.  Some like Ruby you don't declare anything and they're quick to code little stuff in but programs of even a few thousand lines (which isn't large in the software world) become insanely unmanageable.  Then Java at the other extreme which is very typesafe but you have to type in a lot of punctuation, angle brackets, datatypes, repetition, etc. just to be able to get anything done. And some in the middle like Go which are typesafe but with type inference which aim to do the best of both. In the Miller (5.0.0) DSL you get `y=x+1` by default but you can have things like `int y = x+1` etc. so the typesafety is opt-in. See also [Type-checking](reference-dsl-variables.md#type-checking) for more information on type-checking.
 
 ## Related tools
 
-Here's a comprehensive list: https://github.com/dbohdan/structured-text-tools. It doesn't mention [rows](https://github.com/turicas/rows) so here's a plug for that as well.
+Here's a comprehensive list: [https://github.com/dbohdan/structured-text-tools](https://github.com/dbohdan/structured-text-tools). It doesn't mention [rows](https://github.com/turicas/rows) so here's a plug for that as well.
 
 ## Moving forward
 
-I originally aimed Miller at people who already know what ``sed``/``awk``/``cut``/``sort``/``join`` are and wanted some options. But as time goes by I realize that tools like this can be useful to folks who *don't* know what those things are; people who aren't primarily coders; people who are scientists, or data scientists. These days some journalists do data analysis.  So moving forward in terms of docs, I am working on having more cookbook, follow-by-example stuff in addition to the existing language-reference kinds of stuff.  And continuing to seek out input from people who use Miller on where to go next.
+I originally aimed Miller at people who already know what `sed`/`awk`/`cut`/`sort`/`join` are and wanted some options. But as time goes by I realize that tools like this can be useful to folks who *don't* know what those things are; people who aren't primarily coders; people who are scientists, or data scientists. These days some journalists do data analysis.  So moving forward in terms of docs, I am working on having more cookbook, follow-by-example stuff in addition to the existing language-reference kinds of stuff.  And continuing to seek out input from people who use Miller on where to go next.
diff --git a/docs6b/docs/x b/docs6b/docs/x
index c52f6c484..8e271f785 100644
--- a/docs6b/docs/x
+++ b/docs6b/docs/x
@@ -1,18 +1 @@
-o..........o...................o............o.......o.............
-o.................................................................
-o...o...........o................................o.....o..........
-.o...................o.o..........................................
-...o..............................................................
-...o...........o..................................................
-...........o...........................................o..........
-...........o.................o......................o..o..........
-.............o.....................o...................o..........
-..................o.............................o.................
-.....................o.o........................o.................
-.......................o..........................................
-.......................o........................o.................
-.........................o.............................o..........
-.............................o....................................
-..........................................oo......................
-...........................................o......................
-.......................................................o..........
+[type-checking](reference-dsl-variables.md#type-checking)
diff --git a/docs6b/mkdocs.yml b/docs6b/mkdocs.yml
index 38094d6b3..3dcfb8e5d 100644
--- a/docs6b/mkdocs.yml
+++ b/docs6b/mkdocs.yml
@@ -73,9 +73,3 @@ nav:
     - "Installation": "installation.md"
     - "Building from source": "build.md"
 extra_css: ['extra.css']
-markdown_extensions:
-  - pymdownx.superfences
-  - codehilite
-  - admonition
-  - pymdownx.highlight:
-      linenums: true
diff --git a/docs6b/mkdocs.yml.000 b/docs6b/mkdocs.yml.000
deleted file mode 100644
index b6ac4b5e1..000000000
--- a/docs6b/mkdocs.yml.000
+++ /dev/null
@@ -1,9 +0,0 @@
-site_name: Miller Documentation
-theme:
-  name: readthedocs
-  palette:
-    - scheme: slate
-nav:
-  - 'index.md'
-  - '10min.md'
-extra_css: ['extra.css']
diff --git a/docs6b/mkdocs.yml.001 b/docs6b/mkdocs.yml.001
deleted file mode 100644
index 3beca6bca..000000000
--- a/docs6b/mkdocs.yml.001
+++ /dev/null
@@ -1,12 +0,0 @@
-site_name: Miller Documentation
-theme:
-  name: readthedocs
-  palette:
-    - scheme: slate
-nav:
-  - 'index.md'
-  - '10min.md'
-extra_css: ['extra.css']
-markdown_extensions:
-  - pymdownx.highlight
-  - pymdownx.superfences
diff --git a/docs6b/mkdocs.yml.002 b/docs6b/mkdocs.yml.002
deleted file mode 100644
index 3c7404db0..000000000
--- a/docs6b/mkdocs.yml.002
+++ /dev/null
@@ -1,84 +0,0 @@
-site_name: Miller Documentation
-theme:
-  name: readthedocs
-  palette:
-    - scheme: slate
-nav:
-    - 'Getting started':
-        - 'index.md'
-        - '10min.md'
-        - 'keystroke-savers.md'
-        - 'programming-language.md'
-        - 'miller-on-windows.md'
-        - 'community.md'
-    - 'Miller in more detail':
-        - 'features.md'
-        - 'feature-comparison.md'
-        - 'file-formats.md'
-        - 'record-heterogeneity.md'
-        - 'internationalization.md'
-        - 'output-colorization.md'
-        - 'customization.md'
-        - 'repl.md'
-        - 'new-in-miller-6.md'
-        - 'contributing.md'
-    - 'FAQs and recipes':
-        - 'csv-with-and-without-headers.md'
-        - 'shapes-of-data.md'
-        - 'operating-on-all-fields.md'
-        - 'special-symbols-and-formatting.md'
-        - 'dates-and-times.md'
-        - 'then-chaining.md'
-        - 'joins.md'
-        - 'shell-commands.md'
-        - 'data-diving-examples.md'
-        - 'log-processing-examples.md'
-        - 'sql-examples.md'
-        - 'data-cleaning-examples.md'
-        - 'statistics-examples.md'
-        - 'randomizing-examples.md'
-        - 'two-pass-algorithms.md'
-        - 'dkvp-examples.md'
-        - 'programming-examples.md'
-        - 'misc-examples.md'
-    - 'Background':
-        - 'why.md'
-        - 'etymology.md'
-        - 'originality.md'
-        - 'performance.md'
-    - 'Reference':
-        - 'reference-main-overview.md'
-        - 'reference-verbs.md'
-        - 'reference-main-io-options.md'
-        - 'reference-main-then-chaining.md'
-        - 'reference-main-auxiliary-commands.md'
-        - 'reference-main-data-types.md'
-        - 'reference-dsl-arrays.md'
-        - 'reference-main-null-data.md'
-        - 'reference-main-arithmetic.md'
-        - 'reference-main-regular-expressions.md'
-        - 'reference-main-env-vars.md'
-        - 'reference-main-online-help.md'
-        - 'reference-dsl.md'
-        - 'reference-dsl-syntax.md'
-        - 'reference-dsl-variables.md'
-        - 'reference-dsl-operators.md'
-        - 'reference-dsl-control-structures.md'
-        - 'reference-dsl-user-defined-functions.md'
-        - 'reference-dsl-builtin-functions.md'
-        - 'reference-dsl-output-statements.md'
-        - 'reference-dsl-unset-statements.md'
-        - 'reference-dsl-filter-statements.md'
-        - 'reference-dsl-errors.md'
-        - 'reference-dsl-complexity.md'
-        - 'manpage.md'
-        - 'release-docs.md'
-        - 'installation.md'
-        - 'build.md'
-  extra_css: ['extra.css']
-  markdown_extensions:
-    - pymdownx.superfences
-    - codehilite
-    - admonition
-    - pymdownx.highlight:
-        linenums: true
diff --git a/docs6b/setup.txt b/docs6b/setup.txt
deleted file mode 100644
index a3cd884e1..000000000
--- a/docs6b/setup.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-sudo pip3 install mkdocs
-
-sudo pip3 install mkdocs-ivory
-sudo pip3 install pymdown-extensions
-sudo pip3 install pygments
-
-https://www.mkdocs.org/user-guide/customizing-your-theme/#using-the-theme_dir
diff --git a/docs6b/wishlist.txt b/docs6b/wishlist.txt
index 71025a868..38f74571c 100644
--- a/docs6b/wishlist.txt
+++ b/docs6b/wishlist.txt
@@ -1,21 +1,11 @@
-must have:
-k color configs
-  - iteratble ...
+why:
 k ok on desktop & mobile
-  - awesome so far! :D 
-k highlight line 1 (even if hacky in genrst)
-  - wip
-
-nice to have:
-* dark mode
-  ? wtf
 
 to do:
-* figure out internal-link spelling
-* figure out external-link spelling
-* vis-link css
-
-check:
-* data-file links don't break
-* build.md -- nested bullets ...
-* builtin-functions autogen sh
+* re-do the readme.md
+  - include mkdocs serve, mkdocs build, etc
+* overwrite docs6b -> docs6
+* check data-file links don't break
+* to r.md:
+  - sudo pip3 install git+https://github.com/linkchecker/linkchecker.git
+  - cd site; linkchecker .