diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6e56be257..ddc459a4d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,6 +10,13 @@ updates: directory: "/" schedule: interval: "daily" + groups: + # The codeql-analysis.yml workflow pins init/autobuild/analyze to the + # same commit SHA, so they must be bumped together or CI breaks with a + # CodeQL config-version mismatch. + codeql-action: + patterns: + - "github/codeql-action/*" # Maintain dependencies for Go - package-ecosystem: "gomod" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 97a904d18..c76239f6b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -40,7 +40,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a + uses: github/codeql-action/init@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -51,7 +51,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a + uses: github/codeql-action/autobuild@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # ℹ️ Command-line programs to run using the OS shell. # πŸ“š https://git.io/JvXDl @@ -65,4 +65,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a + uses: github/codeql-action/analyze@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 diff --git a/.gitignore b/.gitignore index 155dfb12c..cb2c84416 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ data/nmc?.* docs/src/polyglot-dkvp-io/__pycache__ docs/site/ +docs/miller-docs.epub .cursor .claude diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 3c4dd6dd4..97d79d7ee 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -10,6 +10,17 @@ build: os: ubuntu-24.04 tools: python: "3.12" + apt_packages: + - pandoc + jobs: + # Read the Docs' built-in `formats:` support (below) only produces + # epub/PDF/htmlzip for Sphinx projects, not MkDocs ones. So for issue + # #1835 we build the epub ourselves after the MkDocs HTML build; anything + # placed in $READTHEDOCS_OUTPUT/epub/ is published on the project's + # Downloads page and in the docs' flyout menu. + post_build: + - mkdir -p $READTHEDOCS_OUTPUT/epub + - bash docs/build-epub.sh $READTHEDOCS_OUTPUT/epub/miller.epub python: install: diff --git a/CLAUDE.md b/CLAUDE.md index 52c006462..33edfd736 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,6 +28,24 @@ staticcheck -version If this works, you can use `make staticcheck` without any further setup. +### Setting Up golangci-lint + +CI runs `golangci-lint` (config in `.golangci.yml`) on every push and PR via +`.github/workflows/golangci-lint.yml`. To run it locally, install the version +matching CI (currently v2.12.2) per the +[official instructions](https://golangci-lint.run/welcome/install/#local-installation), e.g.: + +```bash +curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.12.2 +``` + +This also installs to `~/go/bin/`, so the same `PATH` addition above covers it. + +**Verify the installation:** +```bash +golangci-lint --version +``` + ## Build & Test ### Building @@ -48,8 +66,23 @@ make bench # Run benchmarks ```bash make fmt # Format code with go fmt make staticcheck # Run static analysis (see Initial Setup section above) +make lint # Run golangci-lint, same invocation as CI (see Initial Setup section above) ``` +`make lint` runs `golangci-lint run ./cmd/mlr ./pkg/...`, matching +`.github/workflows/golangci-lint.yml` exactly, so a clean local run usually +means CI's lint job will pass too. It is not part of `make check` or +`make dev`, so run it explicitly before pushing. + +**Note:** CI's lint job restores a persistent `golangci-lint` results cache +shared across commits and PRs (via `golangci-lint-action`), which can +occasionally serve stale `staticcheck` results (e.g. spurious `SA5011` +nil-check warnings) unrelated to your diff. The job is `continue-on-error: +true` for this reason, so it won't block merging. If CI lint fails but +`make lint` is clean locally, it's likely a stale cache β€” rerun the job, or a +maintainer can clear it: `gh cache list --key golangci-lint` then +`gh cache delete `. + ### Full Developer Workflow ```bash make dev # Format, build, test, generate docs (comprehensive check before pushing) @@ -125,9 +158,10 @@ Documentation is built from `.md.in` template files that contain live code sampl Always run: ```bash make dev +make lint ``` -This ensures code formatting, builds successfully, passes all tests, and documentation is up to date. +`make dev` ensures code formatting, builds successfully, passes all tests, and documentation is up to date. `make lint` is separate (not part of `make dev`/`make check`) and mirrors the CI lint job. ## Additional Resources diff --git a/Makefile b/Makefile index e9a1a1a56..9f187571d 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,11 @@ fmt format: staticcheck: staticcheck ./pkg/... ./cmd/mlr/... +# Needs first: https://golangci-lint.run/welcome/install/#local-installation +# Config lives in .golangci.yml; same invocation as .github/workflows/golangci-lint.yml +lint: + golangci-lint run ./cmd/mlr ./pkg/... + # ---------------------------------------------------------------- # For developers before pushing to GitHub. # @@ -125,4 +130,4 @@ release_tarball release-tarball: build check # ================================================================ # Go does its own dependency management, outside of make. -.PHONY: build mlr check unit_test regression_test bench fmt staticcheck dev docs man +.PHONY: build mlr check unit_test regression_test bench fmt staticcheck lint dev docs man diff --git a/README-dev.md b/README-dev.md index 610f3e78c..b5aad4b32 100644 --- a/README-dev.md +++ b/README-dev.md @@ -1,6 +1,7 @@ # Quickstart for developers * `make`, `make check`, `make docs`, etc: see [Makefile](Makefile) in the repo base directory. +* Linting: `make lint` runs [golangci-lint](https://golangci-lint.run/) (config in [.golangci.yml](.golangci.yml)) with the same invocation used by [.github/workflows/golangci-lint.yml](.github/workflows/golangci-lint.yml), so a clean local run usually means CI's lint job will pass too. Install golangci-lint locally per the [official instructions](https://golangci-lint.run/welcome/install/#local-installation) (match the CI version, currently v2.12.2). It's a separate step from `make dev`/`make check` -- run it explicitly before pushing. `make staticcheck` (requires [staticcheck](https://staticcheck.io)) is a second, independent static-analysis pass. Note: CI's lint job caches `staticcheck` results across commits/PRs and can occasionally fail on a stale cache (spurious warnings unrelated to your diff) even when `make lint` is clean locally; the job is `continue-on-error: true` so this won't block merging, and a maintainer can clear the cache with `gh cache delete` if needed. * Software-testing methodology: see [./test/README.md](./test/README.md). * Source-code indexing: please see [https://sourcegraph.com/github.com/johnkerl/miller](https://sourcegraph.com/github.com/johnkerl/miller) * Godoc As of September 2021, `godoc` support is minimal: package-level synopses exist; most `func`/`const`/etc content lacks `godoc`-style comments. To view doc material, you can: diff --git a/README.md b/README.md index 5ea2c9bc5..2e2df48cb 100644 --- a/README.md +++ b/README.md @@ -24,29 +24,6 @@ including but not limited to the familiar **CSV**, **TSV**, and **JSON**/**JSON In the above image you can see how Miller embraces the common themes of key-value-pair data in a variety of data formats. -# Getting started - -[![deepwiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/johnkerl/miller) - -* [Miller in 10 minutes](https://miller.readthedocs.io/en/latest/10min) -* [A Guide To Command-Line Data Manipulation](https://www.smashingmagazine.com/2022/12/guide-command-line-data-manipulation-cli-miller) -* [A quick tutorial on Miller](https://www.ict4g.net/adolfo/notes/data-analysis/miller-quick-tutorial.html) -* [Miller Exercises](https://github.com/GuilloteauQ/miller-exercises) -* [Tools to manipulate CSV files from the Command Line](https://www.ict4g.net/adolfo/notes/data-analysis/tools-to-manipulate-csv.html) -* [www.togaware.com/linux/survivor/CSV_Files.html](https://www.togaware.com/linux/survivor/CSV_Files.html) -* [MLR for CSV manipulation](https://guillim.github.io/terminal/2018/06/19/MLR-for-CSV-manipulation.html) -* [Linux Magazine: Process structured text files with Miller](https://www.linux-magazine.com/Issues/2016/187/Miller) -* [Miller: Command Line CSV File Processing](https://onepointzero.app/posts/miller-command-line-csv-file-processing/) -* [Miller - A Swiss Army Chainsaw for CSV Data, Data Science and Data Munging](https://fuzzyblog.io/blog/data_science/2022/05/13/miller-a-swiss-army-chainsaw-for-csv-data-data-science-and-data-munging.html) -* [Pandas Killer: mlr, the Scientist](https://xvzftube.xyz/posts/pandas_killers/#mlr%3A-the-scientist) - -# More documentation links - -* [**Full documentation**](https://miller.readthedocs.io/) -* [Miller's license is two-clause BSD](https://github.com/johnkerl/miller/blob/main/LICENSE.txt) -* [Notes about issue-labeling in the Github repo](https://github.com/johnkerl/miller/wiki/Issue-labeling) -* [Active issues](https://github.com/johnkerl/miller/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc) - # Installing There's a good chance you can get Miller pre-built for your system: [![Ubuntu](https://img.shields.io/badge/distros-ubuntu-db4923.svg)](https://launchpad.net/ubuntu/+source/miller) @@ -75,6 +52,78 @@ See also [README-versions.md](./README-versions.md) for a full list of package v See also [building from source](https://miller.readthedocs.io/en/latest/build.html). +# Features + +* Miller is **multi-purpose**: it's useful for **data cleaning**, +**data reduction**, **statistical reporting**, **devops**, **system +administration**, **log-file processing**, **format conversion**, and +**database-query post-processing**. + +* You can use Miller to snarf and munge **log-file data**, including selecting +out relevant substreams, then produce CSV format and load that into +all-in-memory/data-frame utilities for further statistical and/or graphical +processing. + +* Miller complements **data-analysis tools** such as **R**, **pandas**, etc.: +you can use Miller to **clean** and **prepare** your data. While you can do +**basic statistics** entirely in Miller, its streaming-data feature and +single-pass algorithms enable you to **reduce very large data sets**. + +* Miller complements SQL **databases**: you can slice, dice, and reformat data +on the client side on its way into or out of a database. You can also reap some +of the benefits of databases for quick, setup-free one-off tasks when you just +need to query some data in disk files in a hurry. + +* Miller also goes beyond the classic Unix tools by stepping fully into our +modern, **no-SQL** world: its essential record-heterogeneity property allows +Miller to operate on data where records with different schema (field names) are +interleaved. + +* Miller is **streaming**: most operations need only a single record in +memory at a time, rather than ingesting all input before producing any output. +For those operations which require deeper retention (`sort`, `tac`, `stats1`), +Miller retains only as much data as needed. This means that whenever +functionally possible, you can operate on files which are larger than your +system’s available RAM, and you can use Miller in **tail -f** contexts. + +* Miller is **pipe-friendly** and interoperates with the Unix toolkit. + +* Miller's I/O formats include **tabular pretty-printing**, **positionally + indexed** (Unix-toolkit style), CSV, TSV, JSON, JSON Lines, and others. + +* Miller does **conversion** between formats. + +* Miller's **processing is format-aware**: e.g. CSV `sort` and `tac` keep header lines first. + +* Miller has high-throughput **performance** on par with the Unix toolkit. + +* Miller is written in portable, modern Go, with **zero runtime dependencies**. +You can download or compile a single binary, `scp` it to a faraway machine, +and expect it to work. + +# Getting started + +[![deepwiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/johnkerl/miller) + +* [Miller in 10 minutes](https://miller.readthedocs.io/en/latest/10min) +* [A Guide To Command-Line Data Manipulation](https://www.smashingmagazine.com/2022/12/guide-command-line-data-manipulation-cli-miller) +* [A quick tutorial on Miller](https://www.ict4g.net/adolfo/notes/data-analysis/miller-quick-tutorial.html) +* [Miller Exercises](https://github.com/GuilloteauQ/miller-exercises) +* [Tools to manipulate CSV files from the Command Line](https://www.ict4g.net/adolfo/notes/data-analysis/tools-to-manipulate-csv.html) +* [www.togaware.com/linux/survivor/CSV_Files.html](https://www.togaware.com/linux/survivor/CSV_Files.html) +* [MLR for CSV manipulation](https://guillim.github.io/terminal/2018/06/19/MLR-for-CSV-manipulation.html) +* [Linux Magazine: Process structured text files with Miller](https://www.linux-magazine.com/Issues/2016/187/Miller) +* [Miller: Command Line CSV File Processing](https://onepointzero.app/posts/miller-command-line-csv-file-processing/) +* [Miller - A Swiss Army Chainsaw for CSV Data, Data Science and Data Munging](https://fuzzyblog.io/blog/data_science/2022/05/13/miller-a-swiss-army-chainsaw-for-csv-data-data-science-and-data-munging.html) +* [Pandas Killer: mlr, the Scientist](https://xvzftube.xyz/posts/pandas_killers/#mlr%3A-the-scientist) + +# More documentation links + +* [**Full documentation**](https://miller.readthedocs.io/) +* [Miller's license is two-clause BSD](https://github.com/johnkerl/miller/blob/main/LICENSE.txt) +* [Notes about issue-labeling in the Github repo](https://github.com/johnkerl/miller/wiki/Issue-labeling) +* [Active issues](https://github.com/johnkerl/miller/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc) + # Community [![GitHub stars](https://img.shields.io/github/stars/johnkerl/miller.svg?label=GitHub%20stars)](https://github.com/johnkerl/miller/stargazers) @@ -125,55 +174,6 @@ See also [building from source](https://miller.readthedocs.io/en/latest/build.ht [License: BSD2](https://github.com/johnkerl/miller/blob/main/LICENSE.txt) -# Features - -* Miller is **multi-purpose**: it's useful for **data cleaning**, -**data reduction**, **statistical reporting**, **devops**, **system -administration**, **log-file processing**, **format conversion**, and -**database-query post-processing**. - -* You can use Miller to snarf and munge **log-file data**, including selecting -out relevant substreams, then produce CSV format and load that into -all-in-memory/data-frame utilities for further statistical and/or graphical -processing. - -* Miller complements **data-analysis tools** such as **R**, **pandas**, etc.: -you can use Miller to **clean** and **prepare** your data. While you can do -**basic statistics** entirely in Miller, its streaming-data feature and -single-pass algorithms enable you to **reduce very large data sets**. - -* Miller complements SQL **databases**: you can slice, dice, and reformat data -on the client side on its way into or out of a database. You can also reap some -of the benefits of databases for quick, setup-free one-off tasks when you just -need to query some data in disk files in a hurry. - -* Miller also goes beyond the classic Unix tools by stepping fully into our -modern, **no-SQL** world: its essential record-heterogeneity property allows -Miller to operate on data where records with different schema (field names) are -interleaved. - -* Miller is **streaming**: most operations need only a single record in -memory at a time, rather than ingesting all input before producing any output. -For those operations which require deeper retention (`sort`, `tac`, `stats1`), -Miller retains only as much data as needed. This means that whenever -functionally possible, you can operate on files which are larger than your -system’s available RAM, and you can use Miller in **tail -f** contexts. - -* Miller is **pipe-friendly** and interoperates with the Unix toolkit. - -* Miller's I/O formats include **tabular pretty-printing**, **positionally - indexed** (Unix-toolkit style), CSV, TSV, JSON, JSON Lines, and others. - -* Miller does **conversion** between formats. - -* Miller's **processing is format-aware**: e.g. CSV `sort` and `tac` keep header lines first. - -* Miller has high-throughput **performance** on par with the Unix toolkit. - -* Miller is written in portable, modern Go, with **zero runtime dependencies**. -You can download or compile a single binary, `scp` it to a faraway machine, -and expect it to work. - # What people are saying about Miller

Today I discovered Millerβ€”it's like jq but for CSV: https://t.co/pn5Ni241KM

Also, "Miller complements data-analysis tools such as R, pandas, etc.: you can use Miller to clean and prepare your data." @GreatBlueC @nfmcclure

— Adrien Trouillaud (@adrienjt) September 24, 2020
diff --git a/docs/Makefile b/docs/Makefile index acc03b3d3..b264d060a 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -3,4 +3,10 @@ build: make -C src genmds mkdocs build -.PHONY: build +# Optional: build a single-file epub of the documentation for offline +# reading. Requires pandoc. Not part of the default build; Read the Docs runs +# this via .readthedocs.yaml (see docs/build-epub.sh for details). +epub: + ./build-epub.sh + +.PHONY: build epub diff --git a/docs/build-epub.sh b/docs/build-epub.sh new file mode 100755 index 000000000..f57da2817 --- /dev/null +++ b/docs/build-epub.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# ================================================================ +# Builds a single-file epub of the Miller documentation from the generated +# Markdown files in docs/src, for offline reading (issue #1835). +# +# Usage: docs/build-epub.sh [output-path] +# Default output is ./miller-docs.epub. +# +# Requires pandoc (https://pandoc.org). This script is invoked on Read the +# Docs (see .readthedocs.yaml) to publish an epub download alongside the HTML +# docs; Read the Docs' built-in `formats:` support only produces epub/PDF for +# Sphinx projects, not MkDocs ones, hence this script. It can also be run +# locally by anyone with pandoc installed. It is not part of `make dev` or +# any other default build path. +# ================================================================ + +set -euo pipefail + +output="${1:-miller-docs.epub}" +# Make the output path absolute, since we cd around below. +case "$output" in + /*) ;; + *) output="$PWD/$output" ;; +esac + +docs_dir=$(cd "$(dirname "$0")" && pwd) +src_dir="$docs_dir/src" + +if ! command -v pandoc > /dev/null 2>&1; then + echo "$0: pandoc not found; please install it (https://pandoc.org)." 1>&2 + exit 1 +fi + +# Chapter ordering is the nav order in mkdocs.yml. Nav entries look like +# - "Miller in 10 minutes": "10min.md" +# and these are the only lines in mkdocs.yml ending with a quoted .md name. +chapters=$(sed -n 's/^ *-.*: *"\(.*\.md\)" *$/\1/p' "$docs_dir/mkdocs.yml") +if [ -z "$chapters" ]; then + echo "$0: could not extract any nav entries from $docs_dir/mkdocs.yml." 1>&2 + exit 1 +fi + +tmp_dir=$(mktemp -d) +trap 'rm -rf "$tmp_dir"' EXIT + +# Each generated page starts with a quicklinks navigation block -- raw HTML, +# useful on the website but not in an epub -- which we strip here. It is the +# only
...
pair at column zero in each page. +inputs=() +for chapter in $chapters; do + if [ ! -f "$src_dir/$chapter" ]; then + echo "$0: $src_dir/$chapter is listed in mkdocs.yml nav but does not exist." 1>&2 + echo "$0: perhaps you need to run: make -C $src_dir genmds" 1>&2 + exit 1 + fi + sed '/^
$/,/^<\/div>$/d' "$src_dir/$chapter" > "$tmp_dir/$chapter" + inputs+=("$tmp_dir/$chapter") +done + +# Run from src_dir so relative image paths (pix/*.png) resolve. +cd "$src_dir" +pandoc \ + --toc \ + --toc-depth=2 \ + --split-level=1 \ + --resource-path="$src_dir" \ + --metadata title="Miller Documentation" \ + --metadata author="John Kerl" \ + --metadata lang="en" \ + --output "$output" \ + "${inputs[@]}" + +echo "Wrote $output" diff --git a/docs/src/10min.md b/docs/src/10min.md index eaec2be05..3ca1b276d 100644 --- a/docs/src/10min.md +++ b/docs/src/10min.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/agent-skill.md b/docs/src/agent-skill.md index 793e34b4d..08ca35b7f 100644 --- a/docs/src/agent-skill.md +++ b/docs/src/agent-skill.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/ai-support.md b/docs/src/ai-support.md index 4c01bc5f0..509e1f0f0 100644 --- a/docs/src/ai-support.md +++ b/docs/src/ai-support.md @@ -1,4 +1,4 @@ - +
Quick links: @@ -64,7 +64,7 @@ one-line summary (here trimmed, and then counted, using Miller itself): 
 [
 {
-  "count": 661
+  "count": 665
 }
 ]
diff --git a/docs/src/ai.md b/docs/src/ai.md index 20f218b21..7637b7ea8 100644 --- a/docs/src/ai.md +++ b/docs/src/ai.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/build.md b/docs/src/build.md index b6678282f..73b3c1c9e 100644 --- a/docs/src/build.md +++ b/docs/src/build.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/community.md b/docs/src/community.md index b0562c345..e530b189b 100644 --- a/docs/src/community.md +++ b/docs/src/community.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/contributing.md b/docs/src/contributing.md index 9e4122abc..256270622 100644 --- a/docs/src/contributing.md +++ b/docs/src/contributing.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/cpu.md b/docs/src/cpu.md index 0fe2e1880..3ef38b294 100644 --- a/docs/src/cpu.md +++ b/docs/src/cpu.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/csv-with-and-without-headers.md b/docs/src/csv-with-and-without-headers.md index 944255e55..773ff1dd5 100644 --- a/docs/src/csv-with-and-without-headers.md +++ b/docs/src/csv-with-and-without-headers.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/customization.md b/docs/src/customization.md index cbc69928f..1c90da971 100644 --- a/docs/src/customization.md +++ b/docs/src/customization.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/data-cleaning-examples.md b/docs/src/data-cleaning-examples.md index 77c08e680..5cddccdd0 100644 --- a/docs/src/data-cleaning-examples.md +++ b/docs/src/data-cleaning-examples.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/data-diving-examples.md b/docs/src/data-diving-examples.md index 100716ec2..0a2f27be2 100644 --- a/docs/src/data-diving-examples.md +++ b/docs/src/data-diving-examples.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/data/het/sales-2021.csv b/docs/src/data/het/sales-2021.csv new file mode 100644 index 000000000..e1476218f --- /dev/null +++ b/docs/src/data/het/sales-2021.csv @@ -0,0 +1,3 @@ +id,product,sales +1,pencil,100 +2,eraser,17 diff --git a/docs/src/data/het/sales-2022.csv b/docs/src/data/het/sales-2022.csv new file mode 100644 index 000000000..142bbcae2 --- /dev/null +++ b/docs/src/data/het/sales-2022.csv @@ -0,0 +1,3 @@ +id,product,color,sales +3,pencil,red,120 +4,eraser,green,32 diff --git a/docs/src/data/het/sales-2023.csv b/docs/src/data/het/sales-2023.csv new file mode 100644 index 000000000..1053c178c --- /dev/null +++ b/docs/src/data/het/sales-2023.csv @@ -0,0 +1,3 @@ +id,product,sales,returns +5,pencil,200,6 +6,notebook,41,2 diff --git a/docs/src/data/join-nest-left.csv b/docs/src/data/join-nest-left.csv new file mode 100644 index 000000000..2d25e4937 --- /dev/null +++ b/docs/src/data/join-nest-left.csv @@ -0,0 +1,3 @@ +id,color +1;2,blue +3,green diff --git a/docs/src/data/join-nest-right.csv b/docs/src/data/join-nest-right.csv new file mode 100644 index 000000000..746cd22dd --- /dev/null +++ b/docs/src/data/join-nest-right.csv @@ -0,0 +1,3 @@ +id,shape +1,circle +2;3,square diff --git a/docs/src/data/join-x.csv b/docs/src/data/join-x.csv new file mode 100644 index 000000000..077ad7434 --- /dev/null +++ b/docs/src/data/join-x.csv @@ -0,0 +1,4 @@ +a,b,c +a,t,1 +b,u,2 +c,v,3 diff --git a/docs/src/data/join-y.csv b/docs/src/data/join-y.csv new file mode 100644 index 000000000..d3f7bb401 --- /dev/null +++ b/docs/src/data/join-y.csv @@ -0,0 +1,4 @@ +e,f,g +a,t,3 +b,u,2 +d,w,1 diff --git a/docs/src/data/lazy-quotes.csv b/docs/src/data/lazy-quotes.csv new file mode 100644 index 000000000..13074bd42 --- /dev/null +++ b/docs/src/data/lazy-quotes.csv @@ -0,0 +1,3 @@ +id,name,flag,city +1,"ACME CORP. INC,Q8,Rome +2,BETA,X,Milan diff --git a/docs/src/data/rank-example.csv b/docs/src/data/rank-example.csv new file mode 100644 index 000000000..6d976d274 --- /dev/null +++ b/docs/src/data/rank-example.csv @@ -0,0 +1,8 @@ +g,x +a,10 +a,20 +a,20 +a,30 +b,5 +b,5 +b,9 diff --git a/docs/src/data/sensor-humidity.csv b/docs/src/data/sensor-humidity.csv new file mode 100644 index 000000000..c22ff5d7d --- /dev/null +++ b/docs/src/data/sensor-humidity.csv @@ -0,0 +1,3 @@ +unixTime,minValue,averageValue,maxValue +1740000000,50.1,50.3,50.5 +1740000120,52.3,52.5,52.7 diff --git a/docs/src/data/sensor-pressure.csv b/docs/src/data/sensor-pressure.csv new file mode 100644 index 000000000..341104a14 --- /dev/null +++ b/docs/src/data/sensor-pressure.csv @@ -0,0 +1,4 @@ +unixTime,minValue,averageValue,maxValue +1740000000,1012.2,1012.3,1012.4 +1740000060,1011.8,1011.9,1012.0 +1740000120,1011.5,1011.6,1011.7 diff --git a/docs/src/data/sensor-temp.csv b/docs/src/data/sensor-temp.csv new file mode 100644 index 000000000..a07b02046 --- /dev/null +++ b/docs/src/data/sensor-temp.csv @@ -0,0 +1,4 @@ +unixTime,minValue,averageValue,maxValue +1740000000,37.2,37.4,37.6 +1740000060,37.5,37.6,37.7 +1740000120,37.8,37.9,38.0 diff --git a/docs/src/date-time-examples.md b/docs/src/date-time-examples.md index 5bcbdac01..837e9dc0e 100644 --- a/docs/src/date-time-examples.md +++ b/docs/src/date-time-examples.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/dkvp-examples.md b/docs/src/dkvp-examples.md index 2f3e3b510..9620eedab 100644 --- a/docs/src/dkvp-examples.md +++ b/docs/src/dkvp-examples.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/etymology.md b/docs/src/etymology.md index 9c277924f..0124817c7 100644 --- a/docs/src/etymology.md +++ b/docs/src/etymology.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/features.md b/docs/src/features.md index 048ec2a04..9339cc156 100644 --- a/docs/src/features.md +++ b/docs/src/features.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/file-formats.md b/docs/src/file-formats.md index 8aad25eb4..2074138c6 100644 --- a/docs/src/file-formats.md +++ b/docs/src/file-formats.md @@ -1,4 +1,4 @@ - +
Quick links: @@ -237,6 +237,79 @@ CSV, TSV, CSV-lite, and TSV-lite have in common the `--implicit-csv-header` flag See also the [`--lazy-quotes` flag](reference-main-flag-list.md#csv-only-flags), which can help with CSV files that are not fully compliant with RFC-4180. +### Handling stray quote characters + +The [`--lazy-quotes` flag](reference-main-flag-list.md#csv-only-flags) makes two specific +relaxations to RFC-4180 parsing (following the semantics of the +[Go CSV library](https://pkg.go.dev/encoding/csv)): a quote may appear inside an *unquoted* +field, and a non-doubled quote may appear inside a *quoted* field. + +What it does **not** change is how quoted fields are delimited. A field whose first character +is a double quote is still a quoted field, and its contents extend -- across field separators +and even line endings -- until the next double quote. In particular, if a field has an +unmatched opening quote, everything up to the next quote character in the file (or the end +of the file) becomes part of that field. This matches the behavior of the Go CSV library +with `LazyQuotes`, as well as Python's `csv` module. For example: + +
+cat data/lazy-quotes.csv
+
+
+id,name,flag,city
+1,"ACME CORP. INC,Q8,Rome
+2,BETA,X,Milan
+
+ +Here the second field of the first data line has an opening quote with no matching close, +so without `--lazy-quotes` we get an error: + +
+mlr --icsv --ojson cat data/lazy-quotes.csv
+
+
+mlr: CSV header/data length mismatch 4 != 1 at filename data/lazy-quotes.csv row 2
+
+ +With `--lazy-quotes`, the quoted field silently absorbs the rest of the file -- including +the field separators and the newline -- since there is no closing quote anywhere: + +
+mlr --icsv --ojson --lazy-quotes --allow-ragged-csv-input cat data/lazy-quotes.csv
+
+
+[
+{
+  "id": 1,
+  "name": "ACME CORP. INC,Q8,Rome\n2,BETA,X,Milan\n"
+}
+]
+
+ +If quote characters in your data are really just ordinary data characters -- that is, the +file doesn't use RFC-4180-style quoting at all -- then [CSV-lite](file-formats.md#csvtsvasvusvetc) +is often a better choice, since it splits on the field separator without treating quotes +specially (the literal quote character is retained in the data): + +
+mlr --icsvlite --ojson cat data/lazy-quotes.csv
+
+
+[
+{
+  "id": 1,
+  "name": "\"ACME CORP. INC",
+  "flag": "Q8",
+  "city": "Rome"
+},
+{
+  "id": 2,
+  "name": "BETA",
+  "flag": "X",
+  "city": "Milan"
+}
+]
+
+ ### Troubleshooting CSV and JSON input Please see [this page](troubleshooting-csv-and-json-input.md). @@ -531,6 +604,47 @@ Since Miller 6.11.0, you can use `--barred-input` with pprint input format: ] +Use `--right` to right-align all cells, or `--right-align-numeric` to right-align only the cells +having numeric values, leaving other cells left-aligned. Headers are right-aligned over columns +whose values are all numeric, so that header and data share the same alignment: + +
+mlr --icsv --opprint --right-align-numeric cat example.csv
+
+
+color  shape    flag   k index quantity   rate
+yellow triangle true   1    11  43.6498 9.8870
+red    square   true   2    15  79.2778 0.0130
+red    circle   true   3    16  13.8103 2.9010
+red    square   false  4    48  77.5542 7.4670
+purple triangle false  5    51  81.2290 8.5910
+red    square   false  6    64  77.1991 9.5310
+purple triangle false  7    65  80.1405 5.8240
+yellow circle   true   8    73  63.9785 4.2370
+yellow circle   true   9    87  63.5058 8.3350
+purple square   false 10    91  72.3735 8.2430
+
+ +
+mlr --icsv --opprint --barred --right-align-numeric cat example.csv
+
+
++--------+----------+-------+----+-------+----------+--------+
+| color  | shape    | flag  |  k | index | quantity |   rate |
++--------+----------+-------+----+-------+----------+--------+
+| yellow | triangle | true  |  1 |    11 |  43.6498 | 9.8870 |
+| red    | square   | true  |  2 |    15 |  79.2778 | 0.0130 |
+| red    | circle   | true  |  3 |    16 |  13.8103 | 2.9010 |
+| red    | square   | false |  4 |    48 |  77.5542 | 7.4670 |
+| purple | triangle | false |  5 |    51 |  81.2290 | 8.5910 |
+| red    | square   | false |  6 |    64 |  77.1991 | 9.5310 |
+| purple | triangle | false |  7 |    65 |  80.1405 | 5.8240 |
+| yellow | circle   | true  |  8 |    73 |  63.9785 | 4.2370 |
+| yellow | circle   | true  |  9 |    87 |  63.5058 | 8.3350 |
+| purple | square   | false | 10 |    91 |  72.3735 | 8.2430 |
++--------+----------+-------+----+-------+----------+--------+
+
+ ## Markdown tabular Markdown format looks like this: @@ -580,6 +694,31 @@ do not need to pass `--md` in addition: mlr --md-aligned cat data/small +The `--right-align-numeric` flag also applies to markdown output: numeric columns get a +right-alignment marker (`---:`) in the header-separator line, so they render right-aligned in +Markdown viewers. With `--omd`, since output is streaming, the marker for each column is chosen +from the first record of each same-schema group; with `--omd-aligned`, a column gets the marker +when all its values are numeric, and its header and cell text are right-justified in the raw +markdown as well: + +
+mlr --icsv --omd-aligned --right-align-numeric cat example.csv
+
+
+| color  | shape    | flag  |    k | index | quantity |   rate |
+| ---    | ---      | ---   | ---: |  ---: |     ---: |   ---: |
+| yellow | triangle | true  |    1 |    11 |  43.6498 | 9.8870 |
+| red    | square   | true  |    2 |    15 |  79.2778 | 0.0130 |
+| red    | circle   | true  |    3 |    16 |  13.8103 | 2.9010 |
+| red    | square   | false |    4 |    48 |  77.5542 | 7.4670 |
+| purple | triangle | false |    5 |    51 |  81.2290 | 8.5910 |
+| red    | square   | false |    6 |    64 |  77.1991 | 9.5310 |
+| purple | triangle | false |    7 |    65 |  80.1405 | 5.8240 |
+| yellow | circle   | true  |    8 |    73 |  63.9785 | 4.2370 |
+| yellow | circle   | true  |    9 |    87 |  63.5058 | 8.3350 |
+| purple | square   | false |   10 |    91 |  72.3735 | 8.2430 |
+
+ ## XTAB: Vertical tabular This is perhaps most useful for looking a very wide and/or multi-column data which causes line-wraps on the screen (but see also diff --git a/docs/src/file-formats.md.in b/docs/src/file-formats.md.in index 607f5da7f..4a821710a 100644 --- a/docs/src/file-formats.md.in +++ b/docs/src/file-formats.md.in @@ -79,6 +79,47 @@ CSV, TSV, CSV-lite, and TSV-lite have in common the `--implicit-csv-header` flag See also the [`--lazy-quotes` flag](reference-main-flag-list.md#csv-only-flags), which can help with CSV files that are not fully compliant with RFC-4180. +### Handling stray quote characters + +The [`--lazy-quotes` flag](reference-main-flag-list.md#csv-only-flags) makes two specific +relaxations to RFC-4180 parsing (following the semantics of the +[Go CSV library](https://pkg.go.dev/encoding/csv)): a quote may appear inside an *unquoted* +field, and a non-doubled quote may appear inside a *quoted* field. + +What it does **not** change is how quoted fields are delimited. A field whose first character +is a double quote is still a quoted field, and its contents extend -- across field separators +and even line endings -- until the next double quote. In particular, if a field has an +unmatched opening quote, everything up to the next quote character in the file (or the end +of the file) becomes part of that field. This matches the behavior of the Go CSV library +with `LazyQuotes`, as well as Python's `csv` module. For example: + +GENMD-RUN-COMMAND +cat data/lazy-quotes.csv +GENMD-EOF + +Here the second field of the first data line has an opening quote with no matching close, +so without `--lazy-quotes` we get an error: + +GENMD-RUN-COMMAND-TOLERATING-ERROR +mlr --icsv --ojson cat data/lazy-quotes.csv +GENMD-EOF + +With `--lazy-quotes`, the quoted field silently absorbs the rest of the file -- including +the field separators and the newline -- since there is no closing quote anywhere: + +GENMD-RUN-COMMAND +mlr --icsv --ojson --lazy-quotes --allow-ragged-csv-input cat data/lazy-quotes.csv +GENMD-EOF + +If quote characters in your data are really just ordinary data characters -- that is, the +file doesn't use RFC-4180-style quoting at all -- then [CSV-lite](file-formats.md#csvtsvasvusvetc) +is often a better choice, since it splits on the field separator without treating quotes +specially (the literal quote character is retained in the data): + +GENMD-RUN-COMMAND +mlr --icsvlite --ojson cat data/lazy-quotes.csv +GENMD-EOF + ### Troubleshooting CSV and JSON input Please see [this page](troubleshooting-csv-and-json-input.md). @@ -212,6 +253,18 @@ GENMD-RUN-COMMAND mlr -o pprint --barred cat data/small | mlr -i pprint --barred-input -o json filter '$b == "pan"' GENMD-EOF +Use `--right` to right-align all cells, or `--right-align-numeric` to right-align only the cells +having numeric values, leaving other cells left-aligned. Headers are right-aligned over columns +whose values are all numeric, so that header and data share the same alignment: + +GENMD-RUN-COMMAND +mlr --icsv --opprint --right-align-numeric cat example.csv +GENMD-EOF + +GENMD-RUN-COMMAND +mlr --icsv --opprint --barred --right-align-numeric cat example.csv +GENMD-EOF + ## Markdown tabular Markdown format looks like this: @@ -243,6 +296,17 @@ GENMD-RUN-COMMAND mlr --md-aligned cat data/small GENMD-EOF +The `--right-align-numeric` flag also applies to markdown output: numeric columns get a +right-alignment marker (`---:`) in the header-separator line, so they render right-aligned in +Markdown viewers. With `--omd`, since output is streaming, the marker for each column is chosen +from the first record of each same-schema group; with `--omd-aligned`, a column gets the marker +when all its values are numeric, and its header and cell text are right-justified in the raw +markdown as well: + +GENMD-RUN-COMMAND +mlr --icsv --omd-aligned --right-align-numeric cat example.csv +GENMD-EOF + ## XTAB: Vertical tabular This is perhaps most useful for looking a very wide and/or multi-column data which causes line-wraps on the screen (but see also diff --git a/docs/src/flatten-unflatten.md b/docs/src/flatten-unflatten.md index 9706ddd1d..c0cb6fe6b 100644 --- a/docs/src/flatten-unflatten.md +++ b/docs/src/flatten-unflatten.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/genmd-filter b/docs/src/genmd-filter index 32626326c..6686c4ba8 100755 --- a/docs/src/genmd-filter +++ b/docs/src/genmd-filter @@ -18,7 +18,7 @@ def main input_handle = $stdin output_handle = $stdout - output_handle.puts("") + output_handle.puts("") # The filename on the command line is used for link-generation; file contents # are read from standard input. diff --git a/docs/src/glossary.md b/docs/src/glossary.md index 542f49494..5875d2a47 100644 --- a/docs/src/glossary.md +++ b/docs/src/glossary.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/how-to-release.md b/docs/src/how-to-release.md index f8b680ca2..33f0f6add 100644 --- a/docs/src/how-to-release.md +++ b/docs/src/how-to-release.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/index.md b/docs/src/index.md index a14fddfca..cc36e236e 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/installing-miller.md b/docs/src/installing-miller.md index f24a0007c..feb485de3 100644 --- a/docs/src/installing-miller.md +++ b/docs/src/installing-miller.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/internationalization.md b/docs/src/internationalization.md index 5fadcab1e..dfe8a0969 100644 --- a/docs/src/internationalization.md +++ b/docs/src/internationalization.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/keystroke-savers.md b/docs/src/keystroke-savers.md index e8fbc9073..0ab43b7e5 100644 --- a/docs/src/keystroke-savers.md +++ b/docs/src/keystroke-savers.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/kubectl-and-helm.md b/docs/src/kubectl-and-helm.md index 5f53001be..87e81f643 100644 --- a/docs/src/kubectl-and-helm.md +++ b/docs/src/kubectl-and-helm.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/log-processing-examples.md b/docs/src/log-processing-examples.md index ad0b2a333..dfbba41da 100644 --- a/docs/src/log-processing-examples.md +++ b/docs/src/log-processing-examples.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/manpage.md b/docs/src/manpage.md index 6fd057f77..41b5b3676 100644 --- a/docs/src/manpage.md +++ b/docs/src/manpage.md @@ -1,4 +1,4 @@ - +
Quick links: @@ -228,11 +228,11 @@ This is simply a copy of what you should see on running `man mlr` at a command p count-similar cut decimate describe fill-down fill-empty filter flatten format-values fraction gap grep group-by group-like gsub having-fields head histogram json-parse json-stringify join label latin1-to-utf8 least-frequent - merge-fields most-frequent nest nothing put regularize remove-empty-columns - rename reorder repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle - skip-trivial-records sort sort-within-records sparsify split ssub stats1 - stats2 step sub summary surv tac tail tee template top utf8-to-latin1 - unflatten uniq unspace unsparsify + merge-fields most-frequent nest nothing put rank regularize + remove-empty-columns rename reorder repeat reshape sample sec2gmtdate sec2gmt + seqgen shuffle skip-trivial-records sort sort-within-records sparkline + sparsify split ssub stats1 stats2 step sub summary surv tac tail tee template + top utf8-to-latin1 unflatten uniq unspace unsparsify 1mFUNCTION LIST0m abs acos acosh antimode any append apply arrayify asin asinh asserting_absent @@ -257,14 +257,14 @@ This is simply a copy of what you should see on running `man mlr` at a command p null_count os percentile percentiles pow qnorm reduce regextract regextract_or_else rightpad round roundm rstrip sec2dhms sec2gmt sec2gmtdate sec2hms sec2localdate sec2localtime select sgn sha1 sha256 sha512 sin sinh - skewness sort sort_collection splita splitax splitkv splitkvx splitnv splitnvx - sqrt ssub stat stddev strfntime strfntime_local strftime strftime_local string - strip strlen strmatch strmatchx strpntime strpntime_local strptime - strptime_local sub substr substr0 substr1 sum sum2 sum3 sum4 sysntime system - systime systimeint tan tanh tolower toupper truncate typeof unflatten unformat - unformatx upntime uptime urand urand32 urandelement urandint urandrange - utf8_to_latin1 variance version ! != !=~ % & && * ** + - . .* .+ .- ./ / // < - << <= <=> == =~ > >= >> >>> ?: ?? ??? ^ ^^ | || ~ + skewness sort sort_collection sparkline splita splitax splitkv splitkvx + splitnv splitnvx sqrt ssub stat stddev strfntime strfntime_local strftime + strftime_local string strip strlen strmatch strmatchx strpntime + strpntime_local strptime strptime_local sub substr substr0 substr1 sum sum2 + sum3 sum4 sysntime system systime systimeint tan tanh tolower toupper truncate + typeof unflatten unformat unformatx upntime uptime urand urand32 urandelement + urandint urandrange utf8_to_latin1 variance version ! != !=~ % & && * ** + - . + .* .+ .- ./ / // < << <= <=> == =~ > >= >> >>> ?: ?? ??? ^ ^^ | || ~ 1mCOMMENTS-IN-DATA FLAGS0m Miller lets you put comments in your data, such as @@ -801,6 +801,13 @@ This is simply a copy of what you should see on running `man mlr` at a command p right-align-multi-word --fw {string} Shortcut for --fixed left-align-multi-word --right Right-justifies all fields for PPRINT output. + --right-align-numeric Right-justifies fields with numeric values for PPRINT + output, leaving other fields left-justified. Headers + are right-justified over columns whose values are all + numeric, so that header and data share the same + alignment. Also applies to markdown output, where + numeric columns get right-alignment markers (`---:`) + in the header-separator line. 1mPROFILING FLAGS0m These are flags for profiling Miller performance. @@ -840,6 +847,9 @@ This is simply a copy of what you should see on running `man mlr` at a command p * Default line endings (`--irs` and `--ors`) are newline which is interpreted to accept carriage-return/newline files (e.g. on Windows) for input, and to produce platform-appropriate line endings on output. + * For CSV, CSV-lite, TSV, and TSV-lite output, ORS may be either newline (the + default) or carriage-return/newline: e.g. `--ors crlf` or `--ors '\r\n'` + for RFC-4180-style line endings on any platform. Notes about all other separators: @@ -1452,7 +1462,11 @@ This is simply a copy of what you should see on running `man mlr` at a command p --auto Automatically computes limits, ignoring --lo and --hi. Holds all values in memory before producing any output. -o {prefix} Prefix for output field name. Default: no prefix. + -s Print a one-line Unicode sparkline per field instead of per-bin + counts. -h|--help Show this message. + With -s, output is one record per value-field, with a sparkline field + instead of one record per bin. 1mjson-parse0m Usage: mlr json-parse [options] @@ -1798,6 +1812,34 @@ This is simply a copy of what you should see on running `man mlr` at a command p See also https://miller.readthedocs.io/reference-dsl for more context. + 1mrank0m + Usage: mlr rank [options] + For each record's value in specified fields, computes the standard + competition rank (1,2,2,4,...) of that value among all input records, + optionally within groups. + E.g. with input records x=10, x=20, x=20, and x=30, emits output records + x=10,x_rank=1 x=20,x_rank=2 x=20,x_rank=2 and x=30,x_rank=4. + + Note: by default this is a two-pass algorithm: on the first pass it retains + input records and their values; on the second pass it computes ranks and + emits output records, in original input order. This means it produces no + output until all input is read, but gives correct ranks regardless of input + order. Use --sorted for a single-pass streaming alternative. + + Options: + -f {a,b,c} Field name(s) to rank. + -g {d,e,f} Optional group-by-field name(s). + --sorted Promise that the input is already sorted by the field(s) being ranked + (within each group, if -g is given). This computes rank in a single + streaming pass and O(1) space, by comparing each record's value only + to the immediately preceding one, rather than buffering all records + to compute an order-independent rank. Produces wrong output if the + input is not in fact sorted. + -h|--help Show this message. + Example: mlr rank -f x data/rank-example.csv + Example: mlr rank -f x -g g data/rank-example.csv + Example: mlr sort -f x then rank -f x --sorted data/rank-example.csv + 1mregularize0m Usage: mlr regularize [options] Outputs records sorted lexically ascending by keys. @@ -1840,8 +1882,10 @@ This is simply a copy of what you should see on running `man mlr` at a command p record start. -f {a,b,c} Field names to reorder. -r {a,b,c} Treat field names as regular expressions. Matched fields are moved to - start or end in record order. Example: -r '^YYY,^XXX' puts all YYY- - and XXX-prefixed fields first (in record order), then the rest. + start or end, grouped by the order the regexes are given; within each + group, fields keep their record order. Example: -r '^YYY,^XXX' puts + all YYY-prefixed fields first, then all XXX-prefixed fields, then the + rest. -b {x} Put field names specified with -f before field name specified by {x}, if any. If {x} isn't present in a given record, the specified fields will not be moved. @@ -1853,7 +1897,7 @@ This is simply a copy of what you should see on running `man mlr` at a command p Examples: mlr reorder -f a,b sends input record "d=4,b=2,a=1,c=3" to "a=1,b=2,d=4,c=3". mlr reorder -e -f a,b sends input record "d=4,b=2,a=1,c=3" to "d=4,c=3,a=1,b=2". - mlr reorder -r '^YYY,^XXX' puts YYY- and XXX-prefixed fields first (record order), then rest. + mlr reorder -r '^YYY,^XXX' puts YYY-prefixed fields first, then XXX-prefixed fields, then rest. 1mrepeat0m Usage: mlr repeat [options] @@ -2071,6 +2115,16 @@ This is simply a copy of what you should see on running `man mlr` at a command p -n Sort field names naturally (e.g. 2 before 12). Combines with -f/-r. -h|--help Show this message. + 1msparkline0m + Usage: mlr sparkline [options] + Reduces numeric field(s), across all records in input order, to a compact + Unicode sparkline -- one block character per record -- for visualizing + trends. Emits one output record per field. Holds all records in memory + before producing any output. + Options: + -f {a,b,c} Field names to sparkline. + -h|--help Show this message. + 1msparsify0m Usage: mlr sparsify [options] Unsets fields for which the key is the empty string (or, optionally, another @@ -2312,7 +2366,7 @@ This is simply a copy of what you should see on running `man mlr` at a command p Show summary statistics about the input data. All summarizers: - field_type string, int, etc. -- if a column has mixed types, all encountered types are printed + field_type string, int, etc. -- if a column has mixed types, all encountered types are printed (see notes below) count +1 for every instance of the field across all records in the input record stream null_count count of field values either empty string or JSON null distinct_count count of distinct values for the field @@ -2342,6 +2396,8 @@ This is simply a copy of what you should see on running `man mlr` at a command p * min, p25, median, p75, and max work for strings as well as numbers * Distinct-counts are computed on string representations -- so 4.1 and 4.10 are counted as distinct here. * If the mode is not unique in the input data, the first-encountered value is reported as the mode. + * A field_type of "int-string", "empty-string", etc. means the column contains values of mixed types -- + all types encountered are printed, hyphen-joined, in the order first encountered. Options: -a {mean,sum,etc.} Use only the specified summarizers. @@ -2447,6 +2503,9 @@ This is simply a copy of what you should see on running `man mlr` at a command p count-distinct. For uniq, -f is a synonym for -g. Output fields are written in the order in which they are named with -g or -f, not in the order in which they appear in the input records. + To deduplicate records by one or more fields while keeping all other + fields, use head: e.g. "mlr head -n 1 -g hash" keeps the first record + for each distinct value of the hash field, with all fields intact. Options: -g {d,e,f} Group-by field names for uniq counts. @@ -2738,11 +2797,13 @@ This is simply a copy of what you should see on running `man mlr` at a command p Map example: fold({"a":1, "b":3, "c": 5}, func(acck,accv,ek,ev) {return {"sum": accv+ev**2}}, {"sum":10000}) returns 10035. 1mformat0m - (class=string #args=variadic) Using first argument as format string, interpolate remaining arguments in place of each "{}" in the format string. Too-few arguments are treated as the empty string; too-many arguments are discarded. + (class=string #args=variadic) Using first argument as format string, interpolate remaining arguments in place of each "{}" in the format string. Also supports 1-based positional placeholders such as "{1}", which allow arguments to be reused and/or reordered. Plain "{}" placeholders consume arguments sequentially, independently of any positional placeholders. Too-few arguments are treated as the empty string, as are positional placeholders exceeding the number of arguments; too-many arguments are discarded. "{0}" is an error value, since positional placeholders are 1-based. Examples: - format("{}:{}:{}", 1,2) gives "1:2:". - format("{}:{}:{}", 1,2,3) gives "1:2:3". - format("{}:{}:{}", 1,2,3,4) gives "1:2:3". + format("{}:{}:{}", 1,2) gives "1:2:". + format("{}:{}:{}", 1,2,3) gives "1:2:3". + format("{}:{}:{}", 1,2,3,4) gives "1:2:3". + format("{1}:{2}:{1}", "a","b") gives "a:b:a". + format("{2}{}:{1}{}", 3,4) gives "43:34". 1mfsec2dhms0m (class=time #args=1) Formats floating-point seconds as in fsec2dhms(500000.25) = "5d18h53m20.250000s" @@ -3265,6 +3326,12 @@ This is simply a copy of what you should see on running `man mlr` at a command p 1msort_collection0m (class=stats #args=1) This is a helper function for the percentiles function; please see its online help for details. + 1msparkline0m + (class=stats #args=1) Returns a string of Unicode block characters (one of per element) representing the relative magnitudes of values in an array or map, for a compact ASCII/Unicode bar chart. Returns error for non-array/non-map types. + Examples: + sparkline([1,2,3,4,5,6,7,8]) is "" + sparkline([3,3,3]) is "" + 1msplita0m (class=conversion #args=2) Splits string into array with type inference. First argument is string to split; second is the separator to split on. Example: @@ -4043,5 +4110,5 @@ This is simply a copy of what you should see on running `man mlr` at a command p MIME Type for Comma-Separated Values (CSV) Files, the Miller docsite https://miller.readthedocs.io - 2026-07-05 4mMILLER24m(1) + 2026-07-08 4mMILLER24m(1) diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt index 0e234b965..a5e8bd3b8 100644 --- a/docs/src/manpage.txt +++ b/docs/src/manpage.txt @@ -207,11 +207,11 @@ count-similar cut decimate describe fill-down fill-empty filter flatten format-values fraction gap grep group-by group-like gsub having-fields head histogram json-parse json-stringify join label latin1-to-utf8 least-frequent - merge-fields most-frequent nest nothing put regularize remove-empty-columns - rename reorder repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle - skip-trivial-records sort sort-within-records sparsify split ssub stats1 - stats2 step sub summary surv tac tail tee template top utf8-to-latin1 - unflatten uniq unspace unsparsify + merge-fields most-frequent nest nothing put rank regularize + remove-empty-columns rename reorder repeat reshape sample sec2gmtdate sec2gmt + seqgen shuffle skip-trivial-records sort sort-within-records sparkline + sparsify split ssub stats1 stats2 step sub summary surv tac tail tee template + top utf8-to-latin1 unflatten uniq unspace unsparsify 1mFUNCTION LIST0m abs acos acosh antimode any append apply arrayify asin asinh asserting_absent @@ -236,14 +236,14 @@ null_count os percentile percentiles pow qnorm reduce regextract regextract_or_else rightpad round roundm rstrip sec2dhms sec2gmt sec2gmtdate sec2hms sec2localdate sec2localtime select sgn sha1 sha256 sha512 sin sinh - skewness sort sort_collection splita splitax splitkv splitkvx splitnv splitnvx - sqrt ssub stat stddev strfntime strfntime_local strftime strftime_local string - strip strlen strmatch strmatchx strpntime strpntime_local strptime - strptime_local sub substr substr0 substr1 sum sum2 sum3 sum4 sysntime system - systime systimeint tan tanh tolower toupper truncate typeof unflatten unformat - unformatx upntime uptime urand urand32 urandelement urandint urandrange - utf8_to_latin1 variance version ! != !=~ % & && * ** + - . .* .+ .- ./ / // < - << <= <=> == =~ > >= >> >>> ?: ?? ??? ^ ^^ | || ~ + skewness sort sort_collection sparkline splita splitax splitkv splitkvx + splitnv splitnvx sqrt ssub stat stddev strfntime strfntime_local strftime + strftime_local string strip strlen strmatch strmatchx strpntime + strpntime_local strptime strptime_local sub substr substr0 substr1 sum sum2 + sum3 sum4 sysntime system systime systimeint tan tanh tolower toupper truncate + typeof unflatten unformat unformatx upntime uptime urand urand32 urandelement + urandint urandrange utf8_to_latin1 variance version ! != !=~ % & && * ** + - . + .* .+ .- ./ / // < << <= <=> == =~ > >= >> >>> ?: ?? ??? ^ ^^ | || ~ 1mCOMMENTS-IN-DATA FLAGS0m Miller lets you put comments in your data, such as @@ -780,6 +780,13 @@ right-align-multi-word --fw {string} Shortcut for --fixed left-align-multi-word --right Right-justifies all fields for PPRINT output. + --right-align-numeric Right-justifies fields with numeric values for PPRINT + output, leaving other fields left-justified. Headers + are right-justified over columns whose values are all + numeric, so that header and data share the same + alignment. Also applies to markdown output, where + numeric columns get right-alignment markers (`---:`) + in the header-separator line. 1mPROFILING FLAGS0m These are flags for profiling Miller performance. @@ -819,6 +826,9 @@ * Default line endings (`--irs` and `--ors`) are newline which is interpreted to accept carriage-return/newline files (e.g. on Windows) for input, and to produce platform-appropriate line endings on output. + * For CSV, CSV-lite, TSV, and TSV-lite output, ORS may be either newline (the + default) or carriage-return/newline: e.g. `--ors crlf` or `--ors '\r\n'` + for RFC-4180-style line endings on any platform. Notes about all other separators: @@ -1431,7 +1441,11 @@ --auto Automatically computes limits, ignoring --lo and --hi. Holds all values in memory before producing any output. -o {prefix} Prefix for output field name. Default: no prefix. + -s Print a one-line Unicode sparkline per field instead of per-bin + counts. -h|--help Show this message. + With -s, output is one record per value-field, with a sparkline field + instead of one record per bin. 1mjson-parse0m Usage: mlr json-parse [options] @@ -1777,6 +1791,34 @@ See also https://miller.readthedocs.io/reference-dsl for more context. + 1mrank0m + Usage: mlr rank [options] + For each record's value in specified fields, computes the standard + competition rank (1,2,2,4,...) of that value among all input records, + optionally within groups. + E.g. with input records x=10, x=20, x=20, and x=30, emits output records + x=10,x_rank=1 x=20,x_rank=2 x=20,x_rank=2 and x=30,x_rank=4. + + Note: by default this is a two-pass algorithm: on the first pass it retains + input records and their values; on the second pass it computes ranks and + emits output records, in original input order. This means it produces no + output until all input is read, but gives correct ranks regardless of input + order. Use --sorted for a single-pass streaming alternative. + + Options: + -f {a,b,c} Field name(s) to rank. + -g {d,e,f} Optional group-by-field name(s). + --sorted Promise that the input is already sorted by the field(s) being ranked + (within each group, if -g is given). This computes rank in a single + streaming pass and O(1) space, by comparing each record's value only + to the immediately preceding one, rather than buffering all records + to compute an order-independent rank. Produces wrong output if the + input is not in fact sorted. + -h|--help Show this message. + Example: mlr rank -f x data/rank-example.csv + Example: mlr rank -f x -g g data/rank-example.csv + Example: mlr sort -f x then rank -f x --sorted data/rank-example.csv + 1mregularize0m Usage: mlr regularize [options] Outputs records sorted lexically ascending by keys. @@ -1819,8 +1861,10 @@ record start. -f {a,b,c} Field names to reorder. -r {a,b,c} Treat field names as regular expressions. Matched fields are moved to - start or end in record order. Example: -r '^YYY,^XXX' puts all YYY- - and XXX-prefixed fields first (in record order), then the rest. + start or end, grouped by the order the regexes are given; within each + group, fields keep their record order. Example: -r '^YYY,^XXX' puts + all YYY-prefixed fields first, then all XXX-prefixed fields, then the + rest. -b {x} Put field names specified with -f before field name specified by {x}, if any. If {x} isn't present in a given record, the specified fields will not be moved. @@ -1832,7 +1876,7 @@ Examples: mlr reorder -f a,b sends input record "d=4,b=2,a=1,c=3" to "a=1,b=2,d=4,c=3". mlr reorder -e -f a,b sends input record "d=4,b=2,a=1,c=3" to "d=4,c=3,a=1,b=2". - mlr reorder -r '^YYY,^XXX' puts YYY- and XXX-prefixed fields first (record order), then rest. + mlr reorder -r '^YYY,^XXX' puts YYY-prefixed fields first, then XXX-prefixed fields, then rest. 1mrepeat0m Usage: mlr repeat [options] @@ -2050,6 +2094,16 @@ -n Sort field names naturally (e.g. 2 before 12). Combines with -f/-r. -h|--help Show this message. + 1msparkline0m + Usage: mlr sparkline [options] + Reduces numeric field(s), across all records in input order, to a compact + Unicode sparkline -- one block character per record -- for visualizing + trends. Emits one output record per field. Holds all records in memory + before producing any output. + Options: + -f {a,b,c} Field names to sparkline. + -h|--help Show this message. + 1msparsify0m Usage: mlr sparsify [options] Unsets fields for which the key is the empty string (or, optionally, another @@ -2291,7 +2345,7 @@ Show summary statistics about the input data. All summarizers: - field_type string, int, etc. -- if a column has mixed types, all encountered types are printed + field_type string, int, etc. -- if a column has mixed types, all encountered types are printed (see notes below) count +1 for every instance of the field across all records in the input record stream null_count count of field values either empty string or JSON null distinct_count count of distinct values for the field @@ -2321,6 +2375,8 @@ * min, p25, median, p75, and max work for strings as well as numbers * Distinct-counts are computed on string representations -- so 4.1 and 4.10 are counted as distinct here. * If the mode is not unique in the input data, the first-encountered value is reported as the mode. + * A field_type of "int-string", "empty-string", etc. means the column contains values of mixed types -- + all types encountered are printed, hyphen-joined, in the order first encountered. Options: -a {mean,sum,etc.} Use only the specified summarizers. @@ -2426,6 +2482,9 @@ count-distinct. For uniq, -f is a synonym for -g. Output fields are written in the order in which they are named with -g or -f, not in the order in which they appear in the input records. + To deduplicate records by one or more fields while keeping all other + fields, use head: e.g. "mlr head -n 1 -g hash" keeps the first record + for each distinct value of the hash field, with all fields intact. Options: -g {d,e,f} Group-by field names for uniq counts. @@ -2717,11 +2776,13 @@ Map example: fold({"a":1, "b":3, "c": 5}, func(acck,accv,ek,ev) {return {"sum": accv+ev**2}}, {"sum":10000}) returns 10035. 1mformat0m - (class=string #args=variadic) Using first argument as format string, interpolate remaining arguments in place of each "{}" in the format string. Too-few arguments are treated as the empty string; too-many arguments are discarded. + (class=string #args=variadic) Using first argument as format string, interpolate remaining arguments in place of each "{}" in the format string. Also supports 1-based positional placeholders such as "{1}", which allow arguments to be reused and/or reordered. Plain "{}" placeholders consume arguments sequentially, independently of any positional placeholders. Too-few arguments are treated as the empty string, as are positional placeholders exceeding the number of arguments; too-many arguments are discarded. "{0}" is an error value, since positional placeholders are 1-based. Examples: - format("{}:{}:{}", 1,2) gives "1:2:". - format("{}:{}:{}", 1,2,3) gives "1:2:3". - format("{}:{}:{}", 1,2,3,4) gives "1:2:3". + format("{}:{}:{}", 1,2) gives "1:2:". + format("{}:{}:{}", 1,2,3) gives "1:2:3". + format("{}:{}:{}", 1,2,3,4) gives "1:2:3". + format("{1}:{2}:{1}", "a","b") gives "a:b:a". + format("{2}{}:{1}{}", 3,4) gives "43:34". 1mfsec2dhms0m (class=time #args=1) Formats floating-point seconds as in fsec2dhms(500000.25) = "5d18h53m20.250000s" @@ -3244,6 +3305,12 @@ 1msort_collection0m (class=stats #args=1) This is a helper function for the percentiles function; please see its online help for details. + 1msparkline0m + (class=stats #args=1) Returns a string of Unicode block characters (one of per element) representing the relative magnitudes of values in an array or map, for a compact ASCII/Unicode bar chart. Returns error for non-array/non-map types. + Examples: + sparkline([1,2,3,4,5,6,7,8]) is "" + sparkline([3,3,3]) is "" + 1msplita0m (class=conversion #args=2) Splits string into array with type inference. First argument is string to split; second is the separator to split on. Example: @@ -4022,4 +4089,4 @@ MIME Type for Comma-Separated Values (CSV) Files, the Miller docsite https://miller.readthedocs.io - 2026-07-05 4mMILLER24m(1) + 2026-07-08 4mMILLER24m(1) diff --git a/docs/src/mcp-server.md b/docs/src/mcp-server.md index ce496d5c2..443c46ca7 100644 --- a/docs/src/mcp-server.md +++ b/docs/src/mcp-server.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/miller-as-library.md b/docs/src/miller-as-library.md index 3b09a4bbc..8f452ec44 100644 --- a/docs/src/miller-as-library.md +++ b/docs/src/miller-as-library.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/miller-on-windows.md b/docs/src/miller-on-windows.md index 8ffb6a44b..615b93e5e 100644 --- a/docs/src/miller-on-windows.md +++ b/docs/src/miller-on-windows.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/miller-programming-language.md b/docs/src/miller-programming-language.md index 2b87c5106..24ce2dd27 100644 --- a/docs/src/miller-programming-language.md +++ b/docs/src/miller-programming-language.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/misc-examples.md b/docs/src/misc-examples.md index a1a56317f..96c5d889b 100644 --- a/docs/src/misc-examples.md +++ b/docs/src/misc-examples.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/new-in-miller-6.md b/docs/src/new-in-miller-6.md index 93698d0a4..a41e470d0 100644 --- a/docs/src/new-in-miller-6.md +++ b/docs/src/new-in-miller-6.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/online-help.md b/docs/src/online-help.md index fcdb52a00..4ea0559bf 100644 --- a/docs/src/online-help.md +++ b/docs/src/online-help.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/operating-on-all-fields.md b/docs/src/operating-on-all-fields.md index 476b685dd..9f2266409 100644 --- a/docs/src/operating-on-all-fields.md +++ b/docs/src/operating-on-all-fields.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/operating-on-all-records.md b/docs/src/operating-on-all-records.md index 668dcc367..1be9dbd65 100644 --- a/docs/src/operating-on-all-records.md +++ b/docs/src/operating-on-all-records.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/originality.md b/docs/src/originality.md index 6e7fd8c49..b2a604fdf 100644 --- a/docs/src/originality.md +++ b/docs/src/originality.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/output-colorization.md b/docs/src/output-colorization.md index e94cfe91a..534b2e1db 100644 --- a/docs/src/output-colorization.md +++ b/docs/src/output-colorization.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/parsing-and-formatting-fields.md b/docs/src/parsing-and-formatting-fields.md index 1f2d5426e..1d7f9a925 100644 --- a/docs/src/parsing-and-formatting-fields.md +++ b/docs/src/parsing-and-formatting-fields.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/performance.md b/docs/src/performance.md index 48d3e1d94..47e308c42 100644 --- a/docs/src/performance.md +++ b/docs/src/performance.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/programming-examples.md b/docs/src/programming-examples.md index a8e42db58..aa9d3bd72 100644 --- a/docs/src/programming-examples.md +++ b/docs/src/programming-examples.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/questions-about-joins.md b/docs/src/questions-about-joins.md index ae4085b02..ec43f9bca 100644 --- a/docs/src/questions-about-joins.md +++ b/docs/src/questions-about-joins.md @@ -1,4 +1,4 @@ - +
Quick links: @@ -139,6 +139,84 @@ Thanks to @aborruso for the tip! See also the [record-heterogeneity page](record-heterogeneity.md). +## Doing SQL-style left, right, inner, and full-outer joins + +Miller's `join` verb is defined in terms of _paired_ and _unpaired_ records, rather than SQL-database terminology -- but you can get SQL-style joins using the `--ul` and `--ur` flags (which emit unpaired left-file and right-file records, respectively), along with [`unsparsify`](reference-verbs.md#unsparsify) to fill in empty cells for non-matches. + +Suppose you have the following two data files, where we want to join on the left file's `a` field matching the right file's `e` field: + +
+a,b,c
+a,t,1
+b,u,2
+c,v,3
+
+ +
+e,f,g
+a,t,3
+b,u,2
+d,w,1
+
+ +In all the following examples, the `-f` file (`data/join-x.csv`) is the left file, and the file in the main input stream (`data/join-y.csv`) is the right file. The flags `-j a -r e` say that the left file's `a` field is matched against the right file's `e` field, with the output join column named `a`. + +**Inner join** -- only matching records -- is what Miller's `join` does by default, since only paired records are emitted: + +
+mlr --icsv --ocsv join -j a -r e -f data/join-x.csv data/join-y.csv
+
+
+a,b,c,f,g
+a,t,1,t,3
+b,u,2,u,2
+
+ +**Left join** keeps all records from the left file, with empty cells where the right file has no match. Use `--ul` to also emit unpaired left-file records, then `unsparsify` to square up the output: + +
+mlr --icsv --ocsv join --ul -j a -r e -f data/join-x.csv \
+  then unsparsify --fill-with "" \
+  data/join-y.csv
+
+
+a,b,c,f,g
+a,t,1,t,3
+b,u,2,u,2
+c,v,3,,
+
+ +**Right join** keeps all records from the right file. Use `--ur` to also emit unpaired right-file records: + +
+mlr --icsv --ocsv join --ur -j a -r e -f data/join-x.csv \
+  then unsparsify --fill-with "" \
+  data/join-y.csv
+
+
+a,b,c,f,g
+a,t,1,t,3
+b,u,2,u,2
+d,,,w,1
+
+ +**Full outer join** keeps all records from both files. Use both `--ul` and `--ur`: + +
+mlr --icsv --ocsv join --ul --ur -j a -r e -f data/join-x.csv \
+  then unsparsify --fill-with "" \
+  data/join-y.csv
+
+
+a,b,c,f,g
+a,t,1,t,3
+b,u,2,u,2
+d,,,w,1
+c,v,3,,
+
+ +Note that unpaired records are emitted after all paired records, so the output ordering may differ from what a SQL database would produce; you can pipe the output through [`sort`](reference-verbs.md#sort) if you need a particular ordering. + ## Doing multiple joins Suppose we have the following data: @@ -199,6 +277,178 @@ id status name task 30 occupied Alice clean +## Merging several files on a common key when column names collide + +The previous example worked painlessly because the non-key column names -- `name` and `status` -- were different in each lookup file. Now suppose you have several files, each containing measurements of a _different_ quantity, but all with the _same_ column names -- say, one file each for temperature, humidity, and pressure, keyed by timestamp: + +
+cat data/sensor-temp.csv
+
+
+unixTime,minValue,averageValue,maxValue
+1740000000,37.2,37.4,37.6
+1740000060,37.5,37.6,37.7
+1740000120,37.8,37.9,38.0
+
+ +
+cat data/sensor-humidity.csv
+
+
+unixTime,minValue,averageValue,maxValue
+1740000000,50.1,50.3,50.5
+1740000120,52.3,52.5,52.7
+
+ +
+cat data/sensor-pressure.csv
+
+
+unixTime,minValue,averageValue,maxValue
+1740000000,1012.2,1012.3,1012.4
+1740000060,1011.8,1011.9,1012.0
+1740000120,1011.5,1011.6,1011.7
+
+ +(Note that the humidity file is missing a row for the middle timestamp.) + +If we merge these with a `then`-chain of `join` commands, as in the previous section, columns are lost: since every file's non-key columns have the same names, each join step overwrites the values from the step before, and only one file's values survive: + +
+mlr --icsv --opprint \
+  join -j unixTime -f data/sensor-temp.csv \
+  then join -j unixTime -f data/sensor-humidity.csv \
+  data/sensor-pressure.csv
+
+
+unixTime   minValue averageValue maxValue
+1740000000 1012.2   1012.3       1012.4
+1740000120 1011.5   1011.6       1011.7
+
+ +The fix is `join`'s `--lp` (left-prefix) option, which renames the non-key columns coming from each `-f` file so that nothing collides: + +
+mlr --icsv --opprint \
+  join --lp temp: -j unixTime -f data/sensor-temp.csv \
+  then join --lp humidity: -j unixTime -f data/sensor-humidity.csv \
+  data/sensor-pressure.csv
+
+
+unixTime   humidity:minValue humidity:averageValue humidity:maxValue temp:minValue temp:averageValue temp:maxValue minValue averageValue maxValue
+1740000000 50.1              50.3                  50.5              37.2          37.4              37.6          1012.2   1012.3       1012.4
+1740000120 52.3              52.5                  52.7              37.8          37.9              38.0          1011.5   1011.6       1011.7
+
+ +The columns from the file at the end of the command line -- here, the pressure file -- keep their unprefixed names. + +If you want just one value column per file, you can also use `join`'s `--lk` option to keep only that column from each `-f` file, then use [cut](reference-verbs.md#cut) and [label](reference-verbs.md#label) to arrange and rename the output columns: + +
+mlr --icsv --opprint \
+  join --lp temp: --lk averageValue -j unixTime -f data/sensor-temp.csv \
+  then join --lp humidity: --lk averageValue -j unixTime -f data/sensor-humidity.csv \
+  then cut -o -f unixTime,temp:averageValue,humidity:averageValue,averageValue \
+  then label unixTime,temperature,humidity,pressure \
+  data/sensor-pressure.csv
+
+
+unixTime   temperature humidity pressure
+1740000000 37.4        50.3     1012.3
+1740000120 37.9        52.5     1011.6
+
+ +Note that `join` gives inner-join semantics by default, so the timestamp missing from the humidity file has been dropped from the output. (This is also why the `paste` command is not a substitute for `join` here: `paste` matches rows by position, so a row missing from one file shifts that file's remaining values onto the wrong rows.) If you'd rather keep those rows, with empty cells where a file has no data, add `--ul --ur` to each join step, and [unsparsify](reference-verbs.md#unsparsify) afterward: + +
+mlr --icsv --opprint \
+  join --ul --ur --lp temp: --lk averageValue -j unixTime -f data/sensor-temp.csv \
+  then join --ul --ur --lp humidity: --lk averageValue -j unixTime -f data/sensor-humidity.csv \
+  then unsparsify \
+  then cut -o -f unixTime,temp:averageValue,humidity:averageValue,averageValue \
+  then label unixTime,temperature,humidity,pressure \
+  then sort -t unixTime \
+  data/sensor-pressure.csv
+
+
+unixTime   temperature humidity pressure
+1740000000 37.4        50.3     1012.3
+1740000060 37.6        -        1011.9
+1740000120 37.9        52.5     1011.6
+
+ +The `unsparsify` must come before the `cut`-and-`label` step, since `label` renames columns positionally. The `sort` is there because unpaired records may be emitted out of order relative to paired ones. + +## How to preprocess the left file of a join? + +The left file (the `-f` argument to `join`) is opened by the `join` verb itself, so it doesn't pass through the main record stream: a `then`-chain can preprocess the right file(s), but not the left one. + +For example, suppose both of these files have multi-valued `id` fields which need [nest --explode](reference-verbs.md#nest) before joining: + +
+cat data/join-nest-left.csv
+
+
+id,color
+1;2,blue
+3,green
+
+ +
+cat data/join-nest-right.csv
+
+
+id,shape
+1,circle
+2;3,square
+
+ +Using `nest` in a `then`-chain handles the right file, but the left file still has the unexploded `id` value `1;2`, so only `id=3` pairs up: + +
+mlr --csv nest --evar ';' -f id \
+  then join -j id -f data/join-nest-left.csv \
+  data/join-nest-right.csv
+
+
+id,color,shape
+3,green,square
+
+ +One way to preprocess the left file, without creating an intermediate file, is the main-level `--prepipe` flag. The left file inherits the main input options -- including `--prepipe` -- so the specified command is applied to the left file as well as to the right file(s): + +
+mlr --csv --prepipe 'mlr --csv nest --evar ";" -f id' \
+  join -j id -f data/join-nest-left.csv \
+  data/join-nest-right.csv
+
+
+id,color,shape
+1,blue,circle
+2,blue,square
+3,green,square
+
+ +Note that `--prepipe` applies the same command to _every_ input file -- which is just what's wanted here, since both files need the same `nest`. + +Another way, if your shell supports it -- bash, zsh, and ksh do, although plain POSIX `sh` and Windows `cmd` do not -- is [process substitution](https://en.wikipedia.org/wiki/Process_substitution), which lets you preprocess the left file with any command at all, independently of the right file(s): + +
+mlr --csv nest --evar ';' -f id \
+  then join -j id -f <(mlr --csv nest --evar ';' -f id data/join-nest-left.csv) \
+  data/join-nest-right.csv
+
+
+id,color,shape
+1,blue,circle
+2,blue,square
+3,green,square
+
+ +Note that while `mlr join --help` lists verb-level `--prepipe` and `--prepipex` flags, as of Miller 6 these do not take effect for the left file -- please use one of the recipes above instead. + +Thanks to @sonicdoe for the process-substitution tip! + ## Updating a database file with new values from another Suppose you have a durable "database" file, and you periodically receive a download diff --git a/docs/src/questions-about-joins.md.in b/docs/src/questions-about-joins.md.in index b976459fb..445421d16 100644 --- a/docs/src/questions-about-joins.md.in +++ b/docs/src/questions-about-joins.md.in @@ -60,6 +60,50 @@ Thanks to @aborruso for the tip! See also the [record-heterogeneity page](record-heterogeneity.md). +## Doing SQL-style left, right, inner, and full-outer joins + +Miller's `join` verb is defined in terms of _paired_ and _unpaired_ records, rather than SQL-database terminology -- but you can get SQL-style joins using the `--ul` and `--ur` flags (which emit unpaired left-file and right-file records, respectively), along with [`unsparsify`](reference-verbs.md#unsparsify) to fill in empty cells for non-matches. + +Suppose you have the following two data files, where we want to join on the left file's `a` field matching the right file's `e` field: + +GENMD-INCLUDE-ESCAPED(data/join-x.csv) + +GENMD-INCLUDE-ESCAPED(data/join-y.csv) + +In all the following examples, the `-f` file (`data/join-x.csv`) is the left file, and the file in the main input stream (`data/join-y.csv`) is the right file. The flags `-j a -r e` say that the left file's `a` field is matched against the right file's `e` field, with the output join column named `a`. + +**Inner join** -- only matching records -- is what Miller's `join` does by default, since only paired records are emitted: + +GENMD-RUN-COMMAND +mlr --icsv --ocsv join -j a -r e -f data/join-x.csv data/join-y.csv +GENMD-EOF + +**Left join** keeps all records from the left file, with empty cells where the right file has no match. Use `--ul` to also emit unpaired left-file records, then `unsparsify` to square up the output: + +GENMD-RUN-COMMAND +mlr --icsv --ocsv join --ul -j a -r e -f data/join-x.csv \ + then unsparsify --fill-with "" \ + data/join-y.csv +GENMD-EOF + +**Right join** keeps all records from the right file. Use `--ur` to also emit unpaired right-file records: + +GENMD-RUN-COMMAND +mlr --icsv --ocsv join --ur -j a -r e -f data/join-x.csv \ + then unsparsify --fill-with "" \ + data/join-y.csv +GENMD-EOF + +**Full outer join** keeps all records from both files. Use both `--ul` and `--ur`: + +GENMD-RUN-COMMAND +mlr --icsv --ocsv join --ul --ur -j a -r e -f data/join-x.csv \ + then unsparsify --fill-with "" \ + data/join-y.csv +GENMD-EOF + +Note that unpaired records are emitted after all paired records, so the output ordering may differ from what a SQL database would produce; you can pipe the output through [`sort`](reference-verbs.md#sort) if you need a particular ordering. + ## Doing multiple joins Suppose we have the following data: @@ -86,6 +130,118 @@ mlr --icsv --opprint join -f multi-join/name-lookup.csv -j id \ multi-join/input.csv GENMD-EOF +## Merging several files on a common key when column names collide + +The previous example worked painlessly because the non-key column names -- `name` and `status` -- were different in each lookup file. Now suppose you have several files, each containing measurements of a _different_ quantity, but all with the _same_ column names -- say, one file each for temperature, humidity, and pressure, keyed by timestamp: + +GENMD-RUN-COMMAND +cat data/sensor-temp.csv +GENMD-EOF + +GENMD-RUN-COMMAND +cat data/sensor-humidity.csv +GENMD-EOF + +GENMD-RUN-COMMAND +cat data/sensor-pressure.csv +GENMD-EOF + +(Note that the humidity file is missing a row for the middle timestamp.) + +If we merge these with a `then`-chain of `join` commands, as in the previous section, columns are lost: since every file's non-key columns have the same names, each join step overwrites the values from the step before, and only one file's values survive: + +GENMD-RUN-COMMAND +mlr --icsv --opprint \ + join -j unixTime -f data/sensor-temp.csv \ + then join -j unixTime -f data/sensor-humidity.csv \ + data/sensor-pressure.csv +GENMD-EOF + +The fix is `join`'s `--lp` (left-prefix) option, which renames the non-key columns coming from each `-f` file so that nothing collides: + +GENMD-RUN-COMMAND +mlr --icsv --opprint \ + join --lp temp: -j unixTime -f data/sensor-temp.csv \ + then join --lp humidity: -j unixTime -f data/sensor-humidity.csv \ + data/sensor-pressure.csv +GENMD-EOF + +The columns from the file at the end of the command line -- here, the pressure file -- keep their unprefixed names. + +If you want just one value column per file, you can also use `join`'s `--lk` option to keep only that column from each `-f` file, then use [cut](reference-verbs.md#cut) and [label](reference-verbs.md#label) to arrange and rename the output columns: + +GENMD-RUN-COMMAND +mlr --icsv --opprint \ + join --lp temp: --lk averageValue -j unixTime -f data/sensor-temp.csv \ + then join --lp humidity: --lk averageValue -j unixTime -f data/sensor-humidity.csv \ + then cut -o -f unixTime,temp:averageValue,humidity:averageValue,averageValue \ + then label unixTime,temperature,humidity,pressure \ + data/sensor-pressure.csv +GENMD-EOF + +Note that `join` gives inner-join semantics by default, so the timestamp missing from the humidity file has been dropped from the output. (This is also why the `paste` command is not a substitute for `join` here: `paste` matches rows by position, so a row missing from one file shifts that file's remaining values onto the wrong rows.) If you'd rather keep those rows, with empty cells where a file has no data, add `--ul --ur` to each join step, and [unsparsify](reference-verbs.md#unsparsify) afterward: + +GENMD-RUN-COMMAND +mlr --icsv --opprint \ + join --ul --ur --lp temp: --lk averageValue -j unixTime -f data/sensor-temp.csv \ + then join --ul --ur --lp humidity: --lk averageValue -j unixTime -f data/sensor-humidity.csv \ + then unsparsify \ + then cut -o -f unixTime,temp:averageValue,humidity:averageValue,averageValue \ + then label unixTime,temperature,humidity,pressure \ + then sort -t unixTime \ + data/sensor-pressure.csv +GENMD-EOF + +The `unsparsify` must come before the `cut`-and-`label` step, since `label` renames columns positionally. The `sort` is there because unpaired records may be emitted out of order relative to paired ones. + +## How to preprocess the left file of a join? + +The left file (the `-f` argument to `join`) is opened by the `join` verb itself, so it doesn't pass through the main record stream: a `then`-chain can preprocess the right file(s), but not the left one. + +For example, suppose both of these files have multi-valued `id` fields which need [nest --explode](reference-verbs.md#nest) before joining: + +GENMD-RUN-COMMAND +cat data/join-nest-left.csv +GENMD-EOF + +GENMD-RUN-COMMAND +cat data/join-nest-right.csv +GENMD-EOF + +Using `nest` in a `then`-chain handles the right file, but the left file still has the unexploded `id` value `1;2`, so only `id=3` pairs up: + +GENMD-RUN-COMMAND +mlr --csv nest --evar ';' -f id \ + then join -j id -f data/join-nest-left.csv \ + data/join-nest-right.csv +GENMD-EOF + +One way to preprocess the left file, without creating an intermediate file, is the main-level `--prepipe` flag. The left file inherits the main input options -- including `--prepipe` -- so the specified command is applied to the left file as well as to the right file(s): + +GENMD-RUN-COMMAND +mlr --csv --prepipe 'mlr --csv nest --evar ";" -f id' \ + join -j id -f data/join-nest-left.csv \ + data/join-nest-right.csv +GENMD-EOF + +Note that `--prepipe` applies the same command to _every_ input file -- which is just what's wanted here, since both files need the same `nest`. + +Another way, if your shell supports it -- bash, zsh, and ksh do, although plain POSIX `sh` and Windows `cmd` do not -- is [process substitution](https://en.wikipedia.org/wiki/Process_substitution), which lets you preprocess the left file with any command at all, independently of the right file(s): + +GENMD-CARDIFY-HIGHLIGHT-THREE +mlr --csv nest --evar ';' -f id \ + then join -j id -f <(mlr --csv nest --evar ';' -f id data/join-nest-left.csv) \ + data/join-nest-right.csv +id,color,shape +1,blue,circle +2,blue,square +3,green,square +GENMD-EOF + +Note that while `mlr join --help` lists verb-level `--prepipe` and `--prepipex` flags, as of Miller 6 these do not take effect for the left file -- please use one of the recipes above instead. + +Thanks to @sonicdoe for the process-substitution tip! + ## Updating a database file with new values from another Suppose you have a durable "database" file, and you periodically receive a download diff --git a/docs/src/questions-about-then-chaining.md b/docs/src/questions-about-then-chaining.md index 7f558ee8a..cefdb84b3 100644 --- a/docs/src/questions-about-then-chaining.md +++ b/docs/src/questions-about-then-chaining.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/randomizing-examples.md b/docs/src/randomizing-examples.md index 3a389e9a7..3bf7907ac 100644 --- a/docs/src/randomizing-examples.md +++ b/docs/src/randomizing-examples.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/record-heterogeneity.md b/docs/src/record-heterogeneity.md index fe4126121..e4b104892 100644 --- a/docs/src/record-heterogeneity.md +++ b/docs/src/record-heterogeneity.md @@ -1,4 +1,4 @@ - +
Quick links: @@ -474,6 +474,95 @@ a,b,c 7,8,9,10 +### Combining multiple CSV files with different headers + +A common special case of the above: you have several CSV files whose column names overlap but +don't all match -- say, data exported year by year, where columns were added or reordered over +time -- and you want to concatenate them into a single rectangular table, with empty cells where a +file didn't have a given column. + +
+cat data/het/sales-2021.csv
+
+
+id,product,sales
+1,pencil,100
+2,eraser,17
+
+ +
+cat data/het/sales-2022.csv
+
+
+id,product,color,sales
+3,pencil,red,120
+4,eraser,green,32
+
+ +
+cat data/het/sales-2023.csv
+
+
+id,product,sales,returns
+5,pencil,200,6
+6,notebook,41,2
+
+ +Since each file's records take their schema from that file's header line, simply catting the +files together gives a schema change partway through the record stream -- which RFC-4180 CSV +output doesn't allow: + +
+mlr --csv cat data/het/sales-2021.csv data/het/sales-2022.csv data/het/sales-2023.csv
+
+
+id,product,sales
+1,pencil,100
+2,eraser,17
+mlr: CSV schema change: first keys "id,product,sales"; current keys "id,product,color,sales"
+mlr: exiting due to data error
+
+ +The solution is the [`unsparsify`](reference-verbs.md#unsparsify) verb, which we saw above: it +gives every record the union of all field names, filling in empty values (or, using +`--fill-with`, whatever you like) where a record lacks a field: + +
+mlr --csv unsparsify data/het/sales-2021.csv data/het/sales-2022.csv data/het/sales-2023.csv
+
+
+id,product,sales,color,returns
+1,pencil,100,,
+2,eraser,17,,
+3,pencil,120,red,
+4,eraser,32,green,
+5,pencil,200,,6
+6,notebook,41,,2
+
+ +Note that `unsparsify` (without `-f`) is +[non-streaming](streaming-and-memory.md): it can't know the full set of field names until it has +read all input, so it retains all records in memory before producing any output. If your files +are too large for that, but you know the complete list of field names up front, you can use +`unsparsify -f` which is streaming. Fields filled in by `-f` are appended to each record, so if +the input files order their columns differently, follow up with +[`regularize`](reference-verbs.md#regularize) or +[`sort-within-records`](reference-verbs.md#sort-within-records) (also both streaming) to give all +records the same column ordering: + +
+mlr --csv unsparsify -f id,product,sales,color,returns then regularize data/het/sales-2021.csv data/het/sales-2022.csv data/het/sales-2023.csv
+
+
+id,product,sales,color,returns
+1,pencil,100,,
+2,eraser,17,,
+3,pencil,120,red,
+4,eraser,32,green,
+5,pencil,200,,6
+6,notebook,41,,2
+
+ ## Processing heterogeneous data Above we saw how to make heterogeneous data homogeneous, and then how to print heterogeneous data. diff --git a/docs/src/record-heterogeneity.md.in b/docs/src/record-heterogeneity.md.in index e3c128b57..806784c99 100644 --- a/docs/src/record-heterogeneity.md.in +++ b/docs/src/record-heterogeneity.md.in @@ -205,6 +205,55 @@ GENMD-RUN-COMMAND mlr --csv --allow-ragged-csv-input cat data/het/ragged.csv GENMD-EOF +### Combining multiple CSV files with different headers + +A common special case of the above: you have several CSV files whose column names overlap but +don't all match -- say, data exported year by year, where columns were added or reordered over +time -- and you want to concatenate them into a single rectangular table, with empty cells where a +file didn't have a given column. + +GENMD-RUN-COMMAND +cat data/het/sales-2021.csv +GENMD-EOF + +GENMD-RUN-COMMAND +cat data/het/sales-2022.csv +GENMD-EOF + +GENMD-RUN-COMMAND +cat data/het/sales-2023.csv +GENMD-EOF + +Since each file's records take their schema from that file's header line, simply catting the +files together gives a schema change partway through the record stream -- which RFC-4180 CSV +output doesn't allow: + +GENMD-RUN-COMMAND-TOLERATING-ERROR +mlr --csv cat data/het/sales-2021.csv data/het/sales-2022.csv data/het/sales-2023.csv +GENMD-EOF + +The solution is the [`unsparsify`](reference-verbs.md#unsparsify) verb, which we saw above: it +gives every record the union of all field names, filling in empty values (or, using +`--fill-with`, whatever you like) where a record lacks a field: + +GENMD-RUN-COMMAND +mlr --csv unsparsify data/het/sales-2021.csv data/het/sales-2022.csv data/het/sales-2023.csv +GENMD-EOF + +Note that `unsparsify` (without `-f`) is +[non-streaming](streaming-and-memory.md): it can't know the full set of field names until it has +read all input, so it retains all records in memory before producing any output. If your files +are too large for that, but you know the complete list of field names up front, you can use +`unsparsify -f` which is streaming. Fields filled in by `-f` are appended to each record, so if +the input files order their columns differently, follow up with +[`regularize`](reference-verbs.md#regularize) or +[`sort-within-records`](reference-verbs.md#sort-within-records) (also both streaming) to give all +records the same column ordering: + +GENMD-RUN-COMMAND +mlr --csv unsparsify -f id,product,sales,color,returns then regularize data/het/sales-2021.csv data/het/sales-2022.csv data/het/sales-2023.csv +GENMD-EOF + ## Processing heterogeneous data Above we saw how to make heterogeneous data homogeneous, and then how to print heterogeneous data. diff --git a/docs/src/reference-dsl-builtin-functions.md b/docs/src/reference-dsl-builtin-functions.md index e77395981..c918cb8a5 100644 --- a/docs/src/reference-dsl-builtin-functions.md +++ b/docs/src/reference-dsl-builtin-functions.md @@ -1,4 +1,4 @@ - +
Quick links: @@ -59,7 +59,7 @@ Operators are listed here along with functions. In this case, the argument count * [**Hashing functions**](#hashing-functions): [md5](#md5), [sha1](#sha1), [sha256](#sha256), [sha512](#sha512). * [**Higher-order-functions functions**](#higher-order-functions-functions): [any](#any), [apply](#apply), [every](#every), [fold](#fold), [reduce](#reduce), [select](#select), [sort](#sort). * [**Math functions**](#math-functions): [abs](#abs), [acos](#acos), [acosh](#acosh), [asin](#asin), [asinh](#asinh), [atan](#atan), [atan2](#atan2), [atanh](#atanh), [cbrt](#cbrt), [ceil](#ceil), [cos](#cos), [cosh](#cosh), [erf](#erf), [erfc](#erfc), [exp](#exp), [expm1](#expm1), [floor](#floor), [invqnorm](#invqnorm), [log](#log), [log10](#log10), [log1p](#log1p), [logifit](#logifit), [max](#max), [min](#min), [qnorm](#qnorm), [round](#round), [roundm](#roundm), [sgn](#sgn), [sin](#sin), [sinh](#sinh), [sqrt](#sqrt), [tan](#tan), [tanh](#tanh), [urand](#urand), [urand32](#urand32), [urandelement](#urandelement), [urandint](#urandint), [urandrange](#urandrange). -* [**Stats functions**](#stats-functions): [antimode](#antimode), [count](#count), [distinct_count](#distinct_count), [kurtosis](#kurtosis), [maxlen](#maxlen), [mean](#mean), [meaneb](#meaneb), [median](#median), [minlen](#minlen), [mode](#mode), [null_count](#null_count), [percentile](#percentile), [percentiles](#percentiles), [skewness](#skewness), [sort_collection](#sort_collection), [stddev](#stddev), [sum](#sum), [sum2](#sum2), [sum3](#sum3), [sum4](#sum4), [variance](#variance). +* [**Stats functions**](#stats-functions): [antimode](#antimode), [count](#count), [distinct_count](#distinct_count), [kurtosis](#kurtosis), [maxlen](#maxlen), [mean](#mean), [meaneb](#meaneb), [median](#median), [minlen](#minlen), [mode](#mode), [null_count](#null_count), [percentile](#percentile), [percentiles](#percentiles), [skewness](#skewness), [sort_collection](#sort_collection), [sparkline](#sparkline), [stddev](#stddev), [sum](#sum), [sum2](#sum2), [sum3](#sum3), [sum4](#sum4), [variance](#variance). * [**String functions**](#string-functions): [base64_decode](#base64_decode), [base64_encode](#base64_encode), [capitalize](#capitalize), [clean_whitespace](#clean_whitespace), [collapse_whitespace](#collapse_whitespace), [contains](#contains), [format](#format), [gssub](#gssub), [gsub](#gsub), [hex_decode](#hex_decode), [hex_encode](#hex_encode), [index](#index), [latin1_to_utf8](#latin1_to_utf8), [leftpad](#leftpad), [lstrip](#lstrip), [regextract](#regextract), [regextract_or_else](#regextract_or_else), [rightpad](#rightpad), [rstrip](#rstrip), [ssub](#ssub), [strip](#strip), [strlen](#strlen), [strmatch](#strmatch), [strmatchx](#strmatchx), [sub](#sub), [substr](#substr), [substr0](#substr0), [substr1](#substr1), [tolower](#tolower), [toupper](#toupper), [truncate](#truncate), [unformat](#unformat), [unformatx](#unformatx), [utf8_to_latin1](#utf8_to_latin1), [\.](#dot). * [**System functions**](#system-functions): [exec](#exec), [hostname](#hostname), [next](#next), [os](#os), [stat](#stat), [system](#system), [version](#version). * [**Time functions**](#time-functions): [dhms2fsec](#dhms2fsec), [dhms2sec](#dhms2sec), [fsec2dhms](#fsec2dhms), [fsec2hms](#fsec2hms), [gmt2localtime](#gmt2localtime), [gmt2nsec](#gmt2nsec), [gmt2sec](#gmt2sec), [hms2fsec](#hms2fsec), [hms2sec](#hms2sec), [localtime2gmt](#localtime2gmt), [localtime2nsec](#localtime2nsec), [localtime2sec](#localtime2sec), [nsec2gmt](#nsec2gmt), [nsec2gmtdate](#nsec2gmtdate), [nsec2localdate](#nsec2localdate), [nsec2localtime](#nsec2localtime), [sec2dhms](#sec2dhms), [sec2gmt](#sec2gmt), [sec2gmtdate](#sec2gmtdate), [sec2hms](#sec2hms), [sec2localdate](#sec2localdate), [sec2localtime](#sec2localtime), [strfntime](#strfntime), [strfntime_local](#strfntime_local), [strftime](#strftime), [strftime_local](#strftime_local), [strpntime](#strpntime), [strpntime_local](#strpntime_local), [strptime](#strptime), [strptime_local](#strptime_local), [sysntime](#sysntime), [systime](#systime), [systimeint](#systimeint), [upntime](#upntime), [uptime](#uptime). @@ -1156,6 +1156,15 @@ sort_collection (class=stats #args=1) This is a helper function for the percent +### sparkline +
+sparkline  (class=stats #args=1) Returns a string of Unicode block characters (one of β–β–‚β–ƒβ–„β–…β–†β–‡β–ˆ per element) representing the relative magnitudes of values in an array or map, for a compact ASCII/Unicode bar chart. Returns error for non-array/non-map types.
+Examples:
+sparkline([1,2,3,4,5,6,7,8]) is "β–β–‚β–ƒβ–„β–…β–†β–‡β–ˆ"
+sparkline([3,3,3]) is "▁▁▁"
+
+ + ### stddev 
 stddev  (class=stats #args=1) Returns the sample standard deviation of values in an array or map. Returns empty string AKA void for array/map of length less than two; returns error for non-array/non-map types.
@@ -1256,11 +1265,13 @@ contains([1,2,3], 2) gives (error)
 
 ### format
 
-format  (class=string #args=variadic) Using first argument as format string, interpolate remaining arguments in place of each "{}" in the format string. Too-few arguments are treated as the empty string; too-many arguments are discarded.
+format  (class=string #args=variadic) Using first argument as format string, interpolate remaining arguments in place of each "{}" in the format string. Also supports 1-based positional placeholders such as "{1}", which allow arguments to be reused and/or reordered. Plain "{}" placeholders consume arguments sequentially, independently of any positional placeholders. Too-few arguments are treated as the empty string, as are positional placeholders exceeding the number of arguments; too-many arguments are discarded. "{0}" is an error value, since positional placeholders are 1-based.
 Examples:
-format("{}:{}:{}", 1,2)     gives "1:2:".
-format("{}:{}:{}", 1,2,3)   gives "1:2:3".
-format("{}:{}:{}", 1,2,3,4) gives "1:2:3".
+format("{}:{}:{}", 1,2)        gives "1:2:".
+format("{}:{}:{}", 1,2,3)      gives "1:2:3".
+format("{}:{}:{}", 1,2,3,4)    gives "1:2:3".
+format("{1}:{2}:{1}", "a","b") gives "a:b:a".
+format("{2}{}:{1}{}", 3,4)     gives "43:34".
 

 
 
diff --git a/docs/src/reference-dsl-complexity.md b/docs/src/reference-dsl-complexity.md
index de97fa3f0..78273304c 100644
--- a/docs/src/reference-dsl-complexity.md
+++ b/docs/src/reference-dsl-complexity.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/reference-dsl-control-structures.md b/docs/src/reference-dsl-control-structures.md
index 60bb52d95..fd4c4d977 100644
--- a/docs/src/reference-dsl-control-structures.md
+++ b/docs/src/reference-dsl-control-structures.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/reference-dsl-differences.md b/docs/src/reference-dsl-differences.md
index 7bcd813e7..491b605f9 100644
--- a/docs/src/reference-dsl-differences.md
+++ b/docs/src/reference-dsl-differences.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/reference-dsl-errors.md b/docs/src/reference-dsl-errors.md
index b4d620faa..dafa7d4ef 100644
--- a/docs/src/reference-dsl-errors.md
+++ b/docs/src/reference-dsl-errors.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/reference-dsl-filter-statements.md b/docs/src/reference-dsl-filter-statements.md
index 3d2d733f2..0d99b12df 100644
--- a/docs/src/reference-dsl-filter-statements.md
+++ b/docs/src/reference-dsl-filter-statements.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/reference-dsl-higher-order-functions.md b/docs/src/reference-dsl-higher-order-functions.md
index 2b3f4691b..bd15a1848 100644
--- a/docs/src/reference-dsl-higher-order-functions.md
+++ b/docs/src/reference-dsl-higher-order-functions.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/reference-dsl-operators.md b/docs/src/reference-dsl-operators.md
index cdba1ca55..bb0d002db 100644
--- a/docs/src/reference-dsl-operators.md
+++ b/docs/src/reference-dsl-operators.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/reference-dsl-output-statements.md b/docs/src/reference-dsl-output-statements.md
index 8db3ddfdd..87b8372c6 100644
--- a/docs/src/reference-dsl-output-statements.md
+++ b/docs/src/reference-dsl-output-statements.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/reference-dsl-syntax.md b/docs/src/reference-dsl-syntax.md
index 9b51cdd61..6204ebe61 100644
--- a/docs/src/reference-dsl-syntax.md
+++ b/docs/src/reference-dsl-syntax.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/reference-dsl-time.md b/docs/src/reference-dsl-time.md
index 0a3aa721e..9c492a7d9 100644
--- a/docs/src/reference-dsl-time.md
+++ b/docs/src/reference-dsl-time.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
@@ -298,17 +298,29 @@ Available format strings for `strptime`:
 | Pattern | Description |
 |---------|-------------|
 | `%%` |  A literal '%' character. |
+| `%a` |  Weekday as locale’s abbreviated name. |
+| `%A` |  Weekday as locale’s full name. |
 | `%b` |  Month as locale’s abbreviated name. |
 | `%B` |  Month as locale’s full name. |
+| `%c` |  Locale’s date and time representation. Equivalent to `%a %b %e %H:%M:%S %Y`. |
 | `%d` |  Day of the month as a zero-padded decimal number. |
+| `%D` |  Equivalent to `%m/%d/%y`. |
+| `%e` |  Day of the month as a space-padded decimal number. |
 | `%f` |  Microsecond as a decimal number, zero-padded on the left. |
+| `%F` |  Equivalent to `%Y-%m-%d`. |
+| `%h` |  Same as `%b`. |
 | `%H` |  Hour (24-hour clock) as a zero-padded decimal number. |
 | `%I` |  Hour (12-hour clock) as a zero-padded decimal number. |
 | `%j` |  Three-digit day of year, like 004 or 363. |
 | `%m` |  Month as a zero-padded decimal number. |
 | `%M` |  Minute as a zero-padded decimal number. |
 | `%p` |  Locale’s equivalent of either AM or PM. |
+| `%r` |  Equivalent to `%I:%M:%S %p`. |
+| `%R` |  Equivalent to `%H:%M`. |
 | `%S` |  Second as a zero-padded decimal number. |
+| `%T` |  Equivalent to `%H:%M:%S`. |
+| `%x` |  Locale’s date representation. Equivalent to `%m/%d/%y`. |
+| `%X` |  Locale’s time representation. Equivalent to `%H:%M:%S`. |
 | `%y` |  Year without century as a zero-padded decimal number. |
 | `%Y` |  Year with century as a decimal number. |
 | `%z` |  UTC offset in the form +HHMM or -HHMM. |
@@ -481,6 +493,74 @@ sec2dhms  (class=time #args=1) Formats integer seconds as in sec2dhms(500000) =
 sec2hms  (class=time #args=1) Formats integer seconds as in sec2hms(5000) = "01:23:20"
+## Timezone troubleshooting + +Since Miller applies your `TZ` environment variable at startup, a timezone problem can make _any_ +Miller command fail immediately -- even one that does nothing with times, such as `mlr --csv cat +example.csv`. The symptom is an error message like + +
+mlr: TZ environment variable appears malformed: "..."
+
+ +or, from Miller 6.6.0 and earlier, the underlying error from the Go `time` library: + +
+mlr :  time: invalid location name
+
+ +There are two distinct causes. + +**Cause 1: `TZ` is set to a file path.** The Go `time` library, which Miller uses, accepts only +[IANA timezone names](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) such as +`America/New_York` (plus the special values `UTC` and `Local`). Unlike the C library, it does not +accept file paths such as `/usr/share/zoneinfo/US/Eastern`, nor the POSIX form with a leading colon +such as `:/etc/localtime` -- even though those work fine with other tools on your system (and worked +with Miller 5, which used the C library): + +
+export TZ=/usr/share/zoneinfo/US/Eastern
+mlr -n put 'end { print strftime_local(0, "%Y-%m-%d %H:%M:%S %Z") }'
+
+
+mlr: TZ environment variable appears malformed: "/usr/share/zoneinfo/US/Eastern"
+
+ +The remedy is to set `TZ` to the zone's name rather than its file path -- here, `export +TZ=US/Eastern` -- or to unset it (`unset TZ`, or `export TZ=""`), in which case Miller uses the +system's local-time setting. + +**Cause 2: the timezone database is missing.** Go programs such as Miller normally read timezone +definitions from a database on the host system, and (unlike C programs) don't fall back to any +built-in copy. If that database is absent, even a perfectly valid name fails: + +
+mlr --csv cat example.csv
+
+
+mlr: TZ environment variable appears malformed: "Asia/Taipei"
+
+ +The database is normally present on Linux and macOS (at `/usr/share/zoneinfo`), but is often missing on: + +* Windows -- unless [Go itself is installed](https://go.dev/doc/install), since the Go toolchain ships a copy of the database (you don't need to _use_ Go; merely installing it suffices). +* Cygwin, unless a package providing a full timezone database is installed. +* Minimal Linux environments such as Alpine-based or `scratch` Docker containers, where the database is an optional package. + +Remedies, in decreasing order of thoroughness: + +* Install the timezone database: `apk add tzdata` on Alpine, `apt-get install tzdata` on + Debian/Ubuntu, and so on; on Windows, install Go as noted above. +* Point the `ZONEINFO` environment variable at a zoneinfo directory or zip file -- the Go `time` + library checks it before the default system locations. For example, a Go installation provides + one at `$GOROOT/lib/time/zoneinfo.zip`. +* Avoid timezone-name lookups altogether: leave `TZ` unset (or set to `""` or `UTC`, which need no + database), use the UTC-oriented functions such as + [sec2gmt](reference-dsl-builtin-functions.md#sec2gmt) and + [strftime](reference-dsl-builtin-functions.md#strftime), and use numeric fixed offsets such as + `+0200` (matched by `%z` in [strptime](reference-dsl-builtin-functions.md#strptime) format + strings) rather than timezone names. + ## References * Non-Miller-specific list of formatting characters for `strftime` and `strptime`: [https://devhints.io/strftime](https://devhints.io/strftime) diff --git a/docs/src/reference-dsl-time.md.in b/docs/src/reference-dsl-time.md.in index 869a58495..4ff0c9a7b 100644 --- a/docs/src/reference-dsl-time.md.in +++ b/docs/src/reference-dsl-time.md.in @@ -230,17 +230,29 @@ Available format strings for `strptime`: | Pattern | Description | |---------|-------------| | `%%` | A literal '%' character. | +| `%a` | Weekday as locale’s abbreviated name. | +| `%A` | Weekday as locale’s full name. | | `%b` | Month as locale’s abbreviated name. | | `%B` | Month as locale’s full name. | +| `%c` | Locale’s date and time representation. Equivalent to `%a %b %e %H:%M:%S %Y`. | | `%d` | Day of the month as a zero-padded decimal number. | +| `%D` | Equivalent to `%m/%d/%y`. | +| `%e` | Day of the month as a space-padded decimal number. | | `%f` | Microsecond as a decimal number, zero-padded on the left. | +| `%F` | Equivalent to `%Y-%m-%d`. | +| `%h` | Same as `%b`. | | `%H` | Hour (24-hour clock) as a zero-padded decimal number. | | `%I` | Hour (12-hour clock) as a zero-padded decimal number. | | `%j` | Three-digit day of year, like 004 or 363. | | `%m` | Month as a zero-padded decimal number. | | `%M` | Minute as a zero-padded decimal number. | | `%p` | Locale’s equivalent of either AM or PM. | +| `%r` | Equivalent to `%I:%M:%S %p`. | +| `%R` | Equivalent to `%H:%M`. | | `%S` | Second as a zero-padded decimal number. | +| `%T` | Equivalent to `%H:%M:%S`. | +| `%x` | Locale’s date representation. Equivalent to `%m/%d/%y`. | +| `%X` | Locale’s time representation. Equivalent to `%H:%M:%S`. | | `%y` | Year without century as a zero-padded decimal number. | | `%Y` | Year with century as a decimal number. | | `%z` | UTC offset in the form +HHMM or -HHMM. | @@ -358,6 +370,69 @@ GENMD-RUN-COMMAND mlr -F | grep hms GENMD-EOF +## Timezone troubleshooting + +Since Miller applies your `TZ` environment variable at startup, a timezone problem can make _any_ +Miller command fail immediately -- even one that does nothing with times, such as `mlr --csv cat +example.csv`. The symptom is an error message like + +
+mlr: TZ environment variable appears malformed: "..."
+
+ +or, from Miller 6.6.0 and earlier, the underlying error from the Go `time` library: + +
+mlr :  time: invalid location name
+
+ +There are two distinct causes. + +**Cause 1: `TZ` is set to a file path.** The Go `time` library, which Miller uses, accepts only +[IANA timezone names](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) such as +`America/New_York` (plus the special values `UTC` and `Local`). Unlike the C library, it does not +accept file paths such as `/usr/share/zoneinfo/US/Eastern`, nor the POSIX form with a leading colon +such as `:/etc/localtime` -- even though those work fine with other tools on your system (and worked +with Miller 5, which used the C library): + +GENMD-RUN-COMMAND-TOLERATING-ERROR +export TZ=/usr/share/zoneinfo/US/Eastern +mlr -n put 'end { print strftime_local(0, "%Y-%m-%d %H:%M:%S %Z") }' +GENMD-EOF + +The remedy is to set `TZ` to the zone's name rather than its file path -- here, `export +TZ=US/Eastern` -- or to unset it (`unset TZ`, or `export TZ=""`), in which case Miller uses the +system's local-time setting. + +**Cause 2: the timezone database is missing.** Go programs such as Miller normally read timezone +definitions from a database on the host system, and (unlike C programs) don't fall back to any +built-in copy. If that database is absent, even a perfectly valid name fails: + +GENMD-CARDIFY-HIGHLIGHT-ONE +mlr --csv cat example.csv +mlr: TZ environment variable appears malformed: "Asia/Taipei" +GENMD-EOF + +The database is normally present on Linux and macOS (at `/usr/share/zoneinfo`), but is often missing on: + +* Windows -- unless [Go itself is installed](https://go.dev/doc/install), since the Go toolchain ships a copy of the database (you don't need to _use_ Go; merely installing it suffices). +* Cygwin, unless a package providing a full timezone database is installed. +* Minimal Linux environments such as Alpine-based or `scratch` Docker containers, where the database is an optional package. + +Remedies, in decreasing order of thoroughness: + +* Install the timezone database: `apk add tzdata` on Alpine, `apt-get install tzdata` on + Debian/Ubuntu, and so on; on Windows, install Go as noted above. +* Point the `ZONEINFO` environment variable at a zoneinfo directory or zip file -- the Go `time` + library checks it before the default system locations. For example, a Go installation provides + one at `$GOROOT/lib/time/zoneinfo.zip`. +* Avoid timezone-name lookups altogether: leave `TZ` unset (or set to `""` or `UTC`, which need no + database), use the UTC-oriented functions such as + [sec2gmt](reference-dsl-builtin-functions.md#sec2gmt) and + [strftime](reference-dsl-builtin-functions.md#strftime), and use numeric fixed offsets such as + `+0200` (matched by `%z` in [strptime](reference-dsl-builtin-functions.md#strptime) format + strings) rather than timezone names. + ## References * Non-Miller-specific list of formatting characters for `strftime` and `strptime`: [https://devhints.io/strftime](https://devhints.io/strftime) diff --git a/docs/src/reference-dsl-unset-statements.md b/docs/src/reference-dsl-unset-statements.md index d7ced4177..2271e46ae 100644 --- a/docs/src/reference-dsl-unset-statements.md +++ b/docs/src/reference-dsl-unset-statements.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-dsl-user-defined-functions.md b/docs/src/reference-dsl-user-defined-functions.md index 5197701de..ee86a86d2 100644 --- a/docs/src/reference-dsl-user-defined-functions.md +++ b/docs/src/reference-dsl-user-defined-functions.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-dsl-variables.md b/docs/src/reference-dsl-variables.md index cfa29e298..0d8c296c1 100644 --- a/docs/src/reference-dsl-variables.md +++ b/docs/src/reference-dsl-variables.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-dsl.md b/docs/src/reference-dsl.md index b51ddef4a..280909fca 100644 --- a/docs/src/reference-dsl.md +++ b/docs/src/reference-dsl.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-main-arithmetic.md b/docs/src/reference-main-arithmetic.md index b35f9d807..8622d441e 100644 --- a/docs/src/reference-main-arithmetic.md +++ b/docs/src/reference-main-arithmetic.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-main-arrays.md b/docs/src/reference-main-arrays.md index abdc3bb63..2f8bc4f14 100644 --- a/docs/src/reference-main-arrays.md +++ b/docs/src/reference-main-arrays.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-main-auxiliary-commands.md b/docs/src/reference-main-auxiliary-commands.md index 16fa67c09..11e1508a5 100644 --- a/docs/src/reference-main-auxiliary-commands.md +++ b/docs/src/reference-main-auxiliary-commands.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-main-compressed-data.md b/docs/src/reference-main-compressed-data.md index 729cf5bbc..cef5ae607 100644 --- a/docs/src/reference-main-compressed-data.md +++ b/docs/src/reference-main-compressed-data.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-main-data-types.md b/docs/src/reference-main-data-types.md index aa6bce487..604bcef7d 100644 --- a/docs/src/reference-main-data-types.md +++ b/docs/src/reference-main-data-types.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-main-env-vars.md b/docs/src/reference-main-env-vars.md index 295973d58..327867327 100644 --- a/docs/src/reference-main-env-vars.md +++ b/docs/src/reference-main-env-vars.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-main-flag-list.md b/docs/src/reference-main-flag-list.md index b5b4b6502..8ec6051d2 100644 --- a/docs/src/reference-main-flag-list.md +++ b/docs/src/reference-main-flag-list.md @@ -1,4 +1,4 @@ - +
Quick links: @@ -415,6 +415,7 @@ These are flags which are applicable to PPRINT format. * `--fixed {string}`: Fixed width specification. One of 'widths:,,...', left-align, left-align-multi-word, right-align, right-align-multi-word * `--fw {string}`: Shortcut for --fixed left-align-multi-word * `--right`: Right-justifies all fields for PPRINT output. +* `--right-align-numeric`: Right-justifies fields with numeric values for PPRINT output, leaving other fields left-justified. Headers are right-justified over columns whose values are all numeric, so that header and data share the same alignment. Also applies to markdown output, where numeric columns get right-alignment markers (`---:`) in the header-separator line. ## Profiling flags @@ -451,6 +452,9 @@ Notes about line endings: * Default line endings (`--irs` and `--ors`) are newline which is interpreted to accept carriage-return/newline files (e.g. on Windows) for input, and to produce platform-appropriate line endings on output. +* For CSV, CSV-lite, TSV, and TSV-lite output, ORS may be either newline (the + default) or carriage-return/newline: e.g. `--ors crlf` or `--ors '\r\n'` + for RFC-4180-style line endings on any platform. Notes about all other separators: diff --git a/docs/src/reference-main-in-place-processing.md b/docs/src/reference-main-in-place-processing.md index 9081e0459..0942e7dc9 100644 --- a/docs/src/reference-main-in-place-processing.md +++ b/docs/src/reference-main-in-place-processing.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-main-maps.md b/docs/src/reference-main-maps.md index 4d8942d8d..95729f078 100644 --- a/docs/src/reference-main-maps.md +++ b/docs/src/reference-main-maps.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-main-null-data.md b/docs/src/reference-main-null-data.md index 80a378f7f..1a863046c 100644 --- a/docs/src/reference-main-null-data.md +++ b/docs/src/reference-main-null-data.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-main-number-formatting.md b/docs/src/reference-main-number-formatting.md index 627cb1748..e1a761f29 100644 --- a/docs/src/reference-main-number-formatting.md +++ b/docs/src/reference-main-number-formatting.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-main-overview.md b/docs/src/reference-main-overview.md index de241cf5f..16e62a461 100644 --- a/docs/src/reference-main-overview.md +++ b/docs/src/reference-main-overview.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-main-regular-expressions.md b/docs/src/reference-main-regular-expressions.md index b9ed50389..db195c23c 100644 --- a/docs/src/reference-main-regular-expressions.md +++ b/docs/src/reference-main-regular-expressions.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-main-separators.md b/docs/src/reference-main-separators.md index 4f37b75a2..f133e8a04 100644 --- a/docs/src/reference-main-separators.md +++ b/docs/src/reference-main-separators.md @@ -1,4 +1,4 @@ - +
Quick links: @@ -261,8 +261,8 @@ a:4;b:5;c:6;d:>>>,|||;<<< Notes: -* CSV IRS and ORS must be newline, and CSV IFS must be a single character. (CSV-lite does not have these restrictions.) -* TSV IRS and ORS must be newline, and TSV IFS must be a tab. (TSV-lite does not have these restrictions.) +* CSV IRS must be newline -- CR/LF line endings are accepted on input. CSV ORS must be either newline (the default) or carriage-return/newline -- e.g. `--ors crlf` or `--ors '\r\n'` for RFC-4180-style line endings on any platform. CSV IFS must be a single character. (CSV-lite does not have these restrictions.) +* TSV IRS must be newline -- CR/LF line endings are accepted on input. TSV ORS must be either newline (the default) or carriage-return/newline. TSV IFS must be a tab. (TSV-lite does not have these restrictions.) * See the [CSV section](file-formats.md#csvtsvasvusvetc) for information about ASV and USV. * JSON and YAML: ignore separator flags from the command line. * Headerless CSV overlaps quite a bit with NIDX format using comma for IFS. See also the page on [CSV with and without headers](csv-with-and-without-headers.md). @@ -270,15 +270,15 @@ Notes: | | **RS** | **FS** | **PS** | |------------|---------|---------|----------| -| [**CSV**](file-formats.md#csvtsvasvusvetc) | Always `\n`; not alterable * | Default `,`; must be single-character | None | -| [**TSV**](file-formats.md#csvtsvasvusvetc) | Always `\n`; not alterable * | Default `\t`; must be single-character | None | +| [**CSV**](file-formats.md#csvtsvasvusvetc) | Default `\n`; may be set to `\r\n` * | Default `,`; must be single-character | None | +| [**TSV**](file-formats.md#csvtsvasvusvetc) | Default `\n`; may be set to `\r\n` * | Default `\t`; must be single-character | None | | [**CSV-lite**](file-formats.md#csvtsvasvusvetc) | Default `\n` * | Default `,` | None | | [**TSV-lite**](file-formats.md#csvtsvasvusvetc) | Default `\n` * | Default `\t` | None | | [**JSON**](file-formats.md#json) | N/A; records are between `{` and `}` | Always `,`; not alterable | Always `:`; not alterable | | [**YAML**](file-formats.md#yaml) | N/A; documents separated by `---` or single array | N/A; not alterable | Always `:`; not alterable | | [**DCF**](file-formats.md#dcf-debian-control-file) | N/A; paragraphs separated by blank lines | N/A; not alterable | Always `:`; not alterable | | [**DKVP**](file-formats.md#dkvp-key-value-pairs) | Default `\n` | Default `,` | Default `=` | -| [**DKVPX**](file-formats.md#dkvpx-key-value-pairs-with-csv-style-quoting) | Default `\n` | Default `,` | Default `=` | +| [**DKVPX**](file-formats.md#dkvpx-key-value-pairs-with-csv-style-quoting) | Default `\n` | Default `,`; must be single-character for input | Default `=`; must be single-character for input | | [**NIDX**](file-formats.md#nidx-index-numbered-toolkit-style) | Default `\n` | Default space | None | | [**XTAB**](file-formats.md#xtab-vertical-tabular) | Not used; records are separated by an extra FS | `\n` * | Default: space with repeats | | [**PPRINT**](file-formats.md#pprint-pretty-printed-tabular) | Default `\n` * | Space with repeats | None | diff --git a/docs/src/reference-main-separators.md.in b/docs/src/reference-main-separators.md.in index 2fd128abf..940d24bef 100644 --- a/docs/src/reference-main-separators.md.in +++ b/docs/src/reference-main-separators.md.in @@ -151,8 +151,8 @@ GENMD-EOF Notes: -* CSV IRS and ORS must be newline, and CSV IFS must be a single character. (CSV-lite does not have these restrictions.) -* TSV IRS and ORS must be newline, and TSV IFS must be a tab. (TSV-lite does not have these restrictions.) +* CSV IRS must be newline -- CR/LF line endings are accepted on input. CSV ORS must be either newline (the default) or carriage-return/newline -- e.g. `--ors crlf` or `--ors '\r\n'` for RFC-4180-style line endings on any platform. CSV IFS must be a single character. (CSV-lite does not have these restrictions.) +* TSV IRS must be newline -- CR/LF line endings are accepted on input. TSV ORS must be either newline (the default) or carriage-return/newline. TSV IFS must be a tab. (TSV-lite does not have these restrictions.) * See the [CSV section](file-formats.md#csvtsvasvusvetc) for information about ASV and USV. * JSON and YAML: ignore separator flags from the command line. * Headerless CSV overlaps quite a bit with NIDX format using comma for IFS. See also the page on [CSV with and without headers](csv-with-and-without-headers.md). @@ -160,15 +160,15 @@ Notes: | | **RS** | **FS** | **PS** | |------------|---------|---------|----------| -| [**CSV**](file-formats.md#csvtsvasvusvetc) | Always `\n`; not alterable * | Default `,`; must be single-character | None | -| [**TSV**](file-formats.md#csvtsvasvusvetc) | Always `\n`; not alterable * | Default `\t`; must be single-character | None | +| [**CSV**](file-formats.md#csvtsvasvusvetc) | Default `\n`; may be set to `\r\n` * | Default `,`; must be single-character | None | +| [**TSV**](file-formats.md#csvtsvasvusvetc) | Default `\n`; may be set to `\r\n` * | Default `\t`; must be single-character | None | | [**CSV-lite**](file-formats.md#csvtsvasvusvetc) | Default `\n` * | Default `,` | None | | [**TSV-lite**](file-formats.md#csvtsvasvusvetc) | Default `\n` * | Default `\t` | None | | [**JSON**](file-formats.md#json) | N/A; records are between `{` and `}` | Always `,`; not alterable | Always `:`; not alterable | | [**YAML**](file-formats.md#yaml) | N/A; documents separated by `---` or single array | N/A; not alterable | Always `:`; not alterable | | [**DCF**](file-formats.md#dcf-debian-control-file) | N/A; paragraphs separated by blank lines | N/A; not alterable | Always `:`; not alterable | | [**DKVP**](file-formats.md#dkvp-key-value-pairs) | Default `\n` | Default `,` | Default `=` | -| [**DKVPX**](file-formats.md#dkvpx-key-value-pairs-with-csv-style-quoting) | Default `\n` | Default `,` | Default `=` | +| [**DKVPX**](file-formats.md#dkvpx-key-value-pairs-with-csv-style-quoting) | Default `\n` | Default `,`; must be single-character for input | Default `=`; must be single-character for input | | [**NIDX**](file-formats.md#nidx-index-numbered-toolkit-style) | Default `\n` | Default space | None | | [**XTAB**](file-formats.md#xtab-vertical-tabular) | Not used; records are separated by an extra FS | `\n` * | Default: space with repeats | | [**PPRINT**](file-formats.md#pprint-pretty-printed-tabular) | Default `\n` * | Space with repeats | None | diff --git a/docs/src/reference-main-strings.md b/docs/src/reference-main-strings.md index b16b03483..fdfaea4c0 100644 --- a/docs/src/reference-main-strings.md +++ b/docs/src/reference-main-strings.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-main-then-chaining.md b/docs/src/reference-main-then-chaining.md index cb6502102..65ae90341 100644 --- a/docs/src/reference-main-then-chaining.md +++ b/docs/src/reference-main-then-chaining.md @@ -1,4 +1,4 @@ - +
Quick links: diff --git a/docs/src/reference-verbs.md b/docs/src/reference-verbs.md index eb7aad527..547bb2938 100644 --- a/docs/src/reference-verbs.md +++ b/docs/src/reference-verbs.md @@ -1,4 +1,4 @@ - +
Quick links: @@ -54,7 +54,7 @@ These fall into categories as follows: * `awk`-like functionality: [filter](reference-verbs.md#filter), [put](reference-verbs.md#put), [sec2gmt](reference-verbs.md#sec2gmt), [sec2gmtdate](reference-verbs.md#sec2gmtdate), [step](reference-verbs.md#step), [tee](reference-verbs.md#tee). -* Statistically oriented: [bar](reference-verbs.md#bar), [bootstrap](reference-verbs.md#bootstrap), [decimate](reference-verbs.md#decimate), [histogram](reference-verbs.md#histogram), [least-frequent](reference-verbs.md#least-frequent), [most-frequent](reference-verbs.md#most-frequent), [sample](reference-verbs.md#sample), [shuffle](reference-verbs.md#shuffle), [stats1](reference-verbs.md#stats1), [stats2](reference-verbs.md#stats2). +* Statistically oriented: [bar](reference-verbs.md#bar), [bootstrap](reference-verbs.md#bootstrap), [decimate](reference-verbs.md#decimate), [histogram](reference-verbs.md#histogram), [least-frequent](reference-verbs.md#least-frequent), [most-frequent](reference-verbs.md#most-frequent), [rank](reference-verbs.md#rank), [sample](reference-verbs.md#sample), [shuffle](reference-verbs.md#shuffle), [sparkline](reference-verbs.md#sparkline), [stats1](reference-verbs.md#stats1), [stats2](reference-verbs.md#stats2). * Particularly oriented toward [Record Heterogeneity](record-heterogeneity.md), although all Miller commands can handle heterogeneous records: [group-by](reference-verbs.md#group-by), [group-like](reference-verbs.md#group-like), [having-fields](reference-verbs.md#having-fields). @@ -1787,7 +1787,11 @@ Options: --auto Automatically computes limits, ignoring --lo and --hi. Holds all values in memory before producing any output. -o {prefix} Prefix for output field name. Default: no prefix. +-s Print a one-line Unicode sparkline per field instead of per-bin + counts. -h|--help Show this message. +With -s, output is one record per value-field, with a sparkline field +instead of one record per bin. This is just a histogram; there's not too much to say here. A note about binning, by example: Suppose you use `--lo 0.0 --hi 1.0 --nbins 10 -f x`. The input numbers less than 0 or greater than 1 aren't counted in any bin. Input numbers equal to 1 are counted in the last bin. That is, bin 0 has `0.0 < x < 0.1`, bin 1 has `0.1 < x < 0.2`, etc., but bin 9 has `0.9 < x < 1.0`. @@ -1830,6 +1834,20 @@ my_bin_lo my_bin_hi my_x_count my_x2_count my_x3_count 0.9 1 1013 507 341 +Use `-s` for a compact one-line Unicode sparkline per field, rather than one output record per bin. Note this sparklines the *binned counts* -- i.e. the shape of the value distribution -- not the field's raw values in record order; for the latter, use [sparkline](reference-verbs.md#sparkline) instead: + +
+mlr --opprint put '$x2=$x**2;$x3=$x2*$x' \
+  then histogram -f x,x2,x3 --lo 0 --hi 1 --nbins 10 -s \
+  data/medium
+
+
+field lo hi sparkline
+x     0  1  β–ˆβ–β–†β–„β–‚β–„β–…β–…β–„β–…
+x2    0  1  β–ˆβ–ƒβ–‚β–‚β–‚β–β–β–β–β–
+x3    0  1  β–ˆβ–‚β–‚β–‚β–β–β–β–β–β–
+
+ ## join 
@@ -2610,6 +2628,106 @@ See also https://miller.readthedocs.io/reference-dsl for more context.
 
 Please see the [DSL reference](reference-dsl.md) for more information about the expression language for `mlr put`.
 
+## rank
+
+
+mlr rank --help
+

+
+Usage: mlr rank [options]
+For each record's value in specified fields, computes the standard
+competition rank (1,2,2,4,...) of that value among all input records,
+optionally within groups.
+E.g. with input records x=10, x=20, x=20, and x=30, emits output records
+x=10,x_rank=1  x=20,x_rank=2  x=20,x_rank=2  and  x=30,x_rank=4.
+
+Note: by default this is a two-pass algorithm: on the first pass it retains
+input records and their values; on the second pass it computes ranks and
+emits output records, in original input order. This means it produces no
+output until all input is read, but gives correct ranks regardless of input
+order. Use --sorted for a single-pass streaming alternative.
+
+Options:
+-f {a,b,c} Field name(s) to rank.
+-g {d,e,f} Optional group-by-field name(s).
+--sorted   Promise that the input is already sorted by the field(s) being ranked
+           (within each group, if -g is given). This computes rank in a single
+           streaming pass and O(1) space, by comparing each record's value only
+           to the immediately preceding one, rather than buffering all records
+           to compute an order-independent rank. Produces wrong output if the
+           input is not in fact sorted.
+-h|--help  Show this message.
+Example: mlr rank -f x data/rank-example.csv
+Example: mlr rank -f x -g g data/rank-example.csv
+Example: mlr sort -f x then rank -f x --sorted data/rank-example.csv
+

+
+For example, suppose you have the following CSV file:
+
+
+g,x
+a,10
+a,20
+a,20
+a,30
+b,5
+b,5
+b,9
+

+
+Then we can rank each record's `x` among all records:
+
+
+mlr --icsv --opprint rank -f x data/rank-example.csv
+

+
+g x  x_rank
+a 10 4
+a 20 5
+a 20 5
+a 30 7
+b 5  1
+b 5  1
+b 9  3
+

+
+Using `-g` we can rank within each group instead:
+
+
+mlr --icsv --opprint rank -f x -g g data/rank-example.csv
+

+
+g x  x_rank
+a 10 1
+a 20 2
+a 20 2
+a 30 4
+b 5  1
+b 5  1
+b 9  3
+

+
+By default `rank` is a two-pass algorithm: it retains all input records so that it can compute
+ranks which don't depend on input order, even when same-valued records aren't adjacent to one
+another. If you know your input is already sorted on the field(s) you're ranking by -- e.g. by
+piping through `mlr sort` first -- then `--sorted` computes ranks in a single streaming pass
+and O(1) space instead, by comparing each record only to the one immediately before it (within
+its group, if `-g` is given):
+
+
+mlr --icsv --opprint sort -f g -nf x then rank -f x -g g --sorted data/rank-example.csv
+

+
+g x  x_rank
+a 10 1
+a 20 2
+a 20 2
+a 30 4
+b 5  1
+b 5  1
+b 9  3
+

+
 ## regularize
 
 
@@ -2751,8 +2869,10 @@ Options:
            record start.
 -f {a,b,c} Field names to reorder.
 -r {a,b,c} Treat field names as regular expressions. Matched fields are moved to
-           start or end in record order. Example: -r '^YYY,^XXX' puts all YYY-
-           and XXX-prefixed fields first (in record order), then the rest.
+           start or end, grouped by the order the regexes are given; within each
+           group, fields keep their record order. Example: -r '^YYY,^XXX' puts
+           all YYY-prefixed fields first, then all XXX-prefixed fields, then the
+           rest.
 -b {x}     Put field names specified with -f before field name specified by {x},
            if any. If {x} isn't present in a given record, the specified fields
            will not be moved.
@@ -2764,7 +2884,7 @@ Options:
 Examples:
 mlr reorder    -f a,b sends input record "d=4,b=2,a=1,c=3" to "a=1,b=2,d=4,c=3".
 mlr reorder -e -f a,b sends input record "d=4,b=2,a=1,c=3" to "d=4,c=3,a=1,b=2".
-mlr reorder -r '^YYY,^XXX' puts YYY- and XXX-prefixed fields first (record order), then rest.
+mlr reorder -r '^YYY,^XXX' puts YYY-prefixed fields first, then XXX-prefixed fields, then rest.
 

 
 This pivots specified field names to the start or end of the record -- for
@@ -3390,6 +3510,38 @@ a b c
 9 8 7
+## sparkline + +
+mlr sparkline --help
+
+
+Usage: mlr sparkline [options]
+Reduces numeric field(s), across all records in input order, to a compact
+Unicode sparkline -- one block character per record -- for visualizing
+trends. Emits one output record per field. Holds all records in memory
+before producing any output.
+Options:
+-f {a,b,c} Field names to sparkline.
+-h|--help  Show this message.
+
+ +This reduces one or more numeric fields, in input-record order, to a compact +Unicode sparkline -- a quick way to eyeball a trend across records without +plotting software. Contrast with [histogram](reference-verbs.md#histogram) +`-s`, which sparklines the *distribution* of a field's values (binned by +value, order-independent) rather than the field's values in record order. + +
+mlr --icsv --opprint sparkline -f index,quantity,rate example.csv
+
+
+field    n  lo      hi     sparkline
+index    10 11      91     β–β–β–β–„β–…β–†β–†β–†β–ˆβ–ˆ
+quantity 10 13.8103 81.229 β–„β–ˆβ–β–ˆβ–ˆβ–ˆβ–ˆβ–†β–†β–‡
+rate     10 0.013   9.887  β–ˆβ–β–ƒβ–†β–‡β–ˆβ–…β–„β–‡β–‡
+
+ ## sparsify 
@@ -4045,7 +4197,7 @@ Usage: mlr summary [options]
 Show summary statistics about the input data.
 
 All summarizers:
-  field_type      string, int, etc. -- if a column has mixed types, all encountered types are printed
+  field_type      string, int, etc. -- if a column has mixed types, all encountered types are printed (see notes below)
   count           +1 for every instance of the field across all records in the input record stream
   null_count      count of field values either empty string or JSON null
   distinct_count  count of distinct values for the field
@@ -4075,6 +4227,8 @@ Notes:
 * min, p25, median, p75, and max work for strings as well as numbers
 * Distinct-counts are computed on string representations -- so 4.1 and 4.10 are counted as distinct here.
 * If the mode is not unique in the input data, the first-encountered value is reported as the mode.
+* A field_type of "int-string", "empty-string", etc. means the column contains values of mixed types --
+  all types encountered are printed, hyphen-joined, in the order first encountered.
 
 Options:
 -a {mean,sum,etc.} Use only the specified summarizers.
@@ -4418,6 +4572,9 @@ Prints distinct values for specified field names. With -c, same as
 count-distinct. For uniq, -f is a synonym for -g. Output fields are
 written in the order in which they are named with -g or -f, not in the
 order in which they appear in the input records.
+To deduplicate records by one or more fields while keeping all other
+fields, use head: e.g. "mlr head -n 1 -g hash" keeps the first record
+for each distinct value of the hash field, with all fields intact.
 
 Options:
 -g {d,e,f} Group-by field names for uniq counts.
@@ -4528,6 +4685,64 @@ count
 18
+Note that `mlr uniq -g` outputs only the group-by columns. If you want to deduplicate +records by one or more fields, while keeping all the other columns, you can use +[head](reference-verbs.md#head) with `-n 1` and `-g`, which keeps the first record for +each distinct combination of the group-by fields: + +
+mlr --c2p head -n 1 -g color,shape then sort -f color,shape data/colored-shapes.csv
+
+
+color  shape    flag i     u        v        w        x
+blue   circle   0    1075  0.780359 0.331467 0.042890 5.725366
+blue   square   0    1604  0.656744 0.687258 0.312663 4.783385
+blue   triangle 0    1105  0.441773 0.445977 0.632936 4.306461
+green  circle   1    2102  0.083514 0.545773 0.518673 5.084667
+green  square   0    765   0.668443 0.016056 0.465615 5.434589
+green  triangle 1    632   0.151301 0.403468 0.051213 5.955109
+orange circle   1    23462 0.995404 0.023490 0.615945 4.749993
+orange square   0    8109  0.776857 0.741520 0.300047 6.671697
+orange triangle 0    2935  0.517583 0.089891 0.901171 4.265854
+purple circle   0    1573  0.997098 0.193719 0.466933 6.253743
+purple square   0    458   0.259926 0.824322 0.723735 6.854221
+purple triangle 0    257   0.435535 0.859129 0.812290 5.753095
+red    circle   1    84    0.209017 0.290052 0.138103 5.065034
+red    square   1    80    0.219668 0.001257 0.792778 2.944117
+red    triangle 0    620   0.427818 0.396070 0.466908 6.075594
+yellow circle   1    370   0.603365 0.423708 0.639785 7.006414
+yellow square   1    546   0.997474 0.676003 0.418644 3.901026
+yellow triangle 1    56    0.632170 0.988721 0.436498 5.798188
+
+ +If you sort the input first, you control which record is kept for each group -- for +example, the one with the highest value of the `u` field: + +
+mlr --c2p sort -nr u then head -n 1 -g color,shape then sort -f color,shape data/colored-shapes.csv
+
+
+color  shape    flag i      u        v        w        x
+blue   circle   1    463634 0.999678 0.963520 0.476339 5.499486
+blue   square   0    99778  0.999969 0.660817 0.553475 5.623643
+blue   triangle 1    497983 0.993693 0.578201 0.477333 4.353290
+green  circle   0    279380 0.999908 0.795141 0.496896 5.600738
+green  square   1    214778 0.999936 0.174019 0.512225 3.508542
+green  triangle 0    267800 0.990811 0.046011 0.522084 5.566473
+orange circle   1    23462  0.995404 0.023490 0.615945 4.749993
+orange square   0    268054 0.998885 0.155239 0.488076 2.753851
+orange triangle 0    391556 0.984002 0.725036 0.491335 6.014094
+purple circle   0    1573   0.997098 0.193719 0.466933 6.253743
+purple square   0    5356   0.999647 0.121173 0.656153 4.255321
+purple triangle 0    450611 0.998687 0.303774 0.515493 5.365962
+red    circle   1    459124 0.999461 0.959222 0.505147 7.091866
+red    square   0    206305 0.999882 0.468152 0.458562 4.890052
+red    triangle 0    300634 0.999661 0.072334 0.486170 5.751868
+yellow circle   1    82010  0.999923 0.368765 0.533363 5.969732
+yellow square   1    87091  0.998947 0.809461 0.522049 5.606017
+yellow triangle 1    89217  0.995942 0.494142 0.512964 4.210098
+
+ The second main way to use `mlr uniq` is without group-by columns, using `-a` instead: 
diff --git a/docs/src/reference-verbs.md.in b/docs/src/reference-verbs.md.in
index 6bfb2273d..1d4b74da3 100644
--- a/docs/src/reference-verbs.md.in
+++ b/docs/src/reference-verbs.md.in
@@ -27,7 +27,7 @@ These fall into categories as follows:
 
 * `awk`-like functionality: [filter](reference-verbs.md#filter), [put](reference-verbs.md#put), [sec2gmt](reference-verbs.md#sec2gmt), [sec2gmtdate](reference-verbs.md#sec2gmtdate), [step](reference-verbs.md#step), [tee](reference-verbs.md#tee).
 
-* Statistically oriented: [bar](reference-verbs.md#bar), [bootstrap](reference-verbs.md#bootstrap), [decimate](reference-verbs.md#decimate), [histogram](reference-verbs.md#histogram), [least-frequent](reference-verbs.md#least-frequent), [most-frequent](reference-verbs.md#most-frequent), [sample](reference-verbs.md#sample), [shuffle](reference-verbs.md#shuffle), [stats1](reference-verbs.md#stats1), [stats2](reference-verbs.md#stats2).
+* Statistically oriented: [bar](reference-verbs.md#bar), [bootstrap](reference-verbs.md#bootstrap), [decimate](reference-verbs.md#decimate), [histogram](reference-verbs.md#histogram), [least-frequent](reference-verbs.md#least-frequent), [most-frequent](reference-verbs.md#most-frequent), [rank](reference-verbs.md#rank), [sample](reference-verbs.md#sample), [shuffle](reference-verbs.md#shuffle), [sparkline](reference-verbs.md#sparkline), [stats1](reference-verbs.md#stats1), [stats2](reference-verbs.md#stats2).
 
 * Particularly oriented toward [Record Heterogeneity](record-heterogeneity.md), although all Miller commands can handle heterogeneous records: [group-by](reference-verbs.md#group-by), [group-like](reference-verbs.md#group-like), [having-fields](reference-verbs.md#having-fields).
 
@@ -579,6 +579,14 @@ mlr --opprint put '$x2=$x**2;$x3=$x2*$x' \
   data/medium
 GENMD-EOF
 
+Use `-s` for a compact one-line Unicode sparkline per field, rather than one output record per bin. Note this sparklines the *binned counts* -- i.e. the shape of the value distribution -- not the field's raw values in record order; for the latter, use [sparkline](reference-verbs.md#sparkline) instead:
+
+GENMD-RUN-COMMAND
+mlr --opprint put '$x2=$x**2;$x3=$x2*$x' \
+  then histogram -f x,x2,x3 --lo 0 --hi 1 --nbins 10 -s \
+  data/medium
+GENMD-EOF
+
 ## join
 
 GENMD-RUN-COMMAND
@@ -788,6 +796,39 @@ GENMD-EOF
 
 Please see the [DSL reference](reference-dsl.md) for more information about the expression language for `mlr put`.
 
+## rank
+
+GENMD-RUN-COMMAND
+mlr rank --help
+GENMD-EOF
+
+For example, suppose you have the following CSV file:
+
+GENMD-INCLUDE-ESCAPED(data/rank-example.csv)
+
+Then we can rank each record's `x` among all records:
+
+GENMD-RUN-COMMAND
+mlr --icsv --opprint rank -f x data/rank-example.csv
+GENMD-EOF
+
+Using `-g` we can rank within each group instead:
+
+GENMD-RUN-COMMAND
+mlr --icsv --opprint rank -f x -g g data/rank-example.csv
+GENMD-EOF
+
+By default `rank` is a two-pass algorithm: it retains all input records so that it can compute
+ranks which don't depend on input order, even when same-valued records aren't adjacent to one
+another. If you know your input is already sorted on the field(s) you're ranking by -- e.g. by
+piping through `mlr sort` first -- then `--sorted` computes ranks in a single streaming pass
+and O(1) space instead, by comparing each record only to the one immediately before it (within
+its group, if `-g` is given):
+
+GENMD-RUN-COMMAND
+mlr --icsv --opprint sort -f g -nf x then rank -f x -g g --sorted data/rank-example.csv
+GENMD-EOF
+
 ## regularize
 
 GENMD-RUN-COMMAND
@@ -1017,6 +1058,22 @@ GENMD-RUN-COMMAND
 mlr --ijson --opprint sort-within-records data/sort-within-records.json
 GENMD-EOF
 
+## sparkline
+
+GENMD-RUN-COMMAND
+mlr sparkline --help
+GENMD-EOF
+
+This reduces one or more numeric fields, in input-record order, to a compact
+Unicode sparkline -- a quick way to eyeball a trend across records without
+plotting software. Contrast with [histogram](reference-verbs.md#histogram)
+`-s`, which sparklines the *distribution* of a field's values (binned by
+value, order-independent) rather than the field's values in record order.
+
+GENMD-RUN-COMMAND
+mlr --icsv --opprint sparkline -f index,quantity,rate example.csv
+GENMD-EOF
+
 ## sparsify
 
 GENMD-RUN-COMMAND
@@ -1315,6 +1372,22 @@ GENMD-RUN-COMMAND
 mlr --c2p uniq -n -g color,shape data/colored-shapes.csv
 GENMD-EOF
 
+Note that `mlr uniq -g` outputs only the group-by columns. If you want to deduplicate
+records by one or more fields, while keeping all the other columns, you can use
+[head](reference-verbs.md#head) with `-n 1` and `-g`, which keeps the first record for
+each distinct combination of the group-by fields:
+
+GENMD-RUN-COMMAND
+mlr --c2p head -n 1 -g color,shape then sort -f color,shape data/colored-shapes.csv
+GENMD-EOF
+
+If you sort the input first, you control which record is kept for each group -- for
+example, the one with the highest value of the `u` field:
+
+GENMD-RUN-COMMAND
+mlr --c2p sort -nr u then head -n 1 -g color,shape then sort -f color,shape data/colored-shapes.csv
+GENMD-EOF
+
 The second main way to use `mlr uniq` is without group-by columns, using `-a` instead:
 
 GENMD-RUN-COMMAND
diff --git a/docs/src/release-docs.md b/docs/src/release-docs.md
index fc7e2a396..534b3ddd8 100644
--- a/docs/src/release-docs.md
+++ b/docs/src/release-docs.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/repl.md b/docs/src/repl.md
index 71de28b33..9d04e2d9c 100644
--- a/docs/src/repl.md
+++ b/docs/src/repl.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/scripting.md b/docs/src/scripting.md
index 953687c81..384f16911 100644
--- a/docs/src/scripting.md
+++ b/docs/src/scripting.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/shapes-of-data.md b/docs/src/shapes-of-data.md
index f97040543..491311211 100644
--- a/docs/src/shapes-of-data.md
+++ b/docs/src/shapes-of-data.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
@@ -388,3 +388,89 @@ outer=3,middle=31,inner1=313,inner2=314
 See also the [record-heterogeneity page](record-heterogeneity.md); see in
 particular the [`regularize` verb](reference-verbs.md#regularize) for a way to
 do this with much less keystroking.
+
+## Transposing very wide data
+
+If your data has many columns -- hundreds, say -- then tabular output formats
+such as CSV or [PPRINT](file-formats.md#pprint-pretty-printed-tabular) become
+hard to read on-screen. One option is [XTAB output
+format](file-formats.md#xtab-vertical-tabular), which prints each record
+vertically -- one field per line, with a blank line between records:
+
+
+mlr --icsv --oxtab head -n 2 example.csv
+

+
+color    yellow
+shape    triangle
+flag     true
+k        1
+index    11
+quantity 43.6498
+rate     9.8870
+
+color    red
+shape    square
+flag     true
+k        2
+index    15
+quantity 79.2778
+rate     0.0130
+

+
+Another option is a full transpose -- turning each input column into an output
+row. Since Miller keys data by field name, rather than by positional index,
+there isn't a built-in `transpose` verb -- but you can get the same effect
+using [out-of-stream variables](reference-dsl-variables.md#out-of-stream-variables).
+Here, the `-N` flag (same as `--implicit-csv-header --headerless-csv-output`)
+makes Miller treat the header line as data -- so field keys are the positional
+names `1`, `2`, `3`, etc. on both input and output, and the header line
+transposes right along with the data lines:
+
+
+mlr --csv -N put -q '
+  for (k, v in $*) {
+    @transpose[k][NR] = v;
+  }
+  end {
+    emit @transpose;
+  }
+' example.csv
+

+
+color,yellow,red,red,red,purple,red,purple,yellow,yellow,purple
+shape,triangle,square,circle,square,triangle,square,triangle,circle,circle,square
+flag,true,true,true,false,false,false,false,true,true,false
+k,1,2,3,4,5,6,7,8,9,10
+index,11,15,16,48,51,64,65,73,87,91
+quantity,43.6498,79.2778,13.8103,77.5542,81.2290,77.1991,80.1405,63.9785,63.5058,72.3735
+rate,9.8870,0.0130,2.9010,7.4670,8.5910,9.5310,5.8240,4.2370,8.3350,8.2430
+

+
+For an on-screen view of the same, use `--opprint` to align the columns:
+
+
+mlr --icsv --opprint -N put -q '
+  for (k, v in $*) {
+    @transpose[k][NR] = v;
+  }
+  end {
+    emit @transpose;
+  }
+' example.csv
+

+
+color    yellow   red     red     red     purple   red     purple   yellow  yellow  purple
+shape    triangle square  circle  square  triangle square  triangle circle  circle  square
+flag     true     true    true    false   false    false   false    true    true    false
+k        1        2       3       4       5        6       7        8       9       10
+index    11       15      16      48      51       64      65       73      87      91
+quantity 43.6498  79.2778 13.8103 77.5542 81.2290  77.1991 80.1405  63.9785 63.5058 72.3735
+rate     9.8870   0.0130  2.9010  7.4670  8.5910   9.5310  5.8240   4.2370  8.3350  8.2430
+

+
+Note that this accumulates the entire file in memory before producing any
+output -- as any full transpose must.
+
+Thanks to @Fravadona on [issue 321](https://github.com/johnkerl/miller/issues/321)
+for the original version of this recipe.
diff --git a/docs/src/shapes-of-data.md.in b/docs/src/shapes-of-data.md.in
index 3636f406d..0b81d4c10 100644
--- a/docs/src/shapes-of-data.md.in
+++ b/docs/src/shapes-of-data.md.in
@@ -209,3 +209,54 @@ GENMD-EOF
 See also the [record-heterogeneity page](record-heterogeneity.md); see in
 particular the [`regularize` verb](reference-verbs.md#regularize) for a way to
 do this with much less keystroking.
+
+## Transposing very wide data
+
+If your data has many columns -- hundreds, say -- then tabular output formats
+such as CSV or [PPRINT](file-formats.md#pprint-pretty-printed-tabular) become
+hard to read on-screen. One option is [XTAB output
+format](file-formats.md#xtab-vertical-tabular), which prints each record
+vertically -- one field per line, with a blank line between records:
+
+GENMD-RUN-COMMAND
+mlr --icsv --oxtab head -n 2 example.csv
+GENMD-EOF
+
+Another option is a full transpose -- turning each input column into an output
+row. Since Miller keys data by field name, rather than by positional index,
+there isn't a built-in `transpose` verb -- but you can get the same effect
+using [out-of-stream variables](reference-dsl-variables.md#out-of-stream-variables).
+Here, the `-N` flag (same as `--implicit-csv-header --headerless-csv-output`)
+makes Miller treat the header line as data -- so field keys are the positional
+names `1`, `2`, `3`, etc. on both input and output, and the header line
+transposes right along with the data lines:
+
+GENMD-RUN-COMMAND
+mlr --csv -N put -q '
+  for (k, v in $*) {
+    @transpose[k][NR] = v;
+  }
+  end {
+    emit @transpose;
+  }
+' example.csv
+GENMD-EOF
+
+For an on-screen view of the same, use `--opprint` to align the columns:
+
+GENMD-RUN-COMMAND
+mlr --icsv --opprint -N put -q '
+  for (k, v in $*) {
+    @transpose[k][NR] = v;
+  }
+  end {
+    emit @transpose;
+  }
+' example.csv
+GENMD-EOF
+
+Note that this accumulates the entire file in memory before producing any
+output -- as any full transpose must.
+
+Thanks to @Fravadona on [issue 321](https://github.com/johnkerl/miller/issues/321)
+for the original version of this recipe.
diff --git a/docs/src/shell-commands.md b/docs/src/shell-commands.md
index e22713a90..98c28730e 100644
--- a/docs/src/shell-commands.md
+++ b/docs/src/shell-commands.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/shell-completion.md b/docs/src/shell-completion.md
index 19add90f7..e48e4084d 100644
--- a/docs/src/shell-completion.md
+++ b/docs/src/shell-completion.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/sorting.md b/docs/src/sorting.md
index 7d876eda2..19eef67be 100644
--- a/docs/src/sorting.md
+++ b/docs/src/sorting.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/special-symbols-and-formatting.md b/docs/src/special-symbols-and-formatting.md
index 58f477ee6..0a0d8128a 100644
--- a/docs/src/special-symbols-and-formatting.md
+++ b/docs/src/special-symbols-and-formatting.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/sql-examples.md b/docs/src/sql-examples.md
index 781b0003a..e80d742b7 100644
--- a/docs/src/sql-examples.md
+++ b/docs/src/sql-examples.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/statistics-examples.md b/docs/src/statistics-examples.md
index 2e80e8a39..5f0bea7d9 100644
--- a/docs/src/statistics-examples.md
+++ b/docs/src/statistics-examples.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/streaming-and-memory.md b/docs/src/streaming-and-memory.md
index e1b0ad183..e552d2cde 100644
--- a/docs/src/streaming-and-memory.md
+++ b/docs/src/streaming-and-memory.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
@@ -100,6 +100,7 @@ They are memory-friendly, and they don't wait for end of input to produce their
 * [merge-fields](reference-verbs.md#merge-fields)
 * [nest](reference-verbs.md#nest) -- if not `implode-values-across-records`
 * [nothing](reference-verbs.md#nothing)
+* [rank](reference-verbs.md#rank) -- if `--sorted`
 * [regularize](reference-verbs.md#regularize)
 * [rename](reference-verbs.md#rename)
 * [reorder](reference-verbs.md#reorder)
@@ -131,6 +132,7 @@ They are memory-unfriendly, and they wait for end of input to produce their outp
 * [least-frequent](reference-verbs.md#least-frequent)
 * [most-frequent](reference-verbs.md#most-frequent)
 * [nest](reference-verbs.md#nest) -- if `implode-values-across-records`
+* [rank](reference-verbs.md#rank) -- if not `--sorted`
 * [remove-empty-columns](reference-verbs.md#remove-empty-columns)
 * [reshape](reference-verbs.md#reshape) -- if long-to-wide
 * [sample](reference-verbs.md#sample)
diff --git a/docs/src/streaming-and-memory.md.in b/docs/src/streaming-and-memory.md.in
index 07e3018dc..f55e18b33 100644
--- a/docs/src/streaming-and-memory.md.in
+++ b/docs/src/streaming-and-memory.md.in
@@ -84,6 +84,7 @@ They are memory-friendly, and they don't wait for end of input to produce their
 * [merge-fields](reference-verbs.md#merge-fields)
 * [nest](reference-verbs.md#nest) -- if not `implode-values-across-records`
 * [nothing](reference-verbs.md#nothing)
+* [rank](reference-verbs.md#rank) -- if `--sorted`
 * [regularize](reference-verbs.md#regularize)
 * [rename](reference-verbs.md#rename)
 * [reorder](reference-verbs.md#reorder)
@@ -115,6 +116,7 @@ They are memory-unfriendly, and they wait for end of input to produce their outp
 * [least-frequent](reference-verbs.md#least-frequent)
 * [most-frequent](reference-verbs.md#most-frequent)
 * [nest](reference-verbs.md#nest) -- if `implode-values-across-records`
+* [rank](reference-verbs.md#rank) -- if not `--sorted`
 * [remove-empty-columns](reference-verbs.md#remove-empty-columns)
 * [reshape](reference-verbs.md#reshape) -- if long-to-wide
 * [sample](reference-verbs.md#sample)
diff --git a/docs/src/structure-of-these-documents.md b/docs/src/structure-of-these-documents.md
index cdaeef8a9..4e3f1a5dc 100644
--- a/docs/src/structure-of-these-documents.md
+++ b/docs/src/structure-of-these-documents.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/troubleshooting-csv-and-json-input.md b/docs/src/troubleshooting-csv-and-json-input.md
index 449b4a854..45bbe8589 100644
--- a/docs/src/troubleshooting-csv-and-json-input.md
+++ b/docs/src/troubleshooting-csv-and-json-input.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
@@ -30,7 +30,9 @@ When Miller reports a parse error, it is often useful to first identify whether
 - For CSV generated by spreadsheets or hand-written scripts, `--csv-trim-leading-space` can help
   when fields look like `"foo", "bar"`, where the second field has a leading space before the quote.
 - For non-standard CSV that contains stray quote characters, `--lazy-quotes` can help Miller accept
-  input that is not fully RFC-4180 compliant.
+  input that is not fully RFC-4180 compliant. Note that a field with an unmatched _opening_ quote
+  still absorbs field separators and newlines until the next quote character or end of file; see
+  [Handling stray quote characters](file-formats.md#handling-stray-quote-characters).
 - For headerless CSV, add `--implicit-csv-header` (or `-N` when you also want headerless output) so
   fields are addressed as `$1`, `$2`, and so on. See also
   [CSV, with and without headers](csv-with-and-without-headers.md).
diff --git a/docs/src/troubleshooting-csv-and-json-input.md.in b/docs/src/troubleshooting-csv-and-json-input.md.in
index 1bcdd3074..543e7ab7f 100644
--- a/docs/src/troubleshooting-csv-and-json-input.md.in
+++ b/docs/src/troubleshooting-csv-and-json-input.md.in
@@ -14,7 +14,9 @@ When Miller reports a parse error, it is often useful to first identify whether
 - For CSV generated by spreadsheets or hand-written scripts, `--csv-trim-leading-space` can help
   when fields look like `"foo", "bar"`, where the second field has a leading space before the quote.
 - For non-standard CSV that contains stray quote characters, `--lazy-quotes` can help Miller accept
-  input that is not fully RFC-4180 compliant.
+  input that is not fully RFC-4180 compliant. Note that a field with an unmatched _opening_ quote
+  still absorbs field separators and newlines until the next quote character or end of file; see
+  [Handling stray quote characters](file-formats.md#handling-stray-quote-characters).
 - For headerless CSV, add `--implicit-csv-header` (or `-N` when you also want headerless output) so
   fields are addressed as `$1`, `$2`, and so on. See also
   [CSV, with and without headers](csv-with-and-without-headers.md).
diff --git a/docs/src/two-pass-algorithms.md b/docs/src/two-pass-algorithms.md
index e475aebf3..bb9164eb4 100644
--- a/docs/src/two-pass-algorithms.md
+++ b/docs/src/two-pass-algorithms.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/unix-toolkit-context.md b/docs/src/unix-toolkit-context.md
index ffc8ede78..28a6765a1 100644
--- a/docs/src/unix-toolkit-context.md
+++ b/docs/src/unix-toolkit-context.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/vimrc.md b/docs/src/vimrc.md
index 25c1c4635..74fa1fed9 100644
--- a/docs/src/vimrc.md
+++ b/docs/src/vimrc.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/docs/src/why.md b/docs/src/why.md
index aa00458be..d7bbc7537 100644
--- a/docs/src/why.md
+++ b/docs/src/why.md
@@ -1,4 +1,4 @@
-
+
 

 
 Quick links:
diff --git a/go.mod b/go.mod
index 24e22c7fd..43e2ce895 100644
--- a/go.mod
+++ b/go.mod
@@ -30,9 +30,9 @@ require (
 	github.com/pkg/profile v1.7.0
 	github.com/rivo/uniseg v0.4.7
 	github.com/stretchr/testify v1.11.1
-	golang.org/x/sys v0.46.0
-	golang.org/x/term v0.44.0
-	golang.org/x/text v0.38.0
+	golang.org/x/sys v0.47.0
+	golang.org/x/term v0.45.0
+	golang.org/x/text v0.40.0
 	gopkg.in/yaml.v3 v3.0.1
 	pault.ag/go/debian v0.21.0
 )
@@ -50,9 +50,9 @@ require (
 	github.com/segmentio/asm v1.1.3 // indirect
 	github.com/segmentio/encoding v0.5.4 // indirect
 	github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
-	golang.org/x/crypto v0.45.0 // indirect
+	golang.org/x/crypto v0.52.0 // indirect
 	golang.org/x/oauth2 v0.35.0 // indirect
-	golang.org/x/tools v0.45.0 // indirect
+	golang.org/x/tools v0.47.0 // indirect
 	gonum.org/v1/gonum v0.16.0 // indirect
 	pault.ag/go/topsort v0.1.1 // indirect
 )
diff --git a/go.sum b/go.sum
index 634757f71..d3e52db63 100644
--- a/go.sum
+++ b/go.sum
@@ -63,19 +63,19 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
 github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
+golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
 golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
 golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
-golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
-golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
-golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
-golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
-golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
-golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
+golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
+golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
 gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
diff --git a/man/manpage.txt b/man/manpage.txt
index 0e234b965..a5e8bd3b8 100644
--- a/man/manpage.txt
+++ b/man/manpage.txt
@@ -207,11 +207,11 @@
        count-similar cut decimate describe fill-down fill-empty filter flatten
        format-values fraction gap grep group-by group-like gsub having-fields head
        histogram json-parse json-stringify join label latin1-to-utf8 least-frequent
-       merge-fields most-frequent nest nothing put regularize remove-empty-columns
-       rename reorder repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle
-       skip-trivial-records sort sort-within-records sparsify split ssub stats1
-       stats2 step sub summary surv tac tail tee template top utf8-to-latin1
-       unflatten uniq unspace unsparsify
+       merge-fields most-frequent nest nothing put rank regularize
+       remove-empty-columns rename reorder repeat reshape sample sec2gmtdate sec2gmt
+       seqgen shuffle skip-trivial-records sort sort-within-records sparkline
+       sparsify split ssub stats1 stats2 step sub summary surv tac tail tee template
+       top utf8-to-latin1 unflatten uniq unspace unsparsify
 
 1mFUNCTION LIST0m
        abs acos acosh antimode any append apply arrayify asin asinh asserting_absent
@@ -236,14 +236,14 @@
        null_count os percentile percentiles pow qnorm reduce regextract
        regextract_or_else rightpad round roundm rstrip sec2dhms sec2gmt sec2gmtdate
        sec2hms sec2localdate sec2localtime select sgn sha1 sha256 sha512 sin sinh
-       skewness sort sort_collection splita splitax splitkv splitkvx splitnv splitnvx
-       sqrt ssub stat stddev strfntime strfntime_local strftime strftime_local string
-       strip strlen strmatch strmatchx strpntime strpntime_local strptime
-       strptime_local sub substr substr0 substr1 sum sum2 sum3 sum4 sysntime system
-       systime systimeint tan tanh tolower toupper truncate typeof unflatten unformat
-       unformatx upntime uptime urand urand32 urandelement urandint urandrange
-       utf8_to_latin1 variance version ! != !=~ % & && * ** + - . .* .+ .- ./ / // <
-       << <= <=> == =~ > >= >> >>> ?: ?? ??? ^ ^^ | || ~
+       skewness sort sort_collection sparkline splita splitax splitkv splitkvx
+       splitnv splitnvx sqrt ssub stat stddev strfntime strfntime_local strftime
+       strftime_local string strip strlen strmatch strmatchx strpntime
+       strpntime_local strptime strptime_local sub substr substr0 substr1 sum sum2
+       sum3 sum4 sysntime system systime systimeint tan tanh tolower toupper truncate
+       typeof unflatten unformat unformatx upntime uptime urand urand32 urandelement
+       urandint urandrange utf8_to_latin1 variance version ! != !=~ % & && * ** + - .
+       .* .+ .- ./ / // < << <= <=> == =~ > >= >> >>> ?: ?? ??? ^ ^^ | || ~
 
 1mCOMMENTS-IN-DATA FLAGS0m
        Miller lets you put comments in your data, such as
@@ -780,6 +780,13 @@
                                 right-align-multi-word
        --fw {string}            Shortcut for --fixed left-align-multi-word
        --right                  Right-justifies all fields for PPRINT output.
+       --right-align-numeric    Right-justifies fields with numeric values for PPRINT
+                                output, leaving other fields left-justified. Headers
+                                are right-justified over columns whose values are all
+                                numeric, so that header and data share the same
+                                alignment. Also applies to markdown output, where
+                                numeric columns get right-alignment markers (`---:`)
+                                in the header-separator line.
 
 1mPROFILING FLAGS0m
        These are flags for profiling Miller performance.
@@ -819,6 +826,9 @@
        * Default line endings (`--irs` and `--ors`) are newline
          which is interpreted to accept carriage-return/newline files (e.g. on Windows)
          for input, and to produce platform-appropriate line endings on output.
+       * For CSV, CSV-lite, TSV, and TSV-lite output, ORS may be either newline (the
+         default) or carriage-return/newline: e.g. `--ors crlf` or `--ors '\r\n'`
+         for RFC-4180-style line endings on any platform.
 
        Notes about all other separators:
 
@@ -1431,7 +1441,11 @@
        --auto      Automatically computes limits, ignoring --lo and --hi. Holds all
                    values in memory before producing any output.
        -o {prefix} Prefix for output field name. Default: no prefix.
+       -s          Print a one-line Unicode sparkline per field instead of per-bin
+                   counts.
        -h|--help   Show this message.
+       With -s, output is one record per value-field, with a sparkline field
+       instead of one record per bin.
 
    1mjson-parse0m
        Usage: mlr json-parse [options]
@@ -1777,6 +1791,34 @@
 
        See also https://miller.readthedocs.io/reference-dsl for more context.
 
+   1mrank0m
+       Usage: mlr rank [options]
+       For each record's value in specified fields, computes the standard
+       competition rank (1,2,2,4,...) of that value among all input records,
+       optionally within groups.
+       E.g. with input records x=10, x=20, x=20, and x=30, emits output records
+       x=10,x_rank=1  x=20,x_rank=2  x=20,x_rank=2  and  x=30,x_rank=4.
+
+       Note: by default this is a two-pass algorithm: on the first pass it retains
+       input records and their values; on the second pass it computes ranks and
+       emits output records, in original input order. This means it produces no
+       output until all input is read, but gives correct ranks regardless of input
+       order. Use --sorted for a single-pass streaming alternative.
+
+       Options:
+       -f {a,b,c} Field name(s) to rank.
+       -g {d,e,f} Optional group-by-field name(s).
+       --sorted   Promise that the input is already sorted by the field(s) being ranked
+                  (within each group, if -g is given). This computes rank in a single
+                  streaming pass and O(1) space, by comparing each record's value only
+                  to the immediately preceding one, rather than buffering all records
+                  to compute an order-independent rank. Produces wrong output if the
+                  input is not in fact sorted.
+       -h|--help  Show this message.
+       Example: mlr rank -f x data/rank-example.csv
+       Example: mlr rank -f x -g g data/rank-example.csv
+       Example: mlr sort -f x then rank -f x --sorted data/rank-example.csv
+
    1mregularize0m
        Usage: mlr regularize [options]
        Outputs records sorted lexically ascending by keys.
@@ -1819,8 +1861,10 @@
                   record start.
        -f {a,b,c} Field names to reorder.
        -r {a,b,c} Treat field names as regular expressions. Matched fields are moved to
-                  start or end in record order. Example: -r '^YYY,^XXX' puts all YYY-
-                  and XXX-prefixed fields first (in record order), then the rest.
+                  start or end, grouped by the order the regexes are given; within each
+                  group, fields keep their record order. Example: -r '^YYY,^XXX' puts
+                  all YYY-prefixed fields first, then all XXX-prefixed fields, then the
+                  rest.
        -b {x}     Put field names specified with -f before field name specified by {x},
                   if any. If {x} isn't present in a given record, the specified fields
                   will not be moved.
@@ -1832,7 +1876,7 @@
        Examples:
        mlr reorder    -f a,b sends input record "d=4,b=2,a=1,c=3" to "a=1,b=2,d=4,c=3".
        mlr reorder -e -f a,b sends input record "d=4,b=2,a=1,c=3" to "d=4,c=3,a=1,b=2".
-       mlr reorder -r '^YYY,^XXX' puts YYY- and XXX-prefixed fields first (record order), then rest.
+       mlr reorder -r '^YYY,^XXX' puts YYY-prefixed fields first, then XXX-prefixed fields, then rest.
 
    1mrepeat0m
        Usage: mlr repeat [options]
@@ -2050,6 +2094,16 @@
        -n         Sort field names naturally (e.g. 2 before 12). Combines with -f/-r.
        -h|--help  Show this message.
 
+   1msparkline0m
+       Usage: mlr sparkline [options]
+       Reduces numeric field(s), across all records in input order, to a compact
+       Unicode sparkline -- one block character per record -- for visualizing
+       trends. Emits one output record per field. Holds all records in memory
+       before producing any output.
+       Options:
+       -f {a,b,c} Field names to sparkline.
+       -h|--help  Show this message.
+
    1msparsify0m
        Usage: mlr sparsify [options]
        Unsets fields for which the key is the empty string (or, optionally, another
@@ -2291,7 +2345,7 @@
        Show summary statistics about the input data.
 
        All summarizers:
-         field_type      string, int, etc. -- if a column has mixed types, all encountered types are printed
+         field_type      string, int, etc. -- if a column has mixed types, all encountered types are printed (see notes below)
          count           +1 for every instance of the field across all records in the input record stream
          null_count      count of field values either empty string or JSON null
          distinct_count  count of distinct values for the field
@@ -2321,6 +2375,8 @@
        * min, p25, median, p75, and max work for strings as well as numbers
        * Distinct-counts are computed on string representations -- so 4.1 and 4.10 are counted as distinct here.
        * If the mode is not unique in the input data, the first-encountered value is reported as the mode.
+       * A field_type of "int-string", "empty-string", etc. means the column contains values of mixed types --
+         all types encountered are printed, hyphen-joined, in the order first encountered.
 
        Options:
        -a {mean,sum,etc.} Use only the specified summarizers.
@@ -2426,6 +2482,9 @@
        count-distinct. For uniq, -f is a synonym for -g. Output fields are
        written in the order in which they are named with -g or -f, not in the
        order in which they appear in the input records.
+       To deduplicate records by one or more fields while keeping all other
+       fields, use head: e.g. "mlr head -n 1 -g hash" keeps the first record
+       for each distinct value of the hash field, with all fields intact.
 
        Options:
        -g {d,e,f} Group-by field names for uniq counts.
@@ -2717,11 +2776,13 @@
        Map example: fold({"a":1, "b":3, "c": 5}, func(acck,accv,ek,ev) {return {"sum": accv+ev**2}}, {"sum":10000}) returns 10035.
 
    1mformat0m
-        (class=string #args=variadic) Using first argument as format string, interpolate remaining arguments in place of each "{}" in the format string. Too-few arguments are treated as the empty string; too-many arguments are discarded.
+        (class=string #args=variadic) Using first argument as format string, interpolate remaining arguments in place of each "{}" in the format string. Also supports 1-based positional placeholders such as "{1}", which allow arguments to be reused and/or reordered. Plain "{}" placeholders consume arguments sequentially, independently of any positional placeholders. Too-few arguments are treated as the empty string, as are positional placeholders exceeding the number of arguments; too-many arguments are discarded. "{0}" is an error value, since positional placeholders are 1-based.
        Examples:
-       format("{}:{}:{}", 1,2)     gives "1:2:".
-       format("{}:{}:{}", 1,2,3)   gives "1:2:3".
-       format("{}:{}:{}", 1,2,3,4) gives "1:2:3".
+       format("{}:{}:{}", 1,2)        gives "1:2:".
+       format("{}:{}:{}", 1,2,3)      gives "1:2:3".
+       format("{}:{}:{}", 1,2,3,4)    gives "1:2:3".
+       format("{1}:{2}:{1}", "a","b") gives "a:b:a".
+       format("{2}{}:{1}{}", 3,4)     gives "43:34".
 
    1mfsec2dhms0m
         (class=time #args=1) Formats floating-point seconds as in fsec2dhms(500000.25) = "5d18h53m20.250000s"
@@ -3244,6 +3305,12 @@
    1msort_collection0m
         (class=stats #args=1) This is a helper function for the percentiles function; please see its online help for details.
 
+   1msparkline0m
+        (class=stats #args=1) Returns a string of Unicode block characters (one of  per element) representing the relative magnitudes of values in an array or map, for a compact ASCII/Unicode bar chart. Returns error for non-array/non-map types.
+       Examples:
+       sparkline([1,2,3,4,5,6,7,8]) is ""
+       sparkline([3,3,3]) is ""
+
    1msplita0m
         (class=conversion #args=2) Splits string into array with type inference. First argument is string to split; second is the separator to split on.
        Example:
@@ -4022,4 +4089,4 @@
        MIME Type for Comma-Separated Values (CSV) Files, the Miller docsite
        https://miller.readthedocs.io
 
-                                  2026-07-05                         4mMILLER24m(1)
+                                  2026-07-08                         4mMILLER24m(1)
diff --git a/man/mlr.1 b/man/mlr.1
index b102f149b..272395a6a 100644
--- a/man/mlr.1
+++ b/man/mlr.1
@@ -2,12 +2,12 @@
 .\"     Title: mlr
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: ./mkman.rb
-.\"      Date: 2026-07-05
+.\"      Date: 2026-07-08
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "MILLER" "1" "2026-07-05" "\ \&" "\ \&"
+.TH "MILLER" "1" "2026-07-08" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Portability definitions
 .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -249,11 +249,11 @@ altkv bar bootstrap case cat check clean-whitespace count-distinct count
 count-similar cut decimate describe fill-down fill-empty filter flatten
 format-values fraction gap grep group-by group-like gsub having-fields head
 histogram json-parse json-stringify join label latin1-to-utf8 least-frequent
-merge-fields most-frequent nest nothing put regularize remove-empty-columns
-rename reorder repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle
-skip-trivial-records sort sort-within-records sparsify split ssub stats1
-stats2 step sub summary surv tac tail tee template top utf8-to-latin1
-unflatten uniq unspace unsparsify
+merge-fields most-frequent nest nothing put rank regularize
+remove-empty-columns rename reorder repeat reshape sample sec2gmtdate sec2gmt
+seqgen shuffle skip-trivial-records sort sort-within-records sparkline
+sparsify split ssub stats1 stats2 step sub summary surv tac tail tee template
+top utf8-to-latin1 unflatten uniq unspace unsparsify
 .fi
 .if n \{\
 .RE
@@ -284,14 +284,14 @@ minlen mmul mode msub next nsec2gmt nsec2gmtdate nsec2localdate nsec2localtime
 null_count os percentile percentiles pow qnorm reduce regextract
 regextract_or_else rightpad round roundm rstrip sec2dhms sec2gmt sec2gmtdate
 sec2hms sec2localdate sec2localtime select sgn sha1 sha256 sha512 sin sinh
-skewness sort sort_collection splita splitax splitkv splitkvx splitnv splitnvx
-sqrt ssub stat stddev strfntime strfntime_local strftime strftime_local string
-strip strlen strmatch strmatchx strpntime strpntime_local strptime
-strptime_local sub substr substr0 substr1 sum sum2 sum3 sum4 sysntime system
-systime systimeint tan tanh tolower toupper truncate typeof unflatten unformat
-unformatx upntime uptime urand urand32 urandelement urandint urandrange
-utf8_to_latin1 variance version ! != !=~ % & && * ** + - . .* .+ .- ./ / // <
-<< <= <=> == =~ > >= >> >>> ?: ?? ??? ^ ^^ | || ~
+skewness sort sort_collection sparkline splita splitax splitkv splitkvx
+splitnv splitnvx sqrt ssub stat stddev strfntime strfntime_local strftime
+strftime_local string strip strlen strmatch strmatchx strpntime
+strpntime_local strptime strptime_local sub substr substr0 substr1 sum sum2
+sum3 sum4 sysntime system systime systimeint tan tanh tolower toupper truncate
+typeof unflatten unformat unformatx upntime uptime urand urand32 urandelement
+urandint urandrange utf8_to_latin1 variance version ! != !=~ % & && * ** + - .
+\&.* .+ .- ./ / // < << <= <=> == =~ > >= >> >>> ?: ?? ??? ^ ^^ | || ~
 .fi
 .if n \{\
 .RE
@@ -932,6 +932,13 @@ These are flags which are applicable to PPRINT format.
                          right-align-multi-word
 --fw {string}            Shortcut for --fixed left-align-multi-word
 --right                  Right-justifies all fields for PPRINT output.
+--right-align-numeric    Right-justifies fields with numeric values for PPRINT
+                         output, leaving other fields left-justified. Headers
+                         are right-justified over columns whose values are all
+                         numeric, so that header and data share the same
+                         alignment. Also applies to markdown output, where
+                         numeric columns get right-alignment markers (`---:`)
+                         in the header-separator line.
 .fi
 .if n \{\
 .RE
@@ -987,6 +994,9 @@ Notes about line endings:
 * Default line endings (`--irs` and `--ors`) are newline
   which is interpreted to accept carriage-return/newline files (e.g. on Windows)
   for input, and to produce platform-appropriate line endings on output.
+* For CSV, CSV-lite, TSV, and TSV-lite output, ORS may be either newline (the
+  default) or carriage-return/newline: e.g. `--ors crlf` or `--ors '\er\en'`
+  for RFC-4180-style line endings on any platform.
 
 Notes about all other separators:
 
@@ -1781,7 +1791,11 @@ Options:
 --auto      Automatically computes limits, ignoring --lo and --hi. Holds all
             values in memory before producing any output.
 -o {prefix} Prefix for output field name. Default: no prefix.
+-s          Print a one-line Unicode sparkline per field instead of per-bin
+            counts.
 -h|--help   Show this message.
+With -s, output is one record per value-field, with a sparkline field
+instead of one record per bin.
 .fi
 .if n \{\
 .RE
@@ -2195,6 +2209,40 @@ See also https://miller.readthedocs.io/reference-dsl for more context.
 .fi
 .if n \{\
 .RE
+.SS "rank"
+.if n \{\
+.RS 0
+.\}
+.nf
+Usage: mlr rank [options]
+For each record's value in specified fields, computes the standard
+competition rank (1,2,2,4,...) of that value among all input records,
+optionally within groups.
+E.g. with input records x=10, x=20, x=20, and x=30, emits output records
+x=10,x_rank=1  x=20,x_rank=2  x=20,x_rank=2  and  x=30,x_rank=4.
+
+Note: by default this is a two-pass algorithm: on the first pass it retains
+input records and their values; on the second pass it computes ranks and
+emits output records, in original input order. This means it produces no
+output until all input is read, but gives correct ranks regardless of input
+order. Use --sorted for a single-pass streaming alternative.
+
+Options:
+-f {a,b,c} Field name(s) to rank.
+-g {d,e,f} Optional group-by-field name(s).
+--sorted   Promise that the input is already sorted by the field(s) being ranked
+           (within each group, if -g is given). This computes rank in a single
+           streaming pass and O(1) space, by comparing each record's value only
+           to the immediately preceding one, rather than buffering all records
+           to compute an order-independent rank. Produces wrong output if the
+           input is not in fact sorted.
+-h|--help  Show this message.
+Example: mlr rank -f x data/rank-example.csv
+Example: mlr rank -f x -g g data/rank-example.csv
+Example: mlr sort -f x then rank -f x --sorted data/rank-example.csv
+.fi
+.if n \{\
+.RE
 .SS "regularize"
 .if n \{\
 .RS 0
@@ -2259,8 +2307,10 @@ Options:
            record start.
 -f {a,b,c} Field names to reorder.
 -r {a,b,c} Treat field names as regular expressions. Matched fields are moved to
-           start or end in record order. Example: -r '^YYY,^XXX' puts all YYY-
-           and XXX-prefixed fields first (in record order), then the rest.
+           start or end, grouped by the order the regexes are given; within each
+           group, fields keep their record order. Example: -r '^YYY,^XXX' puts
+           all YYY-prefixed fields first, then all XXX-prefixed fields, then the
+           rest.
 -b {x}     Put field names specified with -f before field name specified by {x},
            if any. If {x} isn't present in a given record, the specified fields
            will not be moved.
@@ -2272,7 +2322,7 @@ Options:
 Examples:
 mlr reorder    -f a,b sends input record "d=4,b=2,a=1,c=3" to "a=1,b=2,d=4,c=3".
 mlr reorder -e -f a,b sends input record "d=4,b=2,a=1,c=3" to "d=4,c=3,a=1,b=2".
-mlr reorder -r '^YYY,^XXX' puts YYY- and XXX-prefixed fields first (record order), then rest.
+mlr reorder -r '^YYY,^XXX' puts YYY-prefixed fields first, then XXX-prefixed fields, then rest.
 .fi
 .if n \{\
 .RE
@@ -2552,6 +2602,22 @@ Options:
 .fi
 .if n \{\
 .RE
+.SS "sparkline"
+.if n \{\
+.RS 0
+.\}
+.nf
+Usage: mlr sparkline [options]
+Reduces numeric field(s), across all records in input order, to a compact
+Unicode sparkline -- one block character per record -- for visualizing
+trends. Emits one output record per field. Holds all records in memory
+before producing any output.
+Options:
+-f {a,b,c} Field names to sparkline.
+-h|--help  Show this message.
+.fi
+.if n \{\
+.RE
 .SS "sparsify"
 .if n \{\
 .RS 0
@@ -2839,7 +2905,7 @@ Usage: mlr summary [options]
 Show summary statistics about the input data.
 
 All summarizers:
-  field_type      string, int, etc. -- if a column has mixed types, all encountered types are printed
+  field_type      string, int, etc. -- if a column has mixed types, all encountered types are printed (see notes below)
   count           +1 for every instance of the field across all records in the input record stream
   null_count      count of field values either empty string or JSON null
   distinct_count  count of distinct values for the field
@@ -2869,6 +2935,8 @@ Notes:
 * min, p25, median, p75, and max work for strings as well as numbers
 * Distinct-counts are computed on string representations -- so 4.1 and 4.10 are counted as distinct here.
 * If the mode is not unique in the input data, the first-encountered value is reported as the mode.
+* A field_type of "int-string", "empty-string", etc. means the column contains values of mixed types --
+  all types encountered are printed, hyphen-joined, in the order first encountered.
 
 Options:
 -a {mean,sum,etc.} Use only the specified summarizers.
@@ -3028,6 +3096,9 @@ Prints distinct values for specified field names. With -c, same as
 count-distinct. For uniq, -f is a synonym for -g. Output fields are
 written in the order in which they are named with -g or -f, not in the
 order in which they appear in the input records.
+To deduplicate records by one or more fields while keeping all other
+fields, use head: e.g. "mlr head -n 1 -g hash" keeps the first record
+for each distinct value of the hash field, with all fields intact.
 
 Options:
 -g {d,e,f} Group-by field names for uniq counts.
@@ -3723,11 +3794,13 @@ Map example: fold({"a":1, "b":3, "c": 5}, func(acck,accv,ek,ev) {return {"sum":
 .RS 0
 .\}
 .nf
- (class=string #args=variadic) Using first argument as format string, interpolate remaining arguments in place of each "{}" in the format string. Too-few arguments are treated as the empty string; too-many arguments are discarded.
+ (class=string #args=variadic) Using first argument as format string, interpolate remaining arguments in place of each "{}" in the format string. Also supports 1-based positional placeholders such as "{1}", which allow arguments to be reused and/or reordered. Plain "{}" placeholders consume arguments sequentially, independently of any positional placeholders. Too-few arguments are treated as the empty string, as are positional placeholders exceeding the number of arguments; too-many arguments are discarded. "{0}" is an error value, since positional placeholders are 1-based.
 Examples:
-format("{}:{}:{}", 1,2)     gives "1:2:".
-format("{}:{}:{}", 1,2,3)   gives "1:2:3".
-format("{}:{}:{}", 1,2,3,4) gives "1:2:3".
+format("{}:{}:{}", 1,2)        gives "1:2:".
+format("{}:{}:{}", 1,2,3)      gives "1:2:3".
+format("{}:{}:{}", 1,2,3,4)    gives "1:2:3".
+format("{1}:{2}:{1}", "a","b") gives "a:b:a".
+format("{2}{}:{1}{}", 3,4)     gives "43:34".
 .fi
 .if n \{\
 .RE
@@ -4912,6 +4985,18 @@ Map without function: sort({"c":2,"a":3,"b":1}, "vnr") returns {"a":3,"c":2,"b":
 .fi
 .if n \{\
 .RE
+.SS "sparkline"
+.if n \{\
+.RS 0
+.\}
+.nf
+ (class=stats #args=1) Returns a string of Unicode block characters (one of β–β–‚β–ƒβ–„β–…β–†β–‡β–ˆ per element) representing the relative magnitudes of values in an array or map, for a compact ASCII/Unicode bar chart. Returns error for non-array/non-map types.
+Examples:
+sparkline([1,2,3,4,5,6,7,8]) is "β–β–‚β–ƒβ–„β–…β–†β–‡β–ˆ"
+sparkline([3,3,3]) is "▁▁▁"
+.fi
+.if n \{\
+.RE
 .SS "splita"
 .if n \{\
 .RS 0
diff --git a/pkg/bifs/datetime_test.go b/pkg/bifs/datetime_test.go
new file mode 100644
index 000000000..ba6ae673a
--- /dev/null
+++ b/pkg/bifs/datetime_test.go
@@ -0,0 +1,98 @@
+package bifs
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/johnkerl/miller/v6/pkg/mlrval"
+)
+
+// TestBIF_strftime_strptime_roundtrip checks that for each of a set of formats
+// exercising the newly-added %a %A %e %h %x %X %c strptime codes,
+// strptime(strftime(t, fmt), fmt) == t. This is the exact failure mode from
+// https://github.com/johnkerl/miller/issues/1518, where strptime accepted fewer
+// format codes than strftime produced.
+func TestBIF_strftime_strptime_roundtrip(t *testing.T) {
+	formats := []string{
+		"%a %b %e %T %Y",
+		"%A %b %e %T %Y",
+		"%a %h %e %T %Y",
+		"%A %h %e %T %Y",
+		"%c",
+		"%x %X",
+		"%Y-%m-%d %H:%M:%S",
+	}
+
+	epochSecondsValues := []int64{
+		0,          // 1970-01-01, single-digit day
+		1709552468, // 2024-03-04 11:41:08, single-digit day
+		1729555200, // 2024-10-22 00:00:00, double-digit day
+		1735732805, // 2025-01-01 12:00:05, single-digit day, new year
+	}
+
+	for _, format := range formats {
+		for _, epochSeconds := range epochSecondsValues {
+			formatted := BIF_strftime(mlrval.FromInt(epochSeconds), mlrval.FromString(format))
+			formattedString, isString := formatted.GetStringValue()
+			assert.True(t, isString, "strftime(%d, %q) produced non-string %v", epochSeconds, format, formatted)
+
+			parsed := BIF_strptime(mlrval.FromString(formattedString), mlrval.FromString(format))
+			parsedSeconds, isNumeric := parsed.GetNumericToFloatValue()
+			assert.True(t, isNumeric, "strptime(%q, %q) produced non-numeric %v", formattedString, format, parsed)
+
+			assert.Equal(
+				t, float64(epochSeconds), parsedSeconds,
+				"round-trip mismatch for format %q: strftime(%d) = %q, strptime(...) = %v",
+				format, epochSeconds, formattedString, parsedSeconds,
+			)
+		}
+	}
+}
+
+// TestBIF_strptime_new_format_codes exercises the newly-supported %a %A %e %h
+// %x %X %c strptime format codes directly, including %e's space-padding edge
+// cases (single-digit day with/without padding, double-digit day, %e at the
+// end of the string, and %e directly adjacent to another format code).
+func TestBIF_strptime_new_format_codes(t *testing.T) {
+	type testCase struct {
+		input  string
+		format string
+		want   float64
+	}
+
+	cases := []testCase{
+		{"Mon Mar  4 11:41:08 2024", "%a %b %e %T %Y", 1709552468},
+		{"Monday Mar  4 11:41:08 2024", "%A %b %e %T %Y", 1709552468},
+		{"Mon Mar  4 11:41:08 2024", "%a %h %e %T %Y", 1709552468},
+		{"Tue Oct 22 00:00:00 2024", "%a %b %e %T %Y", 1729555200},
+		{"Mon Mar  4 11:41:08 2024", "%c", 1709552468},
+		{"03/04/24 11:41:08", "%x %X", 1709552468},
+		// %e with a single (non-padded) space before a single-digit day.
+		{"Mar 4 2024", "%b %e %Y", 1709510400},
+		// %e with two-digit day.
+		{"Mar 14 2024", "%b %e %Y", 1710374400},
+		// %e directly adjacent to another format code, no intervening text.
+		{"4Mar2024", "%e%b%Y", 1709510400},
+		{"14Mar2024", "%e%b%Y", 1710374400},
+		// %e at the very end of the string.
+		{"2024-03-4", "%Y-%m-%e", 1709510400},
+		{"2024-03-14", "%Y-%m-%e", 1710374400},
+	}
+
+	for _, c := range cases {
+		output := BIF_strptime(mlrval.FromString(c.input), mlrval.FromString(c.format))
+		seconds, isNumeric := output.GetNumericToFloatValue()
+		assert.True(t, isNumeric, "strptime(%q, %q) produced non-numeric %v", c.input, c.format, output)
+		assert.Equal(t, c.want, seconds, "strptime(%q, %q)", c.input, c.format)
+	}
+}
+
+// TestBIF_strptime_still_unsupported_format_code checks that a format code
+// which remains unsupported (%U, week-of-year) still errors, i.e. that adding
+// the new codes didn't accidentally cause unsupported codes to be silently
+// ignored.
+func TestBIF_strptime_still_unsupported_format_code(t *testing.T) {
+	output := BIF_strptime(mlrval.FromString("2024-03-04 09"), mlrval.FromString("%Y-%m-%d %U"))
+	assert.True(t, output.IsError())
+}
diff --git a/pkg/bifs/sparkline.go b/pkg/bifs/sparkline.go
new file mode 100644
index 000000000..e791200b7
--- /dev/null
+++ b/pkg/bifs/sparkline.go
@@ -0,0 +1,72 @@
+package bifs
+
+import (
+	"github.com/johnkerl/miller/v6/pkg/mlrval"
+)
+
+// sparklineTicks are the eighth-height Unicode block characters used to
+// render array/map values as a compact one-line bar chart.
+var sparklineTicks = []rune("β–β–‚β–ƒβ–„β–…β–†β–‡β–ˆ")
+
+func BIF_sparkline(collection *mlrval.Mlrval) *mlrval.Mlrval {
+	ok, valueIfNot := check_collection(collection, "sparkline")
+	if !ok {
+		return valueIfNot
+	}
+
+	var floatValues []float64
+	if collection.IsArray() {
+		array := collection.AcquireArrayValue()
+		floatValues = make([]float64, 0, len(array))
+		for _, element := range array {
+			floatValue, isFloat := element.GetNumericToFloatValue()
+			if !isFloat {
+				return mlrval.FromNotNumericError("sparkline", element)
+			}
+			floatValues = append(floatValues, floatValue)
+		}
+	} else {
+		m := collection.AcquireMapValue()
+		floatValues = make([]float64, 0, m.FieldCount)
+		for pe := m.Head; pe != nil; pe = pe.Next {
+			floatValue, isFloat := pe.Value.GetNumericToFloatValue()
+			if !isFloat {
+				return mlrval.FromNotNumericError("sparkline", pe.Value)
+			}
+			floatValues = append(floatValues, floatValue)
+		}
+	}
+
+	if len(floatValues) == 0 {
+		return mlrval.VOID
+	}
+
+	lo := floatValues[0]
+	hi := floatValues[0]
+	for _, floatValue := range floatValues[1:] {
+		if floatValue < lo {
+			lo = floatValue
+		}
+		if floatValue > hi {
+			hi = floatValue
+		}
+	}
+
+	numTicks := len(sparklineTicks)
+	runes := make([]rune, len(floatValues))
+	for i, floatValue := range floatValues {
+		if hi == lo {
+			runes[i] = sparklineTicks[0]
+			continue
+		}
+		tickIndex := int(float64(numTicks-1)*(floatValue-lo)/(hi-lo) + 0.5)
+		if tickIndex < 0 {
+			tickIndex = 0
+		} else if tickIndex >= numTicks {
+			tickIndex = numTicks - 1
+		}
+		runes[i] = sparklineTicks[tickIndex]
+	}
+
+	return mlrval.FromString(string(runes))
+}
diff --git a/pkg/bifs/sparkline_test.go b/pkg/bifs/sparkline_test.go
new file mode 100644
index 000000000..858ab8c16
--- /dev/null
+++ b/pkg/bifs/sparkline_test.go
@@ -0,0 +1,60 @@
+package bifs
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/johnkerl/miller/v6/pkg/mlrval"
+)
+
+func TestBIF_sparkline(t *testing.T) {
+	// Needs array or map
+	input := mlrval.FromInt(3)
+	output := BIF_sparkline(input)
+	assert.True(t, output.IsError())
+
+	// Non-numeric element
+	input = mlrval.FromArray([]*mlrval.Mlrval{
+		mlrval.FromInt(1),
+		mlrval.FromString("abc"),
+	})
+	output = BIF_sparkline(input)
+	assert.True(t, output.IsError())
+
+	// Empty array is void
+	input = mlrval.FromArray([]*mlrval.Mlrval{})
+	output = BIF_sparkline(input)
+	assert.True(t, output.IsVoid())
+
+	// Ascending values span the full tick range, low to high
+	input = mlrval.FromArray([]*mlrval.Mlrval{
+		mlrval.FromInt(1),
+		mlrval.FromInt(2),
+		mlrval.FromInt(3),
+		mlrval.FromInt(4),
+		mlrval.FromInt(5),
+		mlrval.FromInt(6),
+		mlrval.FromInt(7),
+		mlrval.FromInt(8),
+	})
+	output = BIF_sparkline(input)
+	assert.True(t, mlrval.Equals(output, mlrval.FromString("β–β–‚β–ƒβ–„β–…β–†β–‡β–ˆ")))
+
+	// Same value throughout maps to the lowest tick
+	input = mlrval.FromArray([]*mlrval.Mlrval{
+		mlrval.FromInt(3),
+		mlrval.FromInt(3),
+		mlrval.FromInt(3),
+	})
+	output = BIF_sparkline(input)
+	assert.True(t, mlrval.Equals(output, mlrval.FromString("▁▁▁")))
+
+	// Map input follows insertion order
+	input = array_to_map_for_test(mlrval.FromArray([]*mlrval.Mlrval{
+		mlrval.FromInt(1),
+		mlrval.FromInt(8),
+	}))
+	output = BIF_sparkline(input)
+	assert.True(t, mlrval.Equals(output, mlrval.FromString("β–β–ˆ")))
+}
diff --git a/pkg/bifs/strings.go b/pkg/bifs/strings.go
index 4fbc7e1d0..db15097ff 100644
--- a/pkg/bifs/strings.go
+++ b/pkg/bifs/strings.go
@@ -381,6 +381,10 @@ func BIF_clean_whitespace(input1 *mlrval.Mlrval) *mlrval.Mlrval {
 	return mlrval.FromInferredType(mv.String())
 }
 
+// Matches either a sequential placeholder "{}" or a positional placeholder
+// like "{1}", "{2}", etc.
+var _format_placeholder_regexp = regexp.MustCompile(`\{[0-9]*\}`)
+
 func BIF_format(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval {
 	if len(mlrvals) == 0 {
 		return mlrval.VOID
@@ -390,38 +394,62 @@ func BIF_format(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval {
 		return mlrval.FromTypeErrorUnary("format", mlrvals[0])
 	}
 
-	pieces := lib.SplitString(formatString, "{}")
+	// Example: format("{}:{}", 8, 9) gives "8:9".
+	//
+	// Placeholders come in two forms:
+	// * "{}": consumes the next argument in sequence. The sequence counter is
+	//   advanced only by "{}" placeholders, independently of any positional
+	//   placeholders (same as Rust's format!).
+	// * "{N}" with N >= 1: refers to the Nth argument after the format string
+	//   (1-based). Arguments may be repeated and/or reordered this way:
+	//   format("{1}:{2}:{1}", 8, 9) gives "8:9:8".
+	//
+	// Q: What if too few arguments for format, or a positional index exceeds
+	//    the number of arguments?
+	// A: Interpolate the empty string.
+	// Q: What if too many arguments for format?
+	// A: Leave them off.
+	// Q: What about "{0}"?
+	// A: It's an error value, since positional placeholders are 1-based.
 
 	var buffer bytes.Buffer
 
-	// Example: format("{}:{}", 8, 9)
-	//
-	// * piece[0] ""
-	// * piece[1] ":"
-	// * piece[2] ""
-	// * mlrval[1] 8
-	// * mlrval[2] 9
-	//
-	// So:
-	// * Write piece[0]
-	// * Write mlrvals[1]
-	// * Write piece[1]
-	// * Write mlrvals[2]
-	// * Write piece[2]
+	n := len(mlrvals) // arguments are mlrvals[1] .. mlrvals[n-1]
+	position := 1     // implicit counter for "{}" placeholders
+	remaining := formatString
 
-	// Q: What if too few arguments for format?
-	// A: Leave them off
-	// Q: What if too many arguments for format?
-	// A: Leave them off
-
-	n := len(mlrvals)
-	for i, piece := range pieces {
-		if i > 0 {
-			if i < n {
-				buffer.WriteString(mlrvals[i].String())
-			}
+	for {
+		loc := _format_placeholder_regexp.FindStringIndex(remaining)
+		if loc == nil {
+			buffer.WriteString(remaining)
+			break
 		}
-		buffer.WriteString(piece)
+		buffer.WriteString(remaining[:loc[0]])
+		placeholder := remaining[loc[0]:loc[1]]
+		remaining = remaining[loc[1]:]
+
+		var index int
+		if placeholder == "{}" {
+			index = position
+			position++
+		} else {
+			numString := placeholder[1 : len(placeholder)-1]
+			num, err := strconv.Atoi(numString)
+			if err != nil || num < 1 {
+				return mlrval.FromError(
+					fmt.Errorf(
+						"format: positional placeholder %s is invalid: indices are 1-based",
+						placeholder,
+					),
+				)
+			}
+			index = num
+		}
+
+		if index < n {
+			buffer.WriteString(mlrvals[index].String())
+		}
+		// Else, too few arguments: interpolate the empty string.
 	}
 
 	return mlrval.FromString(buffer.String())
diff --git a/pkg/bifs/strings_test.go b/pkg/bifs/strings_test.go
new file mode 100644
index 000000000..3492662f6
--- /dev/null
+++ b/pkg/bifs/strings_test.go
@@ -0,0 +1,76 @@
+package bifs
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/johnkerl/miller/v6/pkg/mlrval"
+)
+
+func TestBIF_format(t *testing.T) {
+	// No arguments at all
+	output := BIF_format([]*mlrval.Mlrval{})
+	assert.True(t, output.IsVoid())
+
+	// Non-string format string
+	output = BIF_format([]*mlrval.Mlrval{mlrval.FromInt(1)})
+	assert.True(t, output.IsError())
+
+	// Sequential placeholders
+	cases := []struct {
+		formatString string
+		args         []string
+		expected     string
+	}{
+		// Plain "{}" placeholders (pre-existing behavior)
+		{"", []string{}, ""},
+		{"abc", []string{}, "abc"},
+		{"{}", []string{}, ""},
+		{"{}", []string{"1"}, "1"},
+		{"{}", []string{"1", "2"}, "1"},
+		{"{}:{}:{}", []string{"1", "2"}, "1:2:"},
+		{"{}:{}:{}", []string{"1", "2", "3"}, "1:2:3"},
+		{"{}:{}:{}", []string{"1", "2", "3", "4"}, "1:2:3"},
+		{"<{}:{}>", []string{"abc"}, ""},
+		{"<{}:{}>", []string{"abc", "def"}, ""},
+
+		// Positional "{N}" placeholders
+		{"{1}", []string{"a"}, "a"},
+		{"{1}:{1}", []string{"a"}, "a:a"},
+		{"{2}:{1}", []string{"a", "b"}, "b:a"},
+		{"{1}/{2}/{1}_{3}.ext", []string{"a", "b", "c"}, "a/b/a_c.ext"},
+		// Out-of-range positional index interpolates the empty string,
+		// consistent with too-few arguments for "{}"
+		{"{3}", []string{"a"}, ""},
+		{"<{4}>", []string{"a", "b"}, "<>"},
+		// Leading zeros are accepted
+		{"{01}", []string{"a"}, "a"},
+
+		// Mixing: "{}" consumes arguments sequentially, independently of
+		// positional placeholders
+		{"{2}{}:{1}{}", []string{"3", "4"}, "43:34"},
+		{"{}{1}{}", []string{"a", "b"}, "aab"},
+
+		// Non-placeholder braces are left alone
+		{"{ }", []string{"a"}, "{ }"},
+		{"{x}", []string{"a"}, "{x}"},
+		{"{", []string{"a"}, "{"},
+		{"}", []string{"a"}, "}"},
+	}
+
+	for _, c := range cases {
+		mlrvals := []*mlrval.Mlrval{mlrval.FromString(c.formatString)}
+		for _, arg := range c.args {
+			mlrvals = append(mlrvals, mlrval.FromString(arg))
+		}
+		output := BIF_format(mlrvals)
+		stringval, ok := output.GetStringValue()
+		assert.True(t, ok, "format(%q, %v)", c.formatString, c.args)
+		assert.Equal(t, c.expected, stringval, "format(%q, %v)", c.formatString, c.args)
+	}
+
+	// "{0}" is an error, since positional placeholders are 1-based
+	output = BIF_format([]*mlrval.Mlrval{mlrval.FromString("{0}"), mlrval.FromString("a")})
+	assert.True(t, output.IsError())
+}
diff --git a/pkg/cli/option_parse.go b/pkg/cli/option_parse.go
index 06c13c010..a1cadab22 100644
--- a/pkg/cli/option_parse.go
+++ b/pkg/cli/option_parse.go
@@ -158,6 +158,9 @@ Notes about line endings:
 * Default line endings (` + "`--irs`" + ` and ` + "`--ors`" + `) are newline
   which is interpreted to accept carriage-return/newline files (e.g. on Windows)
   for input, and to produce platform-appropriate line endings on output.
+* For CSV, CSV-lite, TSV, and TSV-lite output, ORS may be either newline (the
+  default) or carriage-return/newline: e.g. ` + "`--ors crlf`" + ` or ` + "`--ors '\\r\\n'`" + `
+  for RFC-4180-style line endings on any platform.
 
 Notes about all other separators:
 
@@ -531,6 +534,19 @@ var PPRINTOnlyFlagSection = FlagSection{
 			},
 		},
 
+		{
+			name: "--right-align-numeric",
+			help: "Right-justifies fields with numeric values for PPRINT output, leaving " +
+				"other fields left-justified. Headers are right-justified over columns " +
+				"whose values are all numeric, so that header and data share the same " +
+				"alignment. Also applies to markdown output, where numeric columns get " +
+				"right-alignment markers (`---:`) in the header-separator line.",
+			parser: func(args []string, argc int, pargi *int, options *TOptions) {
+				options.WriterOptions.RightAlignNumericOutput = true
+				*pargi += 1
+			},
+		},
+
 		{
 			name:     "--barred",
 			altNames: []string{"--barred-output"},
diff --git a/pkg/cli/option_types.go b/pkg/cli/option_types.go
index 451fc142b..f9670f5b4 100644
--- a/pkg/cli/option_types.go
+++ b/pkg/cli/option_types.go
@@ -51,8 +51,14 @@ type TReaderOptions struct {
 	irsWasSpecified            bool
 	allowRepeatIFSWasSpecified bool
 
-	UseImplicitHeader    bool
-	AllowRaggedCSVInput  bool
+	UseImplicitHeader   bool
+	AllowRaggedCSVInput bool
+	// Not settable by a command-line flag of its own: this is set when the
+	// skip-trivial-records verb is present in the then-chain. It lets the
+	// CSV/TSV record-readers know that trivial records -- e.g. blank lines
+	// at the end of a CSV file -- are to be skipped rather than treated as
+	// fatal header/data length mismatches. See issue #1535.
+	SkipTrivialRecords   bool
 	CSVLazyQuotes        bool
 	CSVTrimLeadingSpace  bool
 	BarredPprintInput    bool
@@ -102,6 +108,9 @@ type TWriterOptions struct {
 	RightAlignedPPRINTOutput bool
 	RightAlignedXTABOutput   bool
 	MarkdownAlignedOutput    bool
+	// Right-align numeric values in PPRINT output; for markdown output,
+	// use `---:` alignment markers for all-numeric columns.
+	RightAlignNumericOutput bool
 
 	// JSON output: --jlistwrap on, --jvstack on
 	// JSON Lines output: --jlistwrap off, --jvstack off
diff --git a/pkg/dkvpx/dkvpx_reader.go b/pkg/dkvpx/dkvpx_reader.go
index 10949c1f7..e2422e4af 100644
--- a/pkg/dkvpx/dkvpx_reader.go
+++ b/pkg/dkvpx/dkvpx_reader.go
@@ -26,6 +26,9 @@ type Reader struct {
 	// Comma is the pair delimiter (default ',').
 	Comma rune
 
+	// Equals is the key-value separator within a pair (default '=').
+	Equals rune
+
 	// Comment, if not 0, causes lines beginning with this character to be skipped.
 	Comment rune
 
@@ -42,8 +45,9 @@ type Reader struct {
 // NewReader returns a new Reader that reads from r.
 func NewReader(r io.Reader) *Reader {
 	return &Reader{
-		Comma: ',',
-		r:     bufio.NewReader(r),
+		Comma:  ',',
+		Equals: '=',
+		r:      bufio.NewReader(r),
 	}
 }
 
@@ -209,7 +213,7 @@ func (r *Reader) readRecord() (*lib.OrderedMap[string], error) {
 			continue
 		}
 
-		if rn == '=' && !haveKey {
+		if rn == r.Equals && !haveKey {
 			haveKey = true
 			line = line[rnLen:]
 			continue
diff --git a/pkg/dkvpx/dkvpx_reader_test.go b/pkg/dkvpx/dkvpx_reader_test.go
index 664cc7b5c..d7552b95e 100644
--- a/pkg/dkvpx/dkvpx_reader_test.go
+++ b/pkg/dkvpx/dkvpx_reader_test.go
@@ -105,6 +105,22 @@ func TestRead_CommentSkipped(t *testing.T) {
 	assert.Equal(t, map[string]string{"a": "1", "b": "2"}, orderedMapToMap(rec))
 }
 
+func TestRead_NonDefaultComma(t *testing.T) {
+	r := NewReader(strings.NewReader("a=1;b=2;c=\"x;y\"\n"))
+	r.Comma = ';'
+	rec, err := r.Read()
+	assert.NoError(t, err)
+	assert.Equal(t, map[string]string{"a": "1", "b": "2", "c": "x;y"}, orderedMapToMap(rec))
+}
+
+func TestRead_NonDefaultEquals(t *testing.T) {
+	r := NewReader(strings.NewReader("a:1,b:2,c:\"x:y\"\n"))
+	r.Equals = ':'
+	rec, err := r.Read()
+	assert.NoError(t, err)
+	assert.Equal(t, map[string]string{"a": "1", "b": "2", "c": "x:y"}, orderedMapToMap(rec))
+}
+
 func TestRead_OrderPreserved(t *testing.T) {
 	r := NewReader(strings.NewReader("z=3,x=1,y=2\n"))
 	rec, err := r.Read()
diff --git a/pkg/dkvpx/dkvpx_writer.go b/pkg/dkvpx/dkvpx_writer.go
index f72a1a3f3..f74287fe7 100644
--- a/pkg/dkvpx/dkvpx_writer.go
+++ b/pkg/dkvpx/dkvpx_writer.go
@@ -7,17 +7,25 @@ import (
 	"strings"
 )
 
-// needsQuoting reports whether the string must be quoted (contains comma,
-// equals, newline, or double-quote).
-func needsQuoting(s string) bool {
-	return strings.ContainsAny(s, ",\n\r=\"")
+// needsQuoting reports whether the string must be quoted (contains the
+// pair separator, the key-value separator, newline, or double-quote).
+func needsQuoting(s, ofs, ops string) bool {
+	return strings.ContainsAny(s, "\n\r\"") || strings.Contains(s, ofs) || strings.Contains(s, ops)
 }
 
-// FormatField returns the string formatted for DKVPX output: quoted and escaped
-// if it contains comma, equals, newline, or quote; otherwise unchanged.
-// Useful for callers that need to apply colorization or other wrapping.
+// FormatField returns the string formatted for DKVPX output with the default
+// separators (comma and equals): quoted and escaped if it contains a
+// separator, newline, or quote; otherwise unchanged.
 func FormatField(s string) string {
-	if !needsQuoting(s) {
+	return FormatFieldWithSeparators(s, ",", "=")
+}
+
+// FormatFieldWithSeparators returns the string formatted for DKVPX output:
+// quoted and escaped if it contains the given pair separator (OFS), the given
+// key-value separator (OPS), newline, or quote; otherwise unchanged.
+// Useful for callers that need to apply colorization or other wrapping.
+func FormatFieldWithSeparators(s, ofs, ops string) string {
+	if !needsQuoting(s, ofs, ops) {
 		return s
 	}
 	var b strings.Builder
diff --git a/pkg/dkvpx/dkvpx_writer_test.go b/pkg/dkvpx/dkvpx_writer_test.go
index e64a6d227..4d8450d51 100644
--- a/pkg/dkvpx/dkvpx_writer_test.go
+++ b/pkg/dkvpx/dkvpx_writer_test.go
@@ -26,6 +26,15 @@ func TestFormatField_EscapedQuotes(t *testing.T) {
 	assert.Equal(t, `"the ""word"""`, FormatField(`the "word"`))
 }
 
+func TestFormatFieldWithSeparators_NonDefault(t *testing.T) {
+	// With OFS=";" and OPS=":", those characters require quoting; comma and equals do not.
+	assert.Equal(t, `"x;y"`, FormatFieldWithSeparators("x;y", ";", ":"))
+	assert.Equal(t, `"u:v"`, FormatFieldWithSeparators("u:v", ";", ":"))
+	assert.Equal(t, "p,q", FormatFieldWithSeparators("p,q", ";", ":"))
+	assert.Equal(t, "b=c", FormatFieldWithSeparators("b=c", ";", ":"))
+	assert.Equal(t, `"the ""word"""`, FormatFieldWithSeparators(`the "word"`, ";", ":"))
+}
+
 func TestFormatField_RoundTrip(t *testing.T) {
 	input := `"x,y"="a,b,c",z=3` + "\n"
 	rdr := NewReader(strings.NewReader(input))
diff --git a/pkg/dsl/cst/builtin_function_manager.go b/pkg/dsl/cst/builtin_function_manager.go
index e58fb34ff..b5130afbd 100644
--- a/pkg/dsl/cst/builtin_function_manager.go
+++ b/pkg/dsl/cst/builtin_function_manager.go
@@ -696,11 +696,17 @@ Arrays are new in Miller 6; the substr function is older.`,
 			name:  "format",
 			class: FUNC_CLASS_STRING,
 			help: `Using first argument as format string, interpolate remaining arguments in place of
-each "{}" in the format string. Too-few arguments are treated as the empty string; too-many arguments are discarded.`,
+each "{}" in the format string. Also supports 1-based positional placeholders such as "{1}", which allow
+arguments to be reused and/or reordered. Plain "{}" placeholders consume arguments sequentially,
+independently of any positional placeholders. Too-few arguments are treated as the empty string, as are
+positional placeholders exceeding the number of arguments; too-many arguments are discarded. "{0}" is an
+error value, since positional placeholders are 1-based.`,
 			examples: []string{
-				`format("{}:{}:{}", 1,2)     gives "1:2:".`,
-				`format("{}:{}:{}", 1,2,3)   gives "1:2:3".`,
-				`format("{}:{}:{}", 1,2,3,4) gives "1:2:3".`,
+				`format("{}:{}:{}", 1,2)        gives "1:2:".`,
+				`format("{}:{}:{}", 1,2,3)      gives "1:2:3".`,
+				`format("{}:{}:{}", 1,2,3,4)    gives "1:2:3".`,
+				`format("{1}:{2}:{1}", "a","b") gives "a:b:a".`,
+				`format("{2}{}:{1}{}", 3,4)     gives "43:34".`,
 			},
 			variadicFunc: bifs.BIF_format,
 		},
@@ -1091,6 +1097,17 @@ is normally distributed.`,
 			},
 		},
 
+		{
+			name:      "sparkline",
+			class:     FUNC_CLASS_STATS,
+			help:      `Returns a string of Unicode block characters (one of β–β–‚β–ƒβ–„β–…β–†β–‡β–ˆ per element) representing the relative magnitudes of values in an array or map, for a compact ASCII/Unicode bar chart. Returns error for non-array/non-map types.`,
+			unaryFunc: bifs.BIF_sparkline,
+			examples: []string{
+				`sparkline([1,2,3,4,5,6,7,8]) is "β–β–‚β–ƒβ–„β–…β–†β–‡β–ˆ"`,
+				`sparkline([3,3,3]) is "▁▁▁"`,
+			},
+		},
+
 		{
 			name:      "mode",
 			class:     FUNC_CLASS_STATS,
diff --git a/pkg/input/record_reader.go b/pkg/input/record_reader.go
index bf0cd91a2..f70138750 100644
--- a/pkg/input/record_reader.go
+++ b/pkg/input/record_reader.go
@@ -20,3 +20,17 @@ type IRecordReader interface {
 		downstreamDoneChannel <-chan bool, // for mlr head
 	)
 }
+
+// hasNonEmptyField returns true if any of the split-out fields is non-empty.
+// A blank input line splits to zero fields or to a single empty field; a line
+// consisting only of field separators splits to all-empty fields. Used by the
+// CSV/TSV readers to support skipping trivial records at read time, when the
+// skip-trivial-records verb is present in the then-chain. See issue #1535.
+func hasNonEmptyField(fields []string) bool {
+	for _, field := range fields {
+		if field != "" {
+			return true
+		}
+	}
+	return false
+}
diff --git a/pkg/input/record_reader_csv.go b/pkg/input/record_reader_csv.go
index d83a2af6c..585f2e6ef 100644
--- a/pkg/input/record_reader_csv.go
+++ b/pkg/input/record_reader_csv.go
@@ -261,6 +261,13 @@ func (reader *RecordReaderCSV) getRecordBatch(
 
 		} else {
 			if !reader.readerOptions.AllowRaggedCSVInput {
+				if reader.readerOptions.SkipTrivialRecords && !hasNonEmptyField(csvRecord) {
+					// The then-chain includes the skip-trivial-records verb,
+					// so trivial records -- e.g. blank lines at the end of
+					// the file -- are to be skipped, not treated as fatal
+					// header/data length mismatches. See issue #1535.
+					continue
+				}
 				err := fmt.Errorf(
 					"CSV header/data length mismatch %d != %d at filename %s row %d",
 					nh, nd, reader.filename, reader.rowNumber,
diff --git a/pkg/input/record_reader_csvlite.go b/pkg/input/record_reader_csvlite.go
index 1c6bed007..580f001bf 100644
--- a/pkg/input/record_reader_csvlite.go
+++ b/pkg/input/record_reader_csvlite.go
@@ -254,7 +254,11 @@ func getRecordBatchExplicitCSVHeader(
 				if nh > nd {
 					// if header longer than data: use "" values
 					for i = nd; i < nh; i++ {
-						record.PutCopy(reader.headerStrings[i], mlrval.VOID)
+						_, err := record.PutReferenceMaybeDedupe(reader.headerStrings[i], mlrval.VOID.Copy(), dedupeFieldNames)
+						if err != nil {
+							errorChannel <- err
+							return
+						}
 					}
 				}
 			}
diff --git a/pkg/input/record_reader_dkvpx.go b/pkg/input/record_reader_dkvpx.go
index 51dcaae69..a2c338b79 100644
--- a/pkg/input/record_reader_dkvpx.go
+++ b/pkg/input/record_reader_dkvpx.go
@@ -1,10 +1,12 @@
-// RecordReaderDKVPX reads DKVPX format: comma-delimited key=value pairs with
-// CSV-style quoting. It uses the dkvpx package for parsing.
+// RecordReaderDKVPX reads DKVPX format: IFS-delimited (default comma) key=value
+// pairs (default pair separator "=") with CSV-style quoting. It uses the dkvpx
+// package for parsing.
 package input
 
 import (
 	"fmt"
 	"io"
+	"unicode/utf8"
 
 	"github.com/johnkerl/miller/v6/pkg/cli"
 	"github.com/johnkerl/miller/v6/pkg/dkvpx"
@@ -16,6 +18,8 @@ import (
 type RecordReaderDKVPX struct {
 	readerOptions   *cli.TReaderOptions
 	recordsPerBatch int64
+	ifs             rune // The dkvpx reader requires single-character IFS
+	ips             rune // The dkvpx reader requires single-character IPS
 }
 
 func NewRecordReaderDKVPX(
@@ -25,9 +29,19 @@ func NewRecordReaderDKVPX(
 	if readerOptions.IRS != "\n" && readerOptions.IRS != "\r\n" {
 		return nil, fmt.Errorf("for DKVPX, IRS cannot be altered; LF vs CR/LF is autodetected")
 	}
+	if utf8.RuneCountInString(readerOptions.IFS) != 1 {
+		return nil, fmt.Errorf("for DKVPX, IFS can only be a single character")
+	}
+	if utf8.RuneCountInString(readerOptions.IPS) != 1 {
+		return nil, fmt.Errorf("for DKVPX, IPS can only be a single character")
+	}
+	ifs, _ := utf8.DecodeRuneInString(readerOptions.IFS)
+	ips, _ := utf8.DecodeRuneInString(readerOptions.IPS)
 	return &RecordReaderDKVPX{
 		readerOptions:   readerOptions,
 		recordsPerBatch: recordsPerBatch,
+		ifs:             ifs,
+		ips:             ips,
 	}, nil
 }
 
@@ -82,7 +96,8 @@ func (reader *RecordReaderDKVPX) processHandle(
 	recordsPerBatch := reader.recordsPerBatch
 
 	dkvpxReader := dkvpx.NewReader(NewBOMStrippingReader(handle))
-	dkvpxReader.Comma = ','
+	dkvpxReader.Comma = reader.ifs
+	dkvpxReader.Equals = reader.ips
 	if reader.readerOptions.CommentHandling != cli.CommentsAreData &&
 		len(reader.readerOptions.CommentString) == 1 {
 		dkvpxReader.Comment = rune(reader.readerOptions.CommentString[0])
diff --git a/pkg/input/record_reader_dkvpx_test.go b/pkg/input/record_reader_dkvpx_test.go
index b0da5161e..bad6ebac7 100644
--- a/pkg/input/record_reader_dkvpx_test.go
+++ b/pkg/input/record_reader_dkvpx_test.go
@@ -20,6 +20,53 @@ func TestNewRecordReaderDKVPX(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestNewRecordReaderDKVPX_MultiCharIFSRejected(t *testing.T) {
+	readerOptions := cli.DefaultReaderOptions()
+	readerOptions.InputFileFormat = "dkvpx"
+	assert.NoError(t, cli.FinalizeReaderOptions(&readerOptions))
+	readerOptions.IFS = ";;"
+
+	reader, err := NewRecordReaderDKVPX(&readerOptions, 1)
+	assert.Nil(t, reader)
+	assert.Error(t, err)
+}
+
+func TestNewRecordReaderDKVPX_MultiCharIPSRejected(t *testing.T) {
+	readerOptions := cli.DefaultReaderOptions()
+	readerOptions.InputFileFormat = "dkvpx"
+	assert.NoError(t, cli.FinalizeReaderOptions(&readerOptions))
+	readerOptions.IPS = "::"
+
+	reader, err := NewRecordReaderDKVPX(&readerOptions, 1)
+	assert.Nil(t, reader)
+	assert.Error(t, err)
+}
+
+func TestRecordReaderDKVPX_NonDefaultSeparators(t *testing.T) {
+	readerOptions := cli.DefaultReaderOptions()
+	readerOptions.InputFileFormat = "dkvpx"
+	assert.NoError(t, cli.FinalizeReaderOptions(&readerOptions))
+	readerOptions.IFS = ";"
+	readerOptions.IPS = ":"
+
+	reader, err := NewRecordReaderDKVPX(&readerOptions, 1)
+	assert.NoError(t, err)
+
+	ctx := types.Context{}
+	readerChannel := make(chan []*types.RecordAndContext, 4)
+	errorChannel := make(chan error, 1)
+
+	input := strings.NewReader("x:1;y:\"a;b\"\n")
+	go reader.processHandle(input, "(test)", &ctx, readerChannel, errorChannel, nil)
+
+	records := <-readerChannel
+	assert.Len(t, records, 1)
+	assert.Equal(t, "x", records[0].Record.Head.Key)
+	assert.Equal(t, "1", records[0].Record.Head.Value.String())
+	assert.Equal(t, "y", records[0].Record.Head.Next.Key)
+	assert.Equal(t, "a;b", records[0].Record.Head.Next.Value.String())
+}
+
 func TestRecordReaderDKVPX_ReadStdin(t *testing.T) {
 	readerOptions := cli.DefaultReaderOptions()
 	readerOptions.InputFileFormat = "dkvpx"
diff --git a/pkg/input/record_reader_pprint.go b/pkg/input/record_reader_pprint.go
index 4b59d69a4..83597f4b3 100644
--- a/pkg/input/record_reader_pprint.go
+++ b/pkg/input/record_reader_pprint.go
@@ -496,7 +496,15 @@ func getRecordBatchExplicitPprintHeader(
 				if nh > nd {
 					// if header longer than data: use "" values
 					for i = nd; i < nh; i++ {
-						record.PutCopy(reader.headerStrings[i], mlrval.VOID)
+						_, err := record.PutReferenceMaybeDedupe(
+							reader.headerStrings[i],
+							mlrval.VOID.Copy(),
+							dedupeFieldNames,
+						)
+						if err != nil {
+							errorChannel <- err
+							return
+						}
 					}
 				}
 			}
diff --git a/pkg/input/record_reader_tsv.go b/pkg/input/record_reader_tsv.go
index 0e3a54e31..e16dc42d3 100644
--- a/pkg/input/record_reader_tsv.go
+++ b/pkg/input/record_reader_tsv.go
@@ -185,6 +185,13 @@ func getRecordBatchExplicitTSVHeader(
 			// Get data lines on subsequent loop iterations
 		} else {
 			if !reader.readerOptions.AllowRaggedCSVInput && len(reader.headerStrings) != len(fields) {
+				if reader.readerOptions.SkipTrivialRecords && !hasNonEmptyField(fields) {
+					// The then-chain includes the skip-trivial-records verb,
+					// so trivial records -- e.g. blank lines at the end of
+					// the file -- are to be skipped, not treated as fatal
+					// header/data length mismatches. See issue #1535.
+					continue
+				}
 				err := fmt.Errorf(
 					"TSV header/data length mismatch %d != %d at filename %s line %d",
 					len(reader.headerStrings), len(fields), filename, reader.inputLineNumber,
@@ -219,7 +226,11 @@ func getRecordBatchExplicitTSVHeader(
 				if nh > nd {
 					// if header longer than data: use "" values
 					for i = nd; i < nh; i++ {
-						record.PutCopy(reader.headerStrings[i], mlrval.VOID)
+						_, err := record.PutReferenceMaybeDedupe(reader.headerStrings[i], mlrval.VOID.Copy(), dedupeFieldNames)
+						if err != nil {
+							errorChannel <- err
+							return
+						}
 					}
 				}
 			}
diff --git a/pkg/output/record_writer_csv.go b/pkg/output/record_writer_csv.go
index bb96bc706..3f0b075c2 100644
--- a/pkg/output/record_writer_csv.go
+++ b/pkg/output/record_writer_csv.go
@@ -32,7 +32,7 @@ func NewRecordWriterCSV(writerOptions *cli.TWriterOptions) (*RecordWriterCSV, er
 		return nil, fmt.Errorf("for CSV, OFS can only be a single character")
 	}
 	if writerOptions.ORS != "\n" && writerOptions.ORS != "\r\n" {
-		return nil, fmt.Errorf("for CSV, ORS cannot be altered")
+		return nil, fmt.Errorf("for CSV, ORS must be newline or carriage-return/newline")
 	}
 	writer := &RecordWriterCSV{
 		writerOptions:     writerOptions,
@@ -59,6 +59,9 @@ func (writer *RecordWriterCSV) Write(
 	if writer.csvWriter == nil {
 		writer.csvWriter = csv.NewWriter(bufferedOutputStream)
 		writer.csvWriter.Comma = rune(writer.writerOptions.OFS[0]) // xxx temp -- needs length-1 OFS
+		// Issues #1810 and #1722: honor `--ors '\r\n'` (or `--ors crlf`) for
+		// CSV output. The default remains "\n".
+		writer.csvWriter.UseCRLF = writer.writerOptions.ORS == "\r\n"
 	}
 
 	if writer.firstRecordKeys == nil {
diff --git a/pkg/output/record_writer_csv_test.go b/pkg/output/record_writer_csv_test.go
new file mode 100644
index 000000000..7e313bdb6
--- /dev/null
+++ b/pkg/output/record_writer_csv_test.go
@@ -0,0 +1,99 @@
+// Tests for https://github.com/johnkerl/miller/issues/1810 and
+// https://github.com/johnkerl/miller/issues/1722: `--ors '\r\n'` (or `--ors
+// crlf`) should be honored for CSV/CSV-lite/TSV output. These are unit tests
+// rather than regression-test cases since the regression-test harness
+// normalizes CR/LF to LF before comparing outputs, and so cannot assert
+// byte-exact line endings.
+
+package output
+
+import (
+	"bufio"
+	"bytes"
+	"testing"
+
+	"github.com/johnkerl/miller/v6/pkg/cli"
+	"github.com/johnkerl/miller/v6/pkg/mlrval"
+)
+
+func makeTestRecord() *mlrval.Mlrmap {
+	record := mlrval.NewMlrmap()
+	record.PutReference("a", mlrval.FromString("1"))
+	record.PutReference("b", mlrval.FromString("2"))
+	return record
+}
+
+// runWriter drives a record-writer with a single a=1,b=2 record and returns
+// the output bytes.
+func runWriter(t *testing.T, writer IRecordWriter) string {
+	t.Helper()
+	var buffer bytes.Buffer
+	bufferedOutputStream := bufio.NewWriter(&buffer)
+	err := writer.Write(makeTestRecord(), nil, bufferedOutputStream, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Write(nil, nil, bufferedOutputStream, false) // end of stream
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := bufferedOutputStream.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	return buffer.String()
+}
+
+func TestCSVWriterLFDefault(t *testing.T) {
+	writer, err := NewRecordWriterCSV(&cli.TWriterOptions{OFS: ",", ORS: "\n"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	output := runWriter(t, writer)
+	expected := "a,b\n1,2\n"
+	if output != expected {
+		t.Fatalf("expected %q, got %q", expected, output)
+	}
+}
+
+func TestCSVWriterCRLF(t *testing.T) {
+	writer, err := NewRecordWriterCSV(&cli.TWriterOptions{OFS: ",", ORS: "\r\n"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	output := runWriter(t, writer)
+	expected := "a,b\r\n1,2\r\n"
+	if output != expected {
+		t.Fatalf("expected %q, got %q", expected, output)
+	}
+}
+
+func TestCSVWriterRejectsOtherORS(t *testing.T) {
+	_, err := NewRecordWriterCSV(&cli.TWriterOptions{OFS: ",", ORS: ";"})
+	if err == nil {
+		t.Fatal("expected error for ORS \";\" but got none")
+	}
+}
+
+func TestCSVLiteWriterCRLF(t *testing.T) {
+	writer, err := NewRecordWriterCSVLite(&cli.TWriterOptions{OFS: ",", ORS: "\r\n"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	output := runWriter(t, writer)
+	expected := "a,b\r\n1,2\r\n"
+	if output != expected {
+		t.Fatalf("expected %q, got %q", expected, output)
+	}
+}
+
+func TestTSVWriterCRLF(t *testing.T) {
+	writer, err := NewRecordWriterTSV(&cli.TWriterOptions{OFS: "\t", ORS: "\r\n"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	output := runWriter(t, writer)
+	expected := "a\tb\r\n1\t2\r\n"
+	if output != expected {
+		t.Fatalf("expected %q, got %q", expected, output)
+	}
+}
diff --git a/pkg/output/record_writer_dkvpx.go b/pkg/output/record_writer_dkvpx.go
index 6be7c2d26..dd9c73ee1 100644
--- a/pkg/output/record_writer_dkvpx.go
+++ b/pkg/output/record_writer_dkvpx.go
@@ -42,8 +42,8 @@ func (writer *RecordWriterDKVPX) Write(
 		}
 		first = false
 
-		keyStr := dkvpx.FormatField(pe.Key)
-		valStr := dkvpx.FormatField(pe.Value.String())
+		keyStr := dkvpx.FormatFieldWithSeparators(pe.Key, writer.writerOptions.OFS, writer.writerOptions.OPS)
+		valStr := dkvpx.FormatFieldWithSeparators(pe.Value.String(), writer.writerOptions.OFS, writer.writerOptions.OPS)
 
 		bufferedOutputStream.WriteString(colorizer.MaybeColorizeKey(keyStr, outputIsStdout))
 		bufferedOutputStream.WriteString(writer.writerOptions.OPS)
diff --git a/pkg/output/record_writer_markdown.go b/pkg/output/record_writer_markdown.go
index cf41bb877..f60fd7a2c 100644
--- a/pkg/output/record_writer_markdown.go
+++ b/pkg/output/record_writer_markdown.go
@@ -79,7 +79,15 @@ func (writer *RecordWriterMarkdown) writeStreaming(
 
 		bufferedOutputStream.WriteString("|")
 		for pe := outrec.Head; pe != nil; pe = pe.Next {
-			bufferedOutputStream.WriteString(" --- |")
+			// In streaming mode the header-separator line must be emitted
+			// before subsequent records are seen, so with
+			// --right-align-numeric the alignment marker is chosen from this
+			// first record's value.
+			if writer.writerOptions.RightAlignNumericOutput && pe.Value.IsNumeric() {
+				bufferedOutputStream.WriteString(" ---: |")
+			} else {
+				bufferedOutputStream.WriteString(" --- |")
+			}
 		}
 		bufferedOutputStream.WriteString(writer.writerOptions.ORS)
 
@@ -138,10 +146,30 @@ func (writer *RecordWriterMarkdown) flushBatch(
 
 	first := writer.batch[0]
 
-	// Floor of 3 so "---" never overflows the column.
+	// With --right-align-numeric, a column gets a right-alignment marker
+	// (`---:`) when every value in the batch's column is numeric.
+	columnRightAligned := make(map[string]bool)
+	if writer.writerOptions.RightAlignNumericOutput {
+		for pe := first.Head; pe != nil; pe = pe.Next {
+			columnRightAligned[pe.Key] = true
+		}
+		for _, rec := range writer.batch {
+			for pe := rec.Head; pe != nil; pe = pe.Next {
+				if !pe.Value.IsNumeric() {
+					columnRightAligned[pe.Key] = false
+				}
+			}
+		}
+	}
+
+	// Floor of 3 so "---" never overflows the column -- or 4 for "---:".
 	maxWidths := make(map[string]int)
 	for pe := first.Head; pe != nil; pe = pe.Next {
-		maxWidths[pe.Key] = max(lib.DisplayWidth(pe.Key), 3)
+		minWidth := 3
+		if columnRightAligned[pe.Key] {
+			minWidth = 4
+		}
+		maxWidths[pe.Key] = max(lib.DisplayWidth(pe.Key), minWidth)
 	}
 	for _, rec := range writer.batch {
 		for pe := rec.Head; pe != nil; pe = pe.Next {
@@ -153,12 +181,19 @@ func (writer *RecordWriterMarkdown) flushBatch(
 		}
 	}
 
-	// Header
+	// Header. Right-aligned columns get right-justified header text so that
+	// header and data share the same alignment in the raw markdown, matching
+	// how Markdown viewers render the `---:` marker.
 	bufferedOutputStream.WriteString("|")
 	for pe := first.Head; pe != nil; pe = pe.Next {
 		bufferedOutputStream.WriteString(" ")
-		bufferedOutputStream.WriteString(colorizer.MaybeColorizeKey(pe.Key, outputIsStdout))
-		writePadding(bufferedOutputStream, maxWidths[pe.Key]-lib.DisplayWidth(pe.Key))
+		if columnRightAligned[pe.Key] {
+			writePadding(bufferedOutputStream, maxWidths[pe.Key]-lib.DisplayWidth(pe.Key))
+			bufferedOutputStream.WriteString(colorizer.MaybeColorizeKey(pe.Key, outputIsStdout))
+		} else {
+			bufferedOutputStream.WriteString(colorizer.MaybeColorizeKey(pe.Key, outputIsStdout))
+			writePadding(bufferedOutputStream, maxWidths[pe.Key]-lib.DisplayWidth(pe.Key))
+		}
 		bufferedOutputStream.WriteString(" |")
 	}
 	bufferedOutputStream.WriteString(writer.writerOptions.ORS)
@@ -166,8 +201,13 @@ func (writer *RecordWriterMarkdown) flushBatch(
 	// Separator
 	bufferedOutputStream.WriteString("|")
 	for pe := first.Head; pe != nil; pe = pe.Next {
-		bufferedOutputStream.WriteString(" ---")
-		writePadding(bufferedOutputStream, maxWidths[pe.Key]-3)
+		if columnRightAligned[pe.Key] {
+			writePadding(bufferedOutputStream, maxWidths[pe.Key]-4)
+			bufferedOutputStream.WriteString(" ---:")
+		} else {
+			bufferedOutputStream.WriteString(" ---")
+			writePadding(bufferedOutputStream, maxWidths[pe.Key]-3)
+		}
 		bufferedOutputStream.WriteString(" |")
 	}
 	bufferedOutputStream.WriteString(writer.writerOptions.ORS)
@@ -178,8 +218,13 @@ func (writer *RecordWriterMarkdown) flushBatch(
 		for pe := rec.Head; pe != nil; pe = pe.Next {
 			value := strings.ReplaceAll(pe.Value.String(), "|", "\\|")
 			bufferedOutputStream.WriteString(" ")
-			bufferedOutputStream.WriteString(colorizer.MaybeColorizeValue(value, outputIsStdout))
-			writePadding(bufferedOutputStream, maxWidths[pe.Key]-lib.DisplayWidth(value))
+			if columnRightAligned[pe.Key] {
+				writePadding(bufferedOutputStream, maxWidths[pe.Key]-lib.DisplayWidth(value))
+				bufferedOutputStream.WriteString(colorizer.MaybeColorizeValue(value, outputIsStdout))
+			} else {
+				bufferedOutputStream.WriteString(colorizer.MaybeColorizeValue(value, outputIsStdout))
+				writePadding(bufferedOutputStream, maxWidths[pe.Key]-lib.DisplayWidth(value))
+			}
 			bufferedOutputStream.WriteString(" |")
 		}
 		bufferedOutputStream.WriteString(writer.writerOptions.ORS)
diff --git a/pkg/output/record_writer_pprint.go b/pkg/output/record_writer_pprint.go
index 332fac389..47d1afa79 100644
--- a/pkg/output/record_writer_pprint.go
+++ b/pkg/output/record_writer_pprint.go
@@ -122,14 +122,42 @@ func (writer *RecordWriterPPRINT) writeHeterogenousList(
 			maxWidths[key] = width
 		}
 	}
+	rightAlignedHeaders := writer.computeRightAlignedHeaders(records)
 	if barred {
-		writer.writeHeterogenousListBarred(records, maxWidths, bufferedOutputStream, outputIsStdout)
+		writer.writeHeterogenousListBarred(records, maxWidths, rightAlignedHeaders,
+			bufferedOutputStream, outputIsStdout)
 	} else {
-		writer.writeHeterogenousListNonBarred(records, maxWidths, bufferedOutputStream, outputIsStdout)
+		writer.writeHeterogenousListNonBarred(records, maxWidths, rightAlignedHeaders,
+			bufferedOutputStream, outputIsStdout)
 	}
 	return true
 }
 
+// computeRightAlignedHeaders returns, for --right-align-numeric, the set of
+// columns whose every value in the batch is numeric. Headers over such
+// columns are right-aligned so that they line up with their data cells;
+// headers over mixed columns stay left-aligned. See issue #380.
+func (writer *RecordWriterPPRINT) computeRightAlignedHeaders(
+	records []*mlrval.Mlrmap,
+) map[string]bool {
+	rightAlignedHeaders := make(map[string]bool)
+	if !writer.writerOptions.RightAlignNumericOutput {
+		return rightAlignedHeaders
+	}
+	for _, outrec := range records {
+		for pe := outrec.Head; pe != nil; pe = pe.Next {
+			isNumeric := pe.Value.IsNumeric()
+			previous, seen := rightAlignedHeaders[pe.Key]
+			if !seen {
+				rightAlignedHeaders[pe.Key] = isNumeric
+			} else {
+				rightAlignedHeaders[pe.Key] = previous && isNumeric
+			}
+		}
+	}
+	return rightAlignedHeaders
+}
+
 // Example:
 //
 // a   b   i  x                    y
@@ -142,6 +170,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousList(
 func (writer *RecordWriterPPRINT) writeHeterogenousListNonBarred(
 	records []*mlrval.Mlrmap,
 	maxWidths map[string]int,
+	rightAlignedHeaders map[string]bool,
 	bufferedOutputStream *bufio.Writer,
 	outputIsStdout bool,
 ) {
@@ -152,7 +181,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousListNonBarred(
 		// Print header line
 		if onFirst && !writer.writerOptions.HeaderlessOutput {
 			for pe := outrec.Head; pe != nil; pe = pe.Next {
-				if !writer.writerOptions.RightAlignedPPRINTOutput { // left-align
+				if !writer.headerIsRightAligned(pe.Key, rightAlignedHeaders) { // left-align
 					if pe.Next != nil {
 						// Header line, left-align, not last column
 						bufferedOutputStream.WriteString(colorizer.MaybeColorizeKey(pe.Key, outputIsStdout))
@@ -185,7 +214,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousListNonBarred(
 			if s == "" {
 				s = "-"
 			}
-			if !writer.writerOptions.RightAlignedPPRINTOutput { // left-align
+			if !writer.cellIsRightAligned(pe.Value) { // left-align
 				if pe.Next != nil {
 					// Data line, left-align, not last column
 					bufferedOutputStream.WriteString(colorizer.MaybeColorizeValue(s, outputIsStdout))
@@ -247,6 +276,7 @@ type barredChars struct {
 func (writer *RecordWriterPPRINT) writeHeterogenousListBarred(
 	records []*mlrval.Mlrmap,
 	maxWidths map[string]int,
+	rightAlignedHeaders map[string]bool,
 	bufferedOutputStream *bufio.Writer,
 	outputIsStdout bool,
 ) {
@@ -311,7 +341,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousListBarred(
 
 			bufferedOutputStream.WriteString(bc.verticalStart)
 			for pe := outrec.Head; pe != nil; pe = pe.Next {
-				if !writer.writerOptions.RightAlignedPPRINTOutput { // left-align
+				if !writer.headerIsRightAligned(pe.Key, rightAlignedHeaders) { // left-align
 					bufferedOutputStream.WriteString(colorizer.MaybeColorizeKey(pe.Key, outputIsStdout))
 					writer.writePadding(pe.Key, maxWidths[pe.Key], bufferedOutputStream)
 				} else { // right-align
@@ -343,7 +373,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousListBarred(
 		bufferedOutputStream.WriteString(bc.verticalStart)
 		for pe := outrec.Head; pe != nil; pe = pe.Next {
 			s := pe.Value.String()
-			if !writer.writerOptions.RightAlignedPPRINTOutput { // left-align
+			if !writer.cellIsRightAligned(pe.Value) { // left-align
 				bufferedOutputStream.WriteString(colorizer.MaybeColorizeValue(s, outputIsStdout))
 				writer.writePadding(s, maxWidths[pe.Key], bufferedOutputStream)
 			} else { // right-align
@@ -379,6 +409,30 @@ func (writer *RecordWriterPPRINT) writeHeterogenousListBarred(
 	}
 }
 
+// cellIsRightAligned decides the alignment of a single data cell: everything
+// with --right; numeric values only with --right-align-numeric. Header cells
+// are not passed through here -- see headerIsRightAligned.
+func (writer *RecordWriterPPRINT) cellIsRightAligned(value *mlrval.Mlrval) bool {
+	if writer.writerOptions.RightAlignedPPRINTOutput {
+		return true
+	}
+	return writer.writerOptions.RightAlignNumericOutput && value.IsNumeric()
+}
+
+// headerIsRightAligned decides the alignment of a header cell: everything
+// with --right; with --right-align-numeric, headers over all-numeric columns
+// (as precomputed by computeRightAlignedHeaders) so that header and data
+// share the same alignment.
+func (writer *RecordWriterPPRINT) headerIsRightAligned(
+	key string,
+	rightAlignedHeaders map[string]bool,
+) bool {
+	if writer.writerOptions.RightAlignedPPRINTOutput {
+		return true
+	}
+	return rightAlignedHeaders[key]
+}
+
 func (writer *RecordWriterPPRINT) writePadding(
 	text string,
 	fieldWidth int,
diff --git a/pkg/output/record_writer_tsv.go b/pkg/output/record_writer_tsv.go
index 3ea66b90a..5582a6d9e 100644
--- a/pkg/output/record_writer_tsv.go
+++ b/pkg/output/record_writer_tsv.go
@@ -24,7 +24,7 @@ func NewRecordWriterTSV(writerOptions *cli.TWriterOptions) (*RecordWriterTSV, er
 		return nil, fmt.Errorf("for TSV, OFS cannot be altered")
 	}
 	if writerOptions.ORS != "\n" && writerOptions.ORS != "\r\n" {
-		return nil, fmt.Errorf("for CSV, ORS cannot be altered")
+		return nil, fmt.Errorf("for TSV, ORS must be newline or carriage-return/newline")
 	}
 	return &RecordWriterTSV{
 		writerOptions:     writerOptions,
diff --git a/pkg/pbnjay-strptime/strptime.go b/pkg/pbnjay-strptime/strptime.go
index 4c9f1949e..36b50df27 100644
--- a/pkg/pbnjay-strptime/strptime.go
+++ b/pkg/pbnjay-strptime/strptime.go
@@ -23,9 +23,13 @@ THE SOFTWARE.
 // Package strptime provides a C-style strptime wrappers for time.Parse.
 //
 // It supports the following subset of format strings (stolen from python docs):
+//     %a  Weekday as locale’s abbreviated name.
+//     %A  Weekday as locale’s full name.
 //     %d  Day of the month as a zero-padded decimal number.
+//     %e  Day of the month as a space-padded decimal number.
 //     %b  Month as locale’s abbreviated name.
 //     %B  Month as locale’s full name.
+//     %h  Month as locale’s abbreviated name (alias of %b).
 //     %m  Month as a zero-padded decimal number.
 //     %y  Year without century as a zero-padded decimal number.
 //     %Y  Year with century as a decimal number.
@@ -40,6 +44,9 @@ THE SOFTWARE.
 //         "2022-04-03 17:38:20.123 UTC" all parse with "%Y-%m-%d %H:%M:%S.%f UTC").
 //     %z  UTC offset in the form +HHMM or -HHMM.
 //     %Z  Time zone name. UTC, EST, CST
+//     %c  Locale’s date and time representation. Shorthand for "%a %b %e %H:%M:%S %Y".
+//     %x  Locale’s date representation. Shorthand for "%m/%d/%y".
+//     %X  Locale’s time representation. Shorthand for "%H:%M:%S".
 //     %%  A literal '%' character.
 //
 // BUG(pbnjay): If an unsupported specifier is used, it may NOT directly precede a
@@ -225,38 +232,46 @@ func strptime_tz(
 		interveningLen := sil
 		// Now sil becomes the offset of this part within the strptime-style input.
 		if sil > 0 {
-			// Optional ".%f" (mixed modes): when format has ".%f" and next format code is 'f',
-			// the literal "." may be absent (e.g. "2022-04-03 17:38:20 UTC" vs "2022-04-03 17:38:20.123 UTC").
-			dotOptional := false
-			if partsIndex+1 < nparts {
-				nextPart := partsBetweenPercentSigns[partsIndex+1]
-				if len(partBetweenPercentSigns) > 0 &&
-					partBetweenPercentSigns[len(partBetweenPercentSigns)-1] == '.' &&
-					len(nextPart) > 0 && nextPart[0] == 'f' {
-					dotOptional = true
-				}
-			}
-			if dotOptional {
-				dotSearch := partBetweenPercentSigns[1:]
-				sil = strings.Index(strptimeInput[inputIdx:], dotSearch)
-				if sil == -1 {
-					// Try without the dot: find the first char of the literal after %f
-					if partsIndex+1 < nparts {
-						nextPart := partsBetweenPercentSigns[partsIndex+1]
-						if len(nextPart) > 1 {
-							fallbackSearch := string(nextPart[1])
-							sil = strings.Index(strptimeInput[inputIdx:], fallbackSearch)
-							if sil != -1 {
-								interveningLen = 0
-							}
-						} else {
-							sil = len(strptimeInput) - inputIdx
-							interveningLen = 0
-						}
+			if formatCode == 'e' {
+				// %e's own optional leading pad space is indistinguishable from a
+				// literal separator space (e.g. "%b %e %T" matching "Mar  4 ..."), so a
+				// plain search for the next literal text would stop at %e's own padding.
+				// Determine the field width directly instead.
+				sil = spacePaddedDayLen(strptimeInput[inputIdx:])
+			} else {
+				// Optional ".%f" (mixed modes): when format has ".%f" and next format code is 'f',
+				// the literal "." may be absent (e.g. "2022-04-03 17:38:20 UTC" vs "2022-04-03 17:38:20.123 UTC").
+				dotOptional := false
+				if partsIndex+1 < nparts {
+					nextPart := partsBetweenPercentSigns[partsIndex+1]
+					if len(partBetweenPercentSigns) > 0 &&
+						partBetweenPercentSigns[len(partBetweenPercentSigns)-1] == '.' &&
+						len(nextPart) > 0 && nextPart[0] == 'f' {
+						dotOptional = true
 					}
 				}
-			} else {
-				sil = strings.Index(strptimeInput[inputIdx:], partBetweenPercentSigns[1:])
+				if dotOptional {
+					dotSearch := partBetweenPercentSigns[1:]
+					sil = strings.Index(strptimeInput[inputIdx:], dotSearch)
+					if sil == -1 {
+						// Try without the dot: find the first char of the literal after %f
+						if partsIndex+1 < nparts {
+							nextPart := partsBetweenPercentSigns[partsIndex+1]
+							if len(nextPart) > 1 {
+								fallbackSearch := string(nextPart[1])
+								sil = strings.Index(strptimeInput[inputIdx:], fallbackSearch)
+								if sil != -1 {
+									interveningLen = 0
+								}
+							} else {
+								sil = len(strptimeInput) - inputIdx
+								interveningLen = 0
+							}
+						}
+					}
+				} else {
+					sil = strings.Index(strptimeInput[inputIdx:], partBetweenPercentSigns[1:])
+				}
 			}
 		}
 		if sil == -1 {
@@ -273,13 +288,22 @@ func strptime_tz(
 		if supported {
 			// Accumulate the go-lib style template and input strings.
 			if sil == 0 { // No intervening text, e.g. "%Y%m%d"
-				if formatCode == 'f' {
+				switch formatCode {
+				case 'f':
 					// %f is optional decimal point + 0-6 digit runes (microseconds).
 					// Supports mixed modes: no fraction ("20 UTC"), dot only ("20. UTC"), or digits ("20.123 UTC").
 					// Do not consume the rest of the string so that %f%z works:
 					// e.g. ".160001+0100" -> %f takes ".160001", %z takes "+0100".
 					sil, fracHasDigits = parseFracLen(strptimeInput[inputIdx:])
-				} else {
+				case 'e':
+					sil = spacePaddedDayLen(strptimeInput[inputIdx:])
+					if sil == -1 {
+						if _debug {
+							fmt.Printf("format/template mismatch 2\n")
+						}
+						return time.Time{}, ErrFormatMismatch
+					}
+				default:
 					want := len(templateComponent)
 					remaining := len(strptimeInput) - inputIdx
 					if remaining == 0 {
@@ -414,17 +438,42 @@ func parseFracLen(s string) (length int, hasDigits bool) {
 	return n, digits > 0
 }
 
+// spacePaddedDayLen returns the byte length of a strptime %e field in s: a day-of-month
+// as either a single digit ("4"), a space followed by a digit (" 4"), or two digits
+// ("14"). Returns -1 if s does not start with a valid %e field.
+func spacePaddedDayLen(s string) int {
+	isDigit := func(b byte) bool { return b >= '0' && b <= '9' }
+	if s == "" {
+		return -1
+	}
+	if s[0] == ' ' {
+		if len(s) > 1 && isDigit(s[1]) {
+			return 2
+		}
+		return -1
+	}
+	if !isDigit(s[0]) {
+		return -1
+	}
+	if len(s) > 1 && isDigit(s[1]) {
+		return 2
+	}
+	return 1
+}
+
 // expandShorthands handles some shorthands that the C library uses, which we can easily
 // replicate -- e.g. "%F" is "%Y-%m-%d".
 func expandShorthands(format string) string {
 	// TODO: mem cache
+	format = strings.ReplaceAll(format, "%c", "%a %b %e %H:%M:%S %Y")
+	format = strings.ReplaceAll(format, "%x", "%m/%d/%y")
+	format = strings.ReplaceAll(format, "%X", "%H:%M:%S")
 	format = strings.ReplaceAll(format, "%T", "%H:%M:%S")
 	format = strings.ReplaceAll(format, "%D", "%m/%d/%y")
 	format = strings.ReplaceAll(format, "%F", "%Y-%m-%d")
 	format = strings.ReplaceAll(format, "%R", "%H:%M")
 	format = strings.ReplaceAll(format, "%r", "%I:%M:%S %p")
 	format = strings.ReplaceAll(format, "%T", "%H:%M:%S")
-	// We've no %e in this package
 	// format = strings.ReplaceAll(format, "%v", "%e-%b-%Y")
 	return format
 }
@@ -437,9 +486,13 @@ var (
 	ErrFormatUnsupported = errors.New("date format contains unsupported percent-encodings")
 
 	formatMap = map[int]string{
+		'a': "Mon",
+		'A': "Monday",
 		'b': "Jan",
 		'B': "January",
+		'h': "Jan",
 		'd': "02",
+		'e': "_2",
 		'f': "999999",
 		'H': "15",
 		'I': "03",
diff --git a/pkg/pbnjay-strptime/strptime_test.go b/pkg/pbnjay-strptime/strptime_test.go
index e158b27bd..fa4e2d8bf 100644
--- a/pkg/pbnjay-strptime/strptime_test.go
+++ b/pkg/pbnjay-strptime/strptime_test.go
@@ -58,7 +58,7 @@ var testData = []testDataType{
 	},
 	{
 		"01/02/70 14:20",
-		"%D %X", // no such format code
+		"%D %U", // no such format code
 		false,
 		0,
 	},
@@ -151,6 +151,81 @@ var testData = []testDataType{
 		true,
 		1649007500, // .1 second = 1649007500.1 in float, but Unix() truncates to 1649007500
 	},
+
+	// %a / %A (weekday name) and %e (space-padded day of month), and %h (alias of
+	// %b). This is the exact repro from https://github.com/johnkerl/miller/issues/1518:
+	// strptime rejected format codes that strftime itself produces.
+	{
+		"Mon Mar  4 11:41:08 2024",
+		"%a %b %e %T %Y",
+		true,
+		1709552468,
+	},
+	{
+		"Monday Mar  4 11:41:08 2024",
+		"%A %b %e %T %Y",
+		true,
+		1709552468,
+	},
+	{
+		"Mon Mar  4 11:41:08 2024",
+		"%a %h %e %T %Y",
+		true,
+		1709552468,
+	},
+	// %e with a double-digit day -- no space padding present in the source.
+	{
+		"Tue Oct 22 00:00:00 2024",
+		"%a %b %e %T %Y",
+		true,
+		1729555200,
+	},
+	// %e with a single (non-padded) space before a single-digit day.
+	{
+		"Mar 4 2024",
+		"%b %e %Y",
+		true,
+		1709510400,
+	},
+	// %e directly adjacent to another format code, no intervening literal text.
+	{
+		"4Mar2024",
+		"%e%b%Y",
+		true,
+		1709510400,
+	},
+	{
+		"14Mar2024",
+		"%e%b%Y",
+		true,
+		1710374400,
+	},
+	// %e at the very end of the string.
+	{
+		"2024-03-4",
+		"%Y-%m-%e",
+		true,
+		1709510400,
+	},
+	{
+		"2024-03-14",
+		"%Y-%m-%e",
+		true,
+		1710374400,
+	},
+	// %c, %x, %X shorthands.
+	{
+		"Mon Mar  4 11:41:08 2024",
+		"%c",
+		true,
+		1709552468,
+	},
+	{
+		"03/04/24 11:41:08",
+		"%x %X",
+		true,
+		1709552468,
+	},
 }
 
 func TestStrptime(t *testing.T) {
diff --git a/pkg/transformers/aaa_transformer_table.go b/pkg/transformers/aaa_transformer_table.go
index a0f4aab5d..b9bbe7f64 100644
--- a/pkg/transformers/aaa_transformer_table.go
+++ b/pkg/transformers/aaa_transformer_table.go
@@ -48,6 +48,7 @@ var TRANSFORMER_LOOKUP_TABLE = []TransformerSetup{
 	NestSetup,
 	NothingSetup,
 	PutSetup,
+	RankSetup,
 	RegularizeSetup,
 	RemoveEmptyColumnsSetup,
 	RenameSetup,
@@ -62,6 +63,7 @@ var TRANSFORMER_LOOKUP_TABLE = []TransformerSetup{
 	SkipTrivialRecordsSetup,
 	SortSetup,
 	SortWithinRecordsSetup,
+	SparklineSetup,
 	SparsifySetup,
 	SplitSetup,
 	SsubSetup,
diff --git a/pkg/transformers/histogram.go b/pkg/transformers/histogram.go
index 53ccdc859..2c71e5474 100644
--- a/pkg/transformers/histogram.go
+++ b/pkg/transformers/histogram.go
@@ -5,6 +5,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/johnkerl/miller/v6/pkg/bifs"
 	"github.com/johnkerl/miller/v6/pkg/cli"
 	"github.com/johnkerl/miller/v6/pkg/lib"
 	"github.com/johnkerl/miller/v6/pkg/mlrval"
@@ -21,6 +22,7 @@ var histogramOptions = []OptionSpec{
 	{Flag: "--nbins", Arg: "{n}", Type: "int", Desc: "Number of histogram bins. Defaults to 20."},
 	{Flag: "--auto", Type: "bool", Desc: "Automatically computes limits, ignoring --lo and --hi. Holds all values in memory before producing any output."},
 	{Flag: "-o", Arg: "{prefix}", Type: "string", Desc: "Prefix for output field name. Default: no prefix."},
+	{Flag: "-s", Type: "bool", Desc: "Print a one-line Unicode sparkline per field instead of per-bin counts."},
 }
 
 var HistogramSetup = TransformerSetup{
@@ -39,6 +41,8 @@ func transformerHistogramUsage(
 	fmt.Fprintf(o, "Just a histogram. Input values < lo or > hi are not counted.\n")
 	fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb)
 	WriteVerbOptions(o, histogramOptions)
+	fmt.Fprintf(o, "With -s, output is one record per value-field, with a sparkline field\n")
+	fmt.Fprintf(o, "instead of one record per bin.\n")
 }
 
 func transformerHistogramParseCLI(
@@ -61,6 +65,7 @@ func transformerHistogramParseCLI(
 	hi := 0.0
 	doAuto := false
 	outputPrefix := ""
+	doSparkline := false
 
 	var err error
 	for argi < argc /* variable increment: 1 or 2 depending on flag */ {
@@ -111,6 +116,9 @@ func transformerHistogramParseCLI(
 				return nil, err
 			}
 
+		case "-s":
+			doSparkline = true
+
 		default:
 			return nil, cli.VerbErrorf(verb, "option \"%s\" not recognized", opt)
 		}
@@ -140,6 +148,7 @@ func transformerHistogramParseCLI(
 		hi,
 		doAuto,
 		outputPrefix,
+		doSparkline,
 	)
 	if err != nil {
 		return nil, err
@@ -160,6 +169,7 @@ type TransformerHistogram struct {
 	countsByField      map[string][]int64
 	vectorsByFieldName map[string][]float64 // For auto-mode
 	outputPrefix       string
+	doSparkline        bool
 
 	recordTransformerFunc RecordTransformerFunc
 }
@@ -171,6 +181,7 @@ func NewTransformerHistogram(
 	hi float64,
 	doAuto bool,
 	outputPrefix string,
+	doSparkline bool,
 ) (*TransformerHistogram, error) {
 
 	countsByField := make(map[string][]int64)
@@ -185,6 +196,7 @@ func NewTransformerHistogram(
 		valueFieldNames: valueFieldNames,
 		countsByField:   countsByField,
 		outputPrefix:    outputPrefix,
+		doSparkline:     doSparkline,
 		nbins:           nbins,
 	}
 
@@ -256,10 +268,40 @@ func (tr *TransformerHistogram) ingestNonAuto(
 	}
 }
 
+// sparklineRecord summarizes a field's binned counts as a single record with
+// a Unicode-block sparkline field, rather than one record per bin.
+func (tr *TransformerHistogram) sparklineRecord(
+	valueFieldName string,
+	lo float64,
+	hi float64,
+) *mlrval.Mlrmap {
+	counts := tr.countsByField[valueFieldName]
+	countMlrvals := make([]*mlrval.Mlrval, len(counts))
+	for i, count := range counts {
+		countMlrvals[i] = mlrval.FromInt(count)
+	}
+	sparkline := bifs.BIF_sparkline(mlrval.FromArray(countMlrvals))
+
+	outrec := mlrval.NewMlrmapAsRecord()
+	outrec.PutReference(tr.outputPrefix+"field", mlrval.FromString(valueFieldName))
+	outrec.PutReference(tr.outputPrefix+"lo", mlrval.FromFloat(lo))
+	outrec.PutReference(tr.outputPrefix+"hi", mlrval.FromFloat(hi))
+	outrec.PutReference(tr.outputPrefix+"sparkline", sparkline)
+	return outrec
+}
+
 func (tr *TransformerHistogram) emitNonAuto(
 	endOfStreamContext *types.Context,
 	outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
 ) {
+	if tr.doSparkline {
+		for _, valueFieldName := range tr.valueFieldNames {
+			outrec := tr.sparklineRecord(valueFieldName, tr.lo, tr.hi)
+			*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, endOfStreamContext))
+		}
+		return
+	}
+
 	countFieldNames := make(map[string]string)
 	for _, valueFieldName := range tr.valueFieldNames {
 		countFieldNames[valueFieldName] = tr.outputPrefix + valueFieldName + "_count"
@@ -363,6 +405,14 @@ func (tr *TransformerHistogram) emitAuto(
 		}
 	}
 
+	if tr.doSparkline {
+		for _, valueFieldName := range tr.valueFieldNames {
+			outrec := tr.sparklineRecord(valueFieldName, lo, hi)
+			*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, endOfStreamContext))
+		}
+		return
+	}
+
 	// Emission pass
 	countFieldNames := make(map[string]string)
 	for _, valueFieldName := range tr.valueFieldNames {
diff --git a/pkg/transformers/join.go b/pkg/transformers/join.go
index b9cb312da..0fba89c3c 100644
--- a/pkg/transformers/join.go
+++ b/pkg/transformers/join.go
@@ -491,8 +491,8 @@ func (tr *TransformerJoin) ingestLeftFile() {
 	// TODO: perhaps increase recordsPerBatch, and/or refactor
 	recordReader, err := input.Create(readerOpts, 1)
 	if err != nil {
-		// TODO: propagate error to caller
-		return
+		fmt.Fprintf(os.Stderr, "mlr: %v\n", err)
+		os.Exit(1)
 	}
 
 	// Set the initial context for the left-file.
@@ -521,9 +521,8 @@ func (tr *TransformerJoin) ingestLeftFile() {
 		select {
 
 		case err := <-errorChannel:
-			// TODO: propagate error to caller
-			_ = err
-			return
+			fmt.Fprintf(os.Stderr, "mlr: %v\n", err)
+			os.Exit(1)
 
 		case leftrecsAndContexts := <-readerChannel:
 			// TODO: temp for batch-reader refactor
@@ -532,6 +531,17 @@ func (tr *TransformerJoin) ingestLeftFile() {
 			leftrecAndContext.Record = utils.KeepLeftFieldNames(leftrecAndContext.Record, tr.leftKeepFieldNameSet)
 
 			if leftrecAndContext.EndOfStream {
+				// The record-reader may have sent an error (e.g. the left file
+				// is missing or unreadable) immediately before its end-of-stream
+				// marker. Since those are separate channels, the select can see
+				// the end-of-stream marker first -- so, check the error channel
+				// before declaring the ingest complete.
+				select {
+				case err := <-errorChannel:
+					fmt.Fprintf(os.Stderr, "mlr: %v\n", err)
+					os.Exit(1)
+				default:
+				}
 				done = true
 				break // breaks the switch, not the for, in Golang
 			}
diff --git a/pkg/transformers/rank.go b/pkg/transformers/rank.go
new file mode 100644
index 000000000..ae2855709
--- /dev/null
+++ b/pkg/transformers/rank.go
@@ -0,0 +1,284 @@
+package transformers
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/johnkerl/miller/v6/pkg/cli"
+	"github.com/johnkerl/miller/v6/pkg/mlrval"
+	"github.com/johnkerl/miller/v6/pkg/transformers/utils"
+	"github.com/johnkerl/miller/v6/pkg/types"
+)
+
+const verbNameRank = "rank"
+
+var rankOptions = []OptionSpec{
+	{Flag: "-f", Arg: "{a,b,c}", Type: "csv-list", Desc: "Field name(s) to rank."},
+	{Flag: "-g", Arg: "{d,e,f}", Type: "csv-list", Desc: "Optional group-by-field name(s)."},
+	{Flag: "--sorted", Type: "bool", Desc: "Promise that the input is already sorted by the field(s) being ranked (within each group, if -g is given). This computes rank in a single streaming pass and O(1) space, by comparing each record's value only to the immediately preceding one, rather than buffering all records to compute an order-independent rank. Produces wrong output if the input is not in fact sorted."},
+}
+
+var RankSetup = TransformerSetup{
+	Verb:         verbNameRank,
+	UsageFunc:    transformerRankUsage,
+	ParseCLIFunc: transformerRankParseCLI,
+	IgnoresInput: false,
+	Options:      rankOptions,
+}
+
+func transformerRankUsage(
+	o *os.File,
+) {
+	argv0 := "mlr"
+	verb := verbNameRank
+	fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb)
+	fmt.Fprintf(o, "For each record's value in specified fields, computes the standard\n")
+	fmt.Fprintf(o, "competition rank (1,2,2,4,...) of that value among all input records,\n")
+	fmt.Fprintf(o, "optionally within groups.\n")
+	fmt.Fprintf(o, "E.g. with input records x=10, x=20, x=20, and x=30, emits output records\n")
+	fmt.Fprintf(o, "x=10,x_rank=1  x=20,x_rank=2  x=20,x_rank=2  and  x=30,x_rank=4.\n")
+	fmt.Fprintf(o, "\n")
+	fmt.Fprintf(o, "Note: by default this is a two-pass algorithm: on the first pass it retains\n")
+	fmt.Fprintf(o, "input records and their values; on the second pass it computes ranks and\n")
+	fmt.Fprintf(o, "emits output records, in original input order. This means it produces no\n")
+	fmt.Fprintf(o, "output until all input is read, but gives correct ranks regardless of input\n")
+	fmt.Fprintf(o, "order. Use --sorted for a single-pass streaming alternative.\n")
+	fmt.Fprintf(o, "\n")
+	WriteVerbOptions(o, rankOptions)
+	fmt.Fprintln(o, "Example: mlr rank -f x data/rank-example.csv")
+	fmt.Fprintln(o, "Example: mlr rank -f x -g g data/rank-example.csv")
+	fmt.Fprintln(o, "Example: mlr sort -f x then rank -f x --sorted data/rank-example.csv")
+}
+
+func transformerRankParseCLI(
+	pargi *int,
+	argc int,
+	args []string,
+	_ *cli.TOptions,
+	doConstruct bool, // false for first pass of CLI-parse, true for second pass
+) (RecordTransformer, error) {
+
+	// Skip the verb name from the current spot in the mlr command line
+	argi := *pargi
+	verb := args[argi]
+	argi++
+
+	var rankFieldNames []string = nil
+	var groupByFieldNames []string = nil
+	doSorted := false
+
+	var err error
+	for argi < argc /* variable increment: 1 or 2 depending on flag */ {
+		opt := args[argi]
+		if !strings.HasPrefix(opt, "-") {
+			break // No more flag options to process
+		}
+		if args[argi] == "--" {
+			break // All transformers must do this so main-flags can follow verb-flags
+		}
+		argi++
+
+		switch opt {
+		case "-h", "--help":
+			transformerRankUsage(os.Stdout)
+			return nil, cli.ErrHelpRequested
+
+		case "-f":
+			rankFieldNames, err = cli.VerbGetStringArrayArg(verb, opt, args, &argi, argc)
+			if err != nil {
+				return nil, err
+			}
+
+		case "-g":
+			groupByFieldNames, err = cli.VerbGetStringArrayArg(verb, opt, args, &argi, argc)
+			if err != nil {
+				return nil, err
+			}
+
+		case "--sorted":
+			doSorted = true
+
+		default:
+			return nil, cli.VerbErrorf(verb, "option \"%s\" not recognized", opt)
+		}
+	}
+
+	if rankFieldNames == nil {
+		return nil, cli.VerbErrorf(verb, "-f field names required")
+	}
+
+	*pargi = argi
+	if !doConstruct { // All transformers must do this for main command-line parsing
+		return nil, nil
+	}
+
+	transformer, err := NewTransformerRank(
+		rankFieldNames,
+		groupByFieldNames,
+		doSorted,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return transformer, nil
+}
+
+type TransformerRank struct {
+	rankFieldNames    []string
+	groupByFieldNames []string
+	doSorted          bool
+
+	// Default (unsorted) mode: two-pass. Records are retained on the first
+	// pass, along with per-group-per-field percentile-keepers; on the
+	// second pass (end of stream) the retained records are decorated with
+	// rank fields and emitted in original input order.
+	recordsAndContexts []*types.RecordAndContext
+	keepers            map[string]map[string]*utils.PercentileKeeper // grouping-key -> field-name -> keeper
+
+	// --sorted mode: single streaming pass, O(1) space. Same shape as the
+	// keepers map above, but holding lightweight adjacency state instead of
+	// buffered/sorted values.
+	sortedStates map[string]map[string]*tRankSortedFieldState // grouping-key -> field-name -> state
+}
+
+type tRankSortedFieldState struct {
+	count               int64
+	rank                int64
+	havePreviousValue   bool
+	previousValueString string
+}
+
+func NewTransformerRank(
+	rankFieldNames []string,
+	groupByFieldNames []string,
+	doSorted bool,
+) (*TransformerRank, error) {
+	return &TransformerRank{
+		rankFieldNames:     rankFieldNames,
+		groupByFieldNames:  groupByFieldNames,
+		doSorted:           doSorted,
+		recordsAndContexts: []*types.RecordAndContext{},
+		keepers:            make(map[string]map[string]*utils.PercentileKeeper),
+		sortedStates:       make(map[string]map[string]*tRankSortedFieldState),
+	}, nil
+}
+
+func (tr *TransformerRank) Transform(
+	inrecAndContext *types.RecordAndContext,
+	outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
+	inputDownstreamDoneChannel <-chan bool,
+	outputDownstreamDoneChannel chan<- bool,
+) {
+	HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel)
+	if tr.doSorted {
+		tr.transformSorted(inrecAndContext, outputRecordsAndContexts)
+	} else {
+		tr.transformUnsorted(inrecAndContext, outputRecordsAndContexts)
+	}
+}
+
+// transformSorted computes rank in a single pass, O(1) space, by comparing
+// each record's value only to the immediately preceding one within its
+// group. This is only correct if the caller has ensured the input is
+// already sorted by the ranked field(s), e.g. via 'mlr sort'.
+func (tr *TransformerRank) transformSorted(
+	inrecAndContext *types.RecordAndContext,
+	outputRecordsAndContexts *[]*types.RecordAndContext,
+) {
+	if !inrecAndContext.EndOfStream {
+		inrec := inrecAndContext.Record
+
+		groupingKey, hasAll := inrec.GetSelectedValuesJoined(tr.groupByFieldNames)
+		if hasAll {
+			statesForGroup := tr.sortedStates[groupingKey]
+			if statesForGroup == nil {
+				statesForGroup = make(map[string]*tRankSortedFieldState)
+				tr.sortedStates[groupingKey] = statesForGroup
+			}
+			for _, rankFieldName := range tr.rankFieldNames {
+				value := inrec.Get(rankFieldName)
+				if value == nil {
+					continue
+				}
+				state := statesForGroup[rankFieldName]
+				if state == nil {
+					state = &tRankSortedFieldState{}
+					statesForGroup[rankFieldName] = state
+				}
+				state.count++
+				valueString := value.String() // 1, 1.0, and 1.000 are distinct
+				if !state.havePreviousValue || valueString != state.previousValueString {
+					state.rank = state.count
+					state.previousValueString = valueString
+					state.havePreviousValue = true
+				}
+				inrec.PutCopy(rankFieldName+"_rank", mlrval.FromInt(state.rank))
+			}
+		}
+	}
+	*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
+}
+
+// transformUnsorted computes order-independent standard competition rank:
+// on the first pass it retains records and accumulates per-group-per-field
+// value sets; on the second pass (end of stream) it decorates the retained
+// records with rank fields and emits them in original input order.
+func (tr *TransformerRank) transformUnsorted(
+	inrecAndContext *types.RecordAndContext,
+	outputRecordsAndContexts *[]*types.RecordAndContext,
+) {
+	if !inrecAndContext.EndOfStream { // Not end of stream; pass 1
+		inrec := inrecAndContext.Record
+
+		// Append records into a single output list (so that this verb is order-preserving).
+		tr.recordsAndContexts = append(tr.recordsAndContexts, inrecAndContext)
+
+		groupingKey, hasAll := inrec.GetSelectedValuesJoined(tr.groupByFieldNames)
+		if hasAll {
+			keepersForGroup := tr.keepers[groupingKey]
+			if keepersForGroup == nil {
+				keepersForGroup = make(map[string]*utils.PercentileKeeper)
+				tr.keepers[groupingKey] = keepersForGroup
+			}
+			for _, rankFieldName := range tr.rankFieldNames {
+				value := inrec.Get(rankFieldName)
+				if value == nil {
+					continue
+				}
+				keeper := keepersForGroup[rankFieldName]
+				if keeper == nil {
+					keeper = utils.NewPercentileKeeper(false)
+					keepersForGroup[rankFieldName] = keeper
+				}
+				keeper.Ingest(value)
+			}
+		}
+
+	} else { // End of stream; pass 2
+		// Iterate over the retained records, decorating them with rank fields.
+		endOfStreamContext := inrecAndContext.Context
+
+		for _, recordAndContext := range tr.recordsAndContexts {
+			outrec := recordAndContext.Record
+
+			groupingKey, hasAll := outrec.GetSelectedValuesJoined(tr.groupByFieldNames)
+			if hasAll {
+				keepersForGroup := tr.keepers[groupingKey]
+				for _, rankFieldName := range tr.rankFieldNames {
+					value := outrec.Get(rankFieldName)
+					if value == nil {
+						continue
+					}
+					keeper := keepersForGroup[rankFieldName]
+					outrec.PutCopy(rankFieldName+"_rank", keeper.EmitRank(value))
+				}
+			}
+
+			*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &endOfStreamContext))
+		}
+		tr.recordsAndContexts = tr.recordsAndContexts[:0]
+		*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
+	}
+}
diff --git a/pkg/transformers/reorder.go b/pkg/transformers/reorder.go
index 78f4e6772..d5892c47c 100644
--- a/pkg/transformers/reorder.go
+++ b/pkg/transformers/reorder.go
@@ -18,7 +18,7 @@ const verbNameReorder = "reorder"
 var reorderOptions = []OptionSpec{
 	{Flag: "-e", Type: "bool", Desc: "Put specified field names at record end: default is to put them at record start."},
 	{Flag: "-f", Arg: "{a,b,c}", Type: "csv-list", Desc: "Field names to reorder."},
-	{Flag: "-r", Arg: "{a,b,c}", Type: "csv-list", Desc: "Treat field names as regular expressions. Matched fields are moved to start or end in record order. Example: -r '^YYY,^XXX' puts all YYY- and XXX-prefixed fields first (in record order), then the rest."},
+	{Flag: "-r", Arg: "{a,b,c}", Type: "csv-list", Desc: "Treat field names as regular expressions. Matched fields are moved to start or end, grouped by the order the regexes are given; within each group, fields keep their record order. Example: -r '^YYY,^XXX' puts all YYY-prefixed fields first, then all XXX-prefixed fields, then the rest."},
 	{Flag: "-b", Arg: "{x}", Type: "string", Desc: "Put field names specified with -f before field name specified by {x}, if any. If {x} isn't present in a given record, the specified fields will not be moved."},
 	{Flag: "-a", Arg: "{x}", Type: "string", Desc: "Put field names specified with -f after field name specified by {x}, if any. If {x} isn't present in a given record, the specified fields will not be moved."},
 }
@@ -45,7 +45,7 @@ func transformerReorderUsage(
 	fmt.Fprintf(o, "Examples:\n")
 	fmt.Fprintf(o, "%s %s    -f a,b sends input record \"d=4,b=2,a=1,c=3\" to \"a=1,b=2,d=4,c=3\".\n", argv0, verb)
 	fmt.Fprintf(o, "%s %s -e -f a,b sends input record \"d=4,b=2,a=1,c=3\" to \"d=4,c=3,a=1,b=2\".\n", argv0, verb)
-	fmt.Fprintf(o, "%s %s -r '^YYY,^XXX' puts YYY- and XXX-prefixed fields first (record order), then rest.\n", argv0, verb)
+	fmt.Fprintf(o, "%s %s -r '^YYY,^XXX' puts YYY-prefixed fields first, then XXX-prefixed fields, then rest.\n", argv0, verb)
 }
 
 func transformerReorderParseCLI(
@@ -238,18 +238,22 @@ func (tr *TransformerReorder) reorderToStartNoRegex(
 	*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
 }
 
-// reorderSplitByRegex splits record fields into matching (any regex) and rest, preserving record order.
+// reorderSplitByRegex splits record fields into matching (any regex) and rest. Matching fields
+// are grouped by the order the regexes were given on the command line; within each group, and
+// within rest, fields keep their record order. A field matching multiple regexes is claimed by
+// the first regex that matches it.
 func (tr *TransformerReorder) reorderSplitByRegex(inrec *mlrval.Mlrmap) (matching []*mlrval.MlrmapEntry, rest []*mlrval.MlrmapEntry) {
-	for pe := inrec.Head; pe != nil; pe = pe.Next {
-		found := false
-		for _, regex := range tr.regexes {
-			if regex.MatchString(pe.Key) {
+	claimed := make(map[string]bool)
+	for _, regex := range tr.regexes {
+		for pe := inrec.Head; pe != nil; pe = pe.Next {
+			if !claimed[pe.Key] && regex.MatchString(pe.Key) {
 				matching = append(matching, pe)
-				found = true
-				break
+				claimed[pe.Key] = true
 			}
 		}
-		if !found {
+	}
+	for pe := inrec.Head; pe != nil; pe = pe.Next {
+		if !claimed[pe.Key] {
 			rest = append(rest, pe)
 		}
 	}
@@ -373,13 +377,13 @@ func (tr *TransformerReorder) reorderBeforeOrAfterWithRegex(
 		return
 	}
 
-	// Build matching set in record order (OrderedMap preserves insertion order)
+	// Build matching set grouped by regex order (OrderedMap preserves insertion order);
+	// within each group, fields keep their record order.
 	matchingFieldNamesSet := lib.NewOrderedMap[*mlrval.Mlrval]()
-	for pe := inrec.Head; pe != nil; pe = pe.Next {
-		for _, regex := range tr.regexes {
-			if regex.MatchString(pe.Key) && pe.Key != tr.centerFieldName {
+	for _, regex := range tr.regexes {
+		for pe := inrec.Head; pe != nil; pe = pe.Next {
+			if pe.Key != tr.centerFieldName && !matchingFieldNamesSet.Has(pe.Key) && regex.MatchString(pe.Key) {
 				matchingFieldNamesSet.Put(pe.Key, pe.Value)
-				break
 			}
 		}
 	}
diff --git a/pkg/transformers/skip_trivial_records.go b/pkg/transformers/skip_trivial_records.go
index 9f24468d8..c4c6c8a3e 100644
--- a/pkg/transformers/skip_trivial_records.go
+++ b/pkg/transformers/skip_trivial_records.go
@@ -34,7 +34,7 @@ func transformerSkipTrivialRecordsParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cli.TOptions,
+	options *cli.TOptions,
 	doConstruct bool, // false for first pass of CLI-parse, true for second pass
 ) (RecordTransformer, error) {
 
@@ -60,6 +60,12 @@ func transformerSkipTrivialRecordsParseCLI(
 		}
 	}
 
+	// Since the user has explicitly asked for trivial records to be skipped,
+	// let the record-readers know that trivial input lines -- e.g. blank
+	// lines at the end of a CSV file -- are to be skipped rather than
+	// treated as fatal header/data length mismatches. See issue #1535.
+	options.ReaderOptions.SkipTrivialRecords = true
+
 	*pargi = argi
 	if !doConstruct { // All transformers must do this for main command-line parsing
 		return nil, nil
diff --git a/pkg/transformers/sparkline.go b/pkg/transformers/sparkline.go
new file mode 100644
index 000000000..5bf0bf802
--- /dev/null
+++ b/pkg/transformers/sparkline.go
@@ -0,0 +1,182 @@
+package transformers
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/johnkerl/miller/v6/pkg/bifs"
+	"github.com/johnkerl/miller/v6/pkg/cli"
+	"github.com/johnkerl/miller/v6/pkg/mlrval"
+	"github.com/johnkerl/miller/v6/pkg/types"
+)
+
+const verbNameSparkline = "sparkline"
+
+var sparklineOptions = []OptionSpec{
+	{Flag: "-f", Arg: "{a,b,c}", Type: "csv-list", Desc: "Field names to sparkline."},
+}
+
+var SparklineSetup = TransformerSetup{
+	Verb:         verbNameSparkline,
+	UsageFunc:    transformerSparklineUsage,
+	ParseCLIFunc: transformerSparklineParseCLI,
+	IgnoresInput: false,
+	Options:      sparklineOptions,
+}
+
+func transformerSparklineUsage(
+	o *os.File,
+) {
+	fmt.Fprintf(o, "Usage: %s %s [options]\n", "mlr", verbNameSparkline)
+	fmt.Fprintf(o, "Reduces numeric field(s), across all records in input order, to a compact\n")
+	fmt.Fprintf(o, "Unicode sparkline -- one block character per record -- for visualizing\n")
+	fmt.Fprintf(o, "trends. Emits one output record per field. Holds all records in memory\n")
+	fmt.Fprintf(o, "before producing any output.\n")
+	WriteVerbOptions(o, sparklineOptions)
+}
+
+func transformerSparklineParseCLI(
+	pargi *int,
+	argc int,
+	args []string,
+	_ *cli.TOptions,
+	doConstruct bool, // false for first pass of CLI-parse, true for second pass
+) (RecordTransformer, error) {
+
+	// Skip the verb name from the current spot in the mlr command line
+	argi := *pargi
+	verb := args[argi]
+	argi++
+
+	// Parse local flags
+	var err error
+	var fieldNames []string = nil
+
+	for argi < argc /* variable increment: 1 or 2 depending on flag */ {
+		opt := args[argi]
+		if !strings.HasPrefix(opt, "-") {
+			break // No more flag options to process
+		}
+		if args[argi] == "--" {
+			break // All transformers must do this so main-flags can follow verb-flags
+		}
+		argi++
+
+		switch opt {
+		case "-h", "--help":
+			transformerSparklineUsage(os.Stdout)
+			return nil, cli.ErrHelpRequested
+
+		case "-f":
+			fieldNames, err = cli.VerbGetStringArrayArg(verb, opt, args, &argi, argc)
+			if err != nil {
+				return nil, err
+			}
+
+		default:
+			return nil, cli.VerbErrorf(verb, "option \"%s\" not recognized", opt)
+		}
+	}
+
+	if fieldNames == nil {
+		return nil, cli.VerbErrorf(verb, "-f field names required")
+	}
+
+	*pargi = argi
+	if !doConstruct { // All transformers must do this for main command-line parsing
+		return nil, nil
+	}
+
+	transformer, err := NewTransformerSparkline(fieldNames)
+	if err != nil {
+		return nil, err
+	}
+
+	return transformer, nil
+}
+
+type TransformerSparkline struct {
+	fieldNames    []string
+	valuesByField map[string][]*mlrval.Mlrval
+}
+
+func NewTransformerSparkline(
+	fieldNames []string,
+) (*TransformerSparkline, error) {
+	valuesByField := make(map[string][]*mlrval.Mlrval)
+	for _, fieldName := range fieldNames {
+		valuesByField[fieldName] = make([]*mlrval.Mlrval, 0)
+	}
+	return &TransformerSparkline{
+		fieldNames:    fieldNames,
+		valuesByField: valuesByField,
+	}, nil
+}
+
+func (tr *TransformerSparkline) Transform(
+	inrecAndContext *types.RecordAndContext,
+	outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
+	inputDownstreamDoneChannel <-chan bool,
+	outputDownstreamDoneChannel chan<- bool,
+) {
+	HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel)
+
+	if !inrecAndContext.EndOfStream {
+		inrec := inrecAndContext.Record
+		for _, fieldName := range tr.fieldNames {
+			mvalue := inrec.Get(fieldName)
+			if mvalue != nil {
+				tr.valuesByField[fieldName] = append(tr.valuesByField[fieldName], mvalue.Copy())
+			}
+		}
+		return
+	}
+
+	// Else, end of stream: emit one summary record per field.
+	for _, fieldName := range tr.fieldNames {
+		values := tr.valuesByField[fieldName]
+
+		outrec := mlrval.NewMlrmapAsRecord()
+		outrec.PutReference("field", mlrval.FromString(fieldName))
+		outrec.PutReference("n", mlrval.FromInt(int64(len(values))))
+
+		sparkline := bifs.BIF_sparkline(mlrval.FromArray(values))
+		if !sparkline.IsError() {
+			lo, hi, haveLoHi := floatRangeOf(values)
+			if haveLoHi {
+				outrec.PutReference("lo", mlrval.FromFloat(lo))
+				outrec.PutReference("hi", mlrval.FromFloat(hi))
+			}
+		}
+		outrec.PutReference("sparkline", sparkline)
+
+		*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
+	}
+
+	*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // Emit the end-of-stream marker
+}
+
+// floatRangeOf returns the min and max of the numeric values, and whether
+// any were found.
+func floatRangeOf(values []*mlrval.Mlrval) (lo float64, hi float64, haveLoHi bool) {
+	for _, value := range values {
+		floatValue, isFloat := value.GetNumericToFloatValue()
+		if !isFloat {
+			continue
+		}
+		if !haveLoHi {
+			lo = floatValue
+			hi = floatValue
+			haveLoHi = true
+		} else {
+			if floatValue < lo {
+				lo = floatValue
+			}
+			if floatValue > hi {
+				hi = floatValue
+			}
+		}
+	}
+	return lo, hi, haveLoHi
+}
diff --git a/pkg/transformers/summary.go b/pkg/transformers/summary.go
index 7c216445a..5d65ad1b7 100644
--- a/pkg/transformers/summary.go
+++ b/pkg/transformers/summary.go
@@ -29,7 +29,7 @@ type tSummarizerInfo struct {
 }
 
 var allSummarizerInfos = []tSummarizerInfo{
-	{"field_type", "string, int, etc. -- if a column has mixed types, all encountered types are printed", stFieldType},
+	{"field_type", "string, int, etc. -- if a column has mixed types, all encountered types are printed (see notes below)", stFieldType},
 
 	{"count", "+1 for every instance of the field across all records in the input record stream", stAccumulator},
 	{"null_count", "count of field values either empty string or JSON null", stAccumulator},
@@ -107,6 +107,8 @@ func transformerSummaryUsage(
 	fmt.Fprintf(o, "* min, p25, median, p75, and max work for strings as well as numbers\n")
 	fmt.Fprintf(o, "* Distinct-counts are computed on string representations -- so 4.1 and 4.10 are counted as distinct here.\n")
 	fmt.Fprintf(o, "* If the mode is not unique in the input data, the first-encountered value is reported as the mode.\n")
+	fmt.Fprintf(o, "* A field_type of \"int-string\", \"empty-string\", etc. means the column contains values of mixed types --\n")
+	fmt.Fprintf(o, "  all types encountered are printed, hyphen-joined, in the order first encountered.\n")
 	fmt.Fprintf(o, "\n")
 	WriteVerbOptions(o, summaryOptions)
 }
diff --git a/pkg/transformers/uniq.go b/pkg/transformers/uniq.go
index d280ef534..37499f8f8 100644
--- a/pkg/transformers/uniq.go
+++ b/pkg/transformers/uniq.go
@@ -169,6 +169,9 @@ func transformerUniqUsage(
 	fmt.Fprintf(o, "count-distinct. For uniq, -f is a synonym for -g. Output fields are\n")
 	fmt.Fprintf(o, "written in the order in which they are named with -g or -f, not in the\n")
 	fmt.Fprintf(o, "order in which they appear in the input records.\n")
+	fmt.Fprintf(o, "To deduplicate records by one or more fields while keeping all other\n")
+	fmt.Fprintf(o, "fields, use head: e.g. \"%s head -n 1 -g hash\" keeps the first record\n", argv0)
+	fmt.Fprintf(o, "for each distinct value of the hash field, with all fields intact.\n")
 	fmt.Fprintf(o, "\n")
 	WriteVerbOptions(o, uniqOptions)
 }
diff --git a/pkg/transformers/utils/join_bucket_keeper.go b/pkg/transformers/utils/join_bucket_keeper.go
index 52c8d47fe..795abf9f0 100644
--- a/pkg/transformers/utils/join_bucket_keeper.go
+++ b/pkg/transformers/utils/join_bucket_keeper.go
@@ -553,6 +553,17 @@ func (keeper *JoinBucketKeeper) readRecord() *types.RecordAndContext {
 		leftrecAndContext := leftrecsAndContexts[0]
 		leftrecAndContext.Record = KeepLeftFieldNames(leftrecAndContext.Record, keeper.leftKeepFieldNameSet)
 		if leftrecAndContext.EndOfStream { // end-of-stream marker
+			// The record-reader may have sent an error (e.g. the left file is
+			// missing or unreadable) immediately before its end-of-stream
+			// marker. Since those are separate channels, the select can see
+			// the end-of-stream marker first -- so, check the error channel
+			// before declaring the left-file read complete.
+			select {
+			case err := <-keeper.errorChannel:
+				fmt.Fprintf(os.Stderr, "mlr: %v\n", err)
+				os.Exit(1)
+			default:
+			}
 			keeper.recordReaderDone = true
 			return nil
 		}
diff --git a/pkg/transformers/utils/percentile_keeper.go b/pkg/transformers/utils/percentile_keeper.go
index 37725e051..811a0586b 100644
--- a/pkg/transformers/utils/percentile_keeper.go
+++ b/pkg/transformers/utils/percentile_keeper.go
@@ -82,6 +82,27 @@ func (keeper *PercentileKeeper) EmitLinearlyInterpolated(percentile float64) *ml
 	return bifs.GetPercentileLinearlyInterpolated(keeper.data, int(len(keeper.data)), percentile)
 }
 
+// EmitRank returns the standard competition rank (1,2,2,4,...) of value
+// among all values ingested so far: one plus the number of ingested values
+// strictly less than it.
+func (keeper *PercentileKeeper) EmitRank(value *mlrval.Mlrval) *mlrval.Mlrval {
+	if len(keeper.data) == 0 {
+		return mlrval.VOID
+	}
+	keeper.sortIfNecessary()
+	n := len(keeper.data)
+	lo, hi := 0, n
+	for lo < hi {
+		mid := (lo + hi) / 2
+		if mlrval.LessThan(keeper.data[mid], value) {
+			lo = mid + 1
+		} else {
+			hi = mid
+		}
+	}
+	return mlrval.FromInt(int64(lo + 1))
+}
+
 // TODO: COMMENT
 func (keeper *PercentileKeeper) EmitNamed(name string) *mlrval.Mlrval {
 	switch name {
diff --git a/test/cases/cli-help/0001/expout b/test/cases/cli-help/0001/expout
index eef97b479..a6a2d3275 100644
--- a/test/cases/cli-help/0001/expout
+++ b/test/cases/cli-help/0001/expout
@@ -474,7 +474,11 @@ Options:
 --auto      Automatically computes limits, ignoring --lo and --hi. Holds all
             values in memory before producing any output.
 -o {prefix} Prefix for output field name. Default: no prefix.
+-s          Print a one-line Unicode sparkline per field instead of per-bin
+            counts.
 -h|--help   Show this message.
+With -s, output is one record per value-field, with a sparkline field
+instead of one record per bin.
 
 ================================================================
 json-parse
@@ -831,6 +835,35 @@ More example put expressions:
 
 See also https://miller.readthedocs.io/reference-dsl for more context.
 
+================================================================
+rank
+Usage: mlr rank [options]
+For each record's value in specified fields, computes the standard
+competition rank (1,2,2,4,...) of that value among all input records,
+optionally within groups.
+E.g. with input records x=10, x=20, x=20, and x=30, emits output records
+x=10,x_rank=1  x=20,x_rank=2  x=20,x_rank=2  and  x=30,x_rank=4.
+
+Note: by default this is a two-pass algorithm: on the first pass it retains
+input records and their values; on the second pass it computes ranks and
+emits output records, in original input order. This means it produces no
+output until all input is read, but gives correct ranks regardless of input
+order. Use --sorted for a single-pass streaming alternative.
+
+Options:
+-f {a,b,c} Field name(s) to rank.
+-g {d,e,f} Optional group-by-field name(s).
+--sorted   Promise that the input is already sorted by the field(s) being ranked
+           (within each group, if -g is given). This computes rank in a single
+           streaming pass and O(1) space, by comparing each record's value only
+           to the immediately preceding one, rather than buffering all records
+           to compute an order-independent rank. Produces wrong output if the
+           input is not in fact sorted.
+-h|--help  Show this message.
+Example: mlr rank -f x data/rank-example.csv
+Example: mlr rank -f x -g g data/rank-example.csv
+Example: mlr sort -f x then rank -f x --sorted data/rank-example.csv
+
 ================================================================
 regularize
 Usage: mlr regularize [options]
@@ -877,8 +910,10 @@ Options:
            record start.
 -f {a,b,c} Field names to reorder.
 -r {a,b,c} Treat field names as regular expressions. Matched fields are moved to
-           start or end in record order. Example: -r '^YYY,^XXX' puts all YYY-
-           and XXX-prefixed fields first (in record order), then the rest.
+           start or end, grouped by the order the regexes are given; within each
+           group, fields keep their record order. Example: -r '^YYY,^XXX' puts
+           all YYY-prefixed fields first, then all XXX-prefixed fields, then the
+           rest.
 -b {x}     Put field names specified with -f before field name specified by {x},
            if any. If {x} isn't present in a given record, the specified fields
            will not be moved.
@@ -890,7 +925,7 @@ Options:
 Examples:
 mlr reorder    -f a,b sends input record "d=4,b=2,a=1,c=3" to "a=1,b=2,d=4,c=3".
 mlr reorder -e -f a,b sends input record "d=4,b=2,a=1,c=3" to "d=4,c=3,a=1,b=2".
-mlr reorder -r '^YYY,^XXX' puts YYY- and XXX-prefixed fields first (record order), then rest.
+mlr reorder -r '^YYY,^XXX' puts YYY-prefixed fields first, then XXX-prefixed fields, then rest.
 
 ================================================================
 repeat
@@ -1118,6 +1153,17 @@ Options:
 -n         Sort field names naturally (e.g. 2 before 12). Combines with -f/-r.
 -h|--help  Show this message.
 
+================================================================
+sparkline
+Usage: mlr sparkline [options]
+Reduces numeric field(s), across all records in input order, to a compact
+Unicode sparkline -- one block character per record -- for visualizing
+trends. Emits one output record per field. Holds all records in memory
+before producing any output.
+Options:
+-f {a,b,c} Field names to sparkline.
+-h|--help  Show this message.
+
 ================================================================
 sparsify
 Usage: mlr sparsify [options]
@@ -1367,7 +1413,7 @@ Usage: mlr summary [options]
 Show summary statistics about the input data.
 
 All summarizers:
-  field_type      string, int, etc. -- if a column has mixed types, all encountered types are printed
+  field_type      string, int, etc. -- if a column has mixed types, all encountered types are printed (see notes below)
   count           +1 for every instance of the field across all records in the input record stream
   null_count      count of field values either empty string or JSON null
   distinct_count  count of distinct values for the field
@@ -1397,6 +1443,8 @@ Notes:
 * min, p25, median, p75, and max work for strings as well as numbers
 * Distinct-counts are computed on string representations -- so 4.1 and 4.10 are counted as distinct here.
 * If the mode is not unique in the input data, the first-encountered value is reported as the mode.
+* A field_type of "int-string", "empty-string", etc. means the column contains values of mixed types --
+  all types encountered are printed, hyphen-joined, in the order first encountered.
 
 Options:
 -a {mean,sum,etc.} Use only the specified summarizers.
@@ -1511,6 +1559,9 @@ Prints distinct values for specified field names. With -c, same as
 count-distinct. For uniq, -f is a synonym for -g. Output fields are
 written in the order in which they are named with -g or -f, not in the
 order in which they appear in the input records.
+To deduplicate records by one or more fields while keeping all other
+fields, use head: e.g. "mlr head -n 1 -g hash" keeps the first record
+for each distinct value of the hash field, with all fields intact.
 
 Options:
 -g {d,e,f} Group-by field names for uniq counts.
diff --git a/test/cases/dsl-format/0011/cmd b/test/cases/dsl-format/0011/cmd
new file mode 100644
index 000000000..6add080d4
--- /dev/null
+++ b/test/cases/dsl-format/0011/cmd
@@ -0,0 +1 @@
+mlr -n put -f ${CASEDIR}/mlr
diff --git a/test/cases/dsl-format/0011/experr b/test/cases/dsl-format/0011/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/dsl-format/0011/expout b/test/cases/dsl-format/0011/expout
new file mode 100644
index 000000000..73c31a83b
--- /dev/null
+++ b/test/cases/dsl-format/0011/expout
@@ -0,0 +1 @@
+a/b/a_c.ext
diff --git a/test/cases/dsl-format/0011/mlr b/test/cases/dsl-format/0011/mlr
new file mode 100644
index 000000000..5564dc098
--- /dev/null
+++ b/test/cases/dsl-format/0011/mlr
@@ -0,0 +1,3 @@
+end {
+  print format("{1}/{2}/{1}_{3}.ext", "a", "b", "c")
+}
diff --git a/test/cases/dsl-format/0012/cmd b/test/cases/dsl-format/0012/cmd
new file mode 100644
index 000000000..6add080d4
--- /dev/null
+++ b/test/cases/dsl-format/0012/cmd
@@ -0,0 +1 @@
+mlr -n put -f ${CASEDIR}/mlr
diff --git a/test/cases/dsl-format/0012/experr b/test/cases/dsl-format/0012/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/dsl-format/0012/expout b/test/cases/dsl-format/0012/expout
new file mode 100644
index 000000000..5473de45b
--- /dev/null
+++ b/test/cases/dsl-format/0012/expout
@@ -0,0 +1 @@
+b:a
diff --git a/test/cases/dsl-format/0012/mlr b/test/cases/dsl-format/0012/mlr
new file mode 100644
index 000000000..f0560f735
--- /dev/null
+++ b/test/cases/dsl-format/0012/mlr
@@ -0,0 +1,3 @@
+end {
+  print format("{2}:{1}", "a", "b")
+}
diff --git a/test/cases/dsl-format/0013/cmd b/test/cases/dsl-format/0013/cmd
new file mode 100644
index 000000000..6add080d4
--- /dev/null
+++ b/test/cases/dsl-format/0013/cmd
@@ -0,0 +1 @@
+mlr -n put -f ${CASEDIR}/mlr
diff --git a/test/cases/dsl-format/0013/experr b/test/cases/dsl-format/0013/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/dsl-format/0013/expout b/test/cases/dsl-format/0013/expout
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/test/cases/dsl-format/0013/expout
@@ -0,0 +1 @@
+
diff --git a/test/cases/dsl-format/0013/mlr b/test/cases/dsl-format/0013/mlr
new file mode 100644
index 000000000..7bb7073d6
--- /dev/null
+++ b/test/cases/dsl-format/0013/mlr
@@ -0,0 +1,3 @@
+end {
+  print format("{3}", 1)
+}
diff --git a/test/cases/dsl-format/0014/cmd b/test/cases/dsl-format/0014/cmd
new file mode 100644
index 000000000..6add080d4
--- /dev/null
+++ b/test/cases/dsl-format/0014/cmd
@@ -0,0 +1 @@
+mlr -n put -f ${CASEDIR}/mlr
diff --git a/test/cases/dsl-format/0014/experr b/test/cases/dsl-format/0014/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/dsl-format/0014/expout b/test/cases/dsl-format/0014/expout
new file mode 100644
index 000000000..ffba14cee
--- /dev/null
+++ b/test/cases/dsl-format/0014/expout
@@ -0,0 +1 @@
+(error)
diff --git a/test/cases/dsl-format/0014/mlr b/test/cases/dsl-format/0014/mlr
new file mode 100644
index 000000000..16b3b0ff4
--- /dev/null
+++ b/test/cases/dsl-format/0014/mlr
@@ -0,0 +1,3 @@
+end {
+  print format("{0}", 1)
+}
diff --git a/test/cases/dsl-format/0015/cmd b/test/cases/dsl-format/0015/cmd
new file mode 100644
index 000000000..6add080d4
--- /dev/null
+++ b/test/cases/dsl-format/0015/cmd
@@ -0,0 +1 @@
+mlr -n put -f ${CASEDIR}/mlr
diff --git a/test/cases/dsl-format/0015/experr b/test/cases/dsl-format/0015/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/dsl-format/0015/expout b/test/cases/dsl-format/0015/expout
new file mode 100644
index 000000000..dd3be91ed
--- /dev/null
+++ b/test/cases/dsl-format/0015/expout
@@ -0,0 +1 @@
+43:34
diff --git a/test/cases/dsl-format/0015/mlr b/test/cases/dsl-format/0015/mlr
new file mode 100644
index 000000000..8c288f8a6
--- /dev/null
+++ b/test/cases/dsl-format/0015/mlr
@@ -0,0 +1,3 @@
+end {
+  print format("{2}{}:{1}{}", 3, 4)
+}
diff --git a/test/cases/dsl-format/0016/cmd b/test/cases/dsl-format/0016/cmd
new file mode 100644
index 000000000..6add080d4
--- /dev/null
+++ b/test/cases/dsl-format/0016/cmd
@@ -0,0 +1 @@
+mlr -n put -f ${CASEDIR}/mlr
diff --git a/test/cases/dsl-format/0016/experr b/test/cases/dsl-format/0016/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/dsl-format/0016/expout b/test/cases/dsl-format/0016/expout
new file mode 100644
index 000000000..4bc88feba
--- /dev/null
+++ b/test/cases/dsl-format/0016/expout
@@ -0,0 +1 @@
+{ }:{x}
diff --git a/test/cases/dsl-format/0016/mlr b/test/cases/dsl-format/0016/mlr
new file mode 100644
index 000000000..5b768760f
--- /dev/null
+++ b/test/cases/dsl-format/0016/mlr
@@ -0,0 +1,3 @@
+end {
+  print format("{ }:{x}", "a")
+}
diff --git a/test/cases/dsl-gmt-date-time-functions/0020/mlr b/test/cases/dsl-gmt-date-time-functions/0020/mlr
index b582dd920..baa570fe0 100644
--- a/test/cases/dsl-gmt-date-time-functions/0020/mlr
+++ b/test/cases/dsl-gmt-date-time-functions/0020/mlr
@@ -7,6 +7,6 @@ end {
   print strptime("1970:363", "%Y:%j");
   print strptime("1970-01-01 10:20:30 PM", "%F %r");
   print strptime("01/02/70 14:20", "%D %R");
-  print strptime("01/02/70 14:20", "%D %X"); # no such format code
+  print strptime("01/02/70 14:20", "%D %U"); # no such format code
   print fmtnum(strptime("1971-01-01T00:00:00.123456Z", "%Y-%m-%dT%H:%M:%S.%fZ"), "%.6f");
 }
diff --git a/test/cases/dsl-stats/sparkline/various/cmd b/test/cases/dsl-stats/sparkline/various/cmd
new file mode 100644
index 000000000..8e64fdff2
--- /dev/null
+++ b/test/cases/dsl-stats/sparkline/various/cmd
@@ -0,0 +1 @@
+mlr -n --ofmtf 6 --xtab put -f ${CASEDIR}/mlr
diff --git a/test/cases/dsl-stats/sparkline/various/experr b/test/cases/dsl-stats/sparkline/various/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/dsl-stats/sparkline/various/expout b/test/cases/dsl-stats/sparkline/various/expout
new file mode 100644
index 000000000..32ab67b85
--- /dev/null
+++ b/test/cases/dsl-stats/sparkline/various/expout
@@ -0,0 +1,22 @@
+sparkline_0                     (error)
+sparkline_0_type                error
+sparkline_null                  (error)
+sparkline_null_type             error
+sparkline_empty_array           
+sparkline_empty_array_type      empty
+sparkline_array_1               ▁
+sparkline_array_1_type          string
+sparkline_array_flat            ▁▁▁
+sparkline_array_flat_type       string
+sparkline_array_ascending       β–β–‚β–ƒβ–„β–…β–†β–‡β–ˆ
+sparkline_array_ascending_type  string
+sparkline_array_descending      β–ˆβ–‡β–†β–…β–„β–ƒβ–‚β–
+sparkline_array_descending_type string
+sparkline_array_nonnumeric      (error)
+sparkline_array_nonnumeric_type error
+sparkline_empty_map             
+sparkline_empty_map_type        empty
+sparkline_map_1                 ▁
+sparkline_map_1_type            string
+sparkline_map_ascending         β–β–„β–ˆ
+sparkline_map_ascending_type    string
diff --git a/test/cases/dsl-stats/sparkline/various/mlr b/test/cases/dsl-stats/sparkline/various/mlr
new file mode 100644
index 000000000..c99e5796f
--- /dev/null
+++ b/test/cases/dsl-stats/sparkline/various/mlr
@@ -0,0 +1,27 @@
+end {
+    outputs = {};
+
+    outputs["sparkline_0"] = sparkline(0);
+    outputs["sparkline_null"] = sparkline(null);
+    outputs["sparkline_nonesuch"] = sparkline(nonesuch);
+
+    outputs["sparkline_empty_array"] = sparkline([]);
+    outputs["sparkline_array_1"] = sparkline([7]);
+    outputs["sparkline_array_flat"] = sparkline([3,3,3]);
+    outputs["sparkline_array_ascending"] = sparkline([1,2,3,4,5,6,7,8]);
+    outputs["sparkline_array_descending"] = sparkline([8,7,6,5,4,3,2,1]);
+    outputs["sparkline_array_nonnumeric"] = sparkline([1,"abc"]);
+
+    outputs["sparkline_empty_map"] = sparkline({});
+    outputs["sparkline_map_1"] = sparkline({ "a" : 7 });
+    outputs["sparkline_map_ascending"] = sparkline({ "a" : 1, "b" : 4, "c" : 8 });
+
+    typed_outputs = {};
+
+    for (k, v in outputs) {
+        typed_outputs[k] = v;
+        typed_outputs[k."_type"] = typeof(v);
+    }
+
+    emit typed_outputs;
+}
diff --git a/test/cases/help/0012/expout b/test/cases/help/0012/expout
index 0a5403b5c..506c4d4de 100644
--- a/test/cases/help/0012/expout
+++ b/test/cases/help/0012/expout
@@ -27,11 +27,13 @@ Options:
 -n                  Coerce field values autodetected as int to float, and then
                     apply the float format.
 -h|--help           Show this message.
-format  (class=string #args=variadic) Using first argument as format string, interpolate remaining arguments in place of each "{}" in the format string. Too-few arguments are treated as the empty string; too-many arguments are discarded.
+format  (class=string #args=variadic) Using first argument as format string, interpolate remaining arguments in place of each "{}" in the format string. Also supports 1-based positional placeholders such as "{1}", which allow arguments to be reused and/or reordered. Plain "{}" placeholders consume arguments sequentially, independently of any positional placeholders. Too-few arguments are treated as the empty string, as are positional placeholders exceeding the number of arguments; too-many arguments are discarded. "{0}" is an error value, since positional placeholders are 1-based.
 Examples:
-format("{}:{}:{}", 1,2)     gives "1:2:".
-format("{}:{}:{}", 1,2,3)   gives "1:2:3".
-format("{}:{}:{}", 1,2,3,4) gives "1:2:3".
+format("{}:{}:{}", 1,2)        gives "1:2:".
+format("{}:{}:{}", 1,2,3)      gives "1:2:3".
+format("{}:{}:{}", 1,2,3,4)    gives "1:2:3".
+format("{1}:{2}:{1}", "a","b") gives "a:b:a".
+format("{2}{}:{1}{}", 3,4)     gives "43:34".
 unformat  (class=string #args=2) Using first argument as format string, unpacks second argument into an array of matches, with type-inference. On non-match, returns error -- use is_error() to check.
 Examples:
 unformat("{}:{}:{}",  "1:2:3") gives [1, 2, 3].
diff --git a/test/cases/io-csv-ors-crlf/0001/cmd b/test/cases/io-csv-ors-crlf/0001/cmd
new file mode 100644
index 000000000..5993e01b1
--- /dev/null
+++ b/test/cases/io-csv-ors-crlf/0001/cmd
@@ -0,0 +1 @@
+mlr --icsv --ocsv --ors crlf cat test/input/line-term-lf.csv
diff --git a/test/cases/io-csv-ors-crlf/0001/experr b/test/cases/io-csv-ors-crlf/0001/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-csv-ors-crlf/0001/expout b/test/cases/io-csv-ors-crlf/0001/expout
new file mode 100644
index 000000000..37eac50e4
--- /dev/null
+++ b/test/cases/io-csv-ors-crlf/0001/expout
@@ -0,0 +1,11 @@
+a,b,i,x,y
+pan,pan,1,0.34679014,0.72680286
+eks,pan,2,0.75867996,0.52215111
+wye,wye,3,0.20460331,0.33831853
+eks,wye,4,0.38139939,0.13418874
+wye,pan,5,0.57328892,0.86362447
+zee,pan,6,0.52712616,0.49322129
+eks,zee,7,0.61178406,0.18788492
+zee,wye,8,0.59855401,0.97618139
+hat,wye,9,0.03144188,0.74955076
+pan,wye,10,0.50262601,0.95261836
diff --git a/test/cases/io-csv-ors-crlf/0002/cmd b/test/cases/io-csv-ors-crlf/0002/cmd
new file mode 100644
index 000000000..a53581049
--- /dev/null
+++ b/test/cases/io-csv-ors-crlf/0002/cmd
@@ -0,0 +1 @@
+mlr --icsv --ocsv --ors crlf cat test/input/line-term-crlf.csv
diff --git a/test/cases/io-csv-ors-crlf/0002/experr b/test/cases/io-csv-ors-crlf/0002/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-csv-ors-crlf/0002/expout b/test/cases/io-csv-ors-crlf/0002/expout
new file mode 100644
index 000000000..37eac50e4
--- /dev/null
+++ b/test/cases/io-csv-ors-crlf/0002/expout
@@ -0,0 +1,11 @@
+a,b,i,x,y
+pan,pan,1,0.34679014,0.72680286
+eks,pan,2,0.75867996,0.52215111
+wye,wye,3,0.20460331,0.33831853
+eks,wye,4,0.38139939,0.13418874
+wye,pan,5,0.57328892,0.86362447
+zee,pan,6,0.52712616,0.49322129
+eks,zee,7,0.61178406,0.18788492
+zee,wye,8,0.59855401,0.97618139
+hat,wye,9,0.03144188,0.74955076
+pan,wye,10,0.50262601,0.95261836
diff --git a/test/cases/io-csv-ors-crlf/0003/cmd b/test/cases/io-csv-ors-crlf/0003/cmd
new file mode 100644
index 000000000..26297f1cc
--- /dev/null
+++ b/test/cases/io-csv-ors-crlf/0003/cmd
@@ -0,0 +1 @@
+mlr --icsv --ocsv --ors lf cat test/input/line-term-crlf.csv
diff --git a/test/cases/io-csv-ors-crlf/0003/experr b/test/cases/io-csv-ors-crlf/0003/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-csv-ors-crlf/0003/expout b/test/cases/io-csv-ors-crlf/0003/expout
new file mode 100644
index 000000000..37eac50e4
--- /dev/null
+++ b/test/cases/io-csv-ors-crlf/0003/expout
@@ -0,0 +1,11 @@
+a,b,i,x,y
+pan,pan,1,0.34679014,0.72680286
+eks,pan,2,0.75867996,0.52215111
+wye,wye,3,0.20460331,0.33831853
+eks,wye,4,0.38139939,0.13418874
+wye,pan,5,0.57328892,0.86362447
+zee,pan,6,0.52712616,0.49322129
+eks,zee,7,0.61178406,0.18788492
+zee,wye,8,0.59855401,0.97618139
+hat,wye,9,0.03144188,0.74955076
+pan,wye,10,0.50262601,0.95261836
diff --git a/test/cases/io-csv-ors-crlf/0004/cmd b/test/cases/io-csv-ors-crlf/0004/cmd
new file mode 100644
index 000000000..c347399d3
--- /dev/null
+++ b/test/cases/io-csv-ors-crlf/0004/cmd
@@ -0,0 +1 @@
+mlr --icsv --otsv --ors crlf cat test/input/line-term-lf.csv
diff --git a/test/cases/io-csv-ors-crlf/0004/experr b/test/cases/io-csv-ors-crlf/0004/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-csv-ors-crlf/0004/expout b/test/cases/io-csv-ors-crlf/0004/expout
new file mode 100644
index 000000000..03ac8f384
--- /dev/null
+++ b/test/cases/io-csv-ors-crlf/0004/expout
@@ -0,0 +1,11 @@
+a	b	i	x	y
+pan	pan	1	0.34679014	0.72680286
+eks	pan	2	0.75867996	0.52215111
+wye	wye	3	0.20460331	0.33831853
+eks	wye	4	0.38139939	0.13418874
+wye	pan	5	0.57328892	0.86362447
+zee	pan	6	0.52712616	0.49322129
+eks	zee	7	0.61178406	0.18788492
+zee	wye	8	0.59855401	0.97618139
+hat	wye	9	0.03144188	0.74955076
+pan	wye	10	0.50262601	0.95261836
diff --git a/test/cases/io-csv-ors-crlf/0005/cmd b/test/cases/io-csv-ors-crlf/0005/cmd
new file mode 100644
index 000000000..04914ef75
--- /dev/null
+++ b/test/cases/io-csv-ors-crlf/0005/cmd
@@ -0,0 +1 @@
+mlr --icsv --ocsv --ors semicolon cat test/input/line-term-lf.csv
diff --git a/test/cases/io-csv-ors-crlf/0005/experr b/test/cases/io-csv-ors-crlf/0005/experr
new file mode 100644
index 000000000..1e7caec8c
--- /dev/null
+++ b/test/cases/io-csv-ors-crlf/0005/experr
@@ -0,0 +1 @@
+mlr: for CSV, ORS must be newline or carriage-return/newline
diff --git a/test/cases/io-csv-ors-crlf/0005/expout b/test/cases/io-csv-ors-crlf/0005/expout
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-csv-ors-crlf/0005/should-fail b/test/cases/io-csv-ors-crlf/0005/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-dedupe-field-names/csvlite-ragged/cmd b/test/cases/io-dedupe-field-names/csvlite-ragged/cmd
new file mode 100644
index 000000000..d8cfcd449
--- /dev/null
+++ b/test/cases/io-dedupe-field-names/csvlite-ragged/cmd
@@ -0,0 +1 @@
+mlr --icsvlite --ojson --ragged cat ${CASEDIR}/input
diff --git a/test/cases/io-dedupe-field-names/csvlite-ragged/experr b/test/cases/io-dedupe-field-names/csvlite-ragged/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-dedupe-field-names/csvlite-ragged/expout b/test/cases/io-dedupe-field-names/csvlite-ragged/expout
new file mode 100644
index 000000000..3134d793b
--- /dev/null
+++ b/test/cases/io-dedupe-field-names/csvlite-ragged/expout
@@ -0,0 +1,12 @@
+[
+{
+  "name": "foo",
+  "4": "nyc",
+  "4_2": "pen"
+},
+{
+  "name": "bar",
+  "4": "sea",
+  "4_2": ""
+}
+]
diff --git a/test/cases/io-dedupe-field-names/csvlite-ragged/input b/test/cases/io-dedupe-field-names/csvlite-ragged/input
new file mode 100644
index 000000000..7f713f3ab
--- /dev/null
+++ b/test/cases/io-dedupe-field-names/csvlite-ragged/input
@@ -0,0 +1,3 @@
+name,4,4
+foo,nyc,pen
+bar,sea
diff --git a/test/cases/io-dedupe-field-names/pprint-barred-ragged/cmd b/test/cases/io-dedupe-field-names/pprint-barred-ragged/cmd
new file mode 100644
index 000000000..f85d994e7
--- /dev/null
+++ b/test/cases/io-dedupe-field-names/pprint-barred-ragged/cmd
@@ -0,0 +1 @@
+mlr --ipprint --barred-input --ojson --ragged cat ${CASEDIR}/input
diff --git a/test/cases/io-dedupe-field-names/pprint-barred-ragged/experr b/test/cases/io-dedupe-field-names/pprint-barred-ragged/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-dedupe-field-names/pprint-barred-ragged/expout b/test/cases/io-dedupe-field-names/pprint-barred-ragged/expout
new file mode 100644
index 000000000..3134d793b
--- /dev/null
+++ b/test/cases/io-dedupe-field-names/pprint-barred-ragged/expout
@@ -0,0 +1,12 @@
+[
+{
+  "name": "foo",
+  "4": "nyc",
+  "4_2": "pen"
+},
+{
+  "name": "bar",
+  "4": "sea",
+  "4_2": ""
+}
+]
diff --git a/test/cases/io-dedupe-field-names/pprint-barred-ragged/input b/test/cases/io-dedupe-field-names/pprint-barred-ragged/input
new file mode 100644
index 000000000..126d31890
--- /dev/null
+++ b/test/cases/io-dedupe-field-names/pprint-barred-ragged/input
@@ -0,0 +1,6 @@
++------+-----+-----+
+| name | 4   | 4   |
++------+-----+-----+
+| foo  | nyc | pen |
+| bar  | sea |
++------+-----+-----+
diff --git a/test/cases/io-dedupe-field-names/tsv-ragged/cmd b/test/cases/io-dedupe-field-names/tsv-ragged/cmd
new file mode 100644
index 000000000..1af3b6f65
--- /dev/null
+++ b/test/cases/io-dedupe-field-names/tsv-ragged/cmd
@@ -0,0 +1 @@
+mlr --itsv --ojson --ragged cat ${CASEDIR}/input
diff --git a/test/cases/io-dedupe-field-names/tsv-ragged/experr b/test/cases/io-dedupe-field-names/tsv-ragged/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-dedupe-field-names/tsv-ragged/expout b/test/cases/io-dedupe-field-names/tsv-ragged/expout
new file mode 100644
index 000000000..3134d793b
--- /dev/null
+++ b/test/cases/io-dedupe-field-names/tsv-ragged/expout
@@ -0,0 +1,12 @@
+[
+{
+  "name": "foo",
+  "4": "nyc",
+  "4_2": "pen"
+},
+{
+  "name": "bar",
+  "4": "sea",
+  "4_2": ""
+}
+]
diff --git a/test/cases/io-dedupe-field-names/tsv-ragged/input b/test/cases/io-dedupe-field-names/tsv-ragged/input
new file mode 100644
index 000000000..72480cb59
--- /dev/null
+++ b/test/cases/io-dedupe-field-names/tsv-ragged/input
@@ -0,0 +1,3 @@
+name	4	4
+foo	nyc	pen
+bar	sea
diff --git a/test/cases/io-dkvpx/0001/cmd b/test/cases/io-dkvpx/0001/cmd
new file mode 100644
index 000000000..120222981
--- /dev/null
+++ b/test/cases/io-dkvpx/0001/cmd
@@ -0,0 +1 @@
+mlr --dkvpx cat ${CASEDIR}/input
diff --git a/test/cases/io-dkvpx/0001/experr b/test/cases/io-dkvpx/0001/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-dkvpx/0001/expout b/test/cases/io-dkvpx/0001/expout
new file mode 100644
index 000000000..d28db6b83
--- /dev/null
+++ b/test/cases/io-dkvpx/0001/expout
@@ -0,0 +1,2 @@
+a=1,b=2,c="x,y"
+d="p=q",e="he said ""hi"""
diff --git a/test/cases/io-dkvpx/0001/input b/test/cases/io-dkvpx/0001/input
new file mode 100644
index 000000000..d28db6b83
--- /dev/null
+++ b/test/cases/io-dkvpx/0001/input
@@ -0,0 +1,2 @@
+a=1,b=2,c="x,y"
+d="p=q",e="he said ""hi"""
diff --git a/test/cases/io-dkvpx/0002/cmd b/test/cases/io-dkvpx/0002/cmd
new file mode 100644
index 000000000..e16b8a05c
--- /dev/null
+++ b/test/cases/io-dkvpx/0002/cmd
@@ -0,0 +1 @@
+mlr -i dkvpx --ifs semicolon --ojson cat ${CASEDIR}/input
diff --git a/test/cases/io-dkvpx/0002/experr b/test/cases/io-dkvpx/0002/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-dkvpx/0002/expout b/test/cases/io-dkvpx/0002/expout
new file mode 100644
index 000000000..e3af32ce9
--- /dev/null
+++ b/test/cases/io-dkvpx/0002/expout
@@ -0,0 +1,12 @@
+[
+{
+  "a": 1,
+  "b": 2,
+  "c": "x;y"
+},
+{
+  "a": 3,
+  "b": 4,
+  "c": "p,q"
+}
+]
diff --git a/test/cases/io-dkvpx/0002/input b/test/cases/io-dkvpx/0002/input
new file mode 100644
index 000000000..c160321ea
--- /dev/null
+++ b/test/cases/io-dkvpx/0002/input
@@ -0,0 +1,2 @@
+a=1;b=2;c="x;y"
+a=3;b=4;c="p,q"
diff --git a/test/cases/io-dkvpx/0003/cmd b/test/cases/io-dkvpx/0003/cmd
new file mode 100644
index 000000000..ef7bd6988
--- /dev/null
+++ b/test/cases/io-dkvpx/0003/cmd
@@ -0,0 +1 @@
+mlr -i dkvpx --ips colon --ojson cat ${CASEDIR}/input
diff --git a/test/cases/io-dkvpx/0003/experr b/test/cases/io-dkvpx/0003/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-dkvpx/0003/expout b/test/cases/io-dkvpx/0003/expout
new file mode 100644
index 000000000..9176cb485
--- /dev/null
+++ b/test/cases/io-dkvpx/0003/expout
@@ -0,0 +1,7 @@
+[
+{
+  "a": 1,
+  "b": 2,
+  "c": "x:y"
+}
+]
diff --git a/test/cases/io-dkvpx/0003/input b/test/cases/io-dkvpx/0003/input
new file mode 100644
index 000000000..a263a020c
--- /dev/null
+++ b/test/cases/io-dkvpx/0003/input
@@ -0,0 +1 @@
+a:1,b:2,c:"x:y"
diff --git a/test/cases/io-dkvpx/0004/cmd b/test/cases/io-dkvpx/0004/cmd
new file mode 100644
index 000000000..c4b5f27c3
--- /dev/null
+++ b/test/cases/io-dkvpx/0004/cmd
@@ -0,0 +1 @@
+mlr --ijson -o dkvpx --ofs semicolon --ops colon cat ${CASEDIR}/input
diff --git a/test/cases/io-dkvpx/0004/experr b/test/cases/io-dkvpx/0004/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-dkvpx/0004/expout b/test/cases/io-dkvpx/0004/expout
new file mode 100644
index 000000000..80f48485b
--- /dev/null
+++ b/test/cases/io-dkvpx/0004/expout
@@ -0,0 +1 @@
+a:"x;y";b:p,q;c:"u:v"
diff --git a/test/cases/io-dkvpx/0004/input b/test/cases/io-dkvpx/0004/input
new file mode 100644
index 000000000..2445bd406
--- /dev/null
+++ b/test/cases/io-dkvpx/0004/input
@@ -0,0 +1 @@
+{"a": "x;y", "b": "p,q", "c": "u:v"}
diff --git a/test/cases/io-dkvpx/0005/cmd b/test/cases/io-dkvpx/0005/cmd
new file mode 100644
index 000000000..4e2c54c51
--- /dev/null
+++ b/test/cases/io-dkvpx/0005/cmd
@@ -0,0 +1 @@
+mlr -i dkvpx --ifs ';;' --ojson cat ${CASEDIR}/input
diff --git a/test/cases/io-dkvpx/0005/experr b/test/cases/io-dkvpx/0005/experr
new file mode 100644
index 000000000..b31e49c6d
--- /dev/null
+++ b/test/cases/io-dkvpx/0005/experr
@@ -0,0 +1 @@
+mlr: for DKVPX, IFS can only be a single character
diff --git a/test/cases/io-dkvpx/0005/expout b/test/cases/io-dkvpx/0005/expout
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-dkvpx/0005/input b/test/cases/io-dkvpx/0005/input
new file mode 100644
index 000000000..5e1e7d542
--- /dev/null
+++ b/test/cases/io-dkvpx/0005/input
@@ -0,0 +1 @@
+a=1;;b=2
diff --git a/test/cases/io-dkvpx/0005/should-fail b/test/cases/io-dkvpx/0005/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-markdown/0003/cmd b/test/cases/io-markdown/0003/cmd
new file mode 100644
index 000000000..e18c3c97d
--- /dev/null
+++ b/test/cases/io-markdown/0003/cmd
@@ -0,0 +1 @@
+mlr --icsv --omd --right-align-numeric cat ${CASEDIR}/input
diff --git a/test/cases/io-markdown/0003/experr b/test/cases/io-markdown/0003/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-markdown/0003/expout b/test/cases/io-markdown/0003/expout
new file mode 100644
index 000000000..bf03eaedf
--- /dev/null
+++ b/test/cases/io-markdown/0003/expout
@@ -0,0 +1,6 @@
+| a | b | i | x |
+| --- | --- | ---: | ---: |
+| pan | pan | 1 | 0.34679000 |
+| eks | wye | 10000 | 0.75868000 |
+| wye | hat | -3 | N/A |
+| hat | zee | 5 |  |
diff --git a/test/cases/io-markdown/0003/input b/test/cases/io-markdown/0003/input
new file mode 100644
index 000000000..983929e6c
--- /dev/null
+++ b/test/cases/io-markdown/0003/input
@@ -0,0 +1,5 @@
+a,b,i,x
+pan,pan,1,0.34679
+eks,wye,10000,0.75868
+wye,hat,-3,N/A
+hat,zee,5,
diff --git a/test/cases/io-markdown/0004/cmd b/test/cases/io-markdown/0004/cmd
new file mode 100644
index 000000000..38e071e0e
--- /dev/null
+++ b/test/cases/io-markdown/0004/cmd
@@ -0,0 +1 @@
+mlr --icsv --omd-aligned --right-align-numeric cat ${CASEDIR}/input
diff --git a/test/cases/io-markdown/0004/experr b/test/cases/io-markdown/0004/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-markdown/0004/expout b/test/cases/io-markdown/0004/expout
new file mode 100644
index 000000000..5b0727dfc
--- /dev/null
+++ b/test/cases/io-markdown/0004/expout
@@ -0,0 +1,6 @@
+| a   | b   |     i | x          |
+| --- | --- |  ---: | ---        |
+| pan | pan |     1 | 0.34679000 |
+| eks | wye | 10000 | 0.75868000 |
+| wye | hat |    -3 | N/A        |
+| hat | zee |     5 |            |
diff --git a/test/cases/io-markdown/0004/input b/test/cases/io-markdown/0004/input
new file mode 100644
index 000000000..983929e6c
--- /dev/null
+++ b/test/cases/io-markdown/0004/input
@@ -0,0 +1,5 @@
+a,b,i,x
+pan,pan,1,0.34679
+eks,wye,10000,0.75868
+wye,hat,-3,N/A
+hat,zee,5,
diff --git a/test/cases/io-pprint-right-align-numeric/0001/cmd b/test/cases/io-pprint-right-align-numeric/0001/cmd
new file mode 100644
index 000000000..56d83155b
--- /dev/null
+++ b/test/cases/io-pprint-right-align-numeric/0001/cmd
@@ -0,0 +1 @@
+mlr --icsv --opprint --right-align-numeric cat ${CASEDIR}/input
diff --git a/test/cases/io-pprint-right-align-numeric/0001/experr b/test/cases/io-pprint-right-align-numeric/0001/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-pprint-right-align-numeric/0001/expout b/test/cases/io-pprint-right-align-numeric/0001/expout
new file mode 100644
index 000000000..1f26a8e08
--- /dev/null
+++ b/test/cases/io-pprint-right-align-numeric/0001/expout
@@ -0,0 +1,5 @@
+a   b       i x
+pan pan     1 0.34679000
+eks wye 10000 0.75868000
+wye hat    -3 N/A
+hat zee     5 -
diff --git a/test/cases/io-pprint-right-align-numeric/0001/input b/test/cases/io-pprint-right-align-numeric/0001/input
new file mode 100644
index 000000000..983929e6c
--- /dev/null
+++ b/test/cases/io-pprint-right-align-numeric/0001/input
@@ -0,0 +1,5 @@
+a,b,i,x
+pan,pan,1,0.34679
+eks,wye,10000,0.75868
+wye,hat,-3,N/A
+hat,zee,5,
diff --git a/test/cases/io-pprint-right-align-numeric/0002/cmd b/test/cases/io-pprint-right-align-numeric/0002/cmd
new file mode 100644
index 000000000..ad12c83e8
--- /dev/null
+++ b/test/cases/io-pprint-right-align-numeric/0002/cmd
@@ -0,0 +1 @@
+mlr --icsv --opprint --barred --right-align-numeric cat ${CASEDIR}/input
diff --git a/test/cases/io-pprint-right-align-numeric/0002/experr b/test/cases/io-pprint-right-align-numeric/0002/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-pprint-right-align-numeric/0002/expout b/test/cases/io-pprint-right-align-numeric/0002/expout
new file mode 100644
index 000000000..b2bf8aa59
--- /dev/null
+++ b/test/cases/io-pprint-right-align-numeric/0002/expout
@@ -0,0 +1,8 @@
++-----+-----+-------+------------+
+| a   | b   |     i | x          |
++-----+-----+-------+------------+
+| pan | pan |     1 | 0.34679000 |
+| eks | wye | 10000 | 0.75868000 |
+| wye | hat |    -3 | N/A        |
+| hat | zee |     5 |            |
++-----+-----+-------+------------+
diff --git a/test/cases/io-pprint-right-align-numeric/0002/input b/test/cases/io-pprint-right-align-numeric/0002/input
new file mode 100644
index 000000000..983929e6c
--- /dev/null
+++ b/test/cases/io-pprint-right-align-numeric/0002/input
@@ -0,0 +1,5 @@
+a,b,i,x
+pan,pan,1,0.34679
+eks,wye,10000,0.75868
+wye,hat,-3,N/A
+hat,zee,5,
diff --git a/test/cases/io-pprint-right-align-numeric/0003/cmd b/test/cases/io-pprint-right-align-numeric/0003/cmd
new file mode 100644
index 000000000..56d83155b
--- /dev/null
+++ b/test/cases/io-pprint-right-align-numeric/0003/cmd
@@ -0,0 +1 @@
+mlr --icsv --opprint --right-align-numeric cat ${CASEDIR}/input
diff --git a/test/cases/io-pprint-right-align-numeric/0003/experr b/test/cases/io-pprint-right-align-numeric/0003/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-pprint-right-align-numeric/0003/expout b/test/cases/io-pprint-right-align-numeric/0003/expout
new file mode 100644
index 000000000..8b16809b4
--- /dev/null
+++ b/test/cases/io-pprint-right-align-numeric/0003/expout
@@ -0,0 +1,3 @@
+name  count      ratio
+alpha     3 0.90000000
+b        14 0.05000000
diff --git a/test/cases/io-pprint-right-align-numeric/0003/input b/test/cases/io-pprint-right-align-numeric/0003/input
new file mode 100644
index 000000000..254256b7c
--- /dev/null
+++ b/test/cases/io-pprint-right-align-numeric/0003/input
@@ -0,0 +1,3 @@
+name,count,ratio
+alpha,3,0.9
+b,14,0.05
diff --git a/test/cases/io-pprint-right-align-numeric/0004/cmd b/test/cases/io-pprint-right-align-numeric/0004/cmd
new file mode 100644
index 000000000..ad12c83e8
--- /dev/null
+++ b/test/cases/io-pprint-right-align-numeric/0004/cmd
@@ -0,0 +1 @@
+mlr --icsv --opprint --barred --right-align-numeric cat ${CASEDIR}/input
diff --git a/test/cases/io-pprint-right-align-numeric/0004/experr b/test/cases/io-pprint-right-align-numeric/0004/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-pprint-right-align-numeric/0004/expout b/test/cases/io-pprint-right-align-numeric/0004/expout
new file mode 100644
index 000000000..64b7fe62f
--- /dev/null
+++ b/test/cases/io-pprint-right-align-numeric/0004/expout
@@ -0,0 +1,6 @@
++-------+-------+------------+
+| name  | count |      ratio |
++-------+-------+------------+
+| alpha |     3 | 0.90000000 |
+| b     |    14 | 0.05000000 |
++-------+-------+------------+
diff --git a/test/cases/io-pprint-right-align-numeric/0004/input b/test/cases/io-pprint-right-align-numeric/0004/input
new file mode 100644
index 000000000..254256b7c
--- /dev/null
+++ b/test/cases/io-pprint-right-align-numeric/0004/input
@@ -0,0 +1,3 @@
+name,count,ratio
+alpha,3,0.9
+b,14,0.05
diff --git a/test/cases/repl-help/0012/expout b/test/cases/repl-help/0012/expout
index 580c754e0..ccb50a216 100644
--- a/test/cases/repl-help/0012/expout
+++ b/test/cases/repl-help/0012/expout
@@ -13,11 +13,13 @@ For-loop over out-of-stream variables:
 C-style for-loop:
 
   Example:  'for (var i = 0, var b = 1; i < 10; i += 1, b *= 2) { ... }'
-format  (class=string #args=variadic) Using first argument as format string, interpolate remaining arguments in place of each "{}" in the format string. Too-few arguments are treated as the empty string; too-many arguments are discarded.
+format  (class=string #args=variadic) Using first argument as format string, interpolate remaining arguments in place of each "{}" in the format string. Also supports 1-based positional placeholders such as "{1}", which allow arguments to be reused and/or reordered. Plain "{}" placeholders consume arguments sequentially, independently of any positional placeholders. Too-few arguments are treated as the empty string, as are positional placeholders exceeding the number of arguments; too-many arguments are discarded. "{0}" is an error value, since positional placeholders are 1-based.
 Examples:
-format("{}:{}:{}", 1,2)     gives "1:2:".
-format("{}:{}:{}", 1,2,3)   gives "1:2:3".
-format("{}:{}:{}", 1,2,3,4) gives "1:2:3".
+format("{}:{}:{}", 1,2)        gives "1:2:".
+format("{}:{}:{}", 1,2,3)      gives "1:2:3".
+format("{}:{}:{}", 1,2,3,4)    gives "1:2:3".
+format("{1}:{2}:{1}", "a","b") gives "a:b:a".
+format("{2}{}:{1}{}", 3,4)     gives "43:34".
 unformat  (class=string #args=2) Using first argument as format string, unpacks second argument into an array of matches, with type-inference. On non-match, returns error -- use is_error() to check.
 Examples:
 unformat("{}:{}:{}",  "1:2:3") gives [1, 2, 3].
diff --git a/test/cases/verb-histogram/0008/cmd b/test/cases/verb-histogram/0008/cmd
new file mode 100644
index 000000000..7f38f998b
--- /dev/null
+++ b/test/cases/verb-histogram/0008/cmd
@@ -0,0 +1 @@
+mlr --icsv --opprint histogram --nbins 9 --auto -f x,y -s test/input/histo2.csv
diff --git a/test/cases/verb-histogram/0008/experr b/test/cases/verb-histogram/0008/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-histogram/0008/expout b/test/cases/verb-histogram/0008/expout
new file mode 100644
index 000000000..fb9591d33
--- /dev/null
+++ b/test/cases/verb-histogram/0008/expout
@@ -0,0 +1,3 @@
+field lo          hi           sparkline
+x     20.00000000 300.00000000 β–β–β–…β–β–β–ˆβ–β–β–…
+y     20.00000000 300.00000000 β–ˆβ–ƒβ–β–β–β–β–β–β–
diff --git a/test/cases/verb-join/left-file-malformed-sorted/cmd b/test/cases/verb-join/left-file-malformed-sorted/cmd
new file mode 100644
index 000000000..340fbb573
--- /dev/null
+++ b/test/cases/verb-join/left-file-malformed-sorted/cmd
@@ -0,0 +1 @@
+mlr --icsv --opprint join -s -j a -f test/input/join-left-ragged.csv test/input/abixy.csv
diff --git a/test/cases/verb-join/left-file-malformed-sorted/experr b/test/cases/verb-join/left-file-malformed-sorted/experr
new file mode 100644
index 000000000..d082c879e
--- /dev/null
+++ b/test/cases/verb-join/left-file-malformed-sorted/experr
@@ -0,0 +1 @@
+mlr: CSV header/data length mismatch 2 != 4 at filename test/input/join-left-ragged.csv row 3
diff --git a/test/cases/verb-join/left-file-malformed-sorted/expout b/test/cases/verb-join/left-file-malformed-sorted/expout
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-join/left-file-malformed-sorted/should-fail b/test/cases/verb-join/left-file-malformed-sorted/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-join/left-file-malformed-unsorted/cmd b/test/cases/verb-join/left-file-malformed-unsorted/cmd
new file mode 100644
index 000000000..6bbb82d41
--- /dev/null
+++ b/test/cases/verb-join/left-file-malformed-unsorted/cmd
@@ -0,0 +1 @@
+mlr --icsv --opprint join -j a -f test/input/join-left-ragged.csv test/input/abixy.csv
diff --git a/test/cases/verb-join/left-file-malformed-unsorted/experr b/test/cases/verb-join/left-file-malformed-unsorted/experr
new file mode 100644
index 000000000..d082c879e
--- /dev/null
+++ b/test/cases/verb-join/left-file-malformed-unsorted/experr
@@ -0,0 +1 @@
+mlr: CSV header/data length mismatch 2 != 4 at filename test/input/join-left-ragged.csv row 3
diff --git a/test/cases/verb-join/left-file-malformed-unsorted/expout b/test/cases/verb-join/left-file-malformed-unsorted/expout
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-join/left-file-malformed-unsorted/should-fail b/test/cases/verb-join/left-file-malformed-unsorted/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-join/non-windows-left-file-not-found-sorted/cmd b/test/cases/verb-join/non-windows-left-file-not-found-sorted/cmd
new file mode 100644
index 000000000..3fc46bd0e
--- /dev/null
+++ b/test/cases/verb-join/non-windows-left-file-not-found-sorted/cmd
@@ -0,0 +1 @@
+mlr --icsv --opprint join -s -j a -f /nonesuch/nope/never test/input/abixy.csv
diff --git a/test/cases/verb-join/non-windows-left-file-not-found-sorted/experr b/test/cases/verb-join/non-windows-left-file-not-found-sorted/experr
new file mode 100644
index 000000000..81b3580c0
--- /dev/null
+++ b/test/cases/verb-join/non-windows-left-file-not-found-sorted/experr
@@ -0,0 +1 @@
+mlr: open /nonesuch/nope/never: no such file or directory
diff --git a/test/cases/verb-join/non-windows-left-file-not-found-sorted/expout b/test/cases/verb-join/non-windows-left-file-not-found-sorted/expout
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-join/non-windows-left-file-not-found-sorted/should-fail b/test/cases/verb-join/non-windows-left-file-not-found-sorted/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-join/non-windows-left-file-not-found-unsorted/cmd b/test/cases/verb-join/non-windows-left-file-not-found-unsorted/cmd
new file mode 100644
index 000000000..20e60c307
--- /dev/null
+++ b/test/cases/verb-join/non-windows-left-file-not-found-unsorted/cmd
@@ -0,0 +1 @@
+mlr --icsv --opprint join -j a -f /nonesuch/nope/never test/input/abixy.csv
diff --git a/test/cases/verb-join/non-windows-left-file-not-found-unsorted/experr b/test/cases/verb-join/non-windows-left-file-not-found-unsorted/experr
new file mode 100644
index 000000000..81b3580c0
--- /dev/null
+++ b/test/cases/verb-join/non-windows-left-file-not-found-unsorted/experr
@@ -0,0 +1 @@
+mlr: open /nonesuch/nope/never: no such file or directory
diff --git a/test/cases/verb-join/non-windows-left-file-not-found-unsorted/expout b/test/cases/verb-join/non-windows-left-file-not-found-unsorted/expout
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-join/non-windows-left-file-not-found-unsorted/should-fail b/test/cases/verb-join/non-windows-left-file-not-found-unsorted/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-rank/0001/cmd b/test/cases/verb-rank/0001/cmd
new file mode 100644
index 000000000..46ddbeaf4
--- /dev/null
+++ b/test/cases/verb-rank/0001/cmd
@@ -0,0 +1 @@
+mlr --icsv --ocsv rank -f x test/input/rank-data.csv
diff --git a/test/cases/verb-rank/0001/experr b/test/cases/verb-rank/0001/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-rank/0001/expout b/test/cases/verb-rank/0001/expout
new file mode 100644
index 000000000..543c233c9
--- /dev/null
+++ b/test/cases/verb-rank/0001/expout
@@ -0,0 +1,5 @@
+x,x_rank
+10,1
+20,2
+20,2
+30,4
diff --git a/test/cases/verb-rank/0002/cmd b/test/cases/verb-rank/0002/cmd
new file mode 100644
index 000000000..d8ef5e099
--- /dev/null
+++ b/test/cases/verb-rank/0002/cmd
@@ -0,0 +1 @@
+mlr --icsv --ocsv rank -f x test/input/rank-data-unsorted.csv
diff --git a/test/cases/verb-rank/0002/experr b/test/cases/verb-rank/0002/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-rank/0002/expout b/test/cases/verb-rank/0002/expout
new file mode 100644
index 000000000..dc98cbe30
--- /dev/null
+++ b/test/cases/verb-rank/0002/expout
@@ -0,0 +1,5 @@
+x,x_rank
+20,2
+10,1
+20,2
+30,4
diff --git a/test/cases/verb-rank/0003/cmd b/test/cases/verb-rank/0003/cmd
new file mode 100644
index 000000000..81c63e31b
--- /dev/null
+++ b/test/cases/verb-rank/0003/cmd
@@ -0,0 +1 @@
+mlr --icsv --ocsv rank -f x -g g test/input/rank-data-grouped.csv
diff --git a/test/cases/verb-rank/0003/experr b/test/cases/verb-rank/0003/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-rank/0003/expout b/test/cases/verb-rank/0003/expout
new file mode 100644
index 000000000..49fcf4769
--- /dev/null
+++ b/test/cases/verb-rank/0003/expout
@@ -0,0 +1,6 @@
+g,x,x_rank
+a,20,2
+b,5,1
+a,10,1
+b,5,1
+a,20,2
diff --git a/test/cases/verb-rank/0004/cmd b/test/cases/verb-rank/0004/cmd
new file mode 100644
index 000000000..71d4149da
--- /dev/null
+++ b/test/cases/verb-rank/0004/cmd
@@ -0,0 +1 @@
+mlr --icsv --ocsv rank -f x --sorted test/input/rank-data.csv
diff --git a/test/cases/verb-rank/0004/experr b/test/cases/verb-rank/0004/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-rank/0004/expout b/test/cases/verb-rank/0004/expout
new file mode 100644
index 000000000..543c233c9
--- /dev/null
+++ b/test/cases/verb-rank/0004/expout
@@ -0,0 +1,5 @@
+x,x_rank
+10,1
+20,2
+20,2
+30,4
diff --git a/test/cases/verb-rank/0005/cmd b/test/cases/verb-rank/0005/cmd
new file mode 100644
index 000000000..9fad3ad0e
--- /dev/null
+++ b/test/cases/verb-rank/0005/cmd
@@ -0,0 +1 @@
+mlr --icsv --ocsv sort -f x then rank -f x --sorted test/input/rank-data-unsorted.csv
diff --git a/test/cases/verb-rank/0005/experr b/test/cases/verb-rank/0005/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-rank/0005/expout b/test/cases/verb-rank/0005/expout
new file mode 100644
index 000000000..543c233c9
--- /dev/null
+++ b/test/cases/verb-rank/0005/expout
@@ -0,0 +1,5 @@
+x,x_rank
+10,1
+20,2
+20,2
+30,4
diff --git a/test/cases/verb-rank/0006/cmd b/test/cases/verb-rank/0006/cmd
new file mode 100644
index 000000000..530a8c3d2
--- /dev/null
+++ b/test/cases/verb-rank/0006/cmd
@@ -0,0 +1 @@
+mlr rank -g g
diff --git a/test/cases/verb-rank/0006/experr b/test/cases/verb-rank/0006/experr
new file mode 100644
index 000000000..3525047c2
--- /dev/null
+++ b/test/cases/verb-rank/0006/experr
@@ -0,0 +1 @@
+mlr rank: -f field names required
diff --git a/test/cases/verb-rank/0006/expout b/test/cases/verb-rank/0006/expout
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-rank/0006/should-fail b/test/cases/verb-rank/0006/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-reorder/regex-after-grouped/cmd b/test/cases/verb-reorder/regex-after-grouped/cmd
new file mode 100644
index 000000000..5ff531da9
--- /dev/null
+++ b/test/cases/verb-reorder/regex-after-grouped/cmd
@@ -0,0 +1 @@
+mlr --icsv --opprint reorder -r '^bbb,^yyy' -a zzz-f1 test/input/reorder-prefix.csv
diff --git a/test/cases/verb-reorder/regex-after-grouped/experr b/test/cases/verb-reorder/regex-after-grouped/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-reorder/regex-after-grouped/expout b/test/cases/verb-reorder/regex-after-grouped/expout
new file mode 100644
index 000000000..523f274e5
--- /dev/null
+++ b/test/cases/verb-reorder/regex-after-grouped/expout
@@ -0,0 +1,2 @@
+xxx-f1 xxx-f2 zzz-f1 bbb-f1 bbb-f2 yyy-f1 yyy-f2 ccc-f1
+1      2      5      6      7      3      4      8
diff --git a/test/cases/verb-reorder/regex-after/expout b/test/cases/verb-reorder/regex-after/expout
index 62cb82ad5..f09c32456 100644
--- a/test/cases/verb-reorder/regex-after/expout
+++ b/test/cases/verb-reorder/regex-after/expout
@@ -4,7 +4,7 @@
 5  e
 6  f
 3  c
-8  h
 9  i
+8  h
 7  g
 10 j
diff --git a/test/cases/verb-reorder/regex-before-grouped/cmd b/test/cases/verb-reorder/regex-before-grouped/cmd
new file mode 100644
index 000000000..0e5ae666e
--- /dev/null
+++ b/test/cases/verb-reorder/regex-before-grouped/cmd
@@ -0,0 +1 @@
+mlr --icsv --opprint reorder -r '^bbb,^yyy' -b zzz-f1 test/input/reorder-prefix.csv
diff --git a/test/cases/verb-reorder/regex-before-grouped/experr b/test/cases/verb-reorder/regex-before-grouped/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-reorder/regex-before-grouped/expout b/test/cases/verb-reorder/regex-before-grouped/expout
new file mode 100644
index 000000000..c034936cd
--- /dev/null
+++ b/test/cases/verb-reorder/regex-before-grouped/expout
@@ -0,0 +1,2 @@
+xxx-f1 xxx-f2 bbb-f1 bbb-f2 yyy-f1 yyy-f2 zzz-f1 ccc-f1
+1      2      6      7      3      4      5      8
diff --git a/test/cases/verb-reorder/regex-before/expout b/test/cases/verb-reorder/regex-before/expout
index ef4d4f166..009ec80a1 100644
--- a/test/cases/verb-reorder/regex-before/expout
+++ b/test/cases/verb-reorder/regex-before/expout
@@ -3,8 +3,8 @@
 4  d
 5  e
 3  c
-8  h
 9  i
+8  h
 6  f
 7  g
 10 j
diff --git a/test/cases/verb-reorder/regex-end-grouped/cmd b/test/cases/verb-reorder/regex-end-grouped/cmd
new file mode 100644
index 000000000..2b816f713
--- /dev/null
+++ b/test/cases/verb-reorder/regex-end-grouped/cmd
@@ -0,0 +1 @@
+mlr --icsv --opprint reorder -e -r '^bbb,^yyy' test/input/reorder-prefix.csv
diff --git a/test/cases/verb-reorder/regex-end-grouped/experr b/test/cases/verb-reorder/regex-end-grouped/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-reorder/regex-end-grouped/expout b/test/cases/verb-reorder/regex-end-grouped/expout
new file mode 100644
index 000000000..10eaad8d2
--- /dev/null
+++ b/test/cases/verb-reorder/regex-end-grouped/expout
@@ -0,0 +1,2 @@
+xxx-f1 xxx-f2 zzz-f1 ccc-f1 bbb-f1 bbb-f2 yyy-f1 yyy-f2
+1      2      5      8      6      7      3      4
diff --git a/test/cases/verb-reorder/regex-end/expout b/test/cases/verb-reorder/regex-end/expout
index 7a7424aa9..b219a399b 100644
--- a/test/cases/verb-reorder/regex-end/expout
+++ b/test/cases/verb-reorder/regex-end/expout
@@ -6,5 +6,5 @@
 7  g
 10 j
 3  c
-8  h
 9  i
+8  h
diff --git a/test/cases/verb-reorder/regex-first-match-wins/cmd b/test/cases/verb-reorder/regex-first-match-wins/cmd
new file mode 100644
index 000000000..5a9461da8
--- /dev/null
+++ b/test/cases/verb-reorder/regex-first-match-wins/cmd
@@ -0,0 +1 @@
+mlr --icsv --opprint reorder -r '^bbb,f1$' test/input/reorder-prefix.csv
diff --git a/test/cases/verb-reorder/regex-first-match-wins/experr b/test/cases/verb-reorder/regex-first-match-wins/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-reorder/regex-first-match-wins/expout b/test/cases/verb-reorder/regex-first-match-wins/expout
new file mode 100644
index 000000000..652913c4c
--- /dev/null
+++ b/test/cases/verb-reorder/regex-first-match-wins/expout
@@ -0,0 +1,2 @@
+bbb-f1 bbb-f2 xxx-f1 yyy-f1 zzz-f1 ccc-f1 xxx-f2 yyy-f2
+6      7      1      3      5      8      2      4
diff --git a/test/cases/verb-reorder/regex-start-grouped/cmd b/test/cases/verb-reorder/regex-start-grouped/cmd
new file mode 100644
index 000000000..2cbf223dc
--- /dev/null
+++ b/test/cases/verb-reorder/regex-start-grouped/cmd
@@ -0,0 +1 @@
+mlr --icsv --opprint reorder -r '^yyy,^xxx' test/input/reorder-prefix.csv
diff --git a/test/cases/verb-reorder/regex-start-grouped/experr b/test/cases/verb-reorder/regex-start-grouped/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-reorder/regex-start-grouped/expout b/test/cases/verb-reorder/regex-start-grouped/expout
new file mode 100644
index 000000000..d355c54d8
--- /dev/null
+++ b/test/cases/verb-reorder/regex-start-grouped/expout
@@ -0,0 +1,2 @@
+yyy-f1 yyy-f2 xxx-f1 xxx-f2 zzz-f1 bbb-f1 bbb-f2 ccc-f1
+3      4      1      2      5      6      7      8
diff --git a/test/cases/verb-reorder/regex-start/expout b/test/cases/verb-reorder/regex-start/expout
index ee16332d9..80ee472f9 100644
--- a/test/cases/verb-reorder/regex-start/expout
+++ b/test/cases/verb-reorder/regex-start/expout
@@ -1,6 +1,6 @@
 3  c
-8  h
 9  i
+8  h
 1  a
 2  b
 4  d
diff --git a/test/cases/verb-skip-trivial-records/0004/cmd b/test/cases/verb-skip-trivial-records/0004/cmd
new file mode 100644
index 000000000..9d5cdad10
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0004/cmd
@@ -0,0 +1 @@
+mlr --icsv --ocsv skip-trivial-records ${CASEDIR}/input.csv
diff --git a/test/cases/verb-skip-trivial-records/0004/experr b/test/cases/verb-skip-trivial-records/0004/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-skip-trivial-records/0004/expout b/test/cases/verb-skip-trivial-records/0004/expout
new file mode 100644
index 000000000..cfa20f810
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0004/expout
@@ -0,0 +1,2 @@
+a,b
+1,2
diff --git a/test/cases/verb-skip-trivial-records/0004/input.csv b/test/cases/verb-skip-trivial-records/0004/input.csv
new file mode 100644
index 000000000..e961f3a69
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0004/input.csv
@@ -0,0 +1,4 @@
+a,b
+1,2
+
+
diff --git a/test/cases/verb-skip-trivial-records/0005/cmd b/test/cases/verb-skip-trivial-records/0005/cmd
new file mode 100644
index 000000000..211699ef0
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0005/cmd
@@ -0,0 +1 @@
+mlr --icsv --ocsv cat ${CASEDIR}/input.csv
diff --git a/test/cases/verb-skip-trivial-records/0005/experr b/test/cases/verb-skip-trivial-records/0005/experr
new file mode 100644
index 000000000..267d382e9
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0005/experr
@@ -0,0 +1 @@
+mlr: CSV header/data length mismatch 2 != 1 at filename test/cases/verb-skip-trivial-records/0005/input.csv row 3
diff --git a/test/cases/verb-skip-trivial-records/0005/expout b/test/cases/verb-skip-trivial-records/0005/expout
new file mode 100644
index 000000000..cfa20f810
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0005/expout
@@ -0,0 +1,2 @@
+a,b
+1,2
diff --git a/test/cases/verb-skip-trivial-records/0005/input.csv b/test/cases/verb-skip-trivial-records/0005/input.csv
new file mode 100644
index 000000000..e961f3a69
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0005/input.csv
@@ -0,0 +1,4 @@
+a,b
+1,2
+
+
diff --git a/test/cases/verb-skip-trivial-records/0005/should-fail b/test/cases/verb-skip-trivial-records/0005/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-skip-trivial-records/0006/cmd b/test/cases/verb-skip-trivial-records/0006/cmd
new file mode 100644
index 000000000..9d5cdad10
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0006/cmd
@@ -0,0 +1 @@
+mlr --icsv --ocsv skip-trivial-records ${CASEDIR}/input.csv
diff --git a/test/cases/verb-skip-trivial-records/0006/experr b/test/cases/verb-skip-trivial-records/0006/experr
new file mode 100644
index 000000000..1bf493268
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0006/experr
@@ -0,0 +1 @@
+mlr: CSV header/data length mismatch 2 != 1 at filename test/cases/verb-skip-trivial-records/0006/input.csv row 3
diff --git a/test/cases/verb-skip-trivial-records/0006/expout b/test/cases/verb-skip-trivial-records/0006/expout
new file mode 100644
index 000000000..cfa20f810
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0006/expout
@@ -0,0 +1,2 @@
+a,b
+1,2
diff --git a/test/cases/verb-skip-trivial-records/0006/input.csv b/test/cases/verb-skip-trivial-records/0006/input.csv
new file mode 100644
index 000000000..2daca4f38
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0006/input.csv
@@ -0,0 +1,3 @@
+a,b
+1,2
+3
diff --git a/test/cases/verb-skip-trivial-records/0006/should-fail b/test/cases/verb-skip-trivial-records/0006/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-skip-trivial-records/0007/cmd b/test/cases/verb-skip-trivial-records/0007/cmd
new file mode 100644
index 000000000..4181e6610
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0007/cmd
@@ -0,0 +1 @@
+mlr --itsv --otsv skip-trivial-records ${CASEDIR}/input.tsv
diff --git a/test/cases/verb-skip-trivial-records/0007/experr b/test/cases/verb-skip-trivial-records/0007/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-skip-trivial-records/0007/expout b/test/cases/verb-skip-trivial-records/0007/expout
new file mode 100644
index 000000000..8b26211c1
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0007/expout
@@ -0,0 +1,2 @@
+a	b
+1	2
diff --git a/test/cases/verb-skip-trivial-records/0007/input.tsv b/test/cases/verb-skip-trivial-records/0007/input.tsv
new file mode 100644
index 000000000..eec1f8fea
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0007/input.tsv
@@ -0,0 +1,3 @@
+a	b
+1	2
+
diff --git a/test/cases/verb-skip-trivial-records/0008/cmd b/test/cases/verb-skip-trivial-records/0008/cmd
new file mode 100644
index 000000000..9d5cdad10
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0008/cmd
@@ -0,0 +1 @@
+mlr --icsv --ocsv skip-trivial-records ${CASEDIR}/input.csv
diff --git a/test/cases/verb-skip-trivial-records/0008/experr b/test/cases/verb-skip-trivial-records/0008/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-skip-trivial-records/0008/expout b/test/cases/verb-skip-trivial-records/0008/expout
new file mode 100644
index 000000000..0099ae937
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0008/expout
@@ -0,0 +1,3 @@
+a,b
+1,2
+3,4
diff --git a/test/cases/verb-skip-trivial-records/0008/input.csv b/test/cases/verb-skip-trivial-records/0008/input.csv
new file mode 100644
index 000000000..58f982b9e
--- /dev/null
+++ b/test/cases/verb-skip-trivial-records/0008/input.csv
@@ -0,0 +1,4 @@
+a,b
+1,2
+
+3,4
diff --git a/test/cases/verb-sparkline/0001/cmd b/test/cases/verb-sparkline/0001/cmd
new file mode 100644
index 000000000..a9f76465a
--- /dev/null
+++ b/test/cases/verb-sparkline/0001/cmd
@@ -0,0 +1 @@
+mlr --ojson sparkline -f i,x,y test/input/abixy
diff --git a/test/cases/verb-sparkline/0001/experr b/test/cases/verb-sparkline/0001/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sparkline/0001/expout b/test/cases/verb-sparkline/0001/expout
new file mode 100644
index 000000000..64dfa4158
--- /dev/null
+++ b/test/cases/verb-sparkline/0001/expout
@@ -0,0 +1,23 @@
+[
+{
+  "field": "i",
+  "n": 10,
+  "lo": 1.00000000,
+  "hi": 10.00000000,
+  "sparkline": "β–β–‚β–ƒβ–ƒβ–„β–…β–†β–†β–‡β–ˆ"
+},
+{
+  "field": "x",
+  "n": 10,
+  "lo": 0.03144188,
+  "hi": 0.75867996,
+  "sparkline": "β–„β–ˆβ–ƒβ–„β–†β–†β–‡β–†β–β–†"
+},
+{
+  "field": "y",
+  "n": 10,
+  "lo": 0.13418874,
+  "hi": 0.97618139,
+  "sparkline": "β–†β–„β–ƒβ–β–‡β–„β–β–ˆβ–†β–ˆ"
+}
+]
diff --git a/test/cases/verb-sparkline/0002/cmd b/test/cases/verb-sparkline/0002/cmd
new file mode 100644
index 000000000..8f0ce6a63
--- /dev/null
+++ b/test/cases/verb-sparkline/0002/cmd
@@ -0,0 +1 @@
+mlr --opprint sparkline -f i test/input/abixy
diff --git a/test/cases/verb-sparkline/0002/experr b/test/cases/verb-sparkline/0002/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sparkline/0002/expout b/test/cases/verb-sparkline/0002/expout
new file mode 100644
index 000000000..5b20af9bc
--- /dev/null
+++ b/test/cases/verb-sparkline/0002/expout
@@ -0,0 +1,2 @@
+field n  lo         hi          sparkline
+i     10 1.00000000 10.00000000 β–β–‚β–ƒβ–ƒβ–„β–…β–†β–†β–‡β–ˆ
diff --git a/test/cases/verb-sparkline/0003/cmd b/test/cases/verb-sparkline/0003/cmd
new file mode 100644
index 000000000..cb9aaf4c2
--- /dev/null
+++ b/test/cases/verb-sparkline/0003/cmd
@@ -0,0 +1 @@
+mlr sparkline --help
diff --git a/test/cases/verb-sparkline/0003/experr b/test/cases/verb-sparkline/0003/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sparkline/0003/expout b/test/cases/verb-sparkline/0003/expout
new file mode 100644
index 000000000..07eea812c
--- /dev/null
+++ b/test/cases/verb-sparkline/0003/expout
@@ -0,0 +1,8 @@
+Usage: mlr sparkline [options]
+Reduces numeric field(s), across all records in input order, to a compact
+Unicode sparkline -- one block character per record -- for visualizing
+trends. Emits one output record per field. Holds all records in memory
+before producing any output.
+Options:
+-f {a,b,c} Field names to sparkline.
+-h|--help  Show this message.
diff --git a/test/cases/verb-sparkline/0004/cmd b/test/cases/verb-sparkline/0004/cmd
new file mode 100644
index 000000000..cd795af18
--- /dev/null
+++ b/test/cases/verb-sparkline/0004/cmd
@@ -0,0 +1 @@
+mlr --ojson sparkline -f nonesuch,a test/input/abixy
diff --git a/test/cases/verb-sparkline/0004/experr b/test/cases/verb-sparkline/0004/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sparkline/0004/expout b/test/cases/verb-sparkline/0004/expout
new file mode 100644
index 000000000..4e6d4f65b
--- /dev/null
+++ b/test/cases/verb-sparkline/0004/expout
@@ -0,0 +1,12 @@
+[
+{
+  "field": "nonesuch",
+  "n": 0,
+  "sparkline": ""
+},
+{
+  "field": "a",
+  "n": 10,
+  "sparkline": (error)
+}
+]
diff --git a/test/input/join-left-ragged.csv b/test/input/join-left-ragged.csv
new file mode 100644
index 000000000..11cb35f9c
--- /dev/null
+++ b/test/input/join-left-ragged.csv
@@ -0,0 +1,3 @@
+a,l
+pan,one
+eks,two,three,four
diff --git a/test/input/rank-data-grouped.csv b/test/input/rank-data-grouped.csv
new file mode 100644
index 000000000..c4d1092ca
--- /dev/null
+++ b/test/input/rank-data-grouped.csv
@@ -0,0 +1,6 @@
+g,x
+a,20
+b,5
+a,10
+b,5
+a,20
diff --git a/test/input/rank-data-unsorted.csv b/test/input/rank-data-unsorted.csv
new file mode 100644
index 000000000..9d68f16b0
--- /dev/null
+++ b/test/input/rank-data-unsorted.csv
@@ -0,0 +1,5 @@
+x
+20
+10
+20
+30
diff --git a/test/input/rank-data.csv b/test/input/rank-data.csv
new file mode 100644
index 000000000..4617ce85c
--- /dev/null
+++ b/test/input/rank-data.csv
@@ -0,0 +1,5 @@
+x
+10
+20
+20
+30
diff --git a/test/input/reorder-prefix.csv b/test/input/reorder-prefix.csv
new file mode 100644
index 000000000..513a622c0
--- /dev/null
+++ b/test/input/reorder-prefix.csv
@@ -0,0 +1,2 @@
+xxx-f1,xxx-f2,yyy-f1,yyy-f2,zzz-f1,bbb-f1,bbb-f2,ccc-f1
+1,2,3,4,5,6,7,8