diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index b3dd4bcec..ca5ad7259 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -36,11 +36,11 @@ jobs:
steps:
- name: Checkout repository
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+ uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
- uses: github/codeql-action/init@181d5eefc20863364f96762470ba6f862bdef56b
+ uses: github/codeql-action/init@cdefb33c0f6224e58673d9004f47f7cb3e328b89
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@@ -51,7 +51,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
- uses: github/codeql-action/autobuild@181d5eefc20863364f96762470ba6f862bdef56b
+ uses: github/codeql-action/autobuild@cdefb33c0f6224e58673d9004f47f7cb3e328b89
# โน๏ธ Command-line programs to run using the OS shell.
# ๐ https://git.io/JvXDl
@@ -65,4 +65,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
- uses: github/codeql-action/analyze@181d5eefc20863364f96762470ba6f862bdef56b
+ uses: github/codeql-action/analyze@cdefb33c0f6224e58673d9004f47f7cb3e328b89
diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml
index fff2e3c40..839eeb43f 100644
--- a/.github/workflows/codespell.yml
+++ b/.github/workflows/codespell.yml
@@ -21,7 +21,7 @@ jobs:
steps:
# Check out the code base
- name: Check out code
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+ uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
with:
# Full git history is needed to get a proper list of changed files within `super-linter`
fetch-depth: 0
@@ -29,7 +29,7 @@ jobs:
# Run linter against code base
# https://github.com/codespell-project/codespell
- name: Codespell
- uses: codespell-project/actions-codespell@406322ec52dd7b488e48c1c4b82e2a8b3a1bf630
+ uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579
with:
check_filenames: true
ignore_words_file: .codespellignore
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 402382e47..98d170d1d 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -15,12 +15,12 @@ jobs:
os: [ubuntu-latest, macos-latest, windows-latest]
steps:
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+ - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
- name: Set up Go
- uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5
+ uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5
with:
- go-version: 1.21
+ go-version: 1.24
- name: Build
run: make build
@@ -41,7 +41,7 @@ jobs:
if: matrix.os == 'windows-latest'
run: mkdir -p bin/${{matrix.os}} && cp mlr.exe bin/${{matrix.os}}
- - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+ - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
with:
name: mlr-${{matrix.os}}
path: bin/${{matrix.os}}/*
diff --git a/.github/workflows/release-snap.yaml b/.github/workflows/release-snap.yaml
new file mode 100644
index 000000000..d0dfada19
--- /dev/null
+++ b/.github/workflows/release-snap.yaml
@@ -0,0 +1,29 @@
+name: Release for Snap
+on:
+ push:
+ tags:
+ - v*
+ workflow_dispatch:
+
+jobs:
+ snap:
+ strategy:
+ matrix:
+ os: [ubuntu-latest, ubuntu-24.04-arm]
+ runs-on: ${{ matrix.os }}
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v6
+
+ - name: Build snap
+ uses: snapcore/action-build@v1
+ id: build
+
+ - name: Publish to Snap Store
+ uses: snapcore/action-publish@v1
+ env:
+ SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.SNAPCRAFT_TOKEN }}
+ with:
+ snap: ${{ steps.build.outputs.snap }}
+ # release: stable # or edge, beta, candidate
+ release: stable
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index a0f0a5178..fa2b59ec5 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,4 +1,4 @@
-name: Release
+name: Release for GitHub
on:
push:
tags:
@@ -6,7 +6,7 @@ on:
workflow_dispatch:
env:
- GO_VERSION: 1.21.1
+ GO_VERSION: 1.24.5
jobs:
release:
@@ -17,19 +17,19 @@ jobs:
runs-on: ${{ matrix.platform }}
steps:
- name: Set up Go
- uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5
+ uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5
with:
go-version: ${{ env.GO_VERSION }}
id: go
- name: Check out code into the Go module directory
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+ uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
with:
fetch-depth: 0
# https://github.com/marketplace/actions/cache
- name: Cache Go modules
- uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684
+ uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7
with:
path: |
~/.cache/go-build
@@ -40,7 +40,7 @@ jobs:
# https://goreleaser.com/ci/actions/
- name: Run GoReleaser
- uses: goreleaser/goreleaser-action@9c156ee8a17a598857849441385a2041ef570552
+ uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a
#if: startsWith(github.ref, 'refs/tags/v')
with:
version: latest
diff --git a/.github/workflows/test-snap-can-build.yml b/.github/workflows/test-snap-can-build.yml
new file mode 100644
index 000000000..c6c197de9
--- /dev/null
+++ b/.github/workflows/test-snap-can-build.yml
@@ -0,0 +1,28 @@
+name: ๐งช Snap Builds
+
+on:
+ push:
+ branches: '*'
+ pull_request:
+ branches: '*'
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ node-version: [20.x]
+
+ steps:
+ - uses: actions/checkout@v6
+
+ - uses: snapcore/action-build@v1
+ id: build
+
+ - uses: diddlesnaps/snapcraft-review-action@v1
+ with:
+ snap: ${{ steps.build.outputs.snap }}
+ isClassic: 'false'
+ # Plugs and Slots declarations to override default denial (requires store assertion to publish)
+ # plugs: ./plug-declaration.json
+ # slots: ./slot-declaration.json
diff --git a/README-dev.md b/README-dev.md
index 0e363db5c..6dd708f95 100644
--- a/README-dev.md
+++ b/README-dev.md
@@ -95,13 +95,14 @@ So, in broad overview, the key packages are:
* Miller dependencies are all in the Go standard library, except two:
* GOCC lexer/parser code-generator from [github.com/goccmack/gocc](https://github.com/goccmack/gocc):
+ * Forked at [github.com/johnkerl/gocc](github.com/johnkerl/gocc).
* This package defines the grammar for Miller's domain-specific language (DSL) for the Miller `put` and `filter` verbs. And, GOCC is a joy to use. :)
* It is used on the terms of its open-source license.
* [golang.org/x/term](https://pkg.go.dev/golang.org/x/term):
* Just a one-line Miller callsite for is-a-terminal checking for the [Miller REPL](./pkg/terminals/repl/README.md).
* It is used on the terms of its open-source license.
* See also [./go.mod](go.mod). Setup:
- * `go get github.com/goccmack/gocc`
+ * `go get github.com/johnkerl/gocc`
* `go get golang.org/x/term`
### Miller per se
diff --git a/README.md b/README.md
index 0819f272e..73d788982 100644
--- a/README.md
+++ b/README.md
@@ -29,6 +29,7 @@ key-value-pair data in a variety of data formats.
* [Miller in 10 minutes](https://miller.readthedocs.io/en/latest/10min)
* [A Guide To Command-Line Data Manipulation](https://www.smashingmagazine.com/2022/12/guide-command-line-data-manipulation-cli-miller)
* [A quick tutorial on Miller](https://www.ict4g.net/adolfo/notes/data-analysis/miller-quick-tutorial.html)
+* [Miller Exercises](https://github.com/GuilloteauQ/miller-exercises)
* [Tools to manipulate CSV files from the Command Line](https://www.ict4g.net/adolfo/notes/data-analysis/tools-to-manipulate-csv.html)
* [www.togaware.com/linux/survivor/CSV_Files.html](https://www.togaware.com/linux/survivor/CSV_Files.html)
* [MLR for CSV manipulation](https://guillim.github.io/terminal/2018/06/19/MLR-for-CSV-manipulation.html)
@@ -45,22 +46,18 @@ key-value-pair data in a variety of data formats.
* [Active issues](https://github.com/johnkerl/miller/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc)
# Installing
-
There's a good chance you can get Miller pre-built for your system:
-
[](https://launchpad.net/ubuntu/+source/miller)
[](https://launchpad.net/ubuntu/xenial/+package/miller)
[](https://packages.fedoraproject.org/pkgs/miller/miller/)
[](https://packages.debian.org/stable/miller)
[](https://packages.gentoo.org/packages/sys-apps/miller)
-
[](http://www.pro-linux.de/cgi-bin/DBApp/check.cgi?ShowApp..20427.100)
[](https://aur.archlinux.org/packages/miller-git)
-
[](http://pkgsrc.se/textproc/miller)
[](https://www.freshports.org/textproc/miller/)
-
[](https://anaconda.org/conda-forge/miller/)
+[](https://snapcraft.io/miller)
[](https://formulae.brew.sh/formula/miller)
[](https://www.macports.org/ports.php?by=name&substr=miller)
[](https://chocolatey.org/packages/miller)
@@ -68,9 +65,9 @@ There's a good chance you can get Miller pre-built for your system:
|OS|Installation command|
|---|---|
-|Linux|`yum install miller`
`apt-get install miller`|
+|Linux|`yum install miller`
`apt-get install miller`
`snap install miller`|
|Mac|`brew install miller`
`port install miller`|
-|Windows|`choco install miller`
`winget install Miller.Miller`|
+|Windows|`choco install miller`
`winget install Miller.Miller`
`scoop install main/miller`|
See also [README-versions.md](./README-versions.md) for a full list of package versions. Note that long-term-support (LtS) releases will likely be on older versions.
@@ -94,6 +91,7 @@ See also [building from source](https://miller.readthedocs.io/en/latest/build.ht
[](https://github.com/johnkerl/miller/actions/workflows/go.yml)
[](https://github.com/johnkerl/miller/actions/workflows/codeql-analysis.yml)
[](https://github.com/johnkerl/miller/actions/workflows/codespell.yml)
+[](https://github.com/johnkerl/miller/actions/workflows/test-snap-can-build.yml)
@@ -112,7 +110,7 @@ See also [building from source](https://miller.readthedocs.io/en/latest/build.ht
* Without `make`:
* To build: `go build github.com/johnkerl/miller/v6/cmd/mlr`.
* To run tests: `go test github.com/johnkerl/miller/v6/pkg/...` and `mlr regtest`.
- * To install: `go install github.com/johnkerl/miller/v6/cmd/mlr` will install to _GOPATH_`/bin/mlr`.
+ * To install: `go install github.com/johnkerl/miller/v6/cmd/mlr@latest` will install to _GOPATH_`/bin/mlr`.
* See also the doc page on [building from source](https://miller.readthedocs.io/en/latest/build).
* For more developer information please see [README-dev.md](./README-dev.md).
diff --git a/cmd/experiments/dsl_parser/one/build b/cmd/experiments/dsl_parser/one/build
index 373184a92..b43d4bc26 100755
--- a/cmd/experiments/dsl_parser/one/build
+++ b/cmd/experiments/dsl_parser/one/build
@@ -28,9 +28,9 @@ mkdir -p $dir
# ----------------------------------------------------------------
# Run the parser-generator
-# Build the bin/gocc executable:
-go get github.com/goccmack/gocc
-#go get github.com/johnkerl/gocc
+# Build the bin/gocc executable (use my fork for performance):
+# get github.com/goccmack/gocc
+go get github.com/johnkerl/gocc
bingocc="$GOPATH/bin/gocc"
if [ ! -x "$bingocc" ]; then
diff --git a/cmd/experiments/dsl_parser/one/go.mod b/cmd/experiments/dsl_parser/one/go.mod
index e4f49daf8..4e81172d6 100644
--- a/cmd/experiments/dsl_parser/one/go.mod
+++ b/cmd/experiments/dsl_parser/one/go.mod
@@ -1,5 +1,5 @@
module one
-go 1.16
+go 1.24
-require github.com/goccmack/gocc v0.0.0-20210322175033-34358ebe5808 // indirect
+toolchain go1.24.5
diff --git a/cmd/experiments/dsl_parser/one/go.sum b/cmd/experiments/dsl_parser/one/go.sum
index dfc52feaf..e69de29bb 100644
--- a/cmd/experiments/dsl_parser/one/go.sum
+++ b/cmd/experiments/dsl_parser/one/go.sum
@@ -1,26 +0,0 @@
-github.com/goccmack/gocc v0.0.0-20210322175033-34358ebe5808 h1:MBgZdx/wBJWTR2Q79mQfP6c8uXdQiu5JowfEz3KhFac=
-github.com/goccmack/gocc v0.0.0-20210322175033-34358ebe5808/go.mod h1:dWhnuKE5wcnGTExA2DH6Iicu21YnWwOPMrc/GyhtbCk=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/cmd/experiments/dsl_parser/two/build b/cmd/experiments/dsl_parser/two/build
index 2cb7893d3..1ea06c916 100755
--- a/cmd/experiments/dsl_parser/two/build
+++ b/cmd/experiments/dsl_parser/two/build
@@ -28,9 +28,9 @@ mkdir -p $dir
# ----------------------------------------------------------------
# Run the parser-generator
-# Build the bin/gocc executable:
-go get github.com/goccmack/gocc
-#go get github.com/johnkerl/gocc
+# Build the bin/gocc executable (use my fork for performance):
+# go get github.com/goccmack/gocc
+go get github.com/johnkerl/gocc
bingocc="$GOPATH/bin/gocc"
if [ ! -x "$bingocc" ]; then
exit 1
diff --git a/cmd/experiments/dsl_parser/two/go.mod b/cmd/experiments/dsl_parser/two/go.mod
index be38de9a3..81c05ea5e 100644
--- a/cmd/experiments/dsl_parser/two/go.mod
+++ b/cmd/experiments/dsl_parser/two/go.mod
@@ -1,5 +1,5 @@
module two
-go 1.16
+go 1.24
-require github.com/goccmack/gocc v0.0.0-20210322175033-34358ebe5808 // indirect
+toolchain go1.24.5
diff --git a/cmd/experiments/dsl_parser/two/go.sum b/cmd/experiments/dsl_parser/two/go.sum
index dfc52feaf..e69de29bb 100644
--- a/cmd/experiments/dsl_parser/two/go.sum
+++ b/cmd/experiments/dsl_parser/two/go.sum
@@ -1,26 +0,0 @@
-github.com/goccmack/gocc v0.0.0-20210322175033-34358ebe5808 h1:MBgZdx/wBJWTR2Q79mQfP6c8uXdQiu5JowfEz3KhFac=
-github.com/goccmack/gocc v0.0.0-20210322175033-34358ebe5808/go.mod h1:dWhnuKE5wcnGTExA2DH6Iicu21YnWwOPMrc/GyhtbCk=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/docs/src/build.md b/docs/src/build.md
index 0bceafb0d..b6678282f 100644
--- a/docs/src/build.md
+++ b/docs/src/build.md
@@ -18,7 +18,7 @@ Quick links:
Please also see [Installation](installing-miller.md) for information about pre-built executables.
-You will need to first install Go version 1.15 or higher: please see [https://go.dev](https://go.dev).
+You will need to first install Go ([this version](https://github.com/johnkerl/miller/blob/main/go.mod#L17)): please see [https://go.dev](https://go.dev).
## Miller license
diff --git a/docs/src/build.md.in b/docs/src/build.md.in
index ef3e4aa7d..3d35ee560 100644
--- a/docs/src/build.md.in
+++ b/docs/src/build.md.in
@@ -2,7 +2,7 @@
Please also see [Installation](installing-miller.md) for information about pre-built executables.
-You will need to first install Go version 1.15 or higher: please see [https://go.dev](https://go.dev).
+You will need to first install Go ([this version](https://github.com/johnkerl/miller/blob/main/go.mod#L17)): please see [https://go.dev](https://go.dev).
## Miller license
diff --git a/docs/src/data-diving-examples.md b/docs/src/data-diving-examples.md
index 100716ec2..297eca211 100644
--- a/docs/src/data-diving-examples.md
+++ b/docs/src/data-diving-examples.md
@@ -26,7 +26,7 @@ Vertical-tabular format is good for a quick look at CSV data layout -- seeing wh
wc -l data/flins.csv
- 36635 data/flins.csv
+36635 data/flins.csv
@@ -227,7 +227,7 @@ Peek at the data:
wc -l data/colored-shapes.dkvp
- 10078 data/colored-shapes.dkvp
+10078 data/colored-shapes.dkvp
diff --git a/docs/src/date-time-examples.md b/docs/src/date-time-examples.md
index 5bcbdac01..cab74de3c 100644
--- a/docs/src/date-time-examples.md
+++ b/docs/src/date-time-examples.md
@@ -68,7 +68,7 @@ date,qoh
wc -l data/miss-date.csv
- 1372 data/miss-date.csv
+1372 data/miss-date.csv
Since there are 1372 lines in the data file, some automation is called for. To find the missing dates, you can convert the dates to seconds since the epoch using `strptime`, then compute adjacent differences (the `cat -n` simply inserts record-counters):
diff --git a/docs/src/file-formats.md b/docs/src/file-formats.md
index 5eaff8b13..8a09dac54 100644
--- a/docs/src/file-formats.md
+++ b/docs/src/file-formats.md
@@ -757,12 +757,14 @@ Notes:
within the input.
--pass-comments-with {string}
Immediately print commented lines within input, with
- specified prefix.
+ specified prefix. For CSV input format, the prefix
+ must be a single character.
--skip-comments Ignore commented lines (prefixed by `#`) within the
input.
--skip-comments-with {string}
Ignore commented lines within input, with specified
- prefix.
+ prefix. For CSV input format, the prefix must be a
+ single character.
Examples:
diff --git a/docs/src/how-to-release.md b/docs/src/how-to-release.md
index 58a445f8d..b19529094 100644
--- a/docs/src/how-to-release.md
+++ b/docs/src/how-to-release.md
@@ -48,7 +48,7 @@ In this example I am using version 6.2.0 to 6.3.0; of course that will change fo
* Thanks to [PR 822](https://github.com/johnkerl/miller/pull/822) which introduces [goreleaser](https://github.com/johnkerl/miller/blob/main/.goreleaser.yml) there are versions for many platforms auto-built and auto-attached to the GitHub release.
* Attach the release tarball and SRPM. Double-check assets were successfully uploaded.
* Publish the release in pre-release mode, until all CI jobs finish successfully. Note that gorelease will create and attach the rest of the binaries.
- * Before marking the release as public, download an executable from among the generated binaries and make sure its `mlr version` prints what you expect -- else, restart this process.
+ * Before marking the release as public, download an executable from among the generated binaries and make sure its `mlr version` prints what you expect -- else, restart this process. MacOS: `xattr -d com.apple.quarantine ./mlr` first.
* Then mark the release as public.
* Build the release-specific docs:
diff --git a/docs/src/how-to-release.md.in b/docs/src/how-to-release.md.in
index e96010f36..522cdbfa9 100644
--- a/docs/src/how-to-release.md.in
+++ b/docs/src/how-to-release.md.in
@@ -32,7 +32,7 @@ In this example I am using version 6.2.0 to 6.3.0; of course that will change fo
* Thanks to [PR 822](https://github.com/johnkerl/miller/pull/822) which introduces [goreleaser](https://github.com/johnkerl/miller/blob/main/.goreleaser.yml) there are versions for many platforms auto-built and auto-attached to the GitHub release.
* Attach the release tarball and SRPM. Double-check assets were successfully uploaded.
* Publish the release in pre-release mode, until all CI jobs finish successfully. Note that gorelease will create and attach the rest of the binaries.
- * Before marking the release as public, download an executable from among the generated binaries and make sure its `mlr version` prints what you expect -- else, restart this process.
+ * Before marking the release as public, download an executable from among the generated binaries and make sure its `mlr version` prints what you expect -- else, restart this process. MacOS: `xattr -d com.apple.quarantine ./mlr` first.
* Then mark the release as public.
* Build the release-specific docs:
diff --git a/docs/src/installing-miller.md b/docs/src/installing-miller.md
index d50b70d31..9de4558ff 100644
--- a/docs/src/installing-miller.md
+++ b/docs/src/installing-miller.md
@@ -30,6 +30,7 @@ Using a package manager:
* MacOS: `brew update` and `brew install miller`, or `sudo port selfupdate` and `sudo port install miller`, depending on your preference of [Homebrew](https://brew.sh) or [MacPorts](https://macports.org).
* Windows: `choco install miller` using [Chocolatey](https://chocolatey.org).
* Note: Miller 6 was released 2022-01-09; [several platforms](https://github.com/johnkerl/miller/blob/main/README-versions.md) may have Miller 5 available.
+* As of Miller 6.16.0, you can do `snap install miller`. Note however that the executable is named `miller`, _not_ `mlr`. See also [https://snapcraft.io/miller](https://snapcraft.io/miller).
See also:
diff --git a/docs/src/installing-miller.md.in b/docs/src/installing-miller.md.in
index b735be725..74e5c9f53 100644
--- a/docs/src/installing-miller.md.in
+++ b/docs/src/installing-miller.md.in
@@ -14,6 +14,7 @@ Using a package manager:
* MacOS: `brew update` and `brew install miller`, or `sudo port selfupdate` and `sudo port install miller`, depending on your preference of [Homebrew](https://brew.sh) or [MacPorts](https://macports.org).
* Windows: `choco install miller` using [Chocolatey](https://chocolatey.org).
* Note: Miller 6 was released 2022-01-09; [several platforms](https://github.com/johnkerl/miller/blob/main/README-versions.md) may have Miller 5 available.
+* As of Miller 6.16.0, you can do `snap install miller`. Note however that the executable is named `miller`, _not_ `mlr`. See also [https://snapcraft.io/miller](https://snapcraft.io/miller).
See also:
diff --git a/docs/src/manpage.md b/docs/src/manpage.md
index 3f4b26f43..39203a0c9 100644
--- a/docs/src/manpage.md
+++ b/docs/src/manpage.md
@@ -48,7 +48,7 @@ This is simply a copy of what you should see on running `man mlr` at a command p
insertion-ordered hash map. This encompasses a variety of data
formats, including but not limited to the familiar CSV, TSV, and JSON.
(Miller can handle positionally-indexed data as a special case.) This
- manpage documents mlr 6.14.0.
+ manpage documents mlr 6.16.0.
1mEXAMPLES0m
mlr --icsv --opprint cat example.csv
@@ -145,6 +145,7 @@ This is simply a copy of what you should see on running `man mlr` at a command p
mlr help comments-in-data-flags
mlr help compressed-data-flags
mlr help csv/tsv-only-flags
+ mlr help dkvp-only-flags
mlr help file-format-flags
mlr help flatten-unflatten-flags
mlr help format-conversion-keystroke-saver-flags
@@ -254,12 +255,14 @@ This is simply a copy of what you should see on running `man mlr` at a command p
within the input.
--pass-comments-with {string}
Immediately print commented lines within input, with
- specified prefix.
+ specified prefix. For CSV input format, the prefix
+ must be a single character.
--skip-comments Ignore commented lines (prefixed by `#`) within the
input.
--skip-comments-with {string}
Ignore commented lines within input, with specified
- prefix.
+ prefix. For CSV input format, the prefix must be a
+ single character.
1mCOMPRESSED-DATA FLAGS0m
Miller offers a few different ways to handle reading data files
@@ -356,6 +359,16 @@ This is simply a copy of what you should see on running `man mlr` at a command p
-N Keystroke-saver for `--implicit-csv-header
--headerless-csv-output`.
+1mDKVP-ONLY FLAGS0m
+ These are flags which are applicable to DKVP format.
+
+ --incr-key Without this option, keyless DKVP fields are keyed by
+ field number. For example: `a=10,b=20,30,d=40,50` is
+ ingested as `$a=10,$b=20,$3=30,$d=40,$5=50`. With
+ this option, they're keyed by a running counter of
+ keyless fields. For example: `a=10,b=20,30,d=40,50`
+ is ingested as `$a=10,$b=20,$1=30,$d=40,$2=50`.
+
1mFILE-FORMAT FLAGS0m
See the File formats doc page, and or `mlr help file-formats`, for more
about file formats Miller supports.
@@ -1275,7 +1288,7 @@ This is simply a copy of what you should see on running `man mlr` at a command p
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
- -a Convert all field names.
+ -a Convert all fields.
-h|--help Show this message.
1mhaving-fields0m
@@ -1837,6 +1850,7 @@ This is simply a copy of what you should see on running `man mlr` at a command p
-nf {comma-separated field names} Same as -n
-nr {comma-separated field names} Numerical descending; nulls sort first
-t {comma-separated field names} Natural ascending
+ -b Move sort fields to start of record, as in reorder -b
-tr|-rt {comma-separated field names} Natural descending
-h|--help Show this message.
@@ -1912,7 +1926,7 @@ This is simply a copy of what you should see on running `man mlr` at a command p
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
- -a Convert all field names.
+ -a Convert all fields.
-h|--help Show this message.
1mstats10m
@@ -2061,7 +2075,7 @@ This is simply a copy of what you should see on running `man mlr` at a command p
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
- -a Convert all field names.
+ -a Convert all fields.
-h|--help Show this message.
1msummary0m
@@ -3745,5 +3759,5 @@ This is simply a copy of what you should see on running `man mlr` at a command p
MIME Type for Comma-Separated Values (CSV) Files, the Miller docsite
https://miller.readthedocs.io
- 2025-07-04 4mMILLER24m(1)
+ 2026-01-02 4mMILLER24m(1)
diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt
index 198610def..90bff3293 100644
--- a/docs/src/manpage.txt
+++ b/docs/src/manpage.txt
@@ -27,7 +27,7 @@
insertion-ordered hash map. This encompasses a variety of data
formats, including but not limited to the familiar CSV, TSV, and JSON.
(Miller can handle positionally-indexed data as a special case.) This
- manpage documents mlr 6.14.0.
+ manpage documents mlr 6.16.0.
1mEXAMPLES0m
mlr --icsv --opprint cat example.csv
@@ -124,6 +124,7 @@
mlr help comments-in-data-flags
mlr help compressed-data-flags
mlr help csv/tsv-only-flags
+ mlr help dkvp-only-flags
mlr help file-format-flags
mlr help flatten-unflatten-flags
mlr help format-conversion-keystroke-saver-flags
@@ -233,12 +234,14 @@
within the input.
--pass-comments-with {string}
Immediately print commented lines within input, with
- specified prefix.
+ specified prefix. For CSV input format, the prefix
+ must be a single character.
--skip-comments Ignore commented lines (prefixed by `#`) within the
input.
--skip-comments-with {string}
Ignore commented lines within input, with specified
- prefix.
+ prefix. For CSV input format, the prefix must be a
+ single character.
1mCOMPRESSED-DATA FLAGS0m
Miller offers a few different ways to handle reading data files
@@ -335,6 +338,16 @@
-N Keystroke-saver for `--implicit-csv-header
--headerless-csv-output`.
+1mDKVP-ONLY FLAGS0m
+ These are flags which are applicable to DKVP format.
+
+ --incr-key Without this option, keyless DKVP fields are keyed by
+ field number. For example: `a=10,b=20,30,d=40,50` is
+ ingested as `$a=10,$b=20,$3=30,$d=40,$5=50`. With
+ this option, they're keyed by a running counter of
+ keyless fields. For example: `a=10,b=20,30,d=40,50`
+ is ingested as `$a=10,$b=20,$1=30,$d=40,$2=50`.
+
1mFILE-FORMAT FLAGS0m
See the File formats doc page, and or `mlr help file-formats`, for more
about file formats Miller supports.
@@ -1254,7 +1267,7 @@
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
- -a Convert all field names.
+ -a Convert all fields.
-h|--help Show this message.
1mhaving-fields0m
@@ -1816,6 +1829,7 @@
-nf {comma-separated field names} Same as -n
-nr {comma-separated field names} Numerical descending; nulls sort first
-t {comma-separated field names} Natural ascending
+ -b Move sort fields to start of record, as in reorder -b
-tr|-rt {comma-separated field names} Natural descending
-h|--help Show this message.
@@ -1891,7 +1905,7 @@
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
- -a Convert all field names.
+ -a Convert all fields.
-h|--help Show this message.
1mstats10m
@@ -2040,7 +2054,7 @@
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
- -a Convert all field names.
+ -a Convert all fields.
-h|--help Show this message.
1msummary0m
@@ -3724,4 +3738,4 @@
MIME Type for Comma-Separated Values (CSV) Files, the Miller docsite
https://miller.readthedocs.io
- 2025-07-04 4mMILLER24m(1)
+ 2026-01-02 4mMILLER24m(1)
diff --git a/docs/src/online-help.md b/docs/src/online-help.md
index 5bbee15a1..bb8185e10 100644
--- a/docs/src/online-help.md
+++ b/docs/src/online-help.md
@@ -55,6 +55,7 @@ Flags:
mlr help comments-in-data-flags
mlr help compressed-data-flags
mlr help csv/tsv-only-flags
+ mlr help dkvp-only-flags
mlr help file-format-flags
mlr help flatten-unflatten-flags
mlr help format-conversion-keystroke-saver-flags
@@ -230,6 +231,7 @@ Options:
-nf {comma-separated field names} Same as -n
-nr {comma-separated field names} Numerical descending; nulls sort first
-t {comma-separated field names} Natural ascending
+-b Move sort fields to start of record, as in reorder -b
-tr|-rt {comma-separated field names} Natural descending
-h|--help Show this message.
diff --git a/docs/src/reference-main-flag-list.md b/docs/src/reference-main-flag-list.md
index 28adc12f6..e0f36f3af 100644
--- a/docs/src/reference-main-flag-list.md
+++ b/docs/src/reference-main-flag-list.md
@@ -63,9 +63,9 @@ Notes:
**Flags:**
* `--pass-comments`: Immediately print commented lines (prefixed by `#`) within the input.
-* `--pass-comments-with {string}`: Immediately print commented lines within input, with specified prefix.
+* `--pass-comments-with {string}`: Immediately print commented lines within input, with specified prefix. For CSV input format, the prefix must be a single character.
* `--skip-comments`: Ignore commented lines (prefixed by `#`) within the input.
-* `--skip-comments-with {string}`: Ignore commented lines within input, with specified prefix.
+* `--skip-comments-with {string}`: Ignore commented lines within input, with specified prefix. For CSV input format, the prefix must be a single character.
## Compressed-data flags
@@ -128,6 +128,15 @@ These are flags which are applicable to CSV format.
* `--quote-all`: Force double-quoting of CSV fields.
* `-N`: Keystroke-saver for `--implicit-csv-header --headerless-csv-output`.
+## DKVP-only flags
+
+These are flags which are applicable to DKVP format.
+
+
+**Flags:**
+
+* `--incr-key`: Without this option, keyless DKVP fields are keyed by field number. For example: `a=10,b=20,30,d=40,50` is ingested as `$a=10,$b=20,$3=30,$d=40,$5=50`. With this option, they're keyed by a running counter of keyless fields. For example: `a=10,b=20,30,d=40,50` is ingested as `$a=10,$b=20,$1=30,$d=40,$2=50`.
+
## File-format flags
See the File formats doc page, and or `mlr help file-formats`, for more
diff --git a/docs/src/reference-main-null-data.md b/docs/src/reference-main-null-data.md
index 63bfffaa9..175ae2ad2 100644
--- a/docs/src/reference-main-null-data.md
+++ b/docs/src/reference-main-null-data.md
@@ -125,7 +125,7 @@ with the exception that the `min` and `max` functions are special: if one argume
x=,y=3,a=3,b=
-Likewise, empty works like 0 for addition and subtraction, and multiplication:
+Likewise, empty works like 0 for addition and subtraction, and like 1 for multiplication:
echo 'x=,y=3' | mlr put '$a = $x + $y; $b = $x - $y; $c = $x * $y'
diff --git a/docs/src/reference-main-null-data.md.in b/docs/src/reference-main-null-data.md.in
index 087edaa78..3ac1051ac 100644
--- a/docs/src/reference-main-null-data.md.in
+++ b/docs/src/reference-main-null-data.md.in
@@ -54,7 +54,7 @@ GENMD-RUN-COMMAND
echo 'x=,y=3' | mlr put '$a=min($x,$y);$b=max($x,$y)'
GENMD-EOF
-Likewise, empty works like 0 for addition and subtraction, and multiplication:
+Likewise, empty works like 0 for addition and subtraction, and like 1 for multiplication:
GENMD-RUN-COMMAND
echo 'x=,y=3' | mlr put '$a = $x + $y; $b = $x - $y; $c = $x * $y'
diff --git a/docs/src/reference-verbs.md b/docs/src/reference-verbs.md
index ab7a599fa..b50c97d7d 100644
--- a/docs/src/reference-verbs.md
+++ b/docs/src/reference-verbs.md
@@ -1465,7 +1465,7 @@ See also the `sub` and `ssub` verbs.
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
--a Convert all field names.
+-a Convert all fields.
-h|--help Show this message.
@@ -2960,6 +2960,7 @@ Options:
-nf {comma-separated field names} Same as -n
-nr {comma-separated field names} Numerical descending; nulls sort first
-t {comma-separated field names} Natural ascending
+-b Move sort fields to start of record, as in reorder -b
-tr|-rt {comma-separated field names} Natural descending
-h|--help Show this message.
@@ -3213,7 +3214,7 @@ the old string, like the `ssub` DSL function. See also the `gsub` and `sub` verb
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
--a Convert all field names.
+-a Convert all fields.
-h|--help Show this message.
@@ -3719,7 +3720,7 @@ See also the `gsub` and `ssub` verbs.
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
--a Convert all field names.
+-a Convert all fields.
-h|--help Show this message.
@@ -4133,7 +4134,7 @@ There are two main ways to use `mlr uniq`: the first way is with `-g` to specify
wc -l data/colored-shapes.csv
- 10079 data/colored-shapes.csv
+10079 data/colored-shapes.csv
@@ -4290,7 +4291,7 @@ color=purple,shape=square,flag=0
wc -l data/repeats.dkvp
- 57 data/repeats.dkvp
+57 data/repeats.dkvp
diff --git a/docs/src/release-docs.md b/docs/src/release-docs.md
index 0e9fcc1c8..22924b141 100644
--- a/docs/src/release-docs.md
+++ b/docs/src/release-docs.md
@@ -24,6 +24,7 @@ If your `mlr version` says something like `Miller v5.10.2` or `mlr 6.0.0`, witho
| Release | Docs | Release notes |
|---------|---------------------------------------------------------------------|---------------|
main | [main branch](https://miller.readthedocs.io/en/main) | N/A |
+6.14.0 | [Miller 6.14.0](https://miller.readthedocs.io/en/6.14.0) | [Survival curve, misc. features and bugfixes](https://github.com/johnkerl/miller/releases/tag/v6.14.0) |
6.13.0 | [Miller 6.13.0](https://miller.readthedocs.io/en/6.13.0) | [File-stat DSL function, new stats accumulator, misc. bugfixes](https://github.com/johnkerl/miller/releases/tag/v6.13.0) |
6.12.0 | [Miller 6.12.0](https://miller.readthedocs.io/en/6.12.0) | [New sparsify verb, wide-table performance improvement, thousands separator for fmtnum function](https://github.com/johnkerl/miller/releases/tag/v6.12.0) |
6.11.0 | [Miller 6.11.0](https://miller.readthedocs.io/en/6.11.0) | [CSV/TSV auto-unsparsify, regex-fieldname support for reorder/sub/ssub/gsub, strmatch DSL function, and more](https://github.com/johnkerl/miller/releases/tag/v6.11.0) |
diff --git a/docs/src/release-docs.md.in b/docs/src/release-docs.md.in
index 7a840c6f0..4b89cf87d 100644
--- a/docs/src/release-docs.md.in
+++ b/docs/src/release-docs.md.in
@@ -8,6 +8,7 @@ If your `mlr version` says something like `Miller v5.10.2` or `mlr 6.0.0`, witho
| Release | Docs | Release notes |
|---------|---------------------------------------------------------------------|---------------|
main | [main branch](https://miller.readthedocs.io/en/main) | N/A |
+6.14.0 | [Miller 6.14.0](https://miller.readthedocs.io/en/6.14.0) | [Survival curve, misc. features and bugfixes](https://github.com/johnkerl/miller/releases/tag/v6.14.0) |
6.13.0 | [Miller 6.13.0](https://miller.readthedocs.io/en/6.13.0) | [File-stat DSL function, new stats accumulator, misc. bugfixes](https://github.com/johnkerl/miller/releases/tag/v6.13.0) |
6.12.0 | [Miller 6.12.0](https://miller.readthedocs.io/en/6.12.0) | [New sparsify verb, wide-table performance improvement, thousands separator for fmtnum function](https://github.com/johnkerl/miller/releases/tag/v6.12.0) |
6.11.0 | [Miller 6.11.0](https://miller.readthedocs.io/en/6.11.0) | [CSV/TSV auto-unsparsify, regex-fieldname support for reorder/sub/ssub/gsub, strmatch DSL function, and more](https://github.com/johnkerl/miller/releases/tag/v6.11.0) |
diff --git a/go.mod b/go.mod
index 526d27725..10b971673 100644
--- a/go.mod
+++ b/go.mod
@@ -14,24 +14,22 @@ module github.com/johnkerl/miller/v6
// Local development:
// replace github.com/johnkerl/lumin => /Users/kerl/git/johnkerl/lumin
-go 1.23.0
-
-toolchain go1.24.2
+go 1.24.0
require (
github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
github.com/johnkerl/lumin v1.0.0
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
- github.com/klauspost/compress v1.17.11
+ github.com/klauspost/compress v1.18.3
github.com/kshedden/statmodel v0.0.0-20210519035403-ee97d3e48df1
- github.com/lestrrat-go/strftime v1.1.0
+ github.com/lestrrat-go/strftime v1.1.1
github.com/mattn/go-isatty v0.0.20
github.com/nine-lives-later/go-windows-terminal-sequences v1.0.4
github.com/pkg/profile v1.7.0
- github.com/stretchr/testify v1.10.0
- golang.org/x/sys v0.33.0
- golang.org/x/term v0.32.0
- golang.org/x/text v0.26.0
+ github.com/stretchr/testify v1.11.1
+ golang.org/x/sys v0.40.0
+ golang.org/x/term v0.39.0
+ golang.org/x/text v0.33.0
)
require (
@@ -41,7 +39,7 @@ require (
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
github.com/kshedden/dstream v0.0.0-20190512025041-c4c410631beb // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
- golang.org/x/tools v0.33.0 // indirect
+ golang.org/x/tools v0.40.0 // indirect
gonum.org/v1/gonum v0.16.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
diff --git a/go.sum b/go.sum
index a902b4c81..0a7bba556 100644
--- a/go.sum
+++ b/go.sum
@@ -17,16 +17,16 @@ github.com/johnkerl/lumin v1.0.0 h1:CV34cHZOJ92Y02RbQ0rd4gA0C06Qck9q8blOyaPoWpU=
github.com/johnkerl/lumin v1.0.0/go.mod h1:eLf5AdQOaLvzZ2zVy4REr/DSeEwG+CZreHwNLICqv9E=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
-github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
+github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/kshedden/dstream v0.0.0-20190512025041-c4c410631beb h1:Z5BVHFk/DLOIUAd2NycF0mLtKfhl7ynm4Uy5+AFhT48=
github.com/kshedden/dstream v0.0.0-20190512025041-c4c410631beb/go.mod h1:+U+6yzfITr4/teU2YhxWhdyw6YzednT/16/UBMjlDrU=
github.com/kshedden/statmodel v0.0.0-20210519035403-ee97d3e48df1 h1:UyIQ1VTQq/0CS/wLYjf3DV6uRKTd1xcsng3BccM4XCY=
github.com/kshedden/statmodel v0.0.0-20210519035403-ee97d3e48df1/go.mod h1:uvVFnikBpVz7S1pdsyUI+BBRlz64vmU6Q+kviiB+fpU=
github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8=
github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is=
-github.com/lestrrat-go/strftime v1.1.0 h1:gMESpZy44/4pXLO/m+sL0yBd1W6LjgjrrD4a68Gapyg=
-github.com/lestrrat-go/strftime v1.1.0/go.mod h1:uzeIB52CeUJenCo1syghlugshMysrqUT51HlxphXVeI=
+github.com/lestrrat-go/strftime v1.1.1 h1:zgf8QCsgj27GlKBy3SU9/8MMgegZ8UCzlCyHYrUF0QU=
+github.com/lestrrat-go/strftime v1.1.1/go.mod h1:YDrzHJAODYQ+xxvrn5SG01uFIQAeDTzpxNVppCz7Nmw=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/nine-lives-later/go-windows-terminal-sequences v1.0.4 h1:NC4H8hewgaktBqMI5yzy6L/Vln5/H7BEziyxaE2fX3Y=
@@ -39,18 +39,18 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
-golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
-golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
-golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
-golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
-golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
+golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
+golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
+golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
+golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
+golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
+golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
+golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
diff --git a/man/manpage.txt b/man/manpage.txt
index 198610def..90bff3293 100644
--- a/man/manpage.txt
+++ b/man/manpage.txt
@@ -27,7 +27,7 @@
insertion-ordered hash map. This encompasses a variety of data
formats, including but not limited to the familiar CSV, TSV, and JSON.
(Miller can handle positionally-indexed data as a special case.) This
- manpage documents mlr 6.14.0.
+ manpage documents mlr 6.16.0.
1mEXAMPLES0m
mlr --icsv --opprint cat example.csv
@@ -124,6 +124,7 @@
mlr help comments-in-data-flags
mlr help compressed-data-flags
mlr help csv/tsv-only-flags
+ mlr help dkvp-only-flags
mlr help file-format-flags
mlr help flatten-unflatten-flags
mlr help format-conversion-keystroke-saver-flags
@@ -233,12 +234,14 @@
within the input.
--pass-comments-with {string}
Immediately print commented lines within input, with
- specified prefix.
+ specified prefix. For CSV input format, the prefix
+ must be a single character.
--skip-comments Ignore commented lines (prefixed by `#`) within the
input.
--skip-comments-with {string}
Ignore commented lines within input, with specified
- prefix.
+ prefix. For CSV input format, the prefix must be a
+ single character.
1mCOMPRESSED-DATA FLAGS0m
Miller offers a few different ways to handle reading data files
@@ -335,6 +338,16 @@
-N Keystroke-saver for `--implicit-csv-header
--headerless-csv-output`.
+1mDKVP-ONLY FLAGS0m
+ These are flags which are applicable to DKVP format.
+
+ --incr-key Without this option, keyless DKVP fields are keyed by
+ field number. For example: `a=10,b=20,30,d=40,50` is
+ ingested as `$a=10,$b=20,$3=30,$d=40,$5=50`. With
+ this option, they're keyed by a running counter of
+ keyless fields. For example: `a=10,b=20,30,d=40,50`
+ is ingested as `$a=10,$b=20,$1=30,$d=40,$2=50`.
+
1mFILE-FORMAT FLAGS0m
See the File formats doc page, and or `mlr help file-formats`, for more
about file formats Miller supports.
@@ -1254,7 +1267,7 @@
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
- -a Convert all field names.
+ -a Convert all fields.
-h|--help Show this message.
1mhaving-fields0m
@@ -1816,6 +1829,7 @@
-nf {comma-separated field names} Same as -n
-nr {comma-separated field names} Numerical descending; nulls sort first
-t {comma-separated field names} Natural ascending
+ -b Move sort fields to start of record, as in reorder -b
-tr|-rt {comma-separated field names} Natural descending
-h|--help Show this message.
@@ -1891,7 +1905,7 @@
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
- -a Convert all field names.
+ -a Convert all fields.
-h|--help Show this message.
1mstats10m
@@ -2040,7 +2054,7 @@
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
- -a Convert all field names.
+ -a Convert all fields.
-h|--help Show this message.
1msummary0m
@@ -3724,4 +3738,4 @@
MIME Type for Comma-Separated Values (CSV) Files, the Miller docsite
https://miller.readthedocs.io
- 2025-07-04 4mMILLER24m(1)
+ 2026-01-02 4mMILLER24m(1)
diff --git a/man/mlr.1 b/man/mlr.1
index a361b0c9c..f36d5e2f0 100644
--- a/man/mlr.1
+++ b/man/mlr.1
@@ -2,12 +2,12 @@
.\" Title: mlr
.\" Author: [see the "AUTHOR" section]
.\" Generator: ./mkman.rb
-.\" Date: 2025-07-04
+.\" Date: 2026-01-02
.\" Manual: \ \&
.\" Source: \ \&
.\" Language: English
.\"
-.TH "MILLER" "1" "2025-07-04" "\ \&" "\ \&"
+.TH "MILLER" "1" "2026-01-02" "\ \&" "\ \&"
.\" -----------------------------------------------------------------
.\" * Portability definitions
.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -47,7 +47,7 @@ on integer-indexed fields: if the natural data structure for the latter is the
array, then Miller's natural data structure is the insertion-ordered hash map.
This encompasses a variety of data formats, including but not limited to the
familiar CSV, TSV, and JSON. (Miller can handle positionally-indexed data as
-a special case.) This manpage documents mlr 6.14.0.
+a special case.) This manpage documents mlr 6.16.0.
.SH "EXAMPLES"
.sp
@@ -161,6 +161,7 @@ Flags:
mlr help comments-in-data-flags
mlr help compressed-data-flags
mlr help csv/tsv-only-flags
+ mlr help dkvp-only-flags
mlr help file-format-flags
mlr help flatten-unflatten-flags
mlr help format-conversion-keystroke-saver-flags
@@ -290,12 +291,14 @@ Notes:
within the input.
--pass-comments-with {string}
Immediately print commented lines within input, with
- specified prefix.
+ specified prefix. For CSV input format, the prefix
+ must be a single character.
--skip-comments Ignore commented lines (prefixed by `#`) within the
input.
--skip-comments-with {string}
Ignore commented lines within input, with specified
- prefix.
+ prefix. For CSV input format, the prefix must be a
+ single character.
.fi
.if n \{\
.RE
@@ -410,6 +413,24 @@ These are flags which are applicable to CSV format.
.fi
.if n \{\
.RE
+.SH "DKVP-ONLY FLAGS"
+.sp
+
+.if n \{\
+.RS 0
+.\}
+.nf
+These are flags which are applicable to DKVP format.
+
+--incr-key Without this option, keyless DKVP fields are keyed by
+ field number. For example: `a=10,b=20,30,d=40,50` is
+ ingested as `$a=10,$b=20,$3=30,$d=40,$5=50`. With
+ this option, they're keyed by a running counter of
+ keyless fields. For example: `a=10,b=20,30,d=40,50`
+ is ingested as `$a=10,$b=20,$1=30,$d=40,$2=50`.
+.fi
+.if n \{\
+.RE
.SH "FILE-FORMAT FLAGS"
.sp
@@ -1565,7 +1586,7 @@ See also the `sub` and `ssub` verbs.
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
--a Convert all field names.
+-a Convert all fields.
-h|--help Show this message.
.fi
.if n \{\
@@ -2289,6 +2310,7 @@ Options:
-nf {comma-separated field names} Same as -n
-nr {comma-separated field names} Numerical descending; nulls sort first
-t {comma-separated field names} Natural ascending
+-b Move sort fields to start of record, as in reorder -b
-tr|-rt {comma-separated field names} Natural descending
-h|--help Show this message.
@@ -2388,7 +2410,7 @@ the old string, like the `ssub` DSL function. See also the `gsub` and `sub` verb
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
--a Convert all field names.
+-a Convert all fields.
-h|--help Show this message.
.fi
.if n \{\
@@ -2561,7 +2583,7 @@ See also the `gsub` and `ssub` verbs.
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
--a Convert all field names.
+-a Convert all fields.
-h|--help Show this message.
.fi
.if n \{\
diff --git a/miller.spec b/miller.spec
index 3a43b6426..166cb35e0 100644
--- a/miller.spec
+++ b/miller.spec
@@ -1,6 +1,6 @@
Summary: Name-indexed data processing tool
Name: miller
-Version: 6.14.0
+Version: 6.16.0
Release: 1%{?dist}
License: BSD
Source: https://github.com/johnkerl/miller/releases/download/%{version}/miller-%{version}.tar.gz
@@ -36,6 +36,12 @@ make install
%{_mandir}/man1/mlr.1*
%changelog
+* Fri Jan 2 2026 John Kerl - 6.16.0-1
+- 6.16.0 release
+
+* Thu Aug 14 2025 John Kerl - 6.15.0-1
+- 6.15.0 release
+
* Fri Jul 4 2025 John Kerl - 6.14.0-1
- 6.14.0 release
diff --git a/pkg/cli/option_parse.go b/pkg/cli/option_parse.go
index 156a056a3..0070b60c8 100644
--- a/pkg/cli/option_parse.go
+++ b/pkg/cli/option_parse.go
@@ -104,6 +104,7 @@ var FLAG_TABLE = FlagTable{
&CSVTSVOnlyFlagSection,
&JSONOnlyFlagSection,
&PPRINTOnlyFlagSection,
+ &DKVPOnlyFlagSection,
&CompressedDataFlagSection,
&CommentsInDataFlagSection,
&OutputColorizationFlagSection,
@@ -523,6 +524,31 @@ var PPRINTOnlyFlagSection = FlagSection{
},
}
+// ================================================================
+// DKVP-ONLY FLAGS
+
+func DKVPOnlyPrintInfo() {
+ fmt.Println("These are flags which are applicable to DKVP format.")
+}
+
+func init() { DKVPOnlyFlagSection.Sort() }
+
+var DKVPOnlyFlagSection = FlagSection{
+ name: "DKVP-only flags",
+ infoPrinter: DKVPOnlyPrintInfo,
+ flags: []Flag{
+
+ {
+ name: "--incr-key",
+ help: "Without this option, keyless DKVP fields are keyed by field number. For example: `a=10,b=20,30,d=40,50` is ingested as `$a=10,$b=20,$3=30,$d=40,$5=50`. With this option, they're keyed by a running counter of keyless fields. For example: `a=10,b=20,30,d=40,50` is ingested as `$a=10,$b=20,$1=30,$d=40,$2=50`.",
+ parser: func(args []string, argc int, pargi *int, options *TOptions) {
+ options.WriterOptions.BarredPprintOutput = true
+ *pargi += 1
+ },
+ },
+ },
+}
+
// ================================================================
// LEGACY FLAGS
@@ -953,9 +979,7 @@ var FileFormatFlagSection = FlagSection{
name: "--ojsonl",
help: "Use JSON Lines format for output data.",
parser: func(args []string, argc int, pargi *int, options *TOptions) {
- options.WriterOptions.OutputFileFormat = "json"
- options.WriterOptions.WrapJSONOutputInOuterList = false
- options.WriterOptions.JSONOutputMultiline = false
+ options.WriterOptions.OutputFileFormat = "jsonl"
*pargi += 1
},
},
@@ -1122,9 +1146,7 @@ var FileFormatFlagSection = FlagSection{
altNames: []string{"--l2l"},
parser: func(args []string, argc int, pargi *int, options *TOptions) {
options.ReaderOptions.InputFileFormat = "json"
- options.WriterOptions.OutputFileFormat = "json"
- options.WriterOptions.WrapJSONOutputInOuterList = false
- options.WriterOptions.JSONOutputMultiline = false
+ options.WriterOptions.OutputFileFormat = "jsonl"
*pargi += 1
},
},
@@ -2652,7 +2674,7 @@ var CommentsInDataFlagSection = FlagSection{
{
name: "--skip-comments-with",
arg: "{string}",
- help: "Ignore commented lines within input, with specified prefix.",
+ help: "Ignore commented lines within input, with specified prefix. For CSV input format, the prefix must be a single character.",
parser: func(args []string, argc int, pargi *int, options *TOptions) {
CheckArgCount(args, *pargi, argc, 2)
options.ReaderOptions.CommentString = args[*pargi+1]
@@ -2674,7 +2696,7 @@ var CommentsInDataFlagSection = FlagSection{
{
name: "--pass-comments-with",
arg: "{string}",
- help: "Immediately print commented lines within input, with specified prefix.",
+ help: "Immediately print commented lines within input, with specified prefix. For CSV input format, the prefix must be a single character.",
parser: func(args []string, argc int, pargi *int, options *TOptions) {
CheckArgCount(args, *pargi, argc, 2)
options.ReaderOptions.CommentString = args[*pargi+1]
diff --git a/pkg/cli/option_types.go b/pkg/cli/option_types.go
index 19227fd73..58917728a 100644
--- a/pkg/cli/option_types.go
+++ b/pkg/cli/option_types.go
@@ -53,11 +53,12 @@ type TReaderOptions struct {
irsWasSpecified bool
allowRepeatIFSWasSpecified bool
- UseImplicitHeader bool
- AllowRaggedCSVInput bool
- CSVLazyQuotes bool
- CSVTrimLeadingSpace bool
- BarredPprintInput bool
+ UseImplicitHeader bool
+ AllowRaggedCSVInput bool
+ CSVLazyQuotes bool
+ CSVTrimLeadingSpace bool
+ BarredPprintInput bool
+ IncrementImplicitKey bool
CommentHandling TCommentHandling
CommentString string
diff --git a/pkg/entrypoint/entrypoint.go b/pkg/entrypoint/entrypoint.go
index 0d2b8d3a9..d8c56c8cf 100644
--- a/pkg/entrypoint/entrypoint.go
+++ b/pkg/entrypoint/entrypoint.go
@@ -50,7 +50,7 @@ func Main() MainReturn {
if !options.DoInPlace {
err = processToStdout(options, recordTransformers)
} else {
- err = processInPlace(options)
+ err = processFilesInPlace(options)
}
if err != nil {
fmt.Fprintf(os.Stderr, "mlr: %v.\n", err)
@@ -73,7 +73,7 @@ func processToStdout(
}
// ----------------------------------------------------------------
-// processInPlace is in-place processing without mlr -I.
+// processFilesInPlace is in-place processing without mlr -I.
//
// For in-place mode, reconstruct the transformers on each input file. E.g.
// 'mlr -I head -n 2 foo bar' should do head -n 2 on foo as well as on bar.
@@ -85,7 +85,7 @@ func processToStdout(
// frequently used code path, this would likely lead to latent bugs. So this
// approach leads to greater code stability.
-func processInPlace(
+func processFilesInPlace(
originalOptions *cli.TOptions,
) error {
// This should have been already checked by the CLI parser when validating
@@ -98,79 +98,104 @@ func processInPlace(
copy(fileNames, originalOptions.FileNames)
for _, fileName := range fileNames {
-
- if _, err := os.Stat(fileName); os.IsNotExist(err) {
- return err
- }
-
- // Reconstruct the transformers for each file name, and allocate
- // reader, mappers, and writer individually for each file name. This
- // way CSV headers appear in each file, head -n 10 puts 10 rows for
- // each output file, and so on.
- options, recordTransformers, err := climain.ParseCommandLine(os.Args)
+ err := processFileInPlace(fileName, originalOptions)
if err != nil {
return err
}
-
- // We can't in-place update http://, https://, etc. Also, anything with
- // --prepipe or --prepipex, we won't try to guess how to invert that
- // command to produce re-compressed output.
- err = lib.IsUpdateableInPlace(fileName, options.ReaderOptions.Prepipe)
- if err != nil {
- return err
- }
-
- containingDirectory := path.Dir(fileName)
- // Names like ./mlr-in-place-2148227797 and ./mlr-in-place-1792078347,
- // as revealed by printing handle.Name().
- handle, err := os.CreateTemp(containingDirectory, "mlr-in-place-")
- if err != nil {
- return err
- }
- tempFileName := handle.Name()
-
- // If the input file is compressed and we'll be doing in-process
- // decompression as we read the input file, try to do in-process
- // compression as we write the output.
- inputFileEncoding := lib.FindInputEncoding(fileName, options.ReaderOptions.FileInputEncoding)
-
- // Get a handle with, perhaps, a recompression wrapper around it.
- wrappedHandle, isNew, err := lib.WrapOutputHandle(handle, inputFileEncoding)
- if err != nil {
- os.Remove(tempFileName)
- return err
- }
-
- // Run the Miller processing stream from the input file to the temp-output file.
- err = stream.Stream([]string{fileName}, options, recordTransformers, wrappedHandle, false)
- if err != nil {
- os.Remove(tempFileName)
- return err
- }
-
- // Close the recompressor handle, if any recompression is being applied.
- if isNew {
- err = wrappedHandle.Close()
- if err != nil {
- os.Remove(tempFileName)
- return err
- }
- }
-
- // Close the handle to the output file. This may force final writes, so
- // it must be error-checked.
- err = handle.Close()
- if err != nil {
- os.Remove(tempFileName)
- return err
- }
-
- // Rename the temp-output file on top of the input file.
- err = os.Rename(tempFileName, fileName)
- if err != nil {
- os.Remove(tempFileName)
- return err
- }
}
return nil
}
+
+func processFileInPlace(
+ fileName string,
+ originalOptions *cli.TOptions,
+) error {
+
+ if _, err := os.Stat(fileName); os.IsNotExist(err) {
+ return err
+ }
+
+ // Reconstruct the transformers for each file name, and allocate
+ // reader, mappers, and writer individually for each file name. This
+ // way CSV headers appear in each file, head -n 10 puts 10 rows for
+ // each output file, and so on.
+ options, recordTransformers, err := climain.ParseCommandLine(os.Args)
+ if err != nil {
+ return err
+ }
+
+ // We can't in-place update http://, https://, etc. Also, anything with
+ // --prepipe or --prepipex, we won't try to guess how to invert that
+ // command to produce re-compressed output.
+ err = lib.IsUpdateableInPlace(fileName, options.ReaderOptions.Prepipe)
+ if err != nil {
+ return err
+ }
+
+ // Get the original file's mode so we can preserve it.
+ fileInfo, err := os.Stat(fileName)
+ if err != nil {
+ return err
+ }
+ originalMode := fileInfo.Mode()
+
+ containingDirectory := path.Dir(fileName)
+ // Names like ./mlr-in-place-2148227797 and ./mlr-in-place-1792078347,
+ // as revealed by printing handle.Name().
+ handle, err := os.CreateTemp(containingDirectory, "mlr-in-place-")
+ if err != nil {
+ return err
+ }
+ tempFileName := handle.Name()
+
+ // If the input file is compressed and we'll be doing in-process
+ // decompression as we read the input file, try to do in-process
+ // compression as we write the output.
+ inputFileEncoding := lib.FindInputEncoding(fileName, options.ReaderOptions.FileInputEncoding)
+
+ // Get a handle with, perhaps, a recompression wrapper around it.
+ wrappedHandle, isNew, err := lib.WrapOutputHandle(handle, inputFileEncoding)
+ if err != nil {
+ os.Remove(tempFileName)
+ return err
+ }
+
+ // Run the Miller processing stream from the input file to the temp-output file.
+ err = stream.Stream([]string{fileName}, options, recordTransformers, wrappedHandle, false)
+ if err != nil {
+ os.Remove(tempFileName)
+ return err
+ }
+
+ // Close the recompressor handle, if any recompression is being applied.
+ if isNew {
+ err = wrappedHandle.Close()
+ if err != nil {
+ os.Remove(tempFileName)
+ return err
+ }
+ }
+
+ // Close the handle to the output file. This may force final writes, so
+ // it must be error-checked.
+ err = handle.Close()
+ if err != nil {
+ os.Remove(tempFileName)
+ return err
+ }
+
+ // Rename the temp-output file on top of the input file.
+ err = os.Rename(tempFileName, fileName)
+ if err != nil {
+ os.Remove(tempFileName)
+ return err
+ }
+
+ // Set the mode to match the original.
+ err = os.Chmod(fileName, originalMode)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/pkg/go-csv/csv_reader.go b/pkg/go-csv/csv_reader.go
index 507e9a94c..5a0820a01 100644
--- a/pkg/go-csv/csv_reader.go
+++ b/pkg/go-csv/csv_reader.go
@@ -311,15 +311,28 @@ func (r *Reader) readRecord(dst []string) ([]string, error) {
var errRead error
for errRead == nil {
line, errRead = r.readLine()
- if r.Comment != 0 && nextRune(line) == r.Comment {
- line = nil
- continue // Skip comment lines
- }
+
+ // MILLER-SPECIFIC UPDATE: DO NOT DO THIS
+ // if r.Comment != 0 && nextRune(line) == r.Comment {
+ // line = nil
+ // continue // Skip comment lines
+ // }
+
// MILLER-SPECIFIC UPDATE: DO NOT DO THIS
// if errRead == nil && len(line) == lengthNL(line) {
- // line = nil
- // continue // Skip empty lines
+ // line = nil
+ // continue // Skip empty lines
// }
+
+ // MILLER-SPECIFIC UPDATE: If the line starts with the comment character,
+ // don't attempt to CSV-parse it -- just hand it back as a single field.
+ // This allows two things:
+ // * User comments get passed through as intended, without being reformatted;
+ // * Users can do things like `# a"b` in their comments without getting an
+ // imbalanced-double-quote error.
+ if r.Comment != 0 && nextRune(line) == r.Comment {
+ return []string{string(line)}, nil
+ }
break
}
if errRead == io.EOF {
diff --git a/pkg/input/record_reader_csv.go b/pkg/input/record_reader_csv.go
index 6ed07250d..aa7dec084 100644
--- a/pkg/input/record_reader_csv.go
+++ b/pkg/input/record_reader_csv.go
@@ -1,7 +1,6 @@
package input
import (
- "bytes"
"container/list"
"fmt"
"io"
@@ -40,6 +39,11 @@ func NewRecordReaderCSV(
if len(readerOptions.IFS) != 1 {
return nil, fmt.Errorf("for CSV, IFS can only be a single character")
}
+ if readerOptions.CommentHandling != cli.CommentsAreData {
+ if len(readerOptions.CommentString) != 1 {
+ return nil, fmt.Errorf("for CSV, the comment prefix must be a single character")
+ }
+ }
return &RecordReaderCSV{
readerOptions: readerOptions,
ifs0: readerOptions.IFS[0],
@@ -109,6 +113,14 @@ func (reader *RecordReaderCSV) processHandle(
csvReader.Comma = rune(reader.ifs0)
csvReader.LazyQuotes = reader.csvLazyQuotes
csvReader.TrimLeadingSpace = reader.csvTrimLeadingSpace
+
+ if reader.readerOptions.CommentHandling != cli.CommentsAreData {
+ if len(reader.readerOptions.CommentString) == 1 {
+ // Use our modified fork of the go-csv package
+ csvReader.Comment = rune(reader.readerOptions.CommentString[0])
+ }
+ }
+
csvRecordsChannel := make(chan *list.List, recordsPerBatch)
go channelizedCSVRecordScanner(csvReader, csvRecordsChannel, downstreamDoneChannel, errorChannel,
recordsPerBatch)
@@ -318,42 +330,17 @@ func (reader *RecordReaderCSV) maybeConsumeComment(
// However, sadly, bytes.Buffer does not implement io.Writer because
// its Write method has pointer receiver. So we have a WorkaroundBuffer
// struct below which has non-pointer receiver.
- buffer := NewWorkaroundBuffer()
- csvWriter := csv.NewWriter(buffer)
- csvWriter.Comma = rune(reader.ifs0)
- csvWriter.Write(csvRecord)
- csvWriter.Flush()
- recordsAndContexts.PushBack(types.NewOutputString(buffer.String(), context))
+
+ // Contract with our fork of the go-csv CSV Reader, and, our own constructor.
+ lib.InternalCodingErrorIf(len(csvRecord) != 1)
+ recordsAndContexts.PushBack(types.NewOutputString(csvRecord[0], context))
+
} else /* reader.readerOptions.CommentHandling == cli.SkipComments */ {
// discard entirely
}
return false
}
-// ----------------------------------------------------------------
-// As noted above: wraps a bytes.Buffer, whose Write method has pointer
-// receiver, in a struct with non-pointer receiver so that it implements
-// io.Writer.
-
-type WorkaroundBuffer struct {
- pbuffer *bytes.Buffer
-}
-
-func NewWorkaroundBuffer() WorkaroundBuffer {
- var buffer bytes.Buffer
- return WorkaroundBuffer{
- pbuffer: &buffer,
- }
-}
-
-func (wb WorkaroundBuffer) Write(p []byte) (n int, err error) {
- return wb.pbuffer.Write(p)
-}
-
-func (wb WorkaroundBuffer) String() string {
- return wb.pbuffer.String()
-}
-
// ----------------------------------------------------------------
// BOM-stripping
//
diff --git a/pkg/input/record_reader_dkvp_nidx.go b/pkg/input/record_reader_dkvp_nidx.go
index efc0ae385..6a53c8c26 100644
--- a/pkg/input/record_reader_dkvp_nidx.go
+++ b/pkg/input/record_reader_dkvp_nidx.go
@@ -14,7 +14,7 @@ import (
"github.com/johnkerl/miller/v6/pkg/types"
)
-// splitter_DKVP_NIDX is a function type for the one bit of code differing
+// line_splitter_DKVP_NIDX is a function type for the one bit of code differing
// between the DKVP reader and the NIDX reader, namely, how it splits lines.
type line_splitter_DKVP_NIDX func(reader *RecordReaderDKVPNIDX, line string) (*mlrval.Mlrmap, error)
@@ -169,25 +169,42 @@ func recordFromDKVPLine(reader *RecordReaderDKVPNIDX, line string) (*mlrval.Mlrm
pairs := reader.fieldSplitter.Split(line)
+ // Without --incr-key:
+ // echo 'a,z=b,c' | mlr cat gives 1=a,z=b,3=c
+ // I.e. implicit keys are taken from the 1-up field counter.
+ // With it:
+ // echo 'a,z=b,c' | mlr cat gives 1=a,z=b,2=c
+ // I.e. implicit keys are taken from a 1-up count of fields lacking explicit keys.
+ incr_key := 0
+
for i, pair := range pairs {
kv := reader.pairSplitter.Split(pair)
if len(kv) == 0 || (len(kv) == 1 && kv[0] == "") {
// Ignore. This is expected when splitting with repeated IFS.
} else if len(kv) == 1 {
- // E.g the pair has no equals sign: "a" rather than "a=1" or
+ // E.g. the pair has no equals sign: "a" rather than "a=1" or
// "a=". Here we use the positional index as the key. This way
// DKVP is a generalization of NIDX.
- key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up
+ //
+ // Also: recall that Miller userspace indices are 1-up.
+ var int_key int
+ if reader.readerOptions.IncrementImplicitKey {
+ int_key = incr_key
+ } else {
+ int_key = i
+ }
+ str_key := strconv.Itoa(int_key + 1)
+ incr_key++
value := mlrval.FromDeferredType(kv[0])
- _, err := record.PutReferenceMaybeDedupe(key, value, dedupeFieldNames)
+ _, err := record.PutReferenceMaybeDedupe(str_key, value, dedupeFieldNames)
if err != nil {
return nil, err
}
} else {
- key := kv[0]
+ str_key := kv[0]
value := mlrval.FromDeferredType(kv[1])
- _, err := record.PutReferenceMaybeDedupe(key, value, dedupeFieldNames)
+ _, err := record.PutReferenceMaybeDedupe(str_key, value, dedupeFieldNames)
if err != nil {
return nil, err
}
@@ -204,9 +221,9 @@ func recordFromNIDXLine(reader *RecordReaderDKVPNIDX, line string) (*mlrval.Mlrm
var i int = 0
for _, value := range values {
i++
- key := strconv.Itoa(i)
+ str_key := strconv.Itoa(i)
mval := mlrval.FromDeferredType(value)
- record.PutReference(key, mval)
+ record.PutReference(str_key, mval)
}
return record, nil
}
diff --git a/pkg/mlrval/mlrval_json.go b/pkg/mlrval/mlrval_json.go
index d1fb880ed..1a193aa14 100644
--- a/pkg/mlrval/mlrval_json.go
+++ b/pkg/mlrval/mlrval_json.go
@@ -13,6 +13,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "strconv"
"github.com/johnkerl/miller/v6/pkg/colorizer"
"github.com/johnkerl/miller/v6/pkg/lib"
@@ -406,7 +407,16 @@ func millerJSONEncodeString(input string) string {
// ----------------------------------------------------------------
func (mv *Mlrval) marshalJSONInt(outputIsStdout bool) (string, error) {
lib.InternalCodingErrorIf(mv.mvtype != MT_INT)
- return colorizer.MaybeColorizeValue(mv.String(), outputIsStdout), nil
+ // Other formats would use mv.String(): for example, if the user used hex
+ // format, we would emit whatever they set. However, for JSON, we are
+ // required to disrespect the user's formatting, and only emit decimal.
+ // See also https://github.com/johnkerl/miller/issues/1761.
+ ival, ok := mv.GetIntValue()
+ if !ok {
+ panic("Internal coding error: int-typed mlrval denied int access")
+ }
+ s := strconv.FormatInt(ival, 10)
+ return colorizer.MaybeColorizeValue(s, outputIsStdout), nil
}
// ----------------------------------------------------------------
diff --git a/pkg/output/record_writer_factory.go b/pkg/output/record_writer_factory.go
index 84ff64cfe..bb6aba5fa 100644
--- a/pkg/output/record_writer_factory.go
+++ b/pkg/output/record_writer_factory.go
@@ -16,6 +16,8 @@ func Create(writerOptions *cli.TWriterOptions) (IRecordWriter, error) {
return NewRecordWriterDKVP(writerOptions)
case "json":
return NewRecordWriterJSON(writerOptions)
+ case "jsonl":
+ return NewRecordWriterJSONLines(writerOptions)
case "md":
return NewRecordWriterMarkdown(writerOptions)
case "markdown":
diff --git a/pkg/output/record_writer_json.go b/pkg/output/record_writer_json_jsonl.go
similarity index 87%
rename from pkg/output/record_writer_json.go
rename to pkg/output/record_writer_json_jsonl.go
index d0be01461..8c43d43ff 100644
--- a/pkg/output/record_writer_json.go
+++ b/pkg/output/record_writer_json_jsonl.go
@@ -35,6 +35,19 @@ func NewRecordWriterJSON(writerOptions *cli.TWriterOptions) (*RecordWriterJSON,
}, nil
}
+// ----------------------------------------------------------------
+func NewRecordWriterJSONLines(writerOptions *cli.TWriterOptions) (*RecordWriterJSON, error) {
+ wopt := *writerOptions
+ wopt.WrapJSONOutputInOuterList = false
+ wopt.JSONOutputMultiline = false
+ return &RecordWriterJSON{
+ writerOptions: &wopt,
+ jsonFormatting: mlrval.JSON_SINGLE_LINE,
+ jvQuoteAll: writerOptions.JVQuoteAll,
+ wroteAnyRecords: false,
+ }, nil
+}
+
// ----------------------------------------------------------------
func (writer *RecordWriterJSON) Write(
outrec *mlrval.Mlrmap,
diff --git a/pkg/parsing/mlr.bnf b/pkg/parsing/mlr.bnf
index 5903cf419..bd9602f81 100644
--- a/pkg/parsing/mlr.bnf
+++ b/pkg/parsing/mlr.bnf
@@ -7,7 +7,7 @@
// GRAMMAR FOR THE MILLER DOMAIN-SPECIFIC LANGUAGE
//
// This is the Miller DSL's BNF grammar, using the awesome GOCC tool framework
-// from https://github.com/goccmack/gocc.
+// from https://github.com/goccmack/gocc (forked at https://github.com/johnkerl/gocc).
//
// The first section is lexical elements and the second section is syntactical
// elements. These are the analogs of lex and yacc, respectively, using a
diff --git a/pkg/transformers/sort.go b/pkg/transformers/sort.go
index 945c6e581..6342192bb 100644
--- a/pkg/transformers/sort.go
+++ b/pkg/transformers/sort.go
@@ -83,6 +83,7 @@ func transformerSortUsage(
fmt.Fprintf(o, "-nf {comma-separated field names} Same as -n\n")
fmt.Fprintf(o, "-nr {comma-separated field names} Numerical descending; nulls sort first\n")
fmt.Fprintf(o, "-t {comma-separated field names} Natural ascending\n")
+ fmt.Fprintf(o, "-b Move sort fields to start of record, as in reorder -b\n")
fmt.Fprintf(o, "-tr|-rt {comma-separated field names} Natural descending\n")
fmt.Fprintf(o, "-h|--help Show this message.\n")
fmt.Fprintf(o, "\n")
@@ -107,6 +108,7 @@ func transformerSortParseCLI(
groupByFieldNames := make([]string, 0)
comparatorFuncs := make([]mlrval.CmpFuncInt, 0)
+ doMoveToHead := false
for argi < argc /* variable increment: 1 or 2 depending on flag */ {
opt := args[argi]
@@ -255,6 +257,9 @@ func transformerSortParseCLI(
comparatorFuncs = append(comparatorFuncs, mlrval.NumericDescendingComparator)
}
+ } else if opt == "-b" {
+ doMoveToHead = true
+
} else {
transformerSortUsage(os.Stderr)
os.Exit(1)
@@ -274,6 +279,7 @@ func transformerSortParseCLI(
transformer, err := NewTransformerSort(
groupByFieldNames,
comparatorFuncs,
+ doMoveToHead,
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
@@ -304,6 +310,7 @@ type TransformerSort struct {
// -- Input
groupByFieldNames []string
comparatorFuncs []mlrval.CmpFuncInt
+ doMoveToHead bool
// -- State
// Map from string to *list.List:
@@ -316,11 +323,13 @@ type TransformerSort struct {
func NewTransformerSort(
groupByFieldNames []string,
comparatorFuncs []mlrval.CmpFuncInt,
+ doMoveToHead bool,
) (*TransformerSort, error) {
tr := &TransformerSort{
groupByFieldNames: groupByFieldNames,
comparatorFuncs: comparatorFuncs,
+ doMoveToHead: doMoveToHead,
recordListsByGroup: lib.NewOrderedMap(),
groupHeads: lib.NewOrderedMap(),
@@ -346,6 +355,13 @@ func (tr *TransformerSort) Transform(
if !inrecAndContext.EndOfStream {
inrec := inrecAndContext.Record
+ if tr.doMoveToHead {
+ n := len(tr.groupByFieldNames)
+ for i := n - 1; i >= 0; i-- {
+ inrec.MoveToHead(tr.groupByFieldNames[i])
+ }
+ }
+
groupingKey, selectedValues, ok := inrec.GetSelectedValuesAndJoined(
tr.groupByFieldNames,
)
diff --git a/pkg/transformers/subs.go b/pkg/transformers/subs.go
index 59a8e92de..b5530bb17 100644
--- a/pkg/transformers/subs.go
+++ b/pkg/transformers/subs.go
@@ -50,7 +50,7 @@ func transformerSubUsage(
fmt.Fprintf(o, "Options:\n")
fmt.Fprintf(o, "-f {a,b,c} Field names to convert.\n")
fmt.Fprintf(o, "-r {regex} Regular expression for field names to convert.\n")
- fmt.Fprintf(o, "-a Convert all field names.\n")
+ fmt.Fprintf(o, "-a Convert all fields.\n")
fmt.Fprintf(o, "-h|--help Show this message.\n")
}
@@ -64,7 +64,7 @@ func transformerGsubUsage(
fmt.Fprintf(o, "Options:\n")
fmt.Fprintf(o, "-f {a,b,c} Field names to convert.\n")
fmt.Fprintf(o, "-r {regex} Regular expression for field names to convert.\n")
- fmt.Fprintf(o, "-a Convert all field names.\n")
+ fmt.Fprintf(o, "-a Convert all fields.\n")
fmt.Fprintf(o, "-h|--help Show this message.\n")
}
@@ -77,7 +77,7 @@ func transformerSsubUsage(
fmt.Fprintf(o, "Options:\n")
fmt.Fprintf(o, "-f {a,b,c} Field names to convert.\n")
fmt.Fprintf(o, "-r {regex} Regular expression for field names to convert.\n")
- fmt.Fprintf(o, "-a Convert all field names.\n")
+ fmt.Fprintf(o, "-a Convert all fields.\n")
fmt.Fprintf(o, "-h|--help Show this message.\n")
}
diff --git a/pkg/version/version.go b/pkg/version/version.go
index 4654d775d..ec9c7208a 100644
--- a/pkg/version/version.go
+++ b/pkg/version/version.go
@@ -4,4 +4,4 @@ package version
// Nominally things like "6.0.0" for a release, then "6.0.0-dev" in between.
// This makes it clear that a given build is on the main dev branch, not a
// particular snapshot tag.
-var STRING string = "6.14.0"
+var STRING string = "6.16.0"
diff --git a/snap/README.md b/snap/README.md
new file mode 100644
index 000000000..2af316410
--- /dev/null
+++ b/snap/README.md
@@ -0,0 +1,150 @@
+# Failed attempts to create a snap interactively
+
+2026-01-02 I used an Ubuntu 24.04 EC2 instance. I followed https://documentation.ubuntu.com/snapcraft/stable/. Error messages said things like
+
+```
+A network related operation failed in a context of no network access.
+Recommended resolution: Verify that the environment has internet connectivity; see https://canonical-craft-providers.readthedocs-hosted.com/en/latest/explanation/ for further reference.
+Full execution log: '/home/ubuntu/.local/state/snapcraft/log/snapcraft-20260102-170252.488632.log'
+```
+
+when there was in fact no network problem. I remained confused.
+
+```
+$ sudo snapcraft pack
+
+$ lxc list
+
+$ snapcraft pack --destructive-mode
+
+$ snapcraft pack --use-multipass
+
+$ sudo snap install multipass
+
+$ snapcraft pack --use-multipass
+
+$ sudo lxd init --auto
+
+$ lxc network list
+
+$ sudo snapcraft pack
+
+$ sudo snap set snapcraft provider=multipass
+
+$ sudo snapcraft pack --destructive-mode
+
+[This created miller_6.15.0_arm64.snap]
+
+$ snapcraft upload --release=stable *.snap
+No keyring found to store or retrieve credentials from.
+Recommended resolution: Ensure the keyring is working or SNAPCRAFT_STORE_CREDENTIALS is correctly exported into the environment
+For more information, check out: https://documentation.ubuntu.com/snapcraft/stable/how-to/publishing/authenticate
+Full execution log: '/home/ubuntu/.local/state/snapcraft/log/snapcraft-20260102-172357.599171.log'
+
+$ ll *.snap
+-rw-r--r-- 1 root root 8994816 Jan 2 17:22 miller_6.15.0_arm64.snap
+
+$ snap install *.snap
+error: access denied (try with sudo)
+
+$ sudo snap install *.snap
+error: cannot find signatures with metadata for snap/component "miller_6.15.0_arm64.snap"
+```
+
+Conclusion:
+
+* I got cryptic error messages with various permutations.
+* Through trial and error I got a `.snap` file with `sudo` and `multipass` and `--destructive-mode`.
+* Even then, I got a `.snap` file only for the current machine's arch, and the resulting `.snap` file was not locally installable.
+* This led me to try a GitHub Action.
+
+# Info from Claude about auto-releasing
+
+Here's how to set up automatic Snap publishing from GitHub releases:
+
+## 1. Create snapcraft.yaml
+
+First, ensure you have a proper `snapcraft.yaml` in your repo root (or in a `snap/` directory):
+
+```yaml
+name: your-app-name
+base: core22 # or core24 for Ubuntu 24.04
+version: git # automatically uses git tags
+summary: Single-line summary
+description: |
+ Longer description of your application
+
+grade: stable # or devel
+confinement: strict # or classic, devmode
+
+apps:
+ your-app-name:
+ command: bin/your-binary
+ plugs:
+ - home
+ - network
+
+parts:
+ your-app:
+ plugin: nil # change based on your build system (go, python, etc.)
+ source: .
+ # Add build steps as needed
+```
+
+## 2. Get Snapcraft credentials
+
+Export your Snapcraft login credentials:
+
+```bash
+snapcraft export-login --snaps=miller --channels=stable,candidate,beta,edge snapcraft-token.txt
+```
+
+This creates a token file with limited permissions for just your snap.
+
+## 3. Add token to GitHub Secrets
+
+1. Go to your GitHub repo โ Settings โ Secrets and variables โ Actions
+2. Click "New repository secret"
+3. Name: `SNAPCRAFT_TOKEN`
+4. Value: Paste the entire contents of `snapcraft-token.txt`
+
+## 4. Create GitHub Action workflow
+
+Create `.github/workflows/release.yml`:
+
+```yaml
+name: Release to Snap Store
+
+on:
+ release:
+ types: [published]
+
+jobs:
+ snap:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Build snap
+ uses: snapcore/action-build@v1
+ id: build
+
+ - name: Publish to Snap Store
+ uses: snapcore/action-publish@v1
+ env:
+ SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.SNAPCRAFT_TOKEN }}
+ with:
+ snap: ${{ steps.build.outputs.snap }}
+ # release: stable # or edge, beta, candidate
+ release: edge
+```
+
+## Tips
+
+- **Version handling**: Using `version: git` in snapcraft.yaml automatically uses your git tag as the version
+- **Channels**: Start with `edge` channel for testing, then promote to `stable` once confident
+- **Multiple architectures**: Add a build matrix if you need to support arm64, etc.
+- **Testing before stable**: Consider publishing to `candidate` or `beta` first, then manually promote to `stable` after testing
+
+Now when you create a GitHub release with a tag (e.g., `v1.0.0`), the workflow will automatically build and publish your snap!
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
new file mode 100644
index 000000000..d374a84b0
--- /dev/null
+++ b/snap/snapcraft.yaml
@@ -0,0 +1,54 @@
+name: miller
+base: core24
+version: git
+summary: Miller is like awk, sed, cut, join and sort
+description: |
+ Miller is like awk, sed, cut, join, and sort for data formats such as CSV, TSV, JSON, JSON Lines, and positionally-indexed.
+
+grade: stable
+confinement: strict
+
+adopt-info: miller
+
+website: https://github.com/johnkerl/miller/issues
+contact: https://github.com/johnkerl/miller/issues
+issues: https://github.com/johnkerl/miller/issues
+source-code: https://github.com/johnkerl/miller
+
+license: BSD-2-Clause
+compression: lzo
+
+platforms:
+ amd64:
+ build-on: [amd64]
+ build-for: [amd64]
+ arm64:
+ build-on: [arm64]
+ build-for: [arm64]
+ armhf:
+ build-on: [armhf]
+ build-for: [armhf]
+ s390x:
+ build-on: [s390x]
+ build-for: [s390x]
+ ppc64el:
+ build-on: [ppc64el]
+ build-for: [ppc64el]
+
+apps:
+ miller:
+ command: usr/local/bin/mlr
+ plugs:
+ - home
+
+parts:
+ miller:
+ source: https://github.com/johnkerl/miller
+ source-type: git
+ plugin: make
+ build-snaps:
+ - go
+
+ override-pull: |
+ craftctl default
+ craftctl set version="$(git describe --tags | sed 's/^v//' | cut -d "-" -f1)"
diff --git a/test/cases/cli-help/0001/expout b/test/cases/cli-help/0001/expout
index a451ac0c5..19a201c62 100644
--- a/test/cases/cli-help/0001/expout
+++ b/test/cases/cli-help/0001/expout
@@ -393,7 +393,7 @@ See also the `sub` and `ssub` verbs.
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
--a Convert all field names.
+-a Convert all fields.
-h|--help Show this message.
================================================================
@@ -982,6 +982,7 @@ Options:
-nf {comma-separated field names} Same as -n
-nr {comma-separated field names} Numerical descending; nulls sort first
-t {comma-separated field names} Natural ascending
+-b Move sort fields to start of record, as in reorder -b
-tr|-rt {comma-separated field names} Natural descending
-h|--help Show this message.
@@ -1061,7 +1062,7 @@ the old string, like the `ssub` DSL function. See also the `gsub` and `sub` verb
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
--a Convert all field names.
+-a Convert all fields.
-h|--help Show this message.
================================================================
@@ -1214,7 +1215,7 @@ See also the `gsub` and `ssub` verbs.
Options:
-f {a,b,c} Field names to convert.
-r {regex} Regular expression for field names to convert.
--a Convert all field names.
+-a Convert all fields.
-h|--help Show this message.
================================================================
diff --git a/test/cases/io-jsonl-io/0004/cmd b/test/cases/io-jsonl-io/0004/cmd
index 380bba0ca..8aa87f37e 100644
--- a/test/cases/io-jsonl-io/0004/cmd
+++ b/test/cases/io-jsonl-io/0004/cmd
@@ -1 +1 @@
-mlr --ojsonl cat test/input/json-output-options.dkvp
+mlr -o jsonl cat test/input/json-output-options.dkvp
diff --git a/test/cases/io-skip-pass-comments/pr-1346/cmd b/test/cases/io-skip-pass-comments/pr-1346/cmd
new file mode 100644
index 000000000..611187612
--- /dev/null
+++ b/test/cases/io-skip-pass-comments/pr-1346/cmd
@@ -0,0 +1 @@
+mlr --skip-comments --csv --pass-comments cat test/input/pr-1346.csv
diff --git a/test/cases/io-skip-pass-comments/pr-1346/experr b/test/cases/io-skip-pass-comments/pr-1346/experr
new file mode 100644
index 000000000..10864f8ab
--- /dev/null
+++ b/test/cases/io-skip-pass-comments/pr-1346/experr
@@ -0,0 +1 @@
+mlr: mlr: CSV header/data length mismatch 2 != 1 at filename test/input/pr-1346.csv row 4.
diff --git a/test/cases/io-skip-pass-comments/pr-1346/expout b/test/cases/io-skip-pass-comments/pr-1346/expout
new file mode 100644
index 000000000..b7872a7a9
--- /dev/null
+++ b/test/cases/io-skip-pass-comments/pr-1346/expout
@@ -0,0 +1,5 @@
+field1,field2
+a,b
+# that was the first record
+c,d
+# that was the second record, and there is no more data
diff --git a/test/cases/io-skip-pass-comments/pr-1346/should-fail b/test/cases/io-skip-pass-comments/pr-1346/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-skip-pass-comments/pr-1787-a/cmd b/test/cases/io-skip-pass-comments/pr-1787-a/cmd
new file mode 100644
index 000000000..8ecdde63e
--- /dev/null
+++ b/test/cases/io-skip-pass-comments/pr-1787-a/cmd
@@ -0,0 +1 @@
+mlr --csv cat test/input/pr-1787.csv
diff --git a/test/cases/io-skip-pass-comments/pr-1787-a/experr b/test/cases/io-skip-pass-comments/pr-1787-a/experr
new file mode 100644
index 000000000..9e02e68bc
--- /dev/null
+++ b/test/cases/io-skip-pass-comments/pr-1787-a/experr
@@ -0,0 +1 @@
+mlr: parse error on line 3, column 4: bare " in non-quoted-field.
diff --git a/test/cases/io-skip-pass-comments/pr-1787-a/expout b/test/cases/io-skip-pass-comments/pr-1787-a/expout
new file mode 100644
index 000000000..bfde6bfa0
--- /dev/null
+++ b/test/cases/io-skip-pass-comments/pr-1787-a/expout
@@ -0,0 +1,2 @@
+a,b,c
+1,2,3
diff --git a/test/cases/io-skip-pass-comments/pr-1787-a/should-fail b/test/cases/io-skip-pass-comments/pr-1787-a/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-skip-pass-comments/pr-1787-b/cmd b/test/cases/io-skip-pass-comments/pr-1787-b/cmd
new file mode 100644
index 000000000..c79588a16
--- /dev/null
+++ b/test/cases/io-skip-pass-comments/pr-1787-b/cmd
@@ -0,0 +1 @@
+mlr --csv --pass-comments cat test/input/pr-1787.csv
diff --git a/test/cases/io-skip-pass-comments/pr-1787-b/experr b/test/cases/io-skip-pass-comments/pr-1787-b/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-skip-pass-comments/pr-1787-b/expout b/test/cases/io-skip-pass-comments/pr-1787-b/expout
new file mode 100644
index 000000000..23b8c638c
--- /dev/null
+++ b/test/cases/io-skip-pass-comments/pr-1787-b/expout
@@ -0,0 +1,4 @@
+a,b,c
+1,2,3
+# x"y
+4,5,6
diff --git a/test/cases/io-skip-pass-comments/pr-1787-c/cmd b/test/cases/io-skip-pass-comments/pr-1787-c/cmd
new file mode 100644
index 000000000..8e17a1f3e
--- /dev/null
+++ b/test/cases/io-skip-pass-comments/pr-1787-c/cmd
@@ -0,0 +1 @@
+mlr --csv --skip-comments cat test/input/pr-1787.csv
diff --git a/test/cases/io-skip-pass-comments/pr-1787-c/experr b/test/cases/io-skip-pass-comments/pr-1787-c/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-skip-pass-comments/pr-1787-c/expout b/test/cases/io-skip-pass-comments/pr-1787-c/expout
new file mode 100644
index 000000000..88700c714
--- /dev/null
+++ b/test/cases/io-skip-pass-comments/pr-1787-c/expout
@@ -0,0 +1,3 @@
+a,b,c
+1,2,3
+4,5,6
diff --git a/test/cases/io-skip-pass-comments/pr-1787-d/cmd b/test/cases/io-skip-pass-comments/pr-1787-d/cmd
new file mode 100644
index 000000000..9db12e96e
--- /dev/null
+++ b/test/cases/io-skip-pass-comments/pr-1787-d/cmd
@@ -0,0 +1 @@
+mlr --csv --skip-comments-with '##' cat test/input/pr-1787.csv
diff --git a/test/cases/io-skip-pass-comments/pr-1787-d/experr b/test/cases/io-skip-pass-comments/pr-1787-d/experr
new file mode 100644
index 000000000..f8b7d1e1a
--- /dev/null
+++ b/test/cases/io-skip-pass-comments/pr-1787-d/experr
@@ -0,0 +1 @@
+mlr: for CSV, the comment prefix must be a single character.
diff --git a/test/cases/io-skip-pass-comments/pr-1787-d/expout b/test/cases/io-skip-pass-comments/pr-1787-d/expout
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-skip-pass-comments/pr-1787-d/should-fail b/test/cases/io-skip-pass-comments/pr-1787-d/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-format-values/0003/expout b/test/cases/verb-format-values/0003/expout
index 06216b5ca..9a45bc186 100644
--- a/test/cases/verb-format-values/0003/expout
+++ b/test/cases/verb-format-values/0003/expout
@@ -1,7 +1,7 @@
[
{
"hostname": "localhost",
- "pid": 0x3039,
+ "pid": 12345,
"req": {
"id": 6789,
"method": "GET",
diff --git a/test/input/pr-1346.csv b/test/input/pr-1346.csv
new file mode 100644
index 000000000..6a46e0994
--- /dev/null
+++ b/test/input/pr-1346.csv
@@ -0,0 +1,6 @@
+field1,field2
+a,b
+# that was the first record
+c,d
+# that was the second record, and there is no more data
+
diff --git a/test/input/pr-1787.csv b/test/input/pr-1787.csv
new file mode 100644
index 000000000..23b8c638c
--- /dev/null
+++ b/test/input/pr-1787.csv
@@ -0,0 +1,4 @@
+a,b,c
+1,2,3
+# x"y
+4,5,6
diff --git a/tools/build-dsl b/tools/build-dsl
index e2a6186d2..4cf70cbf5 100755
--- a/tools/build-dsl
+++ b/tools/build-dsl
@@ -27,8 +27,8 @@ if [ $# -eq 1 ]; then
fi
fi
-# Build the bin/gocc executable:
-go install github.com/goccmack/gocc
+# Build the bin/gocc executable (use my fork for performance):
+go install github.com/johnkerl/gocc
go mod tidy
bingocc="$HOME/go/bin/gocc"
if [ ! -x "$bingocc" ]; then