diff --git a/docs/src/data/join-ignore-empty-left.csv b/docs/src/data/join-ignore-empty-left.csv
new file mode 100644
index 000000000..171fd1d5c
--- /dev/null
+++ b/docs/src/data/join-ignore-empty-left.csv
@@ -0,0 +1,6 @@
+id,code
+3,0000ff
+2,00ff00
+4,ff0000
+,ffffff
+,000000
diff --git a/docs/src/data/join-ignore-empty-right.csv b/docs/src/data/join-ignore-empty-right.csv
new file mode 100644
index 000000000..badd61587
--- /dev/null
+++ b/docs/src/data/join-ignore-empty-right.csv
@@ -0,0 +1,5 @@
+id,color
+4,red
+2,green
+,white
+,black
diff --git a/docs/src/manpage.md b/docs/src/manpage.md
index 6b1d1b71a..782050585 100644
--- a/docs/src/manpage.md
+++ b/docs/src/manpage.md
@@ -1559,6 +1559,13 @@ This is simply a copy of what you should see on running `man mlr` at a command p
--ul Emit unpaired records from the left file.
--ur Emit unpaired records from the right
file(s).
+ --ignore-empty Treat records with empty-string values in
+ any join-field as if that join-field were
+ absent, on both the left and right files.
+ Such records are never paired -- not even
+ with one another -- and are treated as
+ unpaired, subject to --np/--ul/--ur as
+ usual.
-s|--sorted-input Require sorted input: records must be
sorted lexically by their join-field names,
else not all records will be paired. The
diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt
index 3415b2b84..feb351a26 100644
--- a/docs/src/manpage.txt
+++ b/docs/src/manpage.txt
@@ -1538,6 +1538,13 @@
--ul Emit unpaired records from the left file.
--ur Emit unpaired records from the right
file(s).
+ --ignore-empty Treat records with empty-string values in
+ any join-field as if that join-field were
+ absent, on both the left and right files.
+ Such records are never paired -- not even
+ with one another -- and are treated as
+ unpaired, subject to --np/--ul/--ur as
+ usual.
-s|--sorted-input Require sorted input: records must be
sorted lexically by their join-field names,
else not all records will be paired. The
diff --git a/docs/src/reference-verbs.md b/docs/src/reference-verbs.md
index 7680eddbb..30c2ede20 100644
--- a/docs/src/reference-verbs.md
+++ b/docs/src/reference-verbs.md
@@ -1940,6 +1940,13 @@ Options:
--ul Emit unpaired records from the left file.
--ur Emit unpaired records from the right
file(s).
+--ignore-empty Treat records with empty-string values in
+ any join-field as if that join-field were
+ absent, on both the left and right files.
+ Such records are never paired -- not even
+ with one another -- and are treated as
+ unpaired, subject to --np/--ul/--ur as
+ usual.
-s|--sorted-input Require sorted input: records must be
sorted lexically by their join-field names,
else not all records will be paired. The
@@ -2135,6 +2142,55 @@ left_a left_b left_c right_a right_b right_c
1 4 5 1 4 5
+By default, records with an empty-string value in a join field are joined just like any other value -- so two records which are both missing an ID, say, will be paired with one another even though that's rarely what's wanted:
+
+
+mlr --csv cat data/join-ignore-empty-left.csv
+
+
+id,code
+3,0000ff
+2,00ff00
+4,ff0000
+,ffffff
+,000000
+
+
+
+mlr --csv cat data/join-ignore-empty-right.csv
+
+
+id,color
+4,red
+2,green
+,white
+,black
+
+
+
+mlr --csv join -j id -f data/join-ignore-empty-left.csv data/join-ignore-empty-right.csv
+
+
+id,code,color
+4,ff0000,red
+2,00ff00,green
+,ffffff,white
+,000000,white
+,ffffff,black
+,000000,black
+
+
+Use `--ignore-empty` to instead treat an empty-string join-field value as if the field were absent, on both the left and right files. Such records are never paired -- not even with one another:
+
+
+mlr --csv join --ignore-empty -j id -f data/join-ignore-empty-left.csv data/join-ignore-empty-right.csv
+
+
+id,code,color
+4,ff0000,red
+2,00ff00,green
+
+
## json-parse
diff --git a/docs/src/reference-verbs.md.in b/docs/src/reference-verbs.md.in
index 59d274ea1..4282bd80c 100644
--- a/docs/src/reference-verbs.md.in
+++ b/docs/src/reference-verbs.md.in
@@ -669,6 +669,26 @@ GENMD-RUN-COMMAND
mlr --csvlite --opprint join -j "" --lp left_ --rp right_ -f data/self-join.csv data/self-join.csv
GENMD-EOF
+By default, records with an empty-string value in a join field are joined just like any other value -- so two records which are both missing an ID, say, will be paired with one another even though that's rarely what's wanted:
+
+GENMD-RUN-COMMAND
+mlr --csv cat data/join-ignore-empty-left.csv
+GENMD-EOF
+
+GENMD-RUN-COMMAND
+mlr --csv cat data/join-ignore-empty-right.csv
+GENMD-EOF
+
+GENMD-RUN-COMMAND
+mlr --csv join -j id -f data/join-ignore-empty-left.csv data/join-ignore-empty-right.csv
+GENMD-EOF
+
+Use `--ignore-empty` to instead treat an empty-string join-field value as if the field were absent, on both the left and right files. Such records are never paired -- not even with one another:
+
+GENMD-RUN-COMMAND
+mlr --csv join --ignore-empty -j id -f data/join-ignore-empty-left.csv data/join-ignore-empty-right.csv
+GENMD-EOF
+
## json-parse
GENMD-RUN-COMMAND
diff --git a/man/manpage.txt b/man/manpage.txt
index 3415b2b84..feb351a26 100644
--- a/man/manpage.txt
+++ b/man/manpage.txt
@@ -1538,6 +1538,13 @@
--ul Emit unpaired records from the left file.
--ur Emit unpaired records from the right
file(s).
+ --ignore-empty Treat records with empty-string values in
+ any join-field as if that join-field were
+ absent, on both the left and right files.
+ Such records are never paired -- not even
+ with one another -- and are treated as
+ unpaired, subject to --np/--ul/--ur as
+ usual.
-s|--sorted-input Require sorted input: records must be
sorted lexically by their join-field names,
else not all records will be paired. The
diff --git a/man/mlr.1 b/man/mlr.1
index 621f3d71a..e246cde7f 100644
--- a/man/mlr.1
+++ b/man/mlr.1
@@ -1912,6 +1912,13 @@ Options:
--ul Emit unpaired records from the left file.
--ur Emit unpaired records from the right
file(s).
+--ignore-empty Treat records with empty-string values in
+ any join-field as if that join-field were
+ absent, on both the left and right files.
+ Such records are never paired -- not even
+ with one another -- and are treated as
+ unpaired, subject to --np/--ul/--ur as
+ usual.
-s|--sorted-input Require sorted input: records must be
sorted lexically by their join-field names,
else not all records will be paired. The
diff --git a/pkg/stream/stream.go b/pkg/stream/stream.go
index 40cc38bcc..c205b5677 100644
--- a/pkg/stream/stream.go
+++ b/pkg/stream/stream.go
@@ -104,6 +104,26 @@ func Stream(
}
}
+ // An error and the done-writing signal can be ready simultaneously, and
+ // select chooses among ready channels at random -- so an error may still
+ // be sitting in a buffer when the loop above exits. Senders guarantee the
+ // error is buffered before the end-of-stream marker that lets the writer
+ // finish, so a final non-blocking drain is sufficient to pick it up.
+ if retval == nil {
+ select {
+ case ierr := <-inputErrorChannel:
+ retval = ierr
+ default:
+ }
+ }
+ if retval == nil {
+ select {
+ case derr := <-dataProcessingErrorChannel:
+ retval = derr
+ default:
+ }
+ }
+
if err := bufferedOutputStream.Flush(); err != nil && retval == nil {
retval = err
}
diff --git a/pkg/transformers/aaa_chain_transformer.go b/pkg/transformers/aaa_chain_transformer.go
index 134aa11b2..32575ed07 100644
--- a/pkg/transformers/aaa_chain_transformer.go
+++ b/pkg/transformers/aaa_chain_transformer.go
@@ -213,24 +213,21 @@ func runSingleTransformer(
outputRecordChannel,
inputDownstreamDoneChannel,
outputDownstreamDoneChannel,
+ dataProcessingErrorChannel,
options,
)
if err != nil {
- // Surface the error to stream.Stream's select loop. Non-blocking
- // send: if another goroutine errored first, that error wins.
- select {
- case dataProcessingErrorChannel <- err:
- default:
- }
+ // runSingleTransformerBatch has already sent the error to
+ // dataProcessingErrorChannel and then forwarded an end-of-stream
+ // marker downstream, so the record-writer drains and finishes,
+ // upon which stream.Stream returns the error.
+ //
// Tell upstream (transformers and ultimately the record-reader,
// via the mlr-head mechanism) that we'll ignore further input.
select {
case outputDownstreamDoneChannel <- true:
default:
}
- // runSingleTransformerBatch has already forwarded an end-of-stream
- // marker downstream, so the record-writer drains and finishes,
- // upon which stream.Stream returns the error we sent above.
return
}
}
@@ -238,10 +235,10 @@ func runSingleTransformer(
// runSingleTransformerBatch passes one batch of records through the
// transformer. The boolean return is true on end of record stream. A non-nil
-// error is a mid-stream transformer failure: any output produced before the
-// failure, plus an end-of-stream marker, has already been forwarded
-// downstream so the rest of the chain and the record-writer can drain and
-// finish cleanly.
+// error is a mid-stream transformer failure: the error has already been sent
+// to dataProcessingErrorChannel, and any output produced before the failure,
+// plus an end-of-stream marker, has already been forwarded downstream so the
+// rest of the chain and the record-writer can drain and finish cleanly.
func runSingleTransformerBatch(
inputRecordsAndContexts []*types.RecordAndContext, // list of types.RecordAndContext
recordTransformer RecordTransformer,
@@ -249,6 +246,7 @@ func runSingleTransformerBatch(
outputRecordChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
+ dataProcessingErrorChannel chan<- error,
options *cli.TOptions,
) (bool, error) {
outputRecordsAndContexts := make([]*types.RecordAndContext, 0, len(inputRecordsAndContexts))
@@ -292,6 +290,17 @@ func runSingleTransformerBatch(
outputDownstreamDoneChannel,
)
if err != nil {
+ // Surface the error to stream.Stream's select loop.
+ // Non-blocking send: if another goroutine errored first, that
+ // error wins. This must happen before the end-of-stream
+ // marker is forwarded below: the marker lets the
+ // record-writer finish and signal done-writing, and the error
+ // must already be buffered by then or stream.Stream could
+ // return nil, losing the nonzero exit code.
+ select {
+ case dataProcessingErrorChannel <- err:
+ default:
+ }
// Forward what was produced before the failure, plus an
// end-of-stream marker so downstream drains and finishes.
outputRecordsAndContexts = append(outputRecordsAndContexts,
diff --git a/pkg/transformers/join.go b/pkg/transformers/join.go
index 8f23e09eb..2a5c45b0d 100644
--- a/pkg/transformers/join.go
+++ b/pkg/transformers/join.go
@@ -26,6 +26,7 @@ var joinOptions = []OptionSpec{
{Flag: "--np", Type: "bool", Desc: "Do not emit paired records."},
{Flag: "--ul", Type: "bool", Desc: "Emit unpaired records from the left file."},
{Flag: "--ur", Type: "bool", Desc: "Emit unpaired records from the right file(s)."},
+ {Flag: "--ignore-empty", Type: "bool", Desc: "Treat records with empty-string values in any join-field as if that join-field were absent, on both the left and right files. Such records are never paired -- not even with one another -- and are treated as unpaired, subject to --np/--ul/--ur as usual."},
{Flag: "-s", Aliases: []string{"--sorted-input"}, Type: "bool", Desc: "Require sorted input: records must be sorted lexically by their join-field names, else not all records will be paired. The only likely use case for this is with a left file which is too big to fit into system memory otherwise."},
{Flag: "-u", Type: "bool", Desc: "Enable unsorted input. (This is the default even without -u.) In this case, the entire left file will be loaded into memory."},
{Flag: "--prepipe", Arg: "{command}", Type: "string", Desc: "Shell command to prepipe the left-file input through. As in main input options; see mlr --help for details. If you wish to use a prepipe command for the main input as well as here, it must be specified there as well as here."},
@@ -54,10 +55,11 @@ type tJoinOptions struct {
leftJoinFieldNames []string
rightJoinFieldNames []string
- allowUnsortedInput bool
- emitPairables bool
- emitLeftUnpairables bool
- emitRightUnpairables bool
+ allowUnsortedInput bool
+ emitPairables bool
+ emitLeftUnpairables bool
+ emitRightUnpairables bool
+ ignoreEmptyJoinFields bool
leftFileName string
prepipe string
@@ -77,10 +79,11 @@ func newJoinOptions() *tJoinOptions {
leftJoinFieldNames: nil,
rightJoinFieldNames: nil,
- allowUnsortedInput: true,
- emitPairables: true,
- emitLeftUnpairables: false,
- emitRightUnpairables: false,
+ allowUnsortedInput: true,
+ emitPairables: true,
+ emitLeftUnpairables: false,
+ emitRightUnpairables: false,
+ ignoreEmptyJoinFields: false,
leftFileName: "",
prepipe: "",
@@ -222,6 +225,9 @@ func transformerJoinParseCLI(
case "--ur":
opts.emitRightUnpairables = true
+ case "--ignore-empty":
+ opts.ignoreEmptyJoinFields = true
+
case "-u":
opts.allowUnsortedInput = true
@@ -290,6 +296,18 @@ func transformerJoinParseCLI(
return transformer, nil
}
+// anyValueIsEmpty returns true if any of the given values is present but
+// empty-string (mlrval "void"). Used for --ignore-empty, which treats such
+// join-field values as though the field were absent altogether.
+func anyValueIsEmpty(values []*mlrval.Mlrval) bool {
+ for _, value := range values {
+ if value != nil && value.IsVoid() {
+ return true
+ }
+ }
+ return false
+}
+
type TransformerJoin struct {
opts *tJoinOptions
@@ -352,6 +370,7 @@ func NewTransformerJoin(
&opts.joinFlagOptions.ReaderOptions,
opts.leftJoinFieldNames,
tr.leftKeepFieldNameSet,
+ opts.ignoreEmptyJoinFields,
)
if err != nil {
return nil, err
@@ -400,6 +419,12 @@ func (tr *TransformerJoin) transformHalfStreaming(
groupingKey, hasAllJoinKeys := inrec.GetSelectedValuesJoined(
tr.opts.rightJoinFieldNames,
)
+ if hasAllJoinKeys && tr.opts.ignoreEmptyJoinFields {
+ rightFieldValues, _ := inrec.GetSelectedValues(tr.opts.rightJoinFieldNames)
+ if anyValueIsEmpty(rightFieldValues) {
+ hasAllJoinKeys = false
+ }
+ }
if hasAllJoinKeys {
leftBucket := tr.leftBucketsByJoinFieldValues.Get(groupingKey)
if leftBucket == nil {
@@ -447,6 +472,9 @@ func (tr *TransformerJoin) transformDoublyStreaming(
rightFieldValues, hasAllJoinKeys := rightRec.ReferenceSelectedValues(
tr.opts.rightJoinFieldNames,
)
+ if hasAllJoinKeys && tr.opts.ignoreEmptyJoinFields && anyValueIsEmpty(rightFieldValues) {
+ hasAllJoinKeys = false
+ }
if hasAllJoinKeys {
var err error
isPaired, err = keeper.FindJoinBucket(rightFieldValues)
@@ -569,6 +597,9 @@ func (tr *TransformerJoin) ingestLeftFile() error {
groupingKey, leftFieldValues, ok := leftrec.GetSelectedValuesAndJoined(
tr.opts.leftJoinFieldNames,
)
+ if ok && tr.opts.ignoreEmptyJoinFields && anyValueIsEmpty(leftFieldValues) {
+ ok = false
+ }
if ok {
bucket := tr.leftBucketsByJoinFieldValues.Get(groupingKey)
if bucket == nil { // New key-field-value: new bucket and hash-map entry
diff --git a/pkg/transformers/utils/join_bucket_keeper.go b/pkg/transformers/utils/join_bucket_keeper.go
index 0aba766b8..d069a0ced 100644
--- a/pkg/transformers/utils/join_bucket_keeper.go
+++ b/pkg/transformers/utils/join_bucket_keeper.go
@@ -124,8 +124,9 @@ type JoinBucketKeeper struct {
// TODO: merge with leof flag
recordReaderDone bool
- leftJoinFieldNames []string
- leftKeepFieldNameSet map[string]bool
+ leftJoinFieldNames []string
+ leftKeepFieldNameSet map[string]bool
+ ignoreEmptyJoinFields bool
// Given a left-file of the following form (with left-join-field name "L"):
// +-----+
@@ -157,6 +158,7 @@ func NewJoinBucketKeeper(
joinReaderOptions *cli.TReaderOptions,
leftJoinFieldNames []string,
leftKeepFieldNameSet map[string]bool,
+ ignoreEmptyJoinFields bool,
) (*JoinBucketKeeper, error) {
// Instantiate the record-reader
@@ -187,8 +189,9 @@ func NewJoinBucketKeeper(
errorChannel: errorChannel,
recordReaderDone: false,
- leftJoinFieldNames: leftJoinFieldNames,
- leftKeepFieldNameSet: leftKeepFieldNameSet,
+ leftJoinFieldNames: leftJoinFieldNames,
+ leftKeepFieldNameSet: leftKeepFieldNameSet,
+ ignoreEmptyJoinFields: ignoreEmptyJoinFields,
JoinBucket: NewJoinBucket(nil),
peekRecordAndContext: nil,
@@ -337,7 +340,7 @@ func (keeper *JoinBucketKeeper) prepareForFirstJoinBucket() error {
if keeper.peekRecordAndContext == nil { // left EOF
break
}
- if keeper.peekRecordAndContext.Record.HasSelectedKeys(keeper.leftJoinFieldNames) {
+ if recordHasJoinKeys(keeper.peekRecordAndContext.Record, keeper.leftJoinFieldNames, keeper.ignoreEmptyJoinFields) {
break
}
keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext)
@@ -412,7 +415,7 @@ func (keeper *JoinBucketKeeper) prepareForNewJoinBucket(
}
peekRec := keeper.peekRecordAndContext.Record
- if peekRec.HasSelectedKeys(keeper.leftJoinFieldNames) {
+ if recordHasJoinKeys(peekRec, keeper.leftJoinFieldNames, keeper.ignoreEmptyJoinFields) {
break
}
keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext)
@@ -485,6 +488,9 @@ func (keeper *JoinBucketKeeper) fillNextJoinBucket() error {
peekFieldValues, hasAllJoinKeys := peekRec.ReferenceSelectedValues(
keeper.leftJoinFieldNames,
)
+ if hasAllJoinKeys && keeper.ignoreEmptyJoinFields && valuesContainVoid(peekFieldValues) {
+ hasAllJoinKeys = false
+ }
if hasAllJoinKeys {
cmp := compareLexically(
@@ -595,6 +601,37 @@ func moveRecordsAndContexts(
*source = (*source)[:0]
}
+// recordHasJoinKeys reports whether rec has all of the given field names. If
+// ignoreEmptyJoinFields is set, a field holding an empty-string value counts
+// as absent, same as for --ignore-empty on the right-hand side of the join.
+func recordHasJoinKeys(
+ rec *mlrval.Mlrmap,
+ fieldNames []string,
+ ignoreEmptyJoinFields bool,
+) bool {
+ for _, fieldName := range fieldNames {
+ value := rec.Get(fieldName)
+ if value == nil {
+ return false
+ }
+ if ignoreEmptyJoinFields && value.IsVoid() {
+ return false
+ }
+ }
+ return true
+}
+
+// valuesContainVoid returns true if any of the given values is present but
+// empty-string (mlrval "void").
+func valuesContainVoid(values []*mlrval.Mlrval) bool {
+ for _, value := range values {
+ if value != nil && value.IsVoid() {
+ return true
+ }
+ }
+ return false
+}
+
// Returns -1, 0, 1 as left <, ==, > right, using lexical comparison only (even
// for numerical values).
diff --git a/plans/auto-format-input.md b/plans/auto-format-input.md
new file mode 100644
index 000000000..731401164
--- /dev/null
+++ b/plans/auto-format-input.md
@@ -0,0 +1,308 @@
+# Plan: auto-infer input format from file extension
+
+Feature request: [issue #1188](https://github.com/johnkerl/miller/issues/1188) — given
+`mlr ... mydata.csv`, default to `--icsv` without the user typing it; likewise `.tsv` →
+`--itsv`, etc.
+
+This is at heart a refactor of `pkg/stream/stream.go` and the record-reader layer, not a
+CLI-flag tweak. Today a single record-reader is constructed once and handed the full list
+of file names; with this feature, the reader (and its format-dependent option defaults)
+must be chosen **per input file**.
+
+Scope: **input side only.** Output-format inference is discussed under Open Questions but
+is recommended out of scope for v1, with one carve-out for in-place mode (see complication
+C9, which is a data-loss footgun if ignored).
+
+## Current architecture (survey)
+
+Where readers are constructed — one per process, not per file:
+
+- `pkg/stream/stream.go:51` — `input.Create(&options.ReaderOptions, ...)` once, then
+ `go recordReader.Read(fileNames, ...)` at `stream.go:88`.
+- `pkg/input/record_reader_factory.go:9` — `input.Create` switches on
+ `readerOptions.InputFileFormat` (csv, csvlite, dkvp, dkvpx, json, yaml, nidx,
+ markdown, pprint, tsv, xtab, dcf, recutils, gen).
+- Other `input.Create` call sites, each needing its own treatment:
+ - `pkg/transformers/join.go:510` and `pkg/transformers/utils/join_bucket_keeper.go:163`
+ — the join verb's left file, with its own `joinFlagOptions.ReaderOptions`.
+ - `pkg/terminals/repl/session.go:49` — REPL.
+ - `pkg/terminals/script/runner.go:24` — `mlr script` runner.
+
+How readers consume files — each reader owns the whole file loop:
+
+- `IRecordReader.Read(filenames []string, initialContext types.Context, ...)`
+ (`pkg/input/record_reader.go:14`) takes the *list*; every concrete reader duplicates
+ the same boilerplate: `filenames == nil` → no input (`mlr -n`); `len == 0` → stdin via
+ `lib.OpenStdin`; else loop with `lib.OpenFileForRead`, calling a per-file
+ `processHandle(handle, filename, &context, ...)` that resets per-file reader state and
+ calls `context.UpdateForStartOfFile` (see e.g. `pkg/input/record_reader_csv.go:54-108`).
+ Each `Read` sends exactly one end-of-stream marker after all files.
+- AWK-ish variables: `types.Context.UpdateForStartOfFile` (`pkg/types/context.go:132`)
+ increments FILENUM and resets FNR; NR accumulates across files. This continuity must
+ survive the move to per-file readers.
+- `downstreamDoneChannel` (mlr head fast-exit): readers poll it per batch
+ (`record_reader_csv.go:171`, `line_reader.go:202`). The signal is a **one-shot buffered
+ send**; whichever per-file scanner consumes it stops, and the file loop must remember
+ done-ness so it doesn't open remaining files. (Today the loop just proceeds to the next
+ file, whose scanner will never see the already-consumed signal — a latent inefficiency
+ the refactor should fix, not replicate.)
+
+Where format-dependent defaults are applied — once, at CLI-parse time:
+
+- `cli.FinalizeReaderOptions` (`pkg/cli/option_parse.go:31`) fills in IFS/IPS/IRS and
+ AllowRepeatIFS defaults keyed by `InputFileFormat` (with a special NIDX
+ whitespace-regex case), and un-hexes/un-backslashes the separator strings **by mutating
+ the one shared `TReaderOptions`**. Per-file formats mean per-format finalization; the
+ un-escaping steps must not be re-applied to already-processed values.
+- Reader constructors also validate format-specific constraints at construction time
+ (e.g. `NewRecordReaderCSV` at `record_reader_csv.go:34-43`: single-char IFS, IRS
+ restrictions, comment-string length) and cache derived state (`ifs0`).
+
+File opening / compression:
+
+- `lib.OpenFileForRead` (`pkg/lib/file_readers.go:49`): prepipe → popen; else encoding
+ flag; else suffix-sniffed decompression for `.gz`, `.bz2`, `.zst`
+ (`openEncodedHandleForRead`). `lib.PathToHandle` also supports `http://`, `https://`,
+ `file://` URLs.
+- In-place mode (`pkg/entrypoint/entrypoint.go:127-207`) already runs one full
+ `Stream()` per file, re-parsing the command line each time, and infers input
+ *compression* from the file name (`lib.FindInputEncoding`).
+
+## Proposed design
+
+### 1. Extension→format inference (pure, name-based — no content sniffing)
+
+New function, e.g. `input.InferFormatFromFileName(path string) (format string, ok bool)`:
+
+1. If the path is a URL (`http://`, `https://`, `file://`), strip scheme and any
+ `?query`/`#fragment` before looking at the suffix.
+2. Strip one trailing compression suffix (`.gz`, `.bz2`, `.zst`) — mirrors
+ `openEncodedHandleForRead` — so `data.csv.gz` infers csv.
+3. Map the remaining extension, case-insensitively:
+
+ | extension | format |
+ |---|---|
+ | `.csv` | `csv` |
+ | `.tsv` | `tsv` |
+ | `.json`, `.jsonl`, `.ndjson` | `json` (the JSON reader already handles both) |
+ | `.yaml`, `.yml` | `yaml` |
+ | `.md`, `.markdown` | `markdown` |
+ | `.dkvp` | `dkvp` |
+ | `.nidx` | `nidx` |
+ | `.xtab` | `xtab` |
+ | `.pprint` | `pprint` |
+ | `.rec` | `recutils` |
+ | `.dcf` | `dcf` |
+
+ Deliberately unmapped: `.txt`, `.dat`, `.log`, and anything else ambiguous — those
+ return `ok=false` and take the fallback (below). `.csv` maps to full `csv`, not
+ `csvlite` (users who want csvlite say so explicitly).
+
+Because inference is name-based only, **all per-file formats can be resolved up front**,
+before any goroutine starts — so option-validation errors surface before any output is
+produced, not mid-stream after file 3 of 7.
+
+### 2. Policy: stdin and fallback
+
+- **stdin: no inference** (per maintainer). `len(filenames)==0` uses the fallback format.
+- **Fallback** for stdin and unmapped extensions: the default format, `dkvp`. (Open
+ question Q3 discusses erroring instead.)
+- `mlr -n` (nil filenames): unaffected.
+
+### 3. CLI surface: opt-in flag, not a default flip
+
+- Accept `auto` as an `InputFileFormat` value: `-i auto`, plus a dedicated `--iauto` flag
+ in `FileFormatFlagSection` (`pkg/cli/option_parse.go:844`). Last-one-wins with other
+ format flags, per existing CLI semantics: `--icsv --iauto` means auto.
+- `.mlrrc` gives users a "make it my default" path (`iauto` on a line by itself), since
+ .mlrrc lines are flags without leading dashes. This is the adoption path in lieu of
+ changing the built-in default, which would silently break existing scripts that rely on
+ `mlr cat foo.csv` parsing as DKVP (see Q1).
+- `-o auto` / `--io auto` are **errors** in v1 (output can't be name-inferred; it usually
+ goes to stdout). Revisit under Q2.
+
+### 4. The stream.go / record-reader refactor
+
+Two-layer split — this is the bulk of the work, and is a worthwhile cleanup even
+independent of the feature (it deletes ~13 copies of the same stdin/loop/open
+boilerplate):
+
+- **Per-file readers**: each concrete reader keeps its constructor and its
+ `processHandle`-shaped method; the `Read(filenames, ...)` file loop is deleted from
+ all of them. New narrower interface, roughly:
+
+ ```go
+ type IFileRecordReader interface {
+ ProcessHandle(handle io.Reader, filename string, context *types.Context,
+ readerChannel chan<- []*types.RecordAndContext,
+ errorChannel chan error,
+ downstreamDoneChannel <-chan bool)
+ }
+ ```
+
+ (Most readers already have exactly this method; the change is mostly mechanical.
+ Signature detail to settle during implementation: some readers need to report
+ "downstream done" back to the caller so the driver stops opening further files —
+ probably a `bool` return.)
+
+- **Driver**: one new `FileStreamReader` in `pkg/input` implementing the existing
+ `IRecordReader` interface, owning: the nil/stdin/file-list branching, per-file
+ `OpenFileForRead`/`OpenStdin` + close, `context.UpdateForStartOfFile`, the
+ downstream-done latch across files, and the single end-of-stream marker.
+
+ It is constructed with a pre-resolved plan: `[]struct{ fileName string; reader
+ IFileRecordReader }` (plus a stdin entry when applicable). In non-auto mode that's the
+ same reader for every file — behavior identical to today. In auto mode it's built by
+ running inference per file at setup time.
+
+- **`gen` pseudo-reader** (`pseudo_reader_gen.go`) reads no files; it keeps implementing
+ `IRecordReader` directly and bypasses the driver.
+
+- **Factory**: `input.Create` grows a companion, e.g.
+ `input.CreateForFileNames(readerOptions, recordsPerBatch, fileNames) (IRecordReader, error)`,
+ which handles the auto/non-auto/gen dispatch and becomes what `stream.go`, join, REPL,
+ and script-runner call.
+
+### 5. Per-format reader options
+
+Refactor `FinalizeReaderOptions` so format-dependent defaulting is a pure derivation
+rather than a one-shot mutation:
+
+- Split into (a) a once-only un-escaping step on user-supplied separator strings
+ (unhex/unbackslash — must not run twice), done at CLI-parse time as now; and (b)
+ `deriveReaderOptionsForFormat(base *TReaderOptions, format string) (*TReaderOptions, error)`
+ which returns a **copy** with IFS/IPS/IRS/AllowRepeatIFS defaults (and the NIDX
+ whitespace-regex special case) applied for that format, honoring the
+ `*WasSpecified` booleans so explicit `--ifs` etc. still win for every inferred format.
+- In non-auto mode, (b) is called once with the single format — same net behavior as
+ today. In auto mode, called once per distinct inferred format (cache in a
+ `map[string]*TReaderOptions`), then one concrete reader constructed per distinct
+ format (readers already reset per-file state in `processHandle`, so sharing one reader
+ across same-format files is safe and matches current cross-file behavior).
+- When `InputFileFormat == "auto"` reaches finalize-time, skip the format-keyed lookups
+ (they'd fail on the `defaultFSes` map) — defaults are applied per derived format
+ instead.
+
+## Complications inventory
+
+Ones already flagged by the maintainer:
+
+- **C1 — one reader for all files** → per-file (per-format) construction; addressed by §4.
+- **C2 — stdin** → no inference, fallback format; addressed by §2.
+
+Additional ones surfaced by this survey:
+
+- **C3 — format-dependent option defaults are baked in at CLI-parse time.**
+ IFS/IPS/IRS/AllowRepeatIFS defaults differ per format and are applied by mutating the
+ single shared `TReaderOptions` (`option_parse.go:31-84`). `mlr --iauto cat a.csv b.nidx`
+ needs comma-IFS for one file and whitespace-regex-IFS for the other. Needs the
+ derive-per-format refactor (§5), including not double-applying unhex/unbackslash.
+- **C4 — constructor-time validation and cached state.** Reader constructors validate
+ and cache options (`record_reader_csv.go:34-51`). Mitigation: resolve formats and
+ construct all readers eagerly at stream setup (possible because inference is
+ name-based), so `mlr --iauto --ifs ';;' cat a.csv` fails before any records flow.
+- **C5 — AWK-variable continuity across heterogeneous readers.** FILENUM/NR must keep
+ accumulating when consecutive files use different readers. Solved by the driver owning
+ one `types.Context` and passing it into each per-file read; also exactly one
+ end-of-stream marker, sent by the driver.
+- **C6 — `downstreamDoneChannel` is a one-shot signal.** Once a per-file scanner consumes
+ it, later files can't see it. The driver must latch done-ness (via the per-file return,
+ C4's interface note) and stop opening subsequent files. Today's per-reader loops appear
+ to keep reading subsequent files after `head` is satisfied — the refactor should fix
+ this, and it's worth a regression test (`mlr head -n 1 big1.csv big2.csv` should not
+ read big2 to completion).
+- **C7 — compressed and URL inputs.** `data.csv.gz` must infer csv (strip compression
+ suffix, mirroring `openEncodedHandleForRead`); URLs need scheme/query stripping before
+ suffix inspection. `--prepipe 'unzip -qc' foo.zip` → `.zip` unmapped → fallback (fine;
+ prepipe users can state the format explicitly).
+- **C8 — the join verb reads its own file.** `join -f left.csv` via
+ `ingestLeftFile` (`join.go:506`) and the half-streaming
+ `join_bucket_keeper.go:163` use `joinFlagOptions.ReaderOptions`, which inherit main
+ reader options unless overridden by join's own `-i`. If the inherited/derived format is
+ `auto`, these paths must run the same resolve-then-create helper (trivial: single known
+ file name, resolvable up front). Without this, `-i auto` at main level would hit
+ `input.Create`'s `default:` error ("input file format \"auto\" not found") inside join.
+- **C9 — in-place mode (`mlr -I`) is a data-loss footgun with auto.** `-I` writes output
+ back over the input file using the *output* format. `mlr -I --iauto put ... foo.csv`
+ with default DKVP output would silently rewrite a CSV file as DKVP. v1 must do one of:
+ (a) error on `-I` + auto unless an explicit output format is given, or (b) per file, set
+ the writer format to the inferred reader format when no explicit `-o` was given —
+ natural since `processFileInPlace` already re-parses options per file
+ (`entrypoint.go:161`). Recommend (b); (a) is an acceptable stopgap. Either way this
+ must not ship as "whatever falls out".
+- **C10 — REPL and `mlr script`.** `repl/session.go:49` constructs its reader at session
+ start, before any `:open file` — with auto, resolution has to happen per opened file.
+ Simplest v1: REPL/script reject or ignore `auto` with a clear message, or resolve at
+ `:open` time using the same helper. Decide during implementation; don't leave it
+ crashing on the factory `default:` case.
+- **C11 — mixed formats in one run are now easy to trigger.** Heterogeneous records are
+ native to Miller, so `mlr --iauto cat a.csv b.json` "just works", but users will see
+ format-specific side effects (e.g. CSV output emitting new header blocks on schema
+ change). Docs should show a mixed-format example. Also note format-specific input flags
+ (`--allow-ragged-csv-input`, `--csv-trim-leading-space`, implicit-header, etc.) apply
+ whenever the *inferred* format is csv/tsv — harmless for other formats, worth a doc
+ sentence.
+- **C12 — surprise on the output side.** `mlr --iauto cat data.csv` prints DKVP to
+ stdout. Harmless (visible immediately) but guaranteed to generate "it doesn't work"
+ reports from the very users #1188 is for. See Q2.
+- **C13 — factory/validation error paths.** `"auto"` must be handled everywhere
+ `InputFileFormat` is switch/map-keyed: `input.Create` default case,
+ `FinalizeReaderOptions` map lookups, `pkg/cli/flatten_unflatten.go:99` (auto-flatten
+ decides based on input format — with auto, the per-format derived options must carry
+ the resolved format so flatten/unflatten heuristics see `csv`, not `auto`; audit other
+ `InputFileFormat` consumers with `grep -rn InputFileFormat pkg/`).
+- **C14 — the `dkvpx`/fixed-width/barred-pprint variants.** Inference only ever selects
+ canonical formats; variant selection (`BarredPprintInput`, `FixedWidthSpec`, `dkvpx`)
+ stays explicit-flag-only. No `.pprint`-barred inference.
+- **C15 — case/AV edge cases in names.** Uppercase extensions (`.CSV`), files with no
+ extension, dotfiles (`.csv` as an entire filename — treat as extensionless), a literal
+ `-` filename (Miller doesn't special-case it today; keep it that way), Windows path
+ separators. All belong in the inference unit tests.
+
+## Phased implementation
+
+Phases are separately mergeable, each leaving `make check` green.
+
+1. **Reader-loop extraction (pure refactor, no behavior change).**
+ Introduce `IFileRecordReader` + `FileStreamReader` driver; delete the per-reader file
+ loops; keep `input.Create` signature; fix the C6 done-latch as part of the driver.
+ This is the risky/bulky phase — all 13 readers touched mechanically. Regression suite
+ is the safety net; add the C6 test here.
+2. **Inference function + options derivation.**
+ `InferFormatFromFileName` with unit tests (C7, C15 cases);
+ `deriveReaderOptionsForFormat` refactor of `FinalizeReaderOptions` with unit tests
+ proving explicit `--ifs/--ips/--irs` still override per derived format.
+3. **Wire up `-i auto` / `--iauto` for the main stream.**
+ Flag-table entry, `"auto"` handling at all C13 sites, eager per-file resolution in
+ stream setup, fallback policy (§2). Regression cases: per-extension inference,
+ mixed formats, `.csv.gz`, unmapped extension → dkvp, stdin → dkvp, explicit separator
+ overrides under auto.
+4. **Secondary consumers: join, in-place, REPL/script.**
+ C8 (join left file), C9 (in-place policy — implement (b) or (a)), C10 (REPL/script).
+ Regression cases for each, including the C9 "must not rewrite csv as dkvp" case.
+5. **Docs.**
+ `docs/src/file-formats.md.in` section on auto-inference (extension table, stdin/
+ fallback rules, .mlrrc adoption tip, mixed-format example); flag help text (feeds the
+ auto-generated `reference-main-flag-list`); `mlr help` topics; man page regen via
+ `make dev`.
+
+## Open questions for the maintainer
+
+- **Q1 — default-on vs opt-in.** #1188 asks for this as *default* behavior. That flips
+ parsing of `mlr cat foo.csv` from DKVP to CSV — behavior-breaking for scripts (however
+ few) that depend on it. Recommendation: ship opt-in (`--iauto`, .mlrrc-able) now;
+ consider flipping the default in a major release after the machinery has soaked.
+- **Q2 — output-side inference.** Options: (a) none (v1 recommendation, minus the C9
+ in-place carve-out); (b) `--auto` convenience flag = infer input per file *and*, when
+ no explicit output format was given and all inputs infer to a single common format, use
+ that for output too (resolvable up front since inference is name-based; must define
+ behavior for mixed inputs — probably fall back to dkvp or error). (b) is what
+ #1188-style users likely actually want day-to-day; fine as a fast-follow.
+- **Q3 — unmapped extension under auto: fallback to dkvp, or error?** Fallback is
+ forgiving and matches the stdin story; erroring is more predictable ("you asked for
+ auto and I can't tell what `.dat` is"). Recommendation: fallback + document; a strict
+ variant can come later if requested.
+- **Q4 — reuse one reader per distinct format vs one per file.** Plan assumes per-format
+ reuse (matches today's cross-file behavior exactly, since readers reset state per
+ file). Per-file construction is marginally simpler to reason about but re-runs
+ validation redundantly. Low stakes either way; decide in phase 3.
diff --git a/test/cases/cli-help/0001/expout b/test/cases/cli-help/0001/expout
index d513c22ac..51548e827 100644
--- a/test/cases/cli-help/0001/expout
+++ b/test/cases/cli-help/0001/expout
@@ -562,6 +562,13 @@ Options:
--ul Emit unpaired records from the left file.
--ur Emit unpaired records from the right
file(s).
+--ignore-empty Treat records with empty-string values in
+ any join-field as if that join-field were
+ absent, on both the left and right files.
+ Such records are never paired -- not even
+ with one another -- and are treated as
+ unpaired, subject to --np/--ul/--ur as
+ usual.
-s|--sorted-input Require sorted input: records must be
sorted lexically by their join-field names,
else not all records will be paired. The
diff --git a/test/cases/verb-join/ignore-empty-basic/cmd b/test/cases/verb-join/ignore-empty-basic/cmd
new file mode 100644
index 000000000..4d73970e6
--- /dev/null
+++ b/test/cases/verb-join/ignore-empty-basic/cmd
@@ -0,0 +1 @@
+mlr --csv join --ignore-empty -j id -f test/input/join-1194-left.csv test/input/join-1194-right.csv
diff --git a/test/cases/verb-join/ignore-empty-basic/experr b/test/cases/verb-join/ignore-empty-basic/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-join/ignore-empty-basic/expout b/test/cases/verb-join/ignore-empty-basic/expout
new file mode 100644
index 000000000..9e8af04d2
--- /dev/null
+++ b/test/cases/verb-join/ignore-empty-basic/expout
@@ -0,0 +1,3 @@
+id,code,color
+4,ff0000,red
+2,00ff00,green
diff --git a/test/cases/verb-join/ignore-empty-sorted/cmd b/test/cases/verb-join/ignore-empty-sorted/cmd
new file mode 100644
index 000000000..920da6a1b
--- /dev/null
+++ b/test/cases/verb-join/ignore-empty-sorted/cmd
@@ -0,0 +1 @@
+mlr --csv join -s --ignore-empty -j id -f test/input/join-1194-left-sorted.csv test/input/join-1194-right-sorted.csv
diff --git a/test/cases/verb-join/ignore-empty-sorted/experr b/test/cases/verb-join/ignore-empty-sorted/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-join/ignore-empty-sorted/expout b/test/cases/verb-join/ignore-empty-sorted/expout
new file mode 100644
index 000000000..157c4fc20
--- /dev/null
+++ b/test/cases/verb-join/ignore-empty-sorted/expout
@@ -0,0 +1,3 @@
+id,code,color
+2,00ff00,green
+4,ff0000,red
diff --git a/test/cases/verb-join/ignore-empty-unpaired/cmd b/test/cases/verb-join/ignore-empty-unpaired/cmd
new file mode 100644
index 000000000..d805bb4b1
--- /dev/null
+++ b/test/cases/verb-join/ignore-empty-unpaired/cmd
@@ -0,0 +1 @@
+mlr --icsv --ojson join --ignore-empty --ul --ur -j id -f test/input/join-1194-left.csv test/input/join-1194-right.csv
diff --git a/test/cases/verb-join/ignore-empty-unpaired/experr b/test/cases/verb-join/ignore-empty-unpaired/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-join/ignore-empty-unpaired/expout b/test/cases/verb-join/ignore-empty-unpaired/expout
new file mode 100644
index 000000000..738200530
--- /dev/null
+++ b/test/cases/verb-join/ignore-empty-unpaired/expout
@@ -0,0 +1,32 @@
+[
+{
+ "id": 4,
+ "code": "ff0000",
+ "color": "red"
+},
+{
+ "id": 2,
+ "code": "00ff00",
+ "color": "green"
+},
+{
+ "id": "",
+ "color": "white"
+},
+{
+ "id": "",
+ "color": "black"
+},
+{
+ "id": 3,
+ "code": "0000ff"
+},
+{
+ "id": "",
+ "code": "ffffff"
+},
+{
+ "id": "",
+ "code": "000000"
+}
+]
diff --git a/test/input/join-1194-left-sorted.csv b/test/input/join-1194-left-sorted.csv
new file mode 100644
index 000000000..bebe0ddbd
--- /dev/null
+++ b/test/input/join-1194-left-sorted.csv
@@ -0,0 +1,6 @@
+id,code
+,ffffff
+,000000
+2,00ff00
+3,0000ff
+4,ff0000
diff --git a/test/input/join-1194-left.csv b/test/input/join-1194-left.csv
new file mode 100644
index 000000000..171fd1d5c
--- /dev/null
+++ b/test/input/join-1194-left.csv
@@ -0,0 +1,6 @@
+id,code
+3,0000ff
+2,00ff00
+4,ff0000
+,ffffff
+,000000
diff --git a/test/input/join-1194-right-sorted.csv b/test/input/join-1194-right-sorted.csv
new file mode 100644
index 000000000..abbaf64a6
--- /dev/null
+++ b/test/input/join-1194-right-sorted.csv
@@ -0,0 +1,5 @@
+id,color
+,white
+,black
+2,green
+4,red
diff --git a/test/input/join-1194-right.csv b/test/input/join-1194-right.csv
new file mode 100644
index 000000000..badd61587
--- /dev/null
+++ b/test/input/join-1194-right.csv
@@ -0,0 +1,5 @@
+id,color
+4,red
+2,green
+,white
+,black