From b515e6f18a30e8e0d0384405ac6a0dba7fd1b199 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 6 Jul 2026 15:43:57 -0400 Subject: [PATCH] Say so in the error message when a CSV/TSV length-mismatch line is blank (#1572) Trailing blank lines at the end of a CSV/TSV file produce a header/data length mismatch error which, especially for large files, gives no clue that the offending line is simply blank. Since #2146 (issue #1535), the skip-trivial-records verb is an opt-in way to have such lines skipped -- but users hitting the error had no pointer to it. Now, when the mismatching input line is blank, the CSV and TSV readers append a hint to the error message: mlr: CSV header/data length mismatch 2 != 1 at filename tmp.csv row 4; this input line is blank -- use the skip-trivial-records verb to have blank input lines skipped Errors for genuinely ragged, non-blank lines are unchanged, as is all non-error behavior. Also adds a note to the CSV/JSON troubleshooting docs page about trailing blank lines and skip-trivial-records. Addresses #1572. Co-Authored-By: Claude Fable 5 --- docs/src/troubleshooting-csv-and-json-input.md | 4 ++++ docs/src/troubleshooting-csv-and-json-input.md.in | 4 ++++ pkg/input/record_reader.go | 12 ++++++++++++ pkg/input/record_reader_csv.go | 4 ++-- pkg/input/record_reader_tsv.go | 3 ++- test/cases/io-skip-pass-comments/pr-1346/experr | 2 +- test/cases/io-spec-tsv/0004/experr | 2 +- test/cases/verb-skip-trivial-records/0005/experr | 2 +- test/cases/verb-skip-trivial-records/0009/cmd | 1 + test/cases/verb-skip-trivial-records/0009/experr | 1 + test/cases/verb-skip-trivial-records/0009/expout | 2 ++ test/cases/verb-skip-trivial-records/0009/input.tsv | 4 ++++ .../cases/verb-skip-trivial-records/0009/should-fail | 0 13 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 test/cases/verb-skip-trivial-records/0009/cmd create mode 100644 test/cases/verb-skip-trivial-records/0009/experr create mode 100644 test/cases/verb-skip-trivial-records/0009/expout create mode 100644 test/cases/verb-skip-trivial-records/0009/input.tsv create mode 100644 test/cases/verb-skip-trivial-records/0009/should-fail diff --git a/docs/src/troubleshooting-csv-and-json-input.md b/docs/src/troubleshooting-csv-and-json-input.md index 449b4a854..10848b3a1 100644 --- a/docs/src/troubleshooting-csv-and-json-input.md +++ b/docs/src/troubleshooting-csv-and-json-input.md @@ -27,6 +27,10 @@ When Miller reports a parse error, it is often useful to first identify whether `--csvlite` or `--icsvlite` instead. See also [File Formats](file-formats.md). - If a CSV error mentions a "header/data length mismatch", inspect the reported input line for an unquoted comma, a missing closing quote, or an unexpected extra delimiter. +- A common cause of header/data length mismatches is one or more blank lines at the end of the + file -- in that case the error message says the input line is blank. Use the + `skip-trivial-records` verb to have blank input lines skipped, e.g. `mlr --csv + skip-trivial-records then cat file.csv`. - For CSV generated by spreadsheets or hand-written scripts, `--csv-trim-leading-space` can help when fields look like `"foo", "bar"`, where the second field has a leading space before the quote. - For non-standard CSV that contains stray quote characters, `--lazy-quotes` can help Miller accept diff --git a/docs/src/troubleshooting-csv-and-json-input.md.in b/docs/src/troubleshooting-csv-and-json-input.md.in index 1bcdd3074..4a50d05b4 100644 --- a/docs/src/troubleshooting-csv-and-json-input.md.in +++ b/docs/src/troubleshooting-csv-and-json-input.md.in @@ -11,6 +11,10 @@ When Miller reports a parse error, it is often useful to first identify whether `--csvlite` or `--icsvlite` instead. See also [File Formats](file-formats.md). - If a CSV error mentions a "header/data length mismatch", inspect the reported input line for an unquoted comma, a missing closing quote, or an unexpected extra delimiter. +- A common cause of header/data length mismatches is one or more blank lines at the end of the + file -- in that case the error message says the input line is blank. Use the + `skip-trivial-records` verb to have blank input lines skipped, e.g. `mlr --csv + skip-trivial-records then cat file.csv`. - For CSV generated by spreadsheets or hand-written scripts, `--csv-trim-leading-space` can help when fields look like `"foo", "bar"`, where the second field has a leading space before the quote. - For non-standard CSV that contains stray quote characters, `--lazy-quotes` can help Miller accept diff --git a/pkg/input/record_reader.go b/pkg/input/record_reader.go index f70138750..9d19a6de4 100644 --- a/pkg/input/record_reader.go +++ b/pkg/input/record_reader.go @@ -34,3 +34,15 @@ func hasNonEmptyField(fields []string) bool { } return false } + +// blankInputLineHint returns a hint to be appended to CSV/TSV header/data +// length mismatch errors when the offending input line is blank -- e.g. one +// or more blank lines at the end of the file. Without this, the bare +// mismatch error can be hard to act on, especially for large files. See +// issue #1572. +func blankInputLineHint(fields []string) string { + if hasNonEmptyField(fields) { + return "" + } + return "; this input line is blank -- use the skip-trivial-records verb to have blank input lines skipped" +} diff --git a/pkg/input/record_reader_csv.go b/pkg/input/record_reader_csv.go index 585f2e6ef..e3a7fb4d8 100644 --- a/pkg/input/record_reader_csv.go +++ b/pkg/input/record_reader_csv.go @@ -269,8 +269,8 @@ func (reader *RecordReaderCSV) getRecordBatch( continue } err := fmt.Errorf( - "CSV header/data length mismatch %d != %d at filename %s row %d", - nh, nd, reader.filename, reader.rowNumber, + "CSV header/data length mismatch %d != %d at filename %s row %d%s", + nh, nd, reader.filename, reader.rowNumber, blankInputLineHint(csvRecord), ) errorChannel <- err return diff --git a/pkg/input/record_reader_tsv.go b/pkg/input/record_reader_tsv.go index e16dc42d3..258fb3a81 100644 --- a/pkg/input/record_reader_tsv.go +++ b/pkg/input/record_reader_tsv.go @@ -193,8 +193,9 @@ func getRecordBatchExplicitTSVHeader( continue } err := fmt.Errorf( - "TSV header/data length mismatch %d != %d at filename %s line %d", + "TSV header/data length mismatch %d != %d at filename %s line %d%s", len(reader.headerStrings), len(fields), filename, reader.inputLineNumber, + blankInputLineHint(fields), ) errorChannel <- err return diff --git a/test/cases/io-skip-pass-comments/pr-1346/experr b/test/cases/io-skip-pass-comments/pr-1346/experr index 112319123..4f8ce5071 100644 --- a/test/cases/io-skip-pass-comments/pr-1346/experr +++ b/test/cases/io-skip-pass-comments/pr-1346/experr @@ -1 +1 @@ -mlr: CSV header/data length mismatch 2 != 1 at filename test/input/pr-1346.csv row 4 +mlr: CSV header/data length mismatch 2 != 1 at filename test/input/pr-1346.csv row 4; this input line is blank -- use the skip-trivial-records verb to have blank input lines skipped diff --git a/test/cases/io-spec-tsv/0004/experr b/test/cases/io-spec-tsv/0004/experr index 6911642d4..22070eaff 100644 --- a/test/cases/io-spec-tsv/0004/experr +++ b/test/cases/io-spec-tsv/0004/experr @@ -1 +1 @@ -mlr: TSV header/data length mismatch 1 != 0 at filename test/cases/io-spec-tsv/0004/single-column-with-blank.tsv line 4 +mlr: TSV header/data length mismatch 1 != 0 at filename test/cases/io-spec-tsv/0004/single-column-with-blank.tsv line 4; this input line is blank -- use the skip-trivial-records verb to have blank input lines skipped diff --git a/test/cases/verb-skip-trivial-records/0005/experr b/test/cases/verb-skip-trivial-records/0005/experr index 267d382e9..2799839e9 100644 --- a/test/cases/verb-skip-trivial-records/0005/experr +++ b/test/cases/verb-skip-trivial-records/0005/experr @@ -1 +1 @@ -mlr: CSV header/data length mismatch 2 != 1 at filename test/cases/verb-skip-trivial-records/0005/input.csv row 3 +mlr: CSV header/data length mismatch 2 != 1 at filename test/cases/verb-skip-trivial-records/0005/input.csv row 3; this input line is blank -- use the skip-trivial-records verb to have blank input lines skipped diff --git a/test/cases/verb-skip-trivial-records/0009/cmd b/test/cases/verb-skip-trivial-records/0009/cmd new file mode 100644 index 000000000..0d5071d21 --- /dev/null +++ b/test/cases/verb-skip-trivial-records/0009/cmd @@ -0,0 +1 @@ +mlr --itsv --otsv cat ${CASEDIR}/input.tsv diff --git a/test/cases/verb-skip-trivial-records/0009/experr b/test/cases/verb-skip-trivial-records/0009/experr new file mode 100644 index 000000000..1dea30946 --- /dev/null +++ b/test/cases/verb-skip-trivial-records/0009/experr @@ -0,0 +1 @@ +mlr: TSV header/data length mismatch 2 != 0 at filename test/cases/verb-skip-trivial-records/0009/input.tsv line 3; this input line is blank -- use the skip-trivial-records verb to have blank input lines skipped diff --git a/test/cases/verb-skip-trivial-records/0009/expout b/test/cases/verb-skip-trivial-records/0009/expout new file mode 100644 index 000000000..8b26211c1 --- /dev/null +++ b/test/cases/verb-skip-trivial-records/0009/expout @@ -0,0 +1,2 @@ +a b +1 2 diff --git a/test/cases/verb-skip-trivial-records/0009/input.tsv b/test/cases/verb-skip-trivial-records/0009/input.tsv new file mode 100644 index 000000000..575273049 --- /dev/null +++ b/test/cases/verb-skip-trivial-records/0009/input.tsv @@ -0,0 +1,4 @@ +a b +1 2 + + diff --git a/test/cases/verb-skip-trivial-records/0009/should-fail b/test/cases/verb-skip-trivial-records/0009/should-fail new file mode 100644 index 000000000..e69de29bb