diff --git a/docs/src/troubleshooting-csv-and-json-input.md b/docs/src/troubleshooting-csv-and-json-input.md index 449b4a854..10848b3a1 100644 --- a/docs/src/troubleshooting-csv-and-json-input.md +++ b/docs/src/troubleshooting-csv-and-json-input.md @@ -27,6 +27,10 @@ When Miller reports a parse error, it is often useful to first identify whether `--csvlite` or `--icsvlite` instead. See also [File Formats](file-formats.md). - If a CSV error mentions a "header/data length mismatch", inspect the reported input line for an unquoted comma, a missing closing quote, or an unexpected extra delimiter. +- A common cause of header/data length mismatches is one or more blank lines at the end of the + file -- in that case the error message says the input line is blank. Use the + `skip-trivial-records` verb to have blank input lines skipped, e.g. `mlr --csv + skip-trivial-records then cat file.csv`. - For CSV generated by spreadsheets or hand-written scripts, `--csv-trim-leading-space` can help when fields look like `"foo", "bar"`, where the second field has a leading space before the quote. - For non-standard CSV that contains stray quote characters, `--lazy-quotes` can help Miller accept diff --git a/docs/src/troubleshooting-csv-and-json-input.md.in b/docs/src/troubleshooting-csv-and-json-input.md.in index 1bcdd3074..4a50d05b4 100644 --- a/docs/src/troubleshooting-csv-and-json-input.md.in +++ b/docs/src/troubleshooting-csv-and-json-input.md.in @@ -11,6 +11,10 @@ When Miller reports a parse error, it is often useful to first identify whether `--csvlite` or `--icsvlite` instead. See also [File Formats](file-formats.md). - If a CSV error mentions a "header/data length mismatch", inspect the reported input line for an unquoted comma, a missing closing quote, or an unexpected extra delimiter. +- A common cause of header/data length mismatches is one or more blank lines at the end of the + file -- in that case the error message says the input line is blank. Use the + `skip-trivial-records` verb to have blank input lines skipped, e.g. `mlr --csv + skip-trivial-records then cat file.csv`. - For CSV generated by spreadsheets or hand-written scripts, `--csv-trim-leading-space` can help when fields look like `"foo", "bar"`, where the second field has a leading space before the quote. - For non-standard CSV that contains stray quote characters, `--lazy-quotes` can help Miller accept diff --git a/pkg/input/record_reader.go b/pkg/input/record_reader.go index f70138750..9d19a6de4 100644 --- a/pkg/input/record_reader.go +++ b/pkg/input/record_reader.go @@ -34,3 +34,15 @@ func hasNonEmptyField(fields []string) bool { } return false } + +// blankInputLineHint returns a hint to be appended to CSV/TSV header/data +// length mismatch errors when the offending input line is blank -- e.g. one +// or more blank lines at the end of the file. Without this, the bare +// mismatch error can be hard to act on, especially for large files. See +// issue #1572. +func blankInputLineHint(fields []string) string { + if hasNonEmptyField(fields) { + return "" + } + return "; this input line is blank -- use the skip-trivial-records verb to have blank input lines skipped" +} diff --git a/pkg/input/record_reader_csv.go b/pkg/input/record_reader_csv.go index 585f2e6ef..e3a7fb4d8 100644 --- a/pkg/input/record_reader_csv.go +++ b/pkg/input/record_reader_csv.go @@ -269,8 +269,8 @@ func (reader *RecordReaderCSV) getRecordBatch( continue } err := fmt.Errorf( - "CSV header/data length mismatch %d != %d at filename %s row %d", - nh, nd, reader.filename, reader.rowNumber, + "CSV header/data length mismatch %d != %d at filename %s row %d%s", + nh, nd, reader.filename, reader.rowNumber, blankInputLineHint(csvRecord), ) errorChannel <- err return diff --git a/pkg/input/record_reader_tsv.go b/pkg/input/record_reader_tsv.go index e16dc42d3..258fb3a81 100644 --- a/pkg/input/record_reader_tsv.go +++ b/pkg/input/record_reader_tsv.go @@ -193,8 +193,9 @@ func getRecordBatchExplicitTSVHeader( continue } err := fmt.Errorf( - "TSV header/data length mismatch %d != %d at filename %s line %d", + "TSV header/data length mismatch %d != %d at filename %s line %d%s", len(reader.headerStrings), len(fields), filename, reader.inputLineNumber, + blankInputLineHint(fields), ) errorChannel <- err return diff --git a/test/cases/io-skip-pass-comments/pr-1346/experr b/test/cases/io-skip-pass-comments/pr-1346/experr index 112319123..4f8ce5071 100644 --- a/test/cases/io-skip-pass-comments/pr-1346/experr +++ b/test/cases/io-skip-pass-comments/pr-1346/experr @@ -1 +1 @@ -mlr: CSV header/data length mismatch 2 != 1 at filename test/input/pr-1346.csv row 4 +mlr: CSV header/data length mismatch 2 != 1 at filename test/input/pr-1346.csv row 4; this input line is blank -- use the skip-trivial-records verb to have blank input lines skipped diff --git a/test/cases/io-spec-tsv/0004/experr b/test/cases/io-spec-tsv/0004/experr index 6911642d4..22070eaff 100644 --- a/test/cases/io-spec-tsv/0004/experr +++ b/test/cases/io-spec-tsv/0004/experr @@ -1 +1 @@ -mlr: TSV header/data length mismatch 1 != 0 at filename test/cases/io-spec-tsv/0004/single-column-with-blank.tsv line 4 +mlr: TSV header/data length mismatch 1 != 0 at filename test/cases/io-spec-tsv/0004/single-column-with-blank.tsv line 4; this input line is blank -- use the skip-trivial-records verb to have blank input lines skipped diff --git a/test/cases/verb-skip-trivial-records/0005/experr b/test/cases/verb-skip-trivial-records/0005/experr index 267d382e9..2799839e9 100644 --- a/test/cases/verb-skip-trivial-records/0005/experr +++ b/test/cases/verb-skip-trivial-records/0005/experr @@ -1 +1 @@ -mlr: CSV header/data length mismatch 2 != 1 at filename test/cases/verb-skip-trivial-records/0005/input.csv row 3 +mlr: CSV header/data length mismatch 2 != 1 at filename test/cases/verb-skip-trivial-records/0005/input.csv row 3; this input line is blank -- use the skip-trivial-records verb to have blank input lines skipped diff --git a/test/cases/verb-skip-trivial-records/0009/cmd b/test/cases/verb-skip-trivial-records/0009/cmd new file mode 100644 index 000000000..0d5071d21 --- /dev/null +++ b/test/cases/verb-skip-trivial-records/0009/cmd @@ -0,0 +1 @@ +mlr --itsv --otsv cat ${CASEDIR}/input.tsv diff --git a/test/cases/verb-skip-trivial-records/0009/experr b/test/cases/verb-skip-trivial-records/0009/experr new file mode 100644 index 000000000..1dea30946 --- /dev/null +++ b/test/cases/verb-skip-trivial-records/0009/experr @@ -0,0 +1 @@ +mlr: TSV header/data length mismatch 2 != 0 at filename test/cases/verb-skip-trivial-records/0009/input.tsv line 3; this input line is blank -- use the skip-trivial-records verb to have blank input lines skipped diff --git a/test/cases/verb-skip-trivial-records/0009/expout b/test/cases/verb-skip-trivial-records/0009/expout new file mode 100644 index 000000000..8b26211c1 --- /dev/null +++ b/test/cases/verb-skip-trivial-records/0009/expout @@ -0,0 +1,2 @@ +a b +1 2 diff --git a/test/cases/verb-skip-trivial-records/0009/input.tsv b/test/cases/verb-skip-trivial-records/0009/input.tsv new file mode 100644 index 000000000..575273049 --- /dev/null +++ b/test/cases/verb-skip-trivial-records/0009/input.tsv @@ -0,0 +1,4 @@ +a b +1 2 + + diff --git a/test/cases/verb-skip-trivial-records/0009/should-fail b/test/cases/verb-skip-trivial-records/0009/should-fail new file mode 100644 index 000000000..e69de29bb