mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-18 00:45:47 +00:00
b515e6f18a
Trailing blank lines at the end of a CSV/TSV file produce a header/data length mismatch error which, especially for large files, gives no clue that the offending line is simply blank. Since #2146 (issue #1535), the skip-trivial-records verb is an opt-in way to have such lines skipped -- but users hitting the error had no pointer to it. Now, when the mismatching input line is blank, the CSV and TSV readers append a hint to the error message: mlr: CSV header/data length mismatch 2 != 1 at filename tmp.csv row 4; this input line is blank -- use the skip-trivial-records verb to have blank input lines skipped Errors for genuinely ragged, non-blank lines are unchanged, as is all non-error behavior. Also adds a note to the CSV/JSON troubleshooting docs page about trailing blank lines and skip-trivial-records. Addresses #1572. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
48 lines
1.8 KiB
Go
48 lines
1.8 KiB
Go
// This file contains the interface for file-format-specific record-readers, as
|
|
// well as a collection of utility functions.
|
|
|
|
package input
|
|
|
|
import "github.com/johnkerl/miller/v6/pkg/types"
|
|
|
|
// Since Go is concurrent, the context struct (AWK-like variables such as
|
|
// FILENAME, NF, NF, FNR, etc.) needs to be duplicated and passed through the
|
|
// channels along with each record. Hence the initial context, which readers
|
|
// update on each new file/record, and the channel of types.RecordAndContext
|
|
// rather than channel of mlrval.Mlrmap.
|
|
|
|
type IRecordReader interface {
|
|
Read(
|
|
filenames []string,
|
|
initialContext types.Context,
|
|
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
|
|
errorChannel chan error,
|
|
downstreamDoneChannel <-chan bool, // for mlr head
|
|
)
|
|
}
|
|
|
|
// hasNonEmptyField returns true if any of the split-out fields is non-empty.
|
|
// A blank input line splits to zero fields or to a single empty field; a line
|
|
// consisting only of field separators splits to all-empty fields. Used by the
|
|
// CSV/TSV readers to support skipping trivial records at read time, when the
|
|
// skip-trivial-records verb is present in the then-chain. See issue #1535.
|
|
func hasNonEmptyField(fields []string) bool {
|
|
for _, field := range fields {
|
|
if field != "" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// blankInputLineHint returns a hint to be appended to CSV/TSV header/data
|
|
// length mismatch errors when the offending input line is blank -- e.g. one
|
|
// or more blank lines at the end of the file. Without this, the bare
|
|
// mismatch error can be hard to act on, especially for large files. See
|
|
// issue #1572.
|
|
func blankInputLineHint(fields []string) string {
|
|
if hasNonEmptyField(fields) {
|
|
return ""
|
|
}
|
|
return "; this input line is blank -- use the skip-trivial-records verb to have blank input lines skipped"
|
|
}
|