From abf58f4bb55dbee447cb5657cfbd20e328bd3062 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 22 Aug 2022 09:38:53 -0400 Subject: [PATCH] Restore --quote-all for CSV output (#1084) * Restore --quote-all flag for CSV output * make dev: for doc-build artifacts --- docs/src/manpage.md | 3 +-- docs/src/manpage.txt | 3 +-- docs/src/reference-main-flag-list.md | 2 +- internal/pkg/cli/option_parse.go | 14 +++++++++----- internal/pkg/cli/option_types.go | 2 ++ internal/pkg/output/record_writer_csv.go | 7 +++++-- internal/pkg/output/record_writer_csv_colorizer.go | 4 +++- man/manpage.txt | 3 +-- man/mlr.1 | 3 +-- 9 files changed, 24 insertions(+), 17 deletions(-) diff --git a/docs/src/manpage.md b/docs/src/manpage.md index dea0a94fc..b69fa83c7 100644 --- a/docs/src/manpage.md +++ b/docs/src/manpage.md @@ -334,6 +334,7 @@ CSV/TSV-ONLY FLAGS --no-implicit-csv-header -l your-join-in-with-header.csv ... your-headerless.csv`. + --quote-all Force double-quoting of CSV fields. -N Keystroke-saver for `--implicit-csv-header --headerless-csv-output`. @@ -477,8 +478,6 @@ LEGACY FLAGS --no-mmap Miller no longer uses memory-mapping to access data files. --ojsonx The `--jvstack` flag is now default true in Miller 6. - --quote-all Ignored as of version 6. Types are inferred/retained - through the processing flow now. --quote-minimal Ignored as of version 6. Types are inferred/retained through the processing flow now. --quote-none Ignored as of version 6. Types are inferred/retained diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt index 2aed5f5d8..f35a0410c 100644 --- a/docs/src/manpage.txt +++ b/docs/src/manpage.txt @@ -313,6 +313,7 @@ CSV/TSV-ONLY FLAGS --no-implicit-csv-header -l your-join-in-with-header.csv ... your-headerless.csv`. + --quote-all Force double-quoting of CSV fields. -N Keystroke-saver for `--implicit-csv-header --headerless-csv-output`. @@ -456,8 +457,6 @@ LEGACY FLAGS --no-mmap Miller no longer uses memory-mapping to access data files. --ojsonx The `--jvstack` flag is now default true in Miller 6. - --quote-all Ignored as of version 6. Types are inferred/retained - through the processing flow now. --quote-minimal Ignored as of version 6. Types are inferred/retained through the processing flow now. --quote-none Ignored as of version 6. Types are inferred/retained diff --git a/docs/src/reference-main-flag-list.md b/docs/src/reference-main-flag-list.md index b4d6a8411..a71e0aa8e 100644 --- a/docs/src/reference-main-flag-list.md +++ b/docs/src/reference-main-flag-list.md @@ -121,6 +121,7 @@ These are flags which are applicable to CSV format. * `--implicit-csv-header or --headerless-csv-input or --hi or --implicit-tsv-header`: Use 1,2,3,... as field labels, rather than from line 1 of input files. Tip: combine with `label` to recreate missing headers. * `--lazy-quotes`: Accepts quotes appearing in unquoted fields, and non-doubled quotes appearing in quoted fields. * `--no-implicit-csv-header or --no-implicit-tsv-header`: Opposite of `--implicit-csv-header`. This is the default anyway -- the main use is for the flags to `mlr join` if you have main file(s) which are headerless but you want to join in on a file which does have a CSV/TSV header. Then you could use `mlr --csv --implicit-csv-header join --no-implicit-csv-header -l your-join-in-with-header.csv ... your-headerless.csv`. +* `--quote-all`: Force double-quoting of CSV fields. * `-N`: Keystroke-saver for `--implicit-csv-header --headerless-csv-output`. ## File-format flags @@ -249,7 +250,6 @@ They are accepted as no-op flags in order to keep old scripts from breaking. * `--mmap`: Miller no longer uses memory-mapping to access data files. * `--no-mmap`: Miller no longer uses memory-mapping to access data files. * `--ojsonx`: The `--jvstack` flag is now default true in Miller 6. -* `--quote-all`: Ignored as of version 6. Types are inferred/retained through the processing flow now. * `--quote-minimal`: Ignored as of version 6. Types are inferred/retained through the processing flow now. * `--quote-none`: Ignored as of version 6. Types are inferred/retained through the processing flow now. * `--quote-numeric`: Ignored as of version 6. Types are inferred/retained through the processing flow now. diff --git a/internal/pkg/cli/option_parse.go b/internal/pkg/cli/option_parse.go index 734b74803..6e50778eb 100644 --- a/internal/pkg/cli/option_parse.go +++ b/internal/pkg/cli/option_parse.go @@ -576,11 +576,6 @@ var LegacyFlagSection = FlagSection{ parser: NoOpParse1, }, - { - name: "--quote-all", - help: "Ignored as of version 6. Types are inferred/retained through the processing flow now.", - parser: NoOpParse1, - }, { name: "--quote-none", help: "Ignored as of version 6. Types are inferred/retained through the processing flow now.", @@ -2177,6 +2172,15 @@ var CSVTSVOnlyFlagSection = FlagSection{ *pargi += 1 }, }, + + { + name: "--quote-all", + help: "Force double-quoting of CSV fields.", + parser: func(args []string, argc int, pargi *int, options *TOptions) { + options.WriterOptions.CSVQuoteAll = true + *pargi += 1 + }, + }, }, } diff --git a/internal/pkg/cli/option_types.go b/internal/pkg/cli/option_types.go index 26b8f1e4a..63013ec30 100644 --- a/internal/pkg/cli/option_types.go +++ b/internal/pkg/cli/option_types.go @@ -106,6 +106,8 @@ type TWriterOptions struct { JVQuoteAll bool // --jvquoteall // Not using miller/types enum to avoid package cycle + CSVQuoteAll bool // --quote-all + // When we read things like // // x:a=1,x:b=2 diff --git a/internal/pkg/output/record_writer_csv.go b/internal/pkg/output/record_writer_csv.go index b5b864cf2..9c469bedf 100644 --- a/internal/pkg/output/record_writer_csv.go +++ b/internal/pkg/output/record_writer_csv.go @@ -18,6 +18,8 @@ type RecordWriterCSV struct { lastJoinedHeader *string // Only write one blank line for schema changes / blank input lines justWroteEmptyLine bool + // For double-quote around all fields + quoteAll bool } func NewRecordWriterCSV(writerOptions *cli.TWriterOptions) (*RecordWriterCSV, error) { @@ -32,6 +34,7 @@ func NewRecordWriterCSV(writerOptions *cli.TWriterOptions) (*RecordWriterCSV, er csvWriter: nil, // will be set on first Write() wherein we have the output stream lastJoinedHeader: nil, justWroteEmptyLine: false, + quoteAll: writerOptions.CSVQuoteAll, }, nil } @@ -81,7 +84,7 @@ func (writer *RecordWriterCSV) Write( i++ } //////writer.csvWriter.Write(fields) - writer.WriteCSVRecordMaybeColorized(fields, bufferedOutputStream, outputIsStdout, true) + writer.WriteCSVRecordMaybeColorized(fields, bufferedOutputStream, outputIsStdout, true, writer.quoteAll) } fields := make([]string, outrec.FieldCount) @@ -90,6 +93,6 @@ func (writer *RecordWriterCSV) Write( fields[i] = pe.Value.String() i++ } - writer.WriteCSVRecordMaybeColorized(fields, bufferedOutputStream, outputIsStdout, false) + writer.WriteCSVRecordMaybeColorized(fields, bufferedOutputStream, outputIsStdout, false, writer.quoteAll) writer.justWroteEmptyLine = false } diff --git a/internal/pkg/output/record_writer_csv_colorizer.go b/internal/pkg/output/record_writer_csv_colorizer.go index 47a95e727..a602ae790 100644 --- a/internal/pkg/output/record_writer_csv_colorizer.go +++ b/internal/pkg/output/record_writer_csv_colorizer.go @@ -58,6 +58,7 @@ func (writer *RecordWriterCSV) WriteCSVRecordMaybeColorized( bufferedOutputStream *bufio.Writer, outputIsStdout bool, isKey bool, + quoteAll bool, ) error { comma := writer.csvWriter.Comma @@ -82,7 +83,8 @@ func (writer *RecordWriterCSV) WriteCSVRecordMaybeColorized( // If we don't have to have a quoted field then just // write out the field and continue to the next field. - if !fieldNeedsQuotes(field, comma) { + needsQuotes := quoteAll || fieldNeedsQuotes(field, comma) + if !needsQuotes { if _, err := bufferedOutputStream.WriteString(prefix); err != nil { return err } diff --git a/man/manpage.txt b/man/manpage.txt index 2aed5f5d8..f35a0410c 100644 --- a/man/manpage.txt +++ b/man/manpage.txt @@ -313,6 +313,7 @@ CSV/TSV-ONLY FLAGS --no-implicit-csv-header -l your-join-in-with-header.csv ... your-headerless.csv`. + --quote-all Force double-quoting of CSV fields. -N Keystroke-saver for `--implicit-csv-header --headerless-csv-output`. @@ -456,8 +457,6 @@ LEGACY FLAGS --no-mmap Miller no longer uses memory-mapping to access data files. --ojsonx The `--jvstack` flag is now default true in Miller 6. - --quote-all Ignored as of version 6. Types are inferred/retained - through the processing flow now. --quote-minimal Ignored as of version 6. Types are inferred/retained through the processing flow now. --quote-none Ignored as of version 6. Types are inferred/retained diff --git a/man/mlr.1 b/man/mlr.1 index fc5a7cd70..a42261db0 100644 --- a/man/mlr.1 +++ b/man/mlr.1 @@ -384,6 +384,7 @@ These are flags which are applicable to CSV format. --no-implicit-csv-header -l your-join-in-with-header.csv ... your-headerless.csv`. +--quote-all Force double-quoting of CSV fields. -N Keystroke-saver for `--implicit-csv-header --headerless-csv-output`. .fi @@ -567,8 +568,6 @@ They are accepted as no-op flags in order to keep old scripts from breaking. --no-mmap Miller no longer uses memory-mapping to access data files. --ojsonx The `--jvstack` flag is now default true in Miller 6. ---quote-all Ignored as of version 6. Types are inferred/retained - through the processing flow now. --quote-minimal Ignored as of version 6. Types are inferred/retained through the processing flow now. --quote-none Ignored as of version 6. Types are inferred/retained