From 899e706bdaa380e85820214d3954bc6f8e4f51e8 Mon Sep 17 00:00:00 2001
From: John Kerl <kerl.john.r@gmail.com>
Date: Sun, 31 Mar 2024 15:48:38 -0400
Subject: [PATCH] Add --no-auto-unsparsify flag

---
 pkg/cli/option_parse.go             |  9 ++++++++
 pkg/cli/option_types.go             | 34 +++++++++++++++++++++++++++++
 pkg/output/record_writer_csv.go     |  2 +-
 pkg/output/record_writer_csvlite.go |  2 +-
 pkg/output/record_writer_dkvp.go    |  2 +-
 pkg/output/record_writer_nidx.go    |  2 +-
 pkg/output/record_writer_tsv.go     |  2 +-
 pkg/output/record_writer_xtab.go    |  2 +-
 8 files changed, 49 insertions(+), 6 deletions(-)

diff --git a/pkg/cli/option_parse.go b/pkg/cli/option_parse.go
index f4c455366..67f0f8880 100644
--- a/pkg/cli/option_parse.go
+++ b/pkg/cli/option_parse.go
@@ -2392,6 +2392,15 @@ var CSVTSVOnlyFlagSection = FlagSection{
 			},
 		},
 
+		{
+			name:     "--no-auto-unsparsify",
+			help:     "For CSV/TSV output: if the record keys change from one row to another, emit a blank line and a new header line. This is non-compliant with RFC 4180 but it helpful for heterogeneous data.",
+			parser: func(args []string, argc int, pargi *int, options *TOptions) {
+				options.WriterOptions.NoAutoUnsparsify = true
+				*pargi += 1
+			},
+		},
+
 		{
 			name:     "--implicit-csv-header",
 			altNames: []string{"--headerless-csv-input", "--hi", "--implicit-tsv-header"},
diff --git a/pkg/cli/option_types.go b/pkg/cli/option_types.go
index 22f32658f..7cbece965 100644
--- a/pkg/cli/option_types.go
+++ b/pkg/cli/option_types.go
@@ -134,6 +134,40 @@ type TWriterOptions struct {
 	// (all but JSON) -- unless the user explicitly asks to suppress that.
 	AutoFlatten bool
 
+	// Default CSV/TSV:
+	//   a=1,b=2,c=3
+	//   a=4,b=5
+	// leads to
+	//   a,b,c
+	//   1 2,3
+	//   4,5, <-- note trailing empty field
+	// and
+	//   a=1,b=2,c=3
+	//   d=4,e=5
+	// leads to
+	//   fatal error
+	//
+	// With this flag:
+	//   a=1,b=2,c=3
+	//   a=4,b=5
+	// leads to
+	//   a,b,c
+	//   1 2,3
+	//
+	//   a,b
+	//   4,5
+	//
+	// and
+	//   a=1,b=2,c=3
+	//   d=4,e=5
+	// leads to
+	//   a,b,c
+	//   1,2,3
+	//
+	//   d,e
+	//   4,5
+	NoAutoUnsparsify bool
+
 	// For floating-point numbers: "" means use the Go default.
 	FPOFMT string
 
diff --git a/pkg/output/record_writer_csv.go b/pkg/output/record_writer_csv.go
index 032a57f03..efedd64bf 100644
--- a/pkg/output/record_writer_csv.go
+++ b/pkg/output/record_writer_csv.go
@@ -46,8 +46,8 @@ func (writer *RecordWriterCSV) Write(
 	bufferedOutputStream *bufio.Writer,
 	outputIsStdout bool,
 ) error {
-	// End of record stream: nothing special for this output format
 	if outrec == nil {
+		// End of record stream: nothing special for this output format
 		return nil
 	}
 
diff --git a/pkg/output/record_writer_csvlite.go b/pkg/output/record_writer_csvlite.go
index e3ecf9196..280abf38d 100644
--- a/pkg/output/record_writer_csvlite.go
+++ b/pkg/output/record_writer_csvlite.go
@@ -32,8 +32,8 @@ func (writer *RecordWriterCSVLite) Write(
 	bufferedOutputStream *bufio.Writer,
 	outputIsStdout bool,
 ) error {
-	// End of record stream: nothing special for this output format
 	if outrec == nil {
+		// End of record stream: nothing special for this output format
 		return nil
 	}
 
diff --git a/pkg/output/record_writer_dkvp.go b/pkg/output/record_writer_dkvp.go
index d7a516955..79ea8de05 100644
--- a/pkg/output/record_writer_dkvp.go
+++ b/pkg/output/record_writer_dkvp.go
@@ -25,8 +25,8 @@ func (writer *RecordWriterDKVP) Write(
 	bufferedOutputStream *bufio.Writer,
 	outputIsStdout bool,
 ) error {
-	// End of record stream: nothing special for this output format
 	if outrec == nil {
+		// End of record stream: nothing special for this output format
 		return nil
 	}
 
diff --git a/pkg/output/record_writer_nidx.go b/pkg/output/record_writer_nidx.go
index b8a5573c1..ac599e3a7 100644
--- a/pkg/output/record_writer_nidx.go
+++ b/pkg/output/record_writer_nidx.go
@@ -24,8 +24,8 @@ func (writer *RecordWriterNIDX) Write(
 	bufferedOutputStream *bufio.Writer,
 	outputIsStdout bool,
 ) error {
-	// End of record stream: nothing special for this output format
 	if outrec == nil {
+		// End of record stream: nothing special for this output format
 		return nil
 	}
 
diff --git a/pkg/output/record_writer_tsv.go b/pkg/output/record_writer_tsv.go
index 40f89350a..0e845be79 100644
--- a/pkg/output/record_writer_tsv.go
+++ b/pkg/output/record_writer_tsv.go
@@ -40,8 +40,8 @@ func (writer *RecordWriterTSV) Write(
 	bufferedOutputStream *bufio.Writer,
 	outputIsStdout bool,
 ) error {
-	// End of record stream: nothing special for this output format
 	if outrec == nil {
+		// End of record stream: nothing special for this output format
 		return nil
 	}
 
diff --git a/pkg/output/record_writer_xtab.go b/pkg/output/record_writer_xtab.go
index cd014ddce..bfacdde95 100644
--- a/pkg/output/record_writer_xtab.go
+++ b/pkg/output/record_writer_xtab.go
@@ -48,8 +48,8 @@ func (writer *RecordWriterXTAB) Write(
 	bufferedOutputStream *bufio.Writer,
 	outputIsStdout bool,
 ) error {
-	// End of record stream: nothing special for this output format
 	if outrec == nil {
+		// End of record stream: nothing special for this output format
 		return nil
 	}