Port in-place processing from C to Go (#519)

* Port in-place processing from C to Go

go/src/cli/mlrcli.go

@@ -197,7 +197,7 @@ package cli
 //	options.ofmt            = nil;
 //	options.nr_progress_mod = 0LL;
 //
-//	options.do_in_place     = false;
+//	options.DoInPlace     = false;
 //
 //	options.no_input        = false;
 //	options.have_rand_seed  = false;

go/src/cli/mlrcli_parse.go

@@ -139,10 +139,10 @@ func ParseCommandLine(args []string) (
 		options.FileNames = nil
 	}
 
-	//	if (options.do_in_place && (options.FileNames == nil || options.FileNames.length == 0)) {
-	//		fmt.Fprintf(os.Stderr, "%s: -I option (in-place operation) requires input files.\n", lib.MlrExeName());
-	//		os.Exit(1);
-	//	}
+	if options.DoInPlace && (options.FileNames == nil || len(options.FileNames) == 0) {
+		fmt.Fprintf(os.Stderr, "%s: -I option (in-place operation) requires input files.\n", lib.MlrExeName())
+		os.Exit(1)
+	}
 
 	if options.HaveRandSeed {
 		lib.SeedRandom(int64(options.RandSeed))
@@ -404,9 +404,9 @@ func parseTerminalUsage(args []string, argc int, argi int) bool {
 		//	} else if args[argi] == "--usage-numerical-formatting" {
 		//		mainUsageNumericalFormatting(os.Stdout, lib.MlrExeName());
 		//		return true;
-		//	} else if args[argi] == "--usage-other-options" {
-		//		mainUsageOtherOptions(os.Stdout, lib.MlrExeName());
-		//		return true;
+	} else if args[argi] == "--usage-other-options" {
+		mainUsageOtherOptions(os.Stdout, lib.MlrExeName())
+		return true
 	} else if args[argi] == "--usage-then-chaining" {
 		mainUsageThenChaining(os.Stdout, lib.MlrExeName())
 		return true

go/src/cli/mlrcli_usage.go

@@ -78,11 +78,11 @@ func mainUsageLong(o *os.File, argv0 string) {
 	//	fmt.Fprintf(o, "NUMERICAL FORMATTING:\n");
 	//	mainUsageNumericalFormatting(o, argv0);
 	//	fmt.Fprintf(o, "\n");
-	//
-	//	fmt.Fprintf(o, "OTHER OPTIONS:\n");
-	//	mainUsageOtherOptions(o, argv0);
-	//	fmt.Fprintf(o, "\n");
-	//
+
+	fmt.Fprintf(o, "OTHER OPTIONS:\n")
+	mainUsageOtherOptions(o, argv0)
+	fmt.Fprintf(o, "\n")
+
 	fmt.Fprintf(o, "THEN-CHAINING:\n")
 	mainUsageThenChaining(o, argv0)
 	fmt.Fprintf(o, "\n")
@@ -481,26 +481,27 @@ func mainUsageFormatConversionKeystrokeSaverOptions(o *os.File, argv0 string) {
 //	fmt.Fprintf(o, "                     Defaults to %s.\n", DEFAULT_OFMT);
 //}
 
-//func mainUsageOtherOptions(o *os.File, argv0 string) {
-//	fmt.Fprintf(o, "  --seed {n} with n of the form 12345678 or 0xcafefeed. For put/filter\n");
-//	fmt.Fprintf(o, "                     urand()/urandint()/urand32().\n");
-//	fmt.Fprintf(o, "  --nr-progress-mod {m}, with m a positive integer: print filename and record\n");
-//	fmt.Fprintf(o, "                     count to os.Stderr every m input records.\n");
-//	fmt.Fprintf(o, "  --from {filename}  Use this to specify an input file before the verb(s),\n");
-//	fmt.Fprintf(o, "                     rather than after. May be used more than once. Example:\n");
-//	fmt.Fprintf(o, "                     \"%s --from a.dat --from b.dat cat\" is the same as\n", argv0);
-//	fmt.Fprintf(o, "                     \"%s cat a.dat b.dat\".\n", argv0);
-//	fmt.Fprintf(o, "  -n                 Process no input files, nor standard input either. Useful\n");
-//	fmt.Fprintf(o, "                     for %s put with begin/end statements only. (Same as --from\n", argv0);
-//	fmt.Fprintf(o, "                     /dev/null.) Also useful in \"%s -n put -v '...'\" for\n", argv0);
-//	fmt.Fprintf(o, "                     analyzing abstract syntax trees (if that's your thing).\n");
-//	fmt.Fprintf(o, "  -I                 Process files in-place. For each file name on the command\n");
-//	fmt.Fprintf(o, "                     line, output is written to a temp file in the same\n");
-//	fmt.Fprintf(o, "                     directory, which is then renamed over the original. Each\n");
-//	fmt.Fprintf(o, "                     file is processed in isolation: if the output format is\n");
-//	fmt.Fprintf(o, "                     CSV, CSV headers will be present in each output file;\n");
-//	fmt.Fprintf(o, "                     statistics are only over each file's own records; and so on.\n");
-//}
+// TODO
+func mainUsageOtherOptions(o *os.File, argv0 string) {
+	fmt.Fprintf(o, "  --seed {n} with n of the form 12345678 or 0xcafefeed. For put/filter\n")
+	fmt.Fprintf(o, "                     urand()/urandint()/urand32().\n")
+	fmt.Fprintf(o, "  --nr-progress-mod {m}, with m a positive integer: print filename and record\n")
+	fmt.Fprintf(o, "                     count to os.Stderr every m input records.\n")
+	fmt.Fprintf(o, "  --from {filename}  Use this to specify an input file before the verb(s),\n")
+	fmt.Fprintf(o, "                     rather than after. May be used more than once. Example:\n")
+	fmt.Fprintf(o, "                     \"%s --from a.dat --from b.dat cat\" is the same as\n", argv0)
+	fmt.Fprintf(o, "                     \"%s cat a.dat b.dat\".\n", argv0)
+	fmt.Fprintf(o, "  -n                 Process no input files, nor standard input either. Useful\n")
+	fmt.Fprintf(o, "                     for %s put with begin/end statements only. (Same as --from\n", argv0)
+	fmt.Fprintf(o, "                     /dev/null.) Also useful in \"%s -n put -v '...'\" for\n", argv0)
+	fmt.Fprintf(o, "                     analyzing abstract syntax trees (if that's your thing).\n")
+	fmt.Fprintf(o, "  -I                 Process files in-place. For each file name on the command\n")
+	fmt.Fprintf(o, "                     line, output is written to a temp file in the same\n")
+	fmt.Fprintf(o, "                     directory, which is then renamed over the original. Each\n")
+	fmt.Fprintf(o, "                     file is processed in isolation: if the output format is\n")
+	fmt.Fprintf(o, "                     CSV, CSV headers will be present in each output file;\n")
+	fmt.Fprintf(o, "                     statistics are only over each file's own records; and so on.\n")
+}
 
 func mainUsageThenChaining(o *os.File, argv0 string) {
 	fmt.Fprintf(o, "Output of one verb may be chained as input to another using \"then\", e.g.\n")

go/src/cliutil/option_parse.go

@@ -851,10 +851,10 @@ func ParseMiscOptions(
 		options.NoInput = true
 		argi += 1
 
-		//	} else if args[argi] == "-I" {
-		//		options.do_in_place = true;
-		//		argi += 1;
-		//
+	} else if args[argi] == "-I" {
+		options.DoInPlace = true
+		argi += 1
+
 	} else if args[argi] == "--from" {
 		CheckArgCount(args, argi, argc, 2)
 		options.FileNames = append(options.FileNames, args[argi+1])

go/src/cliutil/option_types.go

@@ -110,10 +110,8 @@ type TOptions struct {
 	//
 	//	char* ofmt;
 	//	nr_progress_mod int64u;
-	//
-	//	do_in_place bool;
-	//
-	NoInput bool // mlr -n
+	DoInPlace bool // mlr -I
+	NoInput   bool // mlr -n
 
 	HaveRandSeed bool
 	RandSeed     int

go/src/entrypoint/entrypoint.go

@@ -1,15 +1,23 @@
+// ================================================================
+// All the usual contents of main() are put into this package for ease of
+// testing.
+// ================================================================
+
 package entrypoint
 
 import (
 	"fmt"
 	"os"
+	"path"
 
 	"miller/src/platform"
 
 	"miller/src/auxents"
 	"miller/src/cli"
+	"miller/src/cliutil"
 	"miller/src/lib"
 	"miller/src/stream"
+	"miller/src/transforming"
 )
 
 // ----------------------------------------------------------------
@@ -18,7 +26,8 @@ func Main() {
 	//
 	//   mlr put '$a = $b . "cd \"efg\" hi"' foo.dat
 	//
-	// as on Linux/Unix/MacOS.
+	// as on Linux/Unix/MacOS. (On the latter platforms, this is just os.Args
+	// as-is.)
 	os.Args = platform.GetArgs()
 
 	// Expand "-xyz" into "-x -y -z" while leaving "--xyz" intact. This is a
@@ -41,9 +50,92 @@ func Main() {
 		os.Exit(1)
 	}
 
-	err = stream.Stream(options, recordTransformers)
+	if !options.DoInPlace {
+		processToStdout(options, recordTransformers)
+	} else {
+		processInPlace(options)
+	}
+}
+
+// ----------------------------------------------------------------
+// processToStdout is normal processing without mlr -I.
+
+func processToStdout(
+	options cliutil.TOptions,
+	recordTransformers []transforming.IRecordTransformer,
+) {
+	err := stream.Stream(options.FileNames, options, recordTransformers, os.Stdout)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, os.Args[0], ": ", err)
 		os.Exit(1)
 	}
 }
+
+// ----------------------------------------------------------------
+// processInPlace is in-place processing without mlr -I.
+//
+// For in-place mode, reconstruct the transformers on each input file. E.g.
+// 'mlr -I head -n 2 foo bar' should do head -n 2 on foo as well as on bar.
+//
+// I could have implemented this with a single construction of the transformers
+// and having each transformers implement a Reset() method.  However, having
+// effectively two initalizers per transformers -- constructor and reset method
+// -- I'd surely miss some logic somewhere.  With in-place mode being a less
+// frequently used code path, this would likely lead to latent bugs. So this
+// approach leads to greater code stability.
+
+func processInPlace(
+	originalOptions cliutil.TOptions,
+) {
+	// This should have been already checked by the CLI parser when validating
+	// the -I flag.
+	lib.InternalCodingErrorIf(originalOptions.FileNames == nil)
+	lib.InternalCodingErrorIf(len(originalOptions.FileNames) == 0)
+
+	// Save off the file names from the command line.
+	fileNames := make([]string, len(originalOptions.FileNames))
+	for i, fileName := range originalOptions.FileNames {
+		fileNames[i] = fileName
+	}
+
+	for _, fileName := range fileNames {
+		// Reconstruct the transformers for each file name, and allocate
+		// reader, mappers, and writer individually for each file name.  This
+		// way CSV headers appear in each file, head -n 10 puts 10 rows for
+		// each output file, and so on.
+
+		containingDirectory := path.Dir(fileName)
+		// Names like ./mlr-in-place-2148227797 and ./mlr-in-place-1792078347,
+		// as revealed by printing handle.Name().
+		handle, err := os.CreateTemp(containingDirectory, "mlr-in-place-")
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: %v\n", lib.MlrExeName(), err)
+			os.Exit(1)
+		}
+		tempFileName := handle.Name()
+
+		options, recordTransformers, err := cli.ParseCommandLine(os.Args)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, os.Args[0], ": ", err)
+			os.Exit(1)
+		}
+
+		err = stream.Stream([]string{fileName}, options, recordTransformers, handle)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, os.Args[0], ": ", err)
+			os.Exit(1)
+		}
+
+		err = handle.Close()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: %v\n", lib.MlrExeName(), err)
+			os.Exit(1)
+		}
+
+		err = os.Rename(tempFileName, fileName)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: %v\n", lib.MlrExeName(), err)
+			os.Exit(1)
+		}
+	}
+}

go/src/output/record_writer_csv.go

@@ -3,7 +3,6 @@ package output
 import (
 	"encoding/csv"
 	"io"
-	"os"
 	"strings"
 
 	"miller/src/cliutil"
@@ -41,7 +40,7 @@ func (this *RecordWriterCSV) Write(
 	}
 
 	if this.csvWriter == nil {
-		this.csvWriter = csv.NewWriter(os.Stdout)
+		this.csvWriter = csv.NewWriter(ostream)
 		this.csvWriter.Comma = rune(this.ofs[0]) // xxx temp
 	}
 

go/src/stream/stream.go

@@ -29,8 +29,12 @@ import (
 //   need the contexts.
 
 func Stream(
+	// fileNames argument is separate from options.FileNames for in-place mode,
+	// which sends along only one file name per call to Stream():
+	fileNames []string,
 	options cliutil.TOptions,
 	recordTransformers []transforming.IRecordTransformer,
+	outputStream *os.File,
 ) error {
 
 	// Since Go is concurrent, the context struct needs to be duplicated and
@@ -62,9 +66,9 @@ func Stream(
 	// Start the reader, transformer, and writer. Let them run until fatal input
 	// error or end-of-processing happens.
 
-	go recordReader.Read(options.FileNames, *initialContext, inputChannel, errorChannel)
+	go recordReader.Read(fileNames, *initialContext, inputChannel, errorChannel)
 	go transforming.ChainTransformer(inputChannel, recordTransformers, outputChannel)
-	go output.ChannelWriter(outputChannel, recordWriter, doneChannel, os.Stdout)
+	go output.ChannelWriter(outputChannel, recordWriter, doneChannel, outputStream)
 
 	done := false
 	for !done {