diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0001/abixy.temp1.expect b/go/regtest/cases/io-in-place-processing/0001/abixy.temp1.expect similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0001/abixy.temp1.expect rename to go/regtest/cases/io-in-place-processing/0001/abixy.temp1.expect diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0001/abixy.temp2.expect b/go/regtest/cases/io-in-place-processing/0001/abixy.temp2.expect similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0001/abixy.temp2.expect rename to go/regtest/cases/io-in-place-processing/0001/abixy.temp2.expect diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0001/cmd b/go/regtest/cases/io-in-place-processing/0001/cmd similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0001/cmd rename to go/regtest/cases/io-in-place-processing/0001/cmd diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0001/experr b/go/regtest/cases/io-in-place-processing/0001/experr similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0001/experr rename to go/regtest/cases/io-in-place-processing/0001/experr diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0001/expout b/go/regtest/cases/io-in-place-processing/0001/expout similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0001/expout rename to go/regtest/cases/io-in-place-processing/0001/expout diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0001/postcmp b/go/regtest/cases/io-in-place-processing/0001/postcmp similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0001/postcmp rename to go/regtest/cases/io-in-place-processing/0001/postcmp diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0001/precopy b/go/regtest/cases/io-in-place-processing/0001/precopy similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0001/precopy rename to go/regtest/cases/io-in-place-processing/0001/precopy diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0002/abixy.temp1.expect b/go/regtest/cases/io-in-place-processing/0002/abixy.temp1.expect similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0002/abixy.temp1.expect rename to go/regtest/cases/io-in-place-processing/0002/abixy.temp1.expect diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0002/abixy.temp2.expect b/go/regtest/cases/io-in-place-processing/0002/abixy.temp2.expect similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0002/abixy.temp2.expect rename to go/regtest/cases/io-in-place-processing/0002/abixy.temp2.expect diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0002/cmd b/go/regtest/cases/io-in-place-processing/0002/cmd similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0002/cmd rename to go/regtest/cases/io-in-place-processing/0002/cmd diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0002/experr b/go/regtest/cases/io-in-place-processing/0002/experr similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0002/experr rename to go/regtest/cases/io-in-place-processing/0002/experr diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0002/expout b/go/regtest/cases/io-in-place-processing/0002/expout similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0002/expout rename to go/regtest/cases/io-in-place-processing/0002/expout diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0002/postcmp b/go/regtest/cases/io-in-place-processing/0002/postcmp similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0002/postcmp rename to go/regtest/cases/io-in-place-processing/0002/postcmp diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0002/precopy b/go/regtest/cases/io-in-place-processing/0002/precopy similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0002/precopy rename to go/regtest/cases/io-in-place-processing/0002/precopy diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0002/should-fail b/go/regtest/cases/io-in-place-processing/0002/should-fail similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0002/should-fail rename to go/regtest/cases/io-in-place-processing/0002/should-fail diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0003/abixy.temp1.expect b/go/regtest/cases/io-in-place-processing/0003/abixy.temp1.expect similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0003/abixy.temp1.expect rename to go/regtest/cases/io-in-place-processing/0003/abixy.temp1.expect diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0003/abixy.temp2.expect b/go/regtest/cases/io-in-place-processing/0003/abixy.temp2.expect similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0003/abixy.temp2.expect rename to go/regtest/cases/io-in-place-processing/0003/abixy.temp2.expect diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0003/cmd b/go/regtest/cases/io-in-place-processing/0003/cmd similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0003/cmd rename to go/regtest/cases/io-in-place-processing/0003/cmd diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0003/experr b/go/regtest/cases/io-in-place-processing/0003/experr similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0003/experr rename to go/regtest/cases/io-in-place-processing/0003/experr diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0003/expout b/go/regtest/cases/io-in-place-processing/0003/expout similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0003/expout rename to go/regtest/cases/io-in-place-processing/0003/expout diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0003/postcmp b/go/regtest/cases/io-in-place-processing/0003/postcmp similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0003/postcmp rename to go/regtest/cases/io-in-place-processing/0003/postcmp diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0003/precopy b/go/regtest/cases/io-in-place-processing/0003/precopy similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0003/precopy rename to go/regtest/cases/io-in-place-processing/0003/precopy diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0003/should-fail b/go/regtest/cases/io-in-place-processing/0003/should-fail similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0003/should-fail rename to go/regtest/cases/io-in-place-processing/0003/should-fail diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0004/abixy.temp1.expect b/go/regtest/cases/io-in-place-processing/0004/abixy.temp1.expect similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0004/abixy.temp1.expect rename to go/regtest/cases/io-in-place-processing/0004/abixy.temp1.expect diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0004/abixy.temp2.expect b/go/regtest/cases/io-in-place-processing/0004/abixy.temp2.expect similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0004/abixy.temp2.expect rename to go/regtest/cases/io-in-place-processing/0004/abixy.temp2.expect diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0004/cmd b/go/regtest/cases/io-in-place-processing/0004/cmd similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0004/cmd rename to go/regtest/cases/io-in-place-processing/0004/cmd diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0004/experr b/go/regtest/cases/io-in-place-processing/0004/experr similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0004/experr rename to go/regtest/cases/io-in-place-processing/0004/experr diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0004/expout b/go/regtest/cases/io-in-place-processing/0004/expout similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0004/expout rename to go/regtest/cases/io-in-place-processing/0004/expout diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0004/postcmp b/go/regtest/cases/io-in-place-processing/0004/postcmp similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0004/postcmp rename to go/regtest/cases/io-in-place-processing/0004/postcmp diff --git a/go/regtest/cases-pending-go-port/io-in-place-processing/0004/precopy b/go/regtest/cases/io-in-place-processing/0004/precopy similarity index 100% rename from go/regtest/cases-pending-go-port/io-in-place-processing/0004/precopy rename to go/regtest/cases/io-in-place-processing/0004/precopy diff --git a/go/src/cli/mlrcli.go b/go/src/cli/mlrcli.go index fa2d43c44..ab80976fd 100644 --- a/go/src/cli/mlrcli.go +++ b/go/src/cli/mlrcli.go @@ -197,7 +197,7 @@ package cli // options.ofmt = nil; // options.nr_progress_mod = 0LL; // -// options.do_in_place = false; +// options.DoInPlace = false; // // options.no_input = false; // options.have_rand_seed = false; diff --git a/go/src/cli/mlrcli_parse.go b/go/src/cli/mlrcli_parse.go index b4211554a..2179274f8 100644 --- a/go/src/cli/mlrcli_parse.go +++ b/go/src/cli/mlrcli_parse.go @@ -139,10 +139,10 @@ func ParseCommandLine(args []string) ( options.FileNames = nil } - // if (options.do_in_place && (options.FileNames == nil || options.FileNames.length == 0)) { - // fmt.Fprintf(os.Stderr, "%s: -I option (in-place operation) requires input files.\n", lib.MlrExeName()); - // os.Exit(1); - // } + if options.DoInPlace && (options.FileNames == nil || len(options.FileNames) == 0) { + fmt.Fprintf(os.Stderr, "%s: -I option (in-place operation) requires input files.\n", lib.MlrExeName()) + os.Exit(1) + } if options.HaveRandSeed { lib.SeedRandom(int64(options.RandSeed)) @@ -404,9 +404,9 @@ func parseTerminalUsage(args []string, argc int, argi int) bool { // } else if args[argi] == "--usage-numerical-formatting" { // mainUsageNumericalFormatting(os.Stdout, lib.MlrExeName()); // return true; - // } else if args[argi] == "--usage-other-options" { - // mainUsageOtherOptions(os.Stdout, lib.MlrExeName()); - // return true; + } else if args[argi] == "--usage-other-options" { + mainUsageOtherOptions(os.Stdout, lib.MlrExeName()) + return true } else if args[argi] == "--usage-then-chaining" { mainUsageThenChaining(os.Stdout, lib.MlrExeName()) return true diff --git a/go/src/cli/mlrcli_usage.go b/go/src/cli/mlrcli_usage.go index 52dcb9235..548392712 100644 --- a/go/src/cli/mlrcli_usage.go +++ b/go/src/cli/mlrcli_usage.go @@ -78,11 +78,11 @@ func mainUsageLong(o *os.File, argv0 string) { // fmt.Fprintf(o, "NUMERICAL FORMATTING:\n"); // mainUsageNumericalFormatting(o, argv0); // fmt.Fprintf(o, "\n"); - // - // fmt.Fprintf(o, "OTHER OPTIONS:\n"); - // mainUsageOtherOptions(o, argv0); - // fmt.Fprintf(o, "\n"); - // + + fmt.Fprintf(o, "OTHER OPTIONS:\n") + mainUsageOtherOptions(o, argv0) + fmt.Fprintf(o, "\n") + fmt.Fprintf(o, "THEN-CHAINING:\n") mainUsageThenChaining(o, argv0) fmt.Fprintf(o, "\n") @@ -481,26 +481,27 @@ func mainUsageFormatConversionKeystrokeSaverOptions(o *os.File, argv0 string) { // fmt.Fprintf(o, " Defaults to %s.\n", DEFAULT_OFMT); //} -//func mainUsageOtherOptions(o *os.File, argv0 string) { -// fmt.Fprintf(o, " --seed {n} with n of the form 12345678 or 0xcafefeed. For put/filter\n"); -// fmt.Fprintf(o, " urand()/urandint()/urand32().\n"); -// fmt.Fprintf(o, " --nr-progress-mod {m}, with m a positive integer: print filename and record\n"); -// fmt.Fprintf(o, " count to os.Stderr every m input records.\n"); -// fmt.Fprintf(o, " --from {filename} Use this to specify an input file before the verb(s),\n"); -// fmt.Fprintf(o, " rather than after. May be used more than once. Example:\n"); -// fmt.Fprintf(o, " \"%s --from a.dat --from b.dat cat\" is the same as\n", argv0); -// fmt.Fprintf(o, " \"%s cat a.dat b.dat\".\n", argv0); -// fmt.Fprintf(o, " -n Process no input files, nor standard input either. Useful\n"); -// fmt.Fprintf(o, " for %s put with begin/end statements only. (Same as --from\n", argv0); -// fmt.Fprintf(o, " /dev/null.) Also useful in \"%s -n put -v '...'\" for\n", argv0); -// fmt.Fprintf(o, " analyzing abstract syntax trees (if that's your thing).\n"); -// fmt.Fprintf(o, " -I Process files in-place. For each file name on the command\n"); -// fmt.Fprintf(o, " line, output is written to a temp file in the same\n"); -// fmt.Fprintf(o, " directory, which is then renamed over the original. Each\n"); -// fmt.Fprintf(o, " file is processed in isolation: if the output format is\n"); -// fmt.Fprintf(o, " CSV, CSV headers will be present in each output file;\n"); -// fmt.Fprintf(o, " statistics are only over each file's own records; and so on.\n"); -//} +// TODO +func mainUsageOtherOptions(o *os.File, argv0 string) { + fmt.Fprintf(o, " --seed {n} with n of the form 12345678 or 0xcafefeed. For put/filter\n") + fmt.Fprintf(o, " urand()/urandint()/urand32().\n") + fmt.Fprintf(o, " --nr-progress-mod {m}, with m a positive integer: print filename and record\n") + fmt.Fprintf(o, " count to os.Stderr every m input records.\n") + fmt.Fprintf(o, " --from {filename} Use this to specify an input file before the verb(s),\n") + fmt.Fprintf(o, " rather than after. May be used more than once. Example:\n") + fmt.Fprintf(o, " \"%s --from a.dat --from b.dat cat\" is the same as\n", argv0) + fmt.Fprintf(o, " \"%s cat a.dat b.dat\".\n", argv0) + fmt.Fprintf(o, " -n Process no input files, nor standard input either. Useful\n") + fmt.Fprintf(o, " for %s put with begin/end statements only. (Same as --from\n", argv0) + fmt.Fprintf(o, " /dev/null.) Also useful in \"%s -n put -v '...'\" for\n", argv0) + fmt.Fprintf(o, " analyzing abstract syntax trees (if that's your thing).\n") + fmt.Fprintf(o, " -I Process files in-place. For each file name on the command\n") + fmt.Fprintf(o, " line, output is written to a temp file in the same\n") + fmt.Fprintf(o, " directory, which is then renamed over the original. Each\n") + fmt.Fprintf(o, " file is processed in isolation: if the output format is\n") + fmt.Fprintf(o, " CSV, CSV headers will be present in each output file;\n") + fmt.Fprintf(o, " statistics are only over each file's own records; and so on.\n") +} func mainUsageThenChaining(o *os.File, argv0 string) { fmt.Fprintf(o, "Output of one verb may be chained as input to another using \"then\", e.g.\n") diff --git a/go/src/cliutil/option_parse.go b/go/src/cliutil/option_parse.go index 78e52a025..200c9fb6d 100644 --- a/go/src/cliutil/option_parse.go +++ b/go/src/cliutil/option_parse.go @@ -851,10 +851,10 @@ func ParseMiscOptions( options.NoInput = true argi += 1 - // } else if args[argi] == "-I" { - // options.do_in_place = true; - // argi += 1; - // + } else if args[argi] == "-I" { + options.DoInPlace = true + argi += 1 + } else if args[argi] == "--from" { CheckArgCount(args, argi, argc, 2) options.FileNames = append(options.FileNames, args[argi+1]) diff --git a/go/src/cliutil/option_types.go b/go/src/cliutil/option_types.go index b9869d3bd..7abb8f81f 100644 --- a/go/src/cliutil/option_types.go +++ b/go/src/cliutil/option_types.go @@ -110,10 +110,8 @@ type TOptions struct { // // char* ofmt; // nr_progress_mod int64u; - // - // do_in_place bool; - // - NoInput bool // mlr -n + DoInPlace bool // mlr -I + NoInput bool // mlr -n HaveRandSeed bool RandSeed int diff --git a/go/src/entrypoint/entrypoint.go b/go/src/entrypoint/entrypoint.go index 711bf55fe..e10eb9ddc 100644 --- a/go/src/entrypoint/entrypoint.go +++ b/go/src/entrypoint/entrypoint.go @@ -1,15 +1,23 @@ +// ================================================================ +// All the usual contents of main() are put into this package for ease of +// testing. +// ================================================================ + package entrypoint import ( "fmt" "os" + "path" "miller/src/platform" "miller/src/auxents" "miller/src/cli" + "miller/src/cliutil" "miller/src/lib" "miller/src/stream" + "miller/src/transforming" ) // ---------------------------------------------------------------- @@ -18,7 +26,8 @@ func Main() { // // mlr put '$a = $b . "cd \"efg\" hi"' foo.dat // - // as on Linux/Unix/MacOS. + // as on Linux/Unix/MacOS. (On the latter platforms, this is just os.Args + // as-is.) os.Args = platform.GetArgs() // Expand "-xyz" into "-x -y -z" while leaving "--xyz" intact. This is a @@ -41,9 +50,92 @@ func Main() { os.Exit(1) } - err = stream.Stream(options, recordTransformers) + if !options.DoInPlace { + processToStdout(options, recordTransformers) + } else { + processInPlace(options) + } +} + +// ---------------------------------------------------------------- +// processToStdout is normal processing without mlr -I. + +func processToStdout( + options cliutil.TOptions, + recordTransformers []transforming.IRecordTransformer, +) { + err := stream.Stream(options.FileNames, options, recordTransformers, os.Stdout) if err != nil { fmt.Fprintln(os.Stderr, os.Args[0], ": ", err) os.Exit(1) } } + +// ---------------------------------------------------------------- +// processInPlace is in-place processing without mlr -I. +// +// For in-place mode, reconstruct the transformers on each input file. E.g. +// 'mlr -I head -n 2 foo bar' should do head -n 2 on foo as well as on bar. +// +// I could have implemented this with a single construction of the transformers +// and having each transformers implement a Reset() method. However, having +// effectively two initalizers per transformers -- constructor and reset method +// -- I'd surely miss some logic somewhere. With in-place mode being a less +// frequently used code path, this would likely lead to latent bugs. So this +// approach leads to greater code stability. + +func processInPlace( + originalOptions cliutil.TOptions, +) { + // This should have been already checked by the CLI parser when validating + // the -I flag. + lib.InternalCodingErrorIf(originalOptions.FileNames == nil) + lib.InternalCodingErrorIf(len(originalOptions.FileNames) == 0) + + // Save off the file names from the command line. + fileNames := make([]string, len(originalOptions.FileNames)) + for i, fileName := range originalOptions.FileNames { + fileNames[i] = fileName + } + + for _, fileName := range fileNames { + // Reconstruct the transformers for each file name, and allocate + // reader, mappers, and writer individually for each file name. This + // way CSV headers appear in each file, head -n 10 puts 10 rows for + // each output file, and so on. + + containingDirectory := path.Dir(fileName) + // Names like ./mlr-in-place-2148227797 and ./mlr-in-place-1792078347, + // as revealed by printing handle.Name(). + handle, err := os.CreateTemp(containingDirectory, "mlr-in-place-") + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", lib.MlrExeName(), err) + os.Exit(1) + } + tempFileName := handle.Name() + + options, recordTransformers, err := cli.ParseCommandLine(os.Args) + if err != nil { + fmt.Fprintln(os.Stderr, os.Args[0], ": ", err) + os.Exit(1) + } + + err = stream.Stream([]string{fileName}, options, recordTransformers, handle) + if err != nil { + fmt.Fprintln(os.Stderr, os.Args[0], ": ", err) + os.Exit(1) + } + + err = handle.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", lib.MlrExeName(), err) + os.Exit(1) + } + + err = os.Rename(tempFileName, fileName) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", lib.MlrExeName(), err) + os.Exit(1) + } + } +} diff --git a/go/src/output/record_writer_csv.go b/go/src/output/record_writer_csv.go index 181b5a995..991aaf33f 100644 --- a/go/src/output/record_writer_csv.go +++ b/go/src/output/record_writer_csv.go @@ -3,7 +3,6 @@ package output import ( "encoding/csv" "io" - "os" "strings" "miller/src/cliutil" @@ -41,7 +40,7 @@ func (this *RecordWriterCSV) Write( } if this.csvWriter == nil { - this.csvWriter = csv.NewWriter(os.Stdout) + this.csvWriter = csv.NewWriter(ostream) this.csvWriter.Comma = rune(this.ofs[0]) // xxx temp } diff --git a/go/src/stream/stream.go b/go/src/stream/stream.go index 19ea26e9b..c303eaaeb 100644 --- a/go/src/stream/stream.go +++ b/go/src/stream/stream.go @@ -29,8 +29,12 @@ import ( // need the contexts. func Stream( + // fileNames argument is separate from options.FileNames for in-place mode, + // which sends along only one file name per call to Stream(): + fileNames []string, options cliutil.TOptions, recordTransformers []transforming.IRecordTransformer, + outputStream *os.File, ) error { // Since Go is concurrent, the context struct needs to be duplicated and @@ -62,9 +66,9 @@ func Stream( // Start the reader, transformer, and writer. Let them run until fatal input // error or end-of-processing happens. - go recordReader.Read(options.FileNames, *initialContext, inputChannel, errorChannel) + go recordReader.Read(fileNames, *initialContext, inputChannel, errorChannel) go transforming.ChainTransformer(inputChannel, recordTransformers, outputChannel) - go output.ChannelWriter(outputChannel, recordWriter, doneChannel, os.Stdout) + go output.ChannelWriter(outputChannel, recordWriter, doneChannel, outputStream) done := false for !done {