diff --git a/go/regtest/cases/cli-help/0001/expout b/go/regtest/cases/cli-help/0001/expout index c23146849..e544324bf 100644 --- a/go/regtest/cases/cli-help/0001/expout +++ b/go/regtest/cases/cli-help/0001/expout @@ -359,6 +359,7 @@ Options: --prepipe {command} As in main input options; see mlr --help for details. If you wish to use a prepipe command for the main input as well as here, it must be specified there as well as here. + --prepipex {command} Likewise. File-format options default to those for the right file names on the Miller argument list, but may be overridden for the left file as follows. Please see the main "mlr --help" for more information on syntax for these arguments: diff --git a/go/src/cli/mlrcli_parse.go b/go/src/cli/mlrcli_parse.go index edb4598a7..ab09dbb6b 100644 --- a/go/src/cli/mlrcli_parse.go +++ b/go/src/cli/mlrcli_parse.go @@ -483,7 +483,7 @@ func handleMlrrcLine( argi := 0 argc := len(args) - if args[0] == "--prepipe" { + if args[0] == "--prepipe" || args[0] == "--prepipex" { // Don't allow code execution via .mlrrc return false } else if args[0] == "--load" || args[0] == "--mload" { diff --git a/go/src/cli/mlrcli_usage.go b/go/src/cli/mlrcli_usage.go index a735e64dd..661c3ca34 100644 --- a/go/src/cli/mlrcli_usage.go +++ b/go/src/cli/mlrcli_usage.go @@ -378,6 +378,9 @@ func mainUsageCompressedDataOptions(o *os.File, argv0 string) { fmt.Fprintf(o, " specify an action to be taken on each input file. This prepipe command must\n") fmt.Fprintf(o, " be able to read from standard input; it will be invoked with\n") fmt.Fprintf(o, " {command} < {filename}.\n") + fmt.Fprintf(o, " --prepipex {command} Like --prepipe with one exception: doesn't insert '<' between\n") + fmt.Fprintf(o, " command and filename at runtime. Useful for some commands like 'unzip -qc' which don't\n") + fmt.Fprintf(o, " read standard input.\n") fmt.Fprintf(o, " Examples:\n") fmt.Fprintf(o, " %s --prepipe 'gunzip'\n", argv0) fmt.Fprintf(o, " %s --prepipe 'zcat -cf'\n", argv0) diff --git a/go/src/cliutil/option_parse.go b/go/src/cliutil/option_parse.go index 91282fd15..53f0bf2b7 100644 --- a/go/src/cliutil/option_parse.go +++ b/go/src/cliutil/option_parse.go @@ -202,18 +202,28 @@ func ParseReaderOptions( } else if args[argi] == "--prepipe" { CheckArgCount(args, argi, argc, 2) readerOptions.Prepipe = args[argi+1] + readerOptions.PrepipeIsRaw = false + argi += 2 + + } else if args[argi] == "--prepipex" { + CheckArgCount(args, argi, argc, 2) + readerOptions.Prepipe = args[argi+1] + readerOptions.PrepipeIsRaw = true argi += 2 } else if args[argi] == "--prepipe-gunzip" { readerOptions.Prepipe = "gunzip" + readerOptions.PrepipeIsRaw = false argi += 1 } else if args[argi] == "--prepipe-zcat" { readerOptions.Prepipe = "zcat" + readerOptions.PrepipeIsRaw = false argi += 1 } else if args[argi] == "--prepipe-bz2" { readerOptions.Prepipe = "bz2" + readerOptions.PrepipeIsRaw = false argi += 1 } else if args[argi] == "--gzin" { diff --git a/go/src/cliutil/option_types.go b/go/src/cliutil/option_types.go index 6beca989d..a548b4cd9 100644 --- a/go/src/cliutil/option_types.go +++ b/go/src/cliutil/option_types.go @@ -57,6 +57,11 @@ type TReaderOptions struct { // For out-of-process handling of compressed data, via popen Prepipe string + // For most things like gunzip we do 'gunzip < filename | mlr ...' if + // filename is present, else 'gunzip | mlr ...' if reading from stdin. + // However some commands like 'unzip -qc' are weird so this option lets + // people give the command and we won't insert the '<'. + PrepipeIsRaw bool // For in-process gunzip/bunzip2/zcat (distinct from prepipe) FileInputEncoding lib.TFileInputEncoding } diff --git a/go/src/input/record_reader_csv.go b/go/src/input/record_reader_csv.go index 9b0dd053c..5198d1928 100644 --- a/go/src/input/record_reader_csv.go +++ b/go/src/input/record_reader_csv.go @@ -39,6 +39,7 @@ func (reader *RecordReaderCSV) Read( if len(filenames) == 0 { // read from stdin handle, err := lib.OpenStdin( reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, reader.readerOptions.FileInputEncoding, ) if err != nil { @@ -50,6 +51,7 @@ func (reader *RecordReaderCSV) Read( handle, err := lib.OpenFileForRead( filename, reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, reader.readerOptions.FileInputEncoding, ) if err != nil { diff --git a/go/src/input/record_reader_csvlite.go b/go/src/input/record_reader_csvlite.go index 76a2bbd3f..fc4865191 100644 --- a/go/src/input/record_reader_csvlite.go +++ b/go/src/input/record_reader_csvlite.go @@ -64,6 +64,7 @@ func (reader *RecordReaderCSVLite) Read( if len(filenames) == 0 { // read from stdin handle, err := lib.OpenStdin( reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, reader.readerOptions.FileInputEncoding, ) if err != nil { @@ -91,6 +92,7 @@ func (reader *RecordReaderCSVLite) Read( handle, err := lib.OpenFileForRead( filename, reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, reader.readerOptions.FileInputEncoding, ) if err != nil { diff --git a/go/src/input/record_reader_dkvp.go b/go/src/input/record_reader_dkvp.go index b7ca8ca6f..83175bb8b 100644 --- a/go/src/input/record_reader_dkvp.go +++ b/go/src/input/record_reader_dkvp.go @@ -32,6 +32,7 @@ func (reader *RecordReaderDKVP) Read( if len(filenames) == 0 { // read from stdin handle, err := lib.OpenStdin( reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, reader.readerOptions.FileInputEncoding, ) if err != nil { @@ -43,6 +44,7 @@ func (reader *RecordReaderDKVP) Read( handle, err := lib.OpenFileForRead( filename, reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, reader.readerOptions.FileInputEncoding, ) if err != nil { diff --git a/go/src/input/record_reader_json.go b/go/src/input/record_reader_json.go index da5fd4cc8..d3eb57d00 100644 --- a/go/src/input/record_reader_json.go +++ b/go/src/input/record_reader_json.go @@ -32,6 +32,7 @@ func (reader *RecordReaderJSON) Read( if len(filenames) == 0 { // read from stdin handle, err := lib.OpenStdin( reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, reader.readerOptions.FileInputEncoding, ) if err != nil { @@ -43,6 +44,7 @@ func (reader *RecordReaderJSON) Read( handle, err := lib.OpenFileForRead( filename, reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, reader.readerOptions.FileInputEncoding, ) if err != nil { diff --git a/go/src/input/record_reader_nidx.go b/go/src/input/record_reader_nidx.go index 58f87a595..7a12d00bf 100644 --- a/go/src/input/record_reader_nidx.go +++ b/go/src/input/record_reader_nidx.go @@ -32,6 +32,7 @@ func (reader *RecordReaderNIDX) Read( if len(filenames) == 0 { // read from stdin handle, err := lib.OpenStdin( reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, reader.readerOptions.FileInputEncoding, ) if err != nil { @@ -43,6 +44,7 @@ func (reader *RecordReaderNIDX) Read( handle, err := lib.OpenFileForRead( filename, reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, reader.readerOptions.FileInputEncoding, ) if err != nil { diff --git a/go/src/input/record_reader_xtab.go b/go/src/input/record_reader_xtab.go index ed805eaad..ffc683c39 100644 --- a/go/src/input/record_reader_xtab.go +++ b/go/src/input/record_reader_xtab.go @@ -44,6 +44,7 @@ func (reader *RecordReaderXTAB) Read( if len(filenames) == 0 { // read from stdin handle, err := lib.OpenStdin( reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, reader.readerOptions.FileInputEncoding, ) if err != nil { @@ -55,6 +56,7 @@ func (reader *RecordReaderXTAB) Read( handle, err := lib.OpenFileForRead( filename, reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, reader.readerOptions.FileInputEncoding, ) if err != nil { diff --git a/go/src/lib/file-readers.go b/go/src/lib/file-readers.go index 5edf4b4ae..c2f9cfe9f 100644 --- a/go/src/lib/file-readers.go +++ b/go/src/lib/file-readers.go @@ -47,10 +47,11 @@ const ( func OpenFileForRead( filename string, prepipe string, + prepipeIsRaw bool, encoding TFileInputEncoding, // ignored if prepipe is non-empty ) (io.ReadCloser, error) { if prepipe != "" { - return openPrepipedHandleForRead(filename, prepipe) + return openPrepipedHandleForRead(filename, prepipe, prepipeIsRaw) } else { handle, err := os.Open(filename) if err != nil { @@ -67,10 +68,11 @@ func OpenFileForRead( // a compression encoding, this ends up being simply os.Stdin. func OpenStdin( prepipe string, + prepipeIsRaw bool, encoding TFileInputEncoding, // ignored if prepipe is non-empty ) (io.ReadCloser, error) { if prepipe != "" { - return openPrepipedHandleForRead("", prepipe) + return openPrepipedHandleForRead("", prepipe, prepipeIsRaw) } else { return openEncodedHandleForRead(os.Stdin, encoding, "") } @@ -79,6 +81,7 @@ func OpenStdin( func openPrepipedHandleForRead( filename string, prepipe string, + prepipeIsRaw bool, ) (io.ReadCloser, error) { escapedFilename := escapeFileNameForPopen(filename) @@ -86,7 +89,11 @@ func openPrepipedHandleForRead( if filename == "" { // stdin command = prepipe } else { - command = prepipe + " < " + escapedFilename + if prepipeIsRaw { + command = prepipe + " " + escapedFilename + } else { + command = prepipe + " < " + escapedFilename + } } return OpenInboundHalfPipe(command) diff --git a/go/src/transformers/join.go b/go/src/transformers/join.go index 72b860a4c..7b7b82027 100644 --- a/go/src/transformers/join.go +++ b/go/src/transformers/join.go @@ -46,6 +46,7 @@ type tJoinOptions struct { leftFileName string prepipe string + prepipeIsRaw bool // These allow the joiner to have its own different format/delimiter for the left-file: joinReaderOptions cliutil.TReaderOptions @@ -67,6 +68,7 @@ func newJoinOptions() *tJoinOptions { leftFileName: "", prepipe: "", + prepipeIsRaw: false, } } @@ -105,6 +107,7 @@ func transformerJoinUsage( lib.MlrExeName()) fmt.Fprintf(o, " If you wish to use a prepipe command for the main input as well\n") fmt.Fprintf(o, " as here, it must be specified there as well as here.\n") + fmt.Fprintf(o, " --prepipex {command} Likewise.\n") fmt.Fprintf(o, "File-format options default to those for the right file names on the Miller\n") fmt.Fprintf(o, "argument list, but may be overridden for the left file as follows. Please see\n") fmt.Fprintf(o, "the main \"%s --help\" for more information on syntax for these arguments:\n", lib.MlrExeName()) @@ -163,6 +166,11 @@ func transformerJoinParseCLI( } else if opt == "--prepipe" { opts.prepipe = cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc) + opts.prepipeIsRaw = false + + } else if opt == "--prepipex" { + opts.prepipe = cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc) + opts.prepipeIsRaw = true } else if opt == "-f" { opts.leftFileName = cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc)