Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-08-03 13:03:00 +00:00
Code Issues Projects Releases Wiki Activity

--prepipex option for unzip -qc (#554)

Browse source
This commit is contained in:
John Kerl 2021-06-03 03:31:25 +00:00
parent e70fc08c9b
commit b08633ca82
13 changed files with 50 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

1
go/regtest/cases/cli-help/0001/expout
View file

@ -359,6 +359,7 @@ Options:
--prepipe {command} As in main input options; see mlr --help for details.
If you wish to use a prepipe command for the main input as well
as here, it must be specified there as well as here.
--prepipex {command} Likewise.
File-format options default to those for the right file names on the Miller
argument list, but may be overridden for the left file as follows. Please see
the main "mlr --help" for more information on syntax for these arguments:

2
go/src/cli/mlrcli_parse.go
View file

@ -483,7 +483,7 @@ func handleMlrrcLine(
argi := 0
argc := len(args)
if args[0] == "--prepipe" {
if args[0] == "--prepipe" || args[0] == "--prepipex" {
// Don't allow code execution via .mlrrc
return false
} else if args[0] == "--load" || args[0] == "--mload" {

3
go/src/cli/mlrcli_usage.go
View file

@ -378,6 +378,9 @@ func mainUsageCompressedDataOptions(o *os.File, argv0 string) {
fmt.Fprintf(o, " specify an action to be taken on each input file. This prepipe command must\n")
fmt.Fprintf(o, " be able to read from standard input; it will be invoked with\n")
fmt.Fprintf(o, " {command} < {filename}.\n")
fmt.Fprintf(o, " --prepipex {command} Like --prepipe with one exception: doesn't insert '<' between\n")
fmt.Fprintf(o, " command and filename at runtime. Useful for some commands like 'unzip -qc' which don't\n")
fmt.Fprintf(o, " read standard input.\n")
fmt.Fprintf(o, " Examples:\n")
fmt.Fprintf(o, " %s --prepipe 'gunzip'\n", argv0)
fmt.Fprintf(o, " %s --prepipe 'zcat -cf'\n", argv0)

10
go/src/cliutil/option_parse.go
View file

@ -202,18 +202,28 @@ func ParseReaderOptions(
} else if args[argi] == "--prepipe" {
CheckArgCount(args, argi, argc, 2)
readerOptions.Prepipe = args[argi+1]
readerOptions.PrepipeIsRaw = false
argi += 2
} else if args[argi] == "--prepipex" {
CheckArgCount(args, argi, argc, 2)
readerOptions.Prepipe = args[argi+1]
readerOptions.PrepipeIsRaw = true
argi += 2
} else if args[argi] == "--prepipe-gunzip" {
readerOptions.Prepipe = "gunzip"
readerOptions.PrepipeIsRaw = false
argi += 1
} else if args[argi] == "--prepipe-zcat" {
readerOptions.Prepipe = "zcat"
readerOptions.PrepipeIsRaw = false
argi += 1
} else if args[argi] == "--prepipe-bz2" {
readerOptions.Prepipe = "bz2"
readerOptions.PrepipeIsRaw = false
argi += 1
} else if args[argi] == "--gzin" {

5
go/src/cliutil/option_types.go
View file

@ -57,6 +57,11 @@ type TReaderOptions struct {
// For out-of-process handling of compressed data, via popen
Prepipe string
// For most things like gunzip we do 'gunzip < filename | mlr ...' if
// filename is present, else 'gunzip | mlr ...' if reading from stdin.
// However some commands like 'unzip -qc' are weird so this option lets
// people give the command and we won't insert the '<'.
PrepipeIsRaw bool
// For in-process gunzip/bunzip2/zcat (distinct from prepipe)
FileInputEncoding lib.TFileInputEncoding
}

2
go/src/input/record_reader_csv.go
View file

@ -39,6 +39,7 @@ func (reader *RecordReaderCSV) Read(
if len(filenames) == 0 { // read from stdin
handle, err := lib.OpenStdin(
reader.readerOptions.Prepipe,
reader.readerOptions.PrepipeIsRaw,
reader.readerOptions.FileInputEncoding,
)
if err != nil {
@ -50,6 +51,7 @@ func (reader *RecordReaderCSV) Read(
handle, err := lib.OpenFileForRead(
filename,
reader.readerOptions.Prepipe,
reader.readerOptions.PrepipeIsRaw,
reader.readerOptions.FileInputEncoding,
)
if err != nil {

2
go/src/input/record_reader_csvlite.go
View file

@ -64,6 +64,7 @@ func (reader *RecordReaderCSVLite) Read(
if len(filenames) == 0 { // read from stdin
handle, err := lib.OpenStdin(
reader.readerOptions.Prepipe,
reader.readerOptions.PrepipeIsRaw,
reader.readerOptions.FileInputEncoding,
)
if err != nil {
@ -91,6 +92,7 @@ func (reader *RecordReaderCSVLite) Read(
handle, err := lib.OpenFileForRead(
filename,
reader.readerOptions.Prepipe,
reader.readerOptions.PrepipeIsRaw,
reader.readerOptions.FileInputEncoding,
)
if err != nil {

2
go/src/input/record_reader_dkvp.go
View file

@ -32,6 +32,7 @@ func (reader *RecordReaderDKVP) Read(
if len(filenames) == 0 { // read from stdin
handle, err := lib.OpenStdin(
reader.readerOptions.Prepipe,
reader.readerOptions.PrepipeIsRaw,
reader.readerOptions.FileInputEncoding,
)
if err != nil {
@ -43,6 +44,7 @@ func (reader *RecordReaderDKVP) Read(
handle, err := lib.OpenFileForRead(
filename,
reader.readerOptions.Prepipe,
reader.readerOptions.PrepipeIsRaw,
reader.readerOptions.FileInputEncoding,
)
if err != nil {

2
go/src/input/record_reader_json.go
View file

@ -32,6 +32,7 @@ func (reader *RecordReaderJSON) Read(
if len(filenames) == 0 { // read from stdin
handle, err := lib.OpenStdin(
reader.readerOptions.Prepipe,
reader.readerOptions.PrepipeIsRaw,
reader.readerOptions.FileInputEncoding,
)
if err != nil {
@ -43,6 +44,7 @@ func (reader *RecordReaderJSON) Read(
handle, err := lib.OpenFileForRead(
filename,
reader.readerOptions.Prepipe,
reader.readerOptions.PrepipeIsRaw,
reader.readerOptions.FileInputEncoding,
)
if err != nil {

2
go/src/input/record_reader_nidx.go
View file

@ -32,6 +32,7 @@ func (reader *RecordReaderNIDX) Read(
if len(filenames) == 0 { // read from stdin
handle, err := lib.OpenStdin(
reader.readerOptions.Prepipe,
reader.readerOptions.PrepipeIsRaw,
reader.readerOptions.FileInputEncoding,
)
if err != nil {
@ -43,6 +44,7 @@ func (reader *RecordReaderNIDX) Read(
handle, err := lib.OpenFileForRead(
filename,
reader.readerOptions.Prepipe,
reader.readerOptions.PrepipeIsRaw,
reader.readerOptions.FileInputEncoding,
)
if err != nil {

2
go/src/input/record_reader_xtab.go
View file

@ -44,6 +44,7 @@ func (reader *RecordReaderXTAB) Read(
if len(filenames) == 0 { // read from stdin
handle, err := lib.OpenStdin(
reader.readerOptions.Prepipe,
reader.readerOptions.PrepipeIsRaw,
reader.readerOptions.FileInputEncoding,
)
if err != nil {
@ -55,6 +56,7 @@ func (reader *RecordReaderXTAB) Read(
handle, err := lib.OpenFileForRead(
filename,
reader.readerOptions.Prepipe,
reader.readerOptions.PrepipeIsRaw,
reader.readerOptions.FileInputEncoding,
)
if err != nil {

13
go/src/lib/file-readers.go
View file

@ -47,10 +47,11 @@ const (
func OpenFileForRead(
filename string,
prepipe string,
prepipeIsRaw bool,
encoding TFileInputEncoding, // ignored if prepipe is non-empty
) (io.ReadCloser, error) {
if prepipe != "" {
return openPrepipedHandleForRead(filename, prepipe)
return openPrepipedHandleForRead(filename, prepipe, prepipeIsRaw)
} else {
handle, err := os.Open(filename)
if err != nil {
@ -67,10 +68,11 @@ func OpenFileForRead(
// a compression encoding, this ends up being simply os.Stdin.
func OpenStdin(
prepipe string,
prepipeIsRaw bool,
encoding TFileInputEncoding, // ignored if prepipe is non-empty
) (io.ReadCloser, error) {
if prepipe != "" {
return openPrepipedHandleForRead("", prepipe)
return openPrepipedHandleForRead("", prepipe, prepipeIsRaw)
} else {
return openEncodedHandleForRead(os.Stdin, encoding, "")
}
@ -79,6 +81,7 @@ func OpenStdin(
func openPrepipedHandleForRead(
filename string,
prepipe string,
prepipeIsRaw bool,
) (io.ReadCloser, error) {
escapedFilename := escapeFileNameForPopen(filename)
@ -86,7 +89,11 @@ func openPrepipedHandleForRead(
if filename == "" { // stdin
command = prepipe
} else {
command = prepipe + " < " + escapedFilename
if prepipeIsRaw {
command = prepipe + " " + escapedFilename
} else {
command = prepipe + " < " + escapedFilename
}
}
return OpenInboundHalfPipe(command)

8
go/src/transformers/join.go
View file

@ -46,6 +46,7 @@ type tJoinOptions struct {
leftFileName string
prepipe string
prepipeIsRaw bool
// These allow the joiner to have its own different format/delimiter for the left-file:
joinReaderOptions cliutil.TReaderOptions
@ -67,6 +68,7 @@ func newJoinOptions() *tJoinOptions {
leftFileName: "",
prepipe: "",
prepipeIsRaw: false,
}
}
@ -105,6 +107,7 @@ func transformerJoinUsage(
lib.MlrExeName())
fmt.Fprintf(o, " If you wish to use a prepipe command for the main input as well\n")
fmt.Fprintf(o, " as here, it must be specified there as well as here.\n")
fmt.Fprintf(o, " --prepipex {command} Likewise.\n")
fmt.Fprintf(o, "File-format options default to those for the right file names on the Miller\n")
fmt.Fprintf(o, "argument list, but may be overridden for the left file as follows. Please see\n")
fmt.Fprintf(o, "the main \"%s --help\" for more information on syntax for these arguments:\n", lib.MlrExeName())
@ -163,6 +166,11 @@ func transformerJoinParseCLI(
} else if opt == "--prepipe" {
opts.prepipe = cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc)
opts.prepipeIsRaw = false
} else if opt == "--prepipex" {
opts.prepipe = cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc)
opts.prepipeIsRaw = true
} else if opt == "-f" {
opts.leftFileName = cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc)