diff --git a/go/regtest/cases/cli-help/0001/expout b/go/regtest/cases/cli-help/0001/expout index df7a32971..526c3082a 100644 --- a/go/regtest/cases/cli-help/0001/expout +++ b/go/regtest/cases/cli-help/0001/expout @@ -138,7 +138,8 @@ Options: ================================================================ Usage: mlr filter [options] {DSL expression} Options: --f {file name} File containing a DSL expression. +-f {file name} File containing a DSL expression. If the filename is a directory, + all *.mlr files in that directory are loaded. -e {expression} You can use this after -f to add an expression. Example use case: define functions/subroutines in a file you specify with -f, then call @@ -472,7 +473,8 @@ Options: ================================================================ Usage: mlr put [options] {DSL expression} Options: --f {file name} File containing a DSL expression. +-f {file name} File containing a DSL expression. If the filename is a directory, + all *.mlr files in that directory are loaded. -e {expression} You can use this after -f to add an expression. Example use case: define functions/subroutines in a file you specify with -f, then call diff --git a/go/regtest/cases/repl/0001/cmd b/go/regtest/cases/repl/0001/cmd index 125fb544c..3d715932b 100644 --- a/go/regtest/cases/repl/0001/cmd +++ b/go/regtest/cases/repl/0001/cmd @@ -1 +1 @@ -mlr repl -q -s ./${CASEDIR}/input +mlr repl -q -s < ./${CASEDIR}/input diff --git a/go/regtest/cases/repl/0002/cmd b/go/regtest/cases/repl/0002/cmd index 125fb544c..3d715932b 100644 --- a/go/regtest/cases/repl/0002/cmd +++ b/go/regtest/cases/repl/0002/cmd @@ -1 +1 @@ -mlr repl -q -s ./${CASEDIR}/input +mlr repl -q -s < ./${CASEDIR}/input diff --git a/go/regtest/cases/repl/0002/expout b/go/regtest/cases/repl/0002/expout index e69de29bb..00750edc0 100644 --- a/go/regtest/cases/repl/0002/expout +++ b/go/regtest/cases/repl/0002/expout @@ -0,0 +1 @@ +3 diff --git a/go/regtest/cases/repl/0003/cmd b/go/regtest/cases/repl/0003/cmd index 125fb544c..3d715932b 100644 --- a/go/regtest/cases/repl/0003/cmd +++ b/go/regtest/cases/repl/0003/cmd @@ -1 +1 @@ -mlr repl -q -s ./${CASEDIR}/input +mlr repl -q -s < ./${CASEDIR}/input diff --git a/go/regtest/cases/repl/0003/expout b/go/regtest/cases/repl/0003/expout index e69de29bb..3a6749204 100644 --- a/go/regtest/cases/repl/0003/expout +++ b/go/regtest/cases/repl/0003/expout @@ -0,0 +1,3 @@ +{ + "x": 3 +} diff --git a/go/regtest/cases/repl/0003/input b/go/regtest/cases/repl/0003/input index 9501eaa40..22d403cfe 100644 --- a/go/regtest/cases/repl/0003/input +++ b/go/regtest/cases/repl/0003/input @@ -1,2 +1,2 @@ -\$x=3 -\$* +$x=3 +$* diff --git a/go/regtest/cases/repl/0004/cmd b/go/regtest/cases/repl/0004/cmd index 125fb544c..3d715932b 100644 --- a/go/regtest/cases/repl/0004/cmd +++ b/go/regtest/cases/repl/0004/cmd @@ -1 +1 @@ -mlr repl -q -s ./${CASEDIR}/input +mlr repl -q -s < ./${CASEDIR}/input diff --git a/go/regtest/cases/repl/0004/expout b/go/regtest/cases/repl/0004/expout index e69de29bb..cf72322ee 100644 --- a/go/regtest/cases/repl/0004/expout +++ b/go/regtest/cases/repl/0004/expout @@ -0,0 +1,13 @@ +HOW ARE THINGS? +3 +#begin 3 +#main 2 +#end 3 +In the beginning: +HELLO +WORLD +In ... +... the middle! +At the end: +GOODBYE +WORLD diff --git a/go/regtest/cases/repl/0005/cmd b/go/regtest/cases/repl/0005/cmd index 125fb544c..3d715932b 100644 --- a/go/regtest/cases/repl/0005/cmd +++ b/go/regtest/cases/repl/0005/cmd @@ -1 +1 @@ -mlr repl -q -s ./${CASEDIR}/input +mlr repl -q -s < ./${CASEDIR}/input diff --git a/go/regtest/cases/repl/0005/expout b/go/regtest/cases/repl/0005/expout index e69de29bb..cc2089b8d 100644 --- a/go/regtest/cases/repl/0005/expout +++ b/go/regtest/cases/repl/0005/expout @@ -0,0 +1,53 @@ +FILENAME="(stdin)",FILENUM=0,NR=0,FNR=0 +FILENAME="regtest/input/medium.dkvp",FILENUM=1,NR=10,FNR=10 +{ + "a": "pan", + "b": "wye", + "i": 10, + "x": 0.5026260055412137, + "y": 0.9526183602969864 +} +a=pan,b=pan,i=11,x=0.7930488423451967,y=0.6505816637259333 +a=zee,b=pan,i=12,x=0.3676141320555616,y=0.23614420670296965 +a=eks,b=pan,i=13,x=0.4915175580479536,y=0.7709126592971468 +a=eks,b=zee,i=14,x=0.5207382318405251,y=0.34141681118811673 +a=eks,b=pan,i=15,x=0.07155556372719507,y=0.3596137145616235 +a=pan,b=pan,i=16,x=0.5736853980681922,y=0.7554169353781729 +a=zee,b=eks,i=17,x=0.29081949506712723,y=0.054478717073354166 +a=hat,b=zee,i=18,x=0.05727869223575699,y=0.13343527626645157 +a=zee,b=pan,i=19,x=0.43144132839222604,y=0.8442204830496998 +a=eks,b=wye,i=20,x=0.38245149780530685,y=0.4730652428100751 +FILENAME="regtest/input/medium.dkvp",FILENUM=1,NR=20,FNR=20 +{ + "a": "eks", + "b": "wye", + "i": 20, + "x": 0.38245149780530685, + "y": 0.4730652428100751 +} +FILENAME="regtest/input/medium.dkvp",FILENUM=1,NR=30,FNR=30 +{ + "a": "zee", + "b": "pan", + "i": 30, + "x": 0.7514525581952406, + "y": 0.5869901584461873 +} +a=eks,b=pan,i=31,x=0.5701563454882129,y=0.822178550628332 +a=wye,b=wye,i=32,x=0.5112738997714514,y=0.33107563648490446 +a=hat,b=pan,i=33,x=0.09303747345405,y=0.18404048980291832 +a=hat,b=wye,i=34,x=0.49061881414751796,y=0.4721354893261027 +a=zee,b=zee,i=35,x=0.12565828887119346,y=0.1936186210637818 +a=pan,b=hat,i=36,x=0.9436794292675323,y=0.9111218711707734 +a=wye,b=pan,i=37,x=0.056537252420374995,y=0.5450112991603523 +a=pan,b=zee,i=38,x=0.508250895522605,y=0.12919877119684198 +a=hat,b=pan,i=39,x=0.4591095934684415,y=0.6526252088703994 +a=pan,b=hat,i=40,x=0.30302850095892986,y=0.2884018711352886 +FILENAME="regtest/input/medium.dkvp",FILENUM=1,NR=40,FNR=40 +{ + "a": "pan", + "b": "hat", + "i": 40, + "x": 0.30302850095892986, + "y": 0.2884018711352886 +} diff --git a/go/regtest/cases/repl/0005/input b/go/regtest/cases/repl/0005/input index 3b22f4426..07dca0337 100644 --- a/go/regtest/cases/repl/0005/input +++ b/go/regtest/cases/repl/0005/input @@ -1,19 +1,19 @@ -:open $indir/medium.dkvp +:open regtest/input/medium.dkvp :context :skip 10 :context -\$* +$* :process 10 :context -\$* +$* :skip until NR == 30 :context -\$* +$* :process until NR == 40 :context -\$* +$* diff --git a/go/regtest/cases/repl/0006/cmd b/go/regtest/cases/repl/0006/cmd index b51afff65..7d1169b75 100644 --- a/go/regtest/cases/repl/0006/cmd +++ b/go/regtest/cases/repl/0006/cmd @@ -1 +1 @@ -mlr repl -q -s --j2x regtest/input/flatten-input-2.json ./${CASEDIR}/input +mlr repl -q -s --j2x regtest/input/flatten-input-2.json < ./${CASEDIR}/input diff --git a/go/regtest/cases/repl/0006/expout b/go/regtest/cases/repl/0006/expout index e69de29bb..35245e22a 100644 --- a/go/regtest/cases/repl/0006/expout +++ b/go/regtest/cases/repl/0006/expout @@ -0,0 +1,16 @@ +hostname localhost +pid 12345 +req.id 6789 +req.method GET +req.path api/check +req.host foo.bar +req.headers.host bar.baz +req.headers.user-agent browser +res.status_code 200 +res.header.content-type text +res.header.content-encoding plain +empty1 {} +empty2 [] +wrapper.empty3 {} +wrapper.emtpy4 [] +End of record stream diff --git a/go/regtest/cases/repl/0007/cmd b/go/regtest/cases/repl/0007/cmd index 2f372db12..301ad82cb 100644 --- a/go/regtest/cases/repl/0007/cmd +++ b/go/regtest/cases/repl/0007/cmd @@ -1 +1 @@ -mlr repl -q -s --x2j regtest/input/unflatten-input.xtab ./${CASEDIR}/input +mlr repl -q -s --x2j regtest/input/unflatten-input.xtab < ./${CASEDIR}/input diff --git a/go/regtest/cases/repl/0007/expout b/go/regtest/cases/repl/0007/expout index e69de29bb..5ff356688 100644 --- a/go/regtest/cases/repl/0007/expout +++ b/go/regtest/cases/repl/0007/expout @@ -0,0 +1,28 @@ +{ + "hostname": "localhost", + "pid": 12345, + "req": { + "id": 6789, + "method": "GET", + "path": "api/check", + "host": "foo.bar", + "headers": { + "host": "bar.baz", + "user-agent": "browser" + } + }, + "res": { + "status_code": 200, + "header": { + "content-type": "text", + "content-encoding": "plain" + } + }, + "empty1": {}, + "empty2": [], + "wrapper": { + "empty3": {}, + "emtpy4": [] + } +} +End of record stream diff --git a/go/regtest/cases/repl/0008/cmd b/go/regtest/cases/repl/0008/cmd index 92d037aff..6922a3271 100644 --- a/go/regtest/cases/repl/0008/cmd +++ b/go/regtest/cases/repl/0008/cmd @@ -1 +1 @@ -mlr repl -q -s --xtab regtest/input/unflatten-input.xtab ./${CASEDIR}/input +mlr repl -q -s --xtab regtest/input/unflatten-input.xtab < ./${CASEDIR}/input diff --git a/go/regtest/cases/repl/0008/expout b/go/regtest/cases/repl/0008/expout index e69de29bb..35245e22a 100644 --- a/go/regtest/cases/repl/0008/expout +++ b/go/regtest/cases/repl/0008/expout @@ -0,0 +1,16 @@ +hostname localhost +pid 12345 +req.id 6789 +req.method GET +req.path api/check +req.host foo.bar +req.headers.host bar.baz +req.headers.user-agent browser +res.status_code 200 +res.header.content-type text +res.header.content-encoding plain +empty1 {} +empty2 [] +wrapper.empty3 {} +wrapper.emtpy4 [] +End of record stream diff --git a/go/regtest/cases/repl/0009/cmd b/go/regtest/cases/repl/0009/cmd index 50523fc7d..66a21cebd 100644 --- a/go/regtest/cases/repl/0009/cmd +++ b/go/regtest/cases/repl/0009/cmd @@ -1 +1 @@ -mlr repl -q -s --json regtest/input/flatten-input-2.json ./${CASEDIR}/input +mlr repl -q -s --json regtest/input/flatten-input-2.json < ./${CASEDIR}/input diff --git a/go/regtest/cases/repl/0009/expout b/go/regtest/cases/repl/0009/expout index e69de29bb..5ff356688 100644 --- a/go/regtest/cases/repl/0009/expout +++ b/go/regtest/cases/repl/0009/expout @@ -0,0 +1,28 @@ +{ + "hostname": "localhost", + "pid": 12345, + "req": { + "id": 6789, + "method": "GET", + "path": "api/check", + "host": "foo.bar", + "headers": { + "host": "bar.baz", + "user-agent": "browser" + } + }, + "res": { + "status_code": 200, + "header": { + "content-type": "text", + "content-encoding": "plain" + } + }, + "empty1": {}, + "empty2": [], + "wrapper": { + "empty3": {}, + "emtpy4": [] + } +} +End of record stream diff --git a/go/regtest/cases/repl/0010/cmd b/go/regtest/cases/repl/0010/cmd index 50523fc7d..66a21cebd 100644 --- a/go/regtest/cases/repl/0010/cmd +++ b/go/regtest/cases/repl/0010/cmd @@ -1 +1 @@ -mlr repl -q -s --json regtest/input/flatten-input-2.json ./${CASEDIR}/input +mlr repl -q -s --json regtest/input/flatten-input-2.json < ./${CASEDIR}/input diff --git a/go/regtest/cases/repl/0010/expout b/go/regtest/cases/repl/0010/expout index e69de29bb..9808817df 100644 --- a/go/regtest/cases/repl/0010/expout +++ b/go/regtest/cases/repl/0010/expout @@ -0,0 +1,56 @@ +{ + "hostname": "localhost", + "pid": 12345, + "req": { + "id": 6789, + "method": "GET", + "path": "api/check", + "host": "foo.bar", + "headers": { + "host": "bar.baz", + "user-agent": "browser" + } + }, + "res": { + "status_code": 200, + "header": { + "content-type": "text", + "content-encoding": "plain" + } + }, + "empty1": {}, + "empty2": [], + "wrapper": { + "empty3": {}, + "emtpy4": [] + } +} +End of record stream +{ + "hostname": "localhost", + "pid": 12345, + "req": { + "id": 6789, + "method": "GET", + "path": "api/check", + "host": "foo.bar", + "headers": { + "host": "bar.baz", + "user-agent": "browser" + } + }, + "res": { + "status_code": 200, + "header": { + "content-type": "text", + "content-encoding": "plain" + } + }, + "empty1": {}, + "empty2": [], + "wrapper": { + "empty3": {}, + "emtpy4": [] + } +} +End of record stream diff --git a/go/regtest/cases/repl/0011/cmd b/go/regtest/cases/repl/0011/cmd index 4ddec61fb..889bc18bd 100644 --- a/go/regtest/cases/repl/0011/cmd +++ b/go/regtest/cases/repl/0011/cmd @@ -1 +1 @@ -mlr repl -q -s -d ./${CASEDIR}/input +mlr repl -q -s -d < ./${CASEDIR}/input diff --git a/go/regtest/cases/repl/0011/expout b/go/regtest/cases/repl/0011/expout index e69de29bb..ae5b27d7c 100644 --- a/go/regtest/cases/repl/0011/expout +++ b/go/regtest/cases/repl/0011/expout @@ -0,0 +1,6 @@ +(statement-block + (= + y + (+ x 1) + ) +) diff --git a/go/regtest/cases/repl/0012/cmd b/go/regtest/cases/repl/0012/cmd index e985c1113..ba2e10713 100644 --- a/go/regtest/cases/repl/0012/cmd +++ b/go/regtest/cases/repl/0012/cmd @@ -1 +1 @@ -mlr repl -q -s -D ./${CASEDIR}/input +mlr repl -q -s -D < ./${CASEDIR}/input diff --git a/go/regtest/cases/repl/0012/expout b/go/regtest/cases/repl/0012/expout index e69de29bb..6ce17b012 100644 --- a/go/regtest/cases/repl/0012/expout +++ b/go/regtest/cases/repl/0012/expout @@ -0,0 +1 @@ +(statement-block (= y (+ x 1))) diff --git a/go/regtest/cases/repl/0013/cmd b/go/regtest/cases/repl/0013/cmd index fc2101ac6..82c083645 100644 --- a/go/regtest/cases/repl/0013/cmd +++ b/go/regtest/cases/repl/0013/cmd @@ -1 +1 @@ -mlr repl -q -s -v ./${CASEDIR}/input +mlr repl -q -s -v < ./${CASEDIR}/input diff --git a/go/regtest/cases/repl/0013/expout b/go/regtest/cases/repl/0013/expout index e69de29bb..b21a7de20 100644 --- a/go/regtest/cases/repl/0013/expout +++ b/go/regtest/cases/repl/0013/expout @@ -0,0 +1,6 @@ +* statement block + * assignment "=" + * local variable "y" + * operator "+" + * local variable "x" + * int literal "1" diff --git a/go/regtest/cases/repl/0014/cmd b/go/regtest/cases/repl/0014/cmd new file mode 100644 index 000000000..fa2bc9497 --- /dev/null +++ b/go/regtest/cases/repl/0014/cmd @@ -0,0 +1 @@ +mlr repl -q -s --load u/stdlib < ./${CASEDIR}/input diff --git a/go/regtest/cases/repl/0014/experr b/go/regtest/cases/repl/0014/experr new file mode 100644 index 000000000..e69de29bb diff --git a/go/regtest/cases/repl/0014/expout b/go/regtest/cases/repl/0014/expout new file mode 100644 index 000000000..f15dc3652 --- /dev/null +++ b/go/regtest/cases/repl/0014/expout @@ -0,0 +1,2 @@ +55 +3628800 diff --git a/go/regtest/cases/repl/0014/input b/go/regtest/cases/repl/0014/input new file mode 100644 index 000000000..77ab8d711 --- /dev/null +++ b/go/regtest/cases/repl/0014/input @@ -0,0 +1,2 @@ +fibo(10) +factorial(10) diff --git a/go/regtest/cases/repl/0015/cmd b/go/regtest/cases/repl/0015/cmd new file mode 100644 index 000000000..88a609544 --- /dev/null +++ b/go/regtest/cases/repl/0015/cmd @@ -0,0 +1 @@ +mlr repl -q -s --mload u/stdlib/fibo.mlr u/stdlib/factorial.mlr -- < ./${CASEDIR}/input diff --git a/go/regtest/cases/repl/0015/experr b/go/regtest/cases/repl/0015/experr new file mode 100644 index 000000000..e69de29bb diff --git a/go/regtest/cases/repl/0015/expout b/go/regtest/cases/repl/0015/expout new file mode 100644 index 000000000..f15dc3652 --- /dev/null +++ b/go/regtest/cases/repl/0015/expout @@ -0,0 +1,2 @@ +55 +3628800 diff --git a/go/regtest/cases/repl/0015/input b/go/regtest/cases/repl/0015/input new file mode 100644 index 000000000..77ab8d711 --- /dev/null +++ b/go/regtest/cases/repl/0015/input @@ -0,0 +1,2 @@ +fibo(10) +factorial(10) diff --git a/go/src/auxents/repl/entry.go b/go/src/auxents/repl/entry.go index 76a5957d5..979689519 100644 --- a/go/src/auxents/repl/entry.go +++ b/go/src/auxents/repl/entry.go @@ -60,6 +60,13 @@ func ReplUsage(verbName string, o *os.File, exitCode int) { -s Don't show prompts +--load {DSL script file} Load script file before presenting the prompt. + If the name following --load is a directory, load all "*.mlr" files + in that directory. + +--mload {DSL script files} -- Like --load but works with more than one filename, + e.g. '--mload *.mlr --'. + -h|--help Show this message. Or any --icsv, --ojson, etc. reader/writer options as for the main Miller command line. @@ -111,6 +118,26 @@ func ReplMain(args []string) int { doWarnings = true argi++ + } else if args[argi] == "--load" { + if argc-argi < 2 { + ReplUsage(replName, os.Stderr, 1) + } + options.DSLPreloadFileNames = append(options.DSLPreloadFileNames, args[argi+1]) + argi += 2 + + } else if args[argi] == "--mload" { + if argc-argi < 2 { + ReplUsage(replName, os.Stderr, 1) + } + argi += 1 + for argi < argc && args[argi] != "--" { + options.DSLPreloadFileNames = append(options.DSLPreloadFileNames, args[argi]) + argi += 1 + } + if args[argi] == "--" { + argi += 1 + } + } else if cliutil.ParseReaderWriterOptions( args, argc, &argi, &options.ReaderOptions, &options.WriterOptions, ) { diff --git a/go/src/auxents/repl/session.go b/go/src/auxents/repl/session.go index b4a2e771f..8145382ef 100644 --- a/go/src/auxents/repl/session.go +++ b/go/src/auxents/repl/session.go @@ -29,6 +29,7 @@ import ( "miller/src/cliutil" "miller/src/dsl/cst" "miller/src/input" + "miller/src/lib" "miller/src/output" "miller/src/runtime" "miller/src/types" @@ -75,7 +76,24 @@ func NewRepl( signal.Notify(sysToSignalHandlerChannel, os.Interrupt, syscall.SIGTERM) go controlCHandler(sysToSignalHandlerChannel, appSignalNotificationChannel) - return &Repl{ + cstRootNode := cst.NewEmptyRoot(&options.WriterOptions).WithRedefinableUDFUDS() + + // TODO + + // If there was a --load/--mload on the command line, load those DSL strings here (e.g. + // someone's local function library). + dslStrings := make([]string, 0) + for _, filename := range options.DSLPreloadFileNames { + theseDSLStrings, err := lib.LoadStringsFromFileOrDir(filename, ".mlr") + if err != nil { + fmt.Fprintf(os.Stderr, "%s %s: cannot load DSL expression from \"%s\": ", + exeName, replName, filename) + return nil, err + } + dslStrings = append(dslStrings, theseDSLStrings...) + } + + repl := &Repl{ exeName: exeName, replName: replName, @@ -87,7 +105,7 @@ func NewRepl( astPrintMode: astPrintMode, doWarnings: doWarnings, - cstRootNode: cst.NewEmptyRoot(&options.WriterOptions).WithRedefinableUDFUDS(), + cstRootNode: cstRootNode, options: options, inputChannel: nil, @@ -99,7 +117,17 @@ func NewRepl( runtimeState: runtimeState, sysToSignalHandlerChannel: sysToSignalHandlerChannel, appSignalNotificationChannel: appSignalNotificationChannel, - }, nil + } + + for _, dslString := range dslStrings { + err := repl.handleDSLStringBulk(dslString, doWarnings) + if err != nil { + // Error message already printed out + return nil, err + } + } + + return repl, nil } // When the user types control-C, immediately print something to the screen, diff --git a/go/src/auxents/repl/verbs.go b/go/src/auxents/repl/verbs.go index e441e7d3d..78f0e200c 100644 --- a/go/src/auxents/repl/verbs.go +++ b/go/src/auxents/repl/verbs.go @@ -6,7 +6,6 @@ package repl import ( "fmt" - "io/ioutil" "os" "strings" @@ -100,6 +99,7 @@ func (this *Repl) handleNonDSLLine(trimmedLine string) bool { // ---------------------------------------------------------------- func usageLoad(this *Repl) { fmt.Println(":load {one or more filenames containing Miller DSL statements}") + fmt.Println("If a filename is a directory, all \"*.mlr\" files will be loaded from within it.") fmt.Print( `Any 'begin {...}' / 'end{...}' blocks are parsed and saved. (You can then type ':begin' or ':end', respectively, to execute them.) User-defined functions and @@ -115,18 +115,19 @@ func handleLoad(this *Repl, args []string) bool { return false } for _, filename := range args { - dslBytes, err := ioutil.ReadFile(filename) + dslStrings, err := lib.LoadStringsFromFileOrDir(filename, ".mlr") if err != nil { - fmt.Printf("Cannot load DSL expression from file \"%s\": ", + fmt.Printf("Cannot load DSL expression file \"%s\": ", filename) fmt.Println(err) return true } - dslString := string(dslBytes) - err = this.handleDSLStringBulk(dslString, this.doWarnings) - if err != nil { - fmt.Println(err) + for _, dslString := range dslStrings { + err = this.handleDSLStringBulk(dslString, this.doWarnings) + if err != nil { + fmt.Println(err) + } } } return true diff --git a/go/src/cli/mlrcli_parse.go b/go/src/cli/mlrcli_parse.go index 2179274f8..bd60133ee 100644 --- a/go/src/cli/mlrcli_parse.go +++ b/go/src/cli/mlrcli_parse.go @@ -288,8 +288,7 @@ func parseTransformers( &argi, argc, args, - &options.ReaderOptions, - &options.WriterOptions, + options, ) lib.InternalCodingErrorIf(transformer == nil) diff --git a/go/src/cli/mlrcli_usage.go b/go/src/cli/mlrcli_usage.go index a903892a9..32573537b 100644 --- a/go/src/cli/mlrcli_usage.go +++ b/go/src/cli/mlrcli_usage.go @@ -496,6 +496,13 @@ func mainUsageOtherOptions(o *os.File, argv0 string) { fmt.Fprintf(o, " The list of filename must end with \"--\". This is useful\n") fmt.Fprintf(o, " for example since \"--from *.csv\" doesn't do what you might\n") fmt.Fprintf(o, " hope but \"--mfrom *.csv --\" does.\n") + fmt.Fprintf(o, " --load {filename} Load DSL script file for all put/filter operations on the command line.\n") + fmt.Fprintf(o, " If the name following --load is a directory, load all \"*.mlr\" files\n") + fmt.Fprintf(o, " in that directory. This is just like \"put -f\" and \"filter -f\"\n") + fmt.Fprintf(o, " except it's up-front on the command line, so you can do something like\n") + fmt.Fprintf(o, " alias mlr='mlr --load ~/myscripts' if you like.\n") + fmt.Fprintf(o, " --mload {names} -- Like --load but works with more than one filename,\n") + fmt.Fprintf(o, " e.g. '--mload *.mlr --'.\n") fmt.Fprintf(o, " -n Process no input files, nor standard input either. Useful\n") fmt.Fprintf(o, " for %s put with begin/end statements only. (Same as --from\n", argv0) fmt.Fprintf(o, " /dev/null.) Also useful in \"%s -n put -v '...'\" for\n", argv0) diff --git a/go/src/cliutil/option_parse.go b/go/src/cliutil/option_parse.go index f5e10d775..d549b688a 100644 --- a/go/src/cliutil/option_parse.go +++ b/go/src/cliutil/option_parse.go @@ -871,6 +871,22 @@ func ParseMiscOptions( argi += 1 } + } else if args[argi] == "--load" { + CheckArgCount(args, argi, argc, 2) + options.DSLPreloadFileNames = append(options.DSLPreloadFileNames, args[argi+1]) + argi += 2 + + } else if args[argi] == "--mload" { + CheckArgCount(args, argi, argc, 2) + argi += 1 + for argi < argc && args[argi] != "--" { + options.DSLPreloadFileNames = append(options.DSLPreloadFileNames, args[argi]) + argi += 1 + } + if args[argi] == "--" { + argi += 1 + } + // } else if args[argi] == "--ofmt" { // CheckArgCount(args, argi, argc, 2); // options.ofmt = args[argi+1]; diff --git a/go/src/cliutil/option_types.go b/go/src/cliutil/option_types.go index 7abb8f81f..efed7240a 100644 --- a/go/src/cliutil/option_types.go +++ b/go/src/cliutil/option_types.go @@ -95,8 +95,20 @@ type TOptions struct { ReaderOptions TReaderOptions WriterOptions TWriterOptions + // Data files to be operated on: e.g. given 'mlr cat foo.dat bar.dat', this + // is ["foo.dat", "bar.dat"]. FileNames []string + // DSL files to be loaded for every put/filter operation -- like 'put -f' + // or 'filter -f' but specified up front on the command line, suitable for + // .mlrrc. Use-case is someone has DSL functions they always want to be + // defined. + // + // TODO: risk of CVE?!? :( -- exclude from .mlrrc & only maybe suggest + // people use an alias -- ? + // TODO: comment re if-dir then load all *.mlr + DSLPreloadFileNames []string + // These are used to construct the transformer list. In particular, for in-place mode // they're reconstructed for each file. We make copies since each pass through a // CLI-parser operates destructively, principally by running strtok over @@ -123,8 +135,9 @@ func DefaultOptions() TOptions { ReaderOptions: DefaultReaderOptions(), WriterOptions: DefaultWriterOptions(), - FileNames: make([]string, 0), - NoInput: false, + FileNames: make([]string, 0), + DSLPreloadFileNames: make([]string, 0), + NoInput: false, } } diff --git a/go/src/lib/readfiles.go b/go/src/lib/readfiles.go new file mode 100644 index 000000000..6c72e8882 --- /dev/null +++ b/go/src/lib/readfiles.go @@ -0,0 +1,74 @@ +// ================================================================ +// Routines for loading strings from files. Nominally for the put/filter verbs +// to load DSL strings from .mlr files. +// ================================================================ + +package lib + +import ( + "io/ioutil" + "os" + "strings" +) + +// LoadStringsFromFileOrDir calls LoadStringFromFile if path exists and is a +// file, or LoadStringsFromDir if path exists and is a directory. In the +// former case the extension is ignored; in the latter case it's used as a +// filter on the directory entries. +func LoadStringsFromFileOrDir(path string, extension string) ([]string, error) { + fileInfo, err := os.Stat(path) + if err != nil { + return nil, err + } + if fileInfo.IsDir() { + return LoadStringsFromDir(path, extension) + } else { + dslString, err := LoadStringFromFile(path) + if err != nil { + return nil, err + } else { + return []string{dslString}, nil + } + } +} + +// LoadStringFromFile is just a wrapper around ioutil.ReadFile, +// with a cast from []byte to string. +func LoadStringFromFile(filename string) (string, error) { + data, err := ioutil.ReadFile(filename) + if err != nil { + return "", err + } + return string(data), nil +} + +// LoadStringsFromDir loads all file contents for files in the given directory +// having the given extension. E.g. LoadStringsFromDir("/u/myfiles", ".mlr") +// will load /u/myfiles/foo.mlr and /u/myfiles/bar.mlr but will skip over +// /u/myfiles/data.csv and /u/myfiles/todo.txt. +func LoadStringsFromDir(dirname string, extension string) ([]string, error) { + dslStrings := make([]string, 0) + + entries, err := os.ReadDir(dirname) + if err != nil { + return nil, err + } + + for i := range entries { + entry := &entries[i] + name := (*entry).Name() + if !strings.HasSuffix(name, extension) { + continue + } + + path := dirname + "/" + name + dslString, err := LoadStringFromFile(path) + if err != nil { + return nil, err + } + + dslStrings = append(dslStrings, dslString) + } + + return dslStrings, nil +} diff --git a/go/src/transformers/altkv.go b/go/src/transformers/altkv.go index 120a56411..fb132edba 100644 --- a/go/src/transformers/altkv.go +++ b/go/src/transformers/altkv.go @@ -40,8 +40,7 @@ func transformerAltkvParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/bar.go b/go/src/transformers/bar.go index 9a3ce3213..7ad0ef51c 100644 --- a/go/src/transformers/bar.go +++ b/go/src/transformers/bar.go @@ -61,8 +61,7 @@ func transformerBarParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/bootstrap.go b/go/src/transformers/bootstrap.go index 397e16ec9..0ff5823e6 100644 --- a/go/src/transformers/bootstrap.go +++ b/go/src/transformers/bootstrap.go @@ -48,8 +48,7 @@ func transformerBootstrapParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/cat.go b/go/src/transformers/cat.go index 130d2708e..6d8d61b15 100644 --- a/go/src/transformers/cat.go +++ b/go/src/transformers/cat.go @@ -43,8 +43,7 @@ func transformerCatParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/check.go b/go/src/transformers/check.go index 2b3b3325d..85e9642ad 100644 --- a/go/src/transformers/check.go +++ b/go/src/transformers/check.go @@ -41,8 +41,7 @@ func transformerCheckParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/clean-whitespace.go b/go/src/transformers/clean-whitespace.go index 513172b57..bc091c7f4 100644 --- a/go/src/transformers/clean-whitespace.go +++ b/go/src/transformers/clean-whitespace.go @@ -49,8 +49,7 @@ func transformerCleanWhitespaceParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { doKeys := true diff --git a/go/src/transformers/count-similar.go b/go/src/transformers/count-similar.go index 09728335b..5441c42e6 100644 --- a/go/src/transformers/count-similar.go +++ b/go/src/transformers/count-similar.go @@ -44,8 +44,7 @@ func transformerCountSimilarParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/count.go b/go/src/transformers/count.go index 27ca2ff1c..9f8679953 100644 --- a/go/src/transformers/count.go +++ b/go/src/transformers/count.go @@ -45,8 +45,7 @@ func transformerCountParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/cut.go b/go/src/transformers/cut.go index 38819faa7..8a2f2ef92 100644 --- a/go/src/transformers/cut.go +++ b/go/src/transformers/cut.go @@ -55,8 +55,7 @@ func transformerCutParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/decimate.go b/go/src/transformers/decimate.go index 5a5f0ff9c..ad300502e 100644 --- a/go/src/transformers/decimate.go +++ b/go/src/transformers/decimate.go @@ -44,8 +44,7 @@ func transformerDecimateParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/fill-down.go b/go/src/transformers/fill-down.go index 94ec9628a..a8e4dcbf9 100644 --- a/go/src/transformers/fill-down.go +++ b/go/src/transformers/fill-down.go @@ -50,8 +50,7 @@ func transformerFillDownParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/fill-empty.go b/go/src/transformers/fill-empty.go index 95f7af528..27f07f830 100644 --- a/go/src/transformers/fill-empty.go +++ b/go/src/transformers/fill-empty.go @@ -41,8 +41,7 @@ func transformerFillEmptyParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/flatten.go b/go/src/transformers/flatten.go index f043d787f..1bad72066 100644 --- a/go/src/transformers/flatten.go +++ b/go/src/transformers/flatten.go @@ -45,8 +45,7 @@ func transformerFlattenParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/fraction.go b/go/src/transformers/fraction.go index 097d5f337..2efdc589d 100644 --- a/go/src/transformers/fraction.go +++ b/go/src/transformers/fraction.go @@ -59,8 +59,7 @@ func transformerFractionParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/gap.go b/go/src/transformers/gap.go index 24fd7aeac..d192092b9 100644 --- a/go/src/transformers/gap.go +++ b/go/src/transformers/gap.go @@ -46,8 +46,7 @@ func transformerGapParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/grep.go b/go/src/transformers/grep.go index 5b7bf8403..814ad9008 100644 --- a/go/src/transformers/grep.go +++ b/go/src/transformers/grep.go @@ -57,8 +57,7 @@ func transformerGrepParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/group-by.go b/go/src/transformers/group-by.go index d72965f92..3c23e4b1d 100644 --- a/go/src/transformers/group-by.go +++ b/go/src/transformers/group-by.go @@ -41,8 +41,7 @@ func transformerGroupByParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/group-like.go b/go/src/transformers/group-like.go index dd1f43193..ca118b285 100644 --- a/go/src/transformers/group-like.go +++ b/go/src/transformers/group-like.go @@ -41,8 +41,7 @@ func transformerGroupLikeParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/having-fields.go b/go/src/transformers/having-fields.go index ea6b2d62e..02cf64653 100644 --- a/go/src/transformers/having-fields.go +++ b/go/src/transformers/having-fields.go @@ -67,8 +67,7 @@ func transformerHavingFieldsParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { havingFieldsCriterion := havingFieldsCriterionUnspecified diff --git a/go/src/transformers/head.go b/go/src/transformers/head.go index 785aa23a5..94879c677 100644 --- a/go/src/transformers/head.go +++ b/go/src/transformers/head.go @@ -49,8 +49,7 @@ func transformerHeadParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/histogram.go b/go/src/transformers/histogram.go index f74912ff6..25d435394 100644 --- a/go/src/transformers/histogram.go +++ b/go/src/transformers/histogram.go @@ -48,8 +48,7 @@ func transformerHistogramParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/join.go b/go/src/transformers/join.go index 01002d9b0..6dfed8b89 100644 --- a/go/src/transformers/join.go +++ b/go/src/transformers/join.go @@ -67,9 +67,6 @@ func newJoinOptions() *tJoinOptions { leftFileName: "", prepipe: "", - - // TODO - // readerOptions: readerOptions, } } @@ -134,8 +131,7 @@ func transformerJoinParseCLI( pargi *int, argc int, args []string, - mainReaderOptions *cliutil.TReaderOptions, // Options for the right-files - __ *cliutil.TWriterOptions, + mainOptions *cliutil.TOptions, // Options for the right-files ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -146,9 +142,9 @@ func transformerJoinParseCLI( // Parse local flags opts := newJoinOptions() - if mainReaderOptions != nil { // for 'mlr --usage-all-verbs', it's nil + if mainOptions != nil { // for 'mlr --usage-all-verbs', it's nil // TODO: make sure this is a full nested-struct copy. - opts.joinReaderOptions = *mainReaderOptions // struct copy + opts.joinReaderOptions = mainOptions.ReaderOptions // struct copy } for argi < argc /* variable increment: 1 or 2 depending on flag */ { diff --git a/go/src/transformers/json-parse.go b/go/src/transformers/json-parse.go index 41671a3ad..298463846 100644 --- a/go/src/transformers/json-parse.go +++ b/go/src/transformers/json-parse.go @@ -44,8 +44,7 @@ func transformerJSONParseParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/json-stringify.go b/go/src/transformers/json-stringify.go index 19606f544..c6619ac0f 100644 --- a/go/src/transformers/json-stringify.go +++ b/go/src/transformers/json-stringify.go @@ -45,8 +45,7 @@ func transformerJSONStringifyParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/label.go b/go/src/transformers/label.go index 042ef9608..5e5892e67 100644 --- a/go/src/transformers/label.go +++ b/go/src/transformers/label.go @@ -45,8 +45,7 @@ func transformerLabelParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/merge-fields.go b/go/src/transformers/merge-fields.go index b6be1e76c..91beb0e69 100644 --- a/go/src/transformers/merge-fields.go +++ b/go/src/transformers/merge-fields.go @@ -82,8 +82,7 @@ func transformerMergeFieldsParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/most-or-least-frequent.go b/go/src/transformers/most-or-least-frequent.go index 1987196f7..8cdf9a270 100644 --- a/go/src/transformers/most-or-least-frequent.go +++ b/go/src/transformers/most-or-least-frequent.go @@ -79,8 +79,7 @@ func transformerMostFrequentParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { return transformerMostOrLeastFrequentParseCLI(pargi, argc, args, true, transformerMostFrequentUsage) } @@ -89,8 +88,7 @@ func transformerLeastFrequentParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { return transformerMostOrLeastFrequentParseCLI(pargi, argc, args, false, transformerLeastFrequentUsage) } diff --git a/go/src/transformers/nest.go b/go/src/transformers/nest.go index 4ab64be80..b1f9f3541 100644 --- a/go/src/transformers/nest.go +++ b/go/src/transformers/nest.go @@ -97,8 +97,7 @@ func transformerNestParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/nothing.go b/go/src/transformers/nothing.go index c421b05e5..3c6b75420 100644 --- a/go/src/transformers/nothing.go +++ b/go/src/transformers/nothing.go @@ -41,8 +41,7 @@ func transformerNothingParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/put-or-filter.go b/go/src/transformers/put-or-filter.go index ea81580b2..cd89456fa 100644 --- a/go/src/transformers/put-or-filter.go +++ b/go/src/transformers/put-or-filter.go @@ -3,7 +3,6 @@ package transformers import ( "errors" "fmt" - "io/ioutil" "os" "strings" @@ -63,7 +62,8 @@ func transformerPutOrFilterUsage( fmt.Fprintf(o, "Usage: %s %s [options] {DSL expression}\n", lib.MlrExeName(), verb) fmt.Fprintf(o, "Options:\n") fmt.Fprintf(o, - `-f {file name} File containing a DSL expression. + `-f {file name} File containing a DSL expression. If the filename is a directory, + all *.mlr files in that directory are loaded. -e {expression} You can use this after -f to add an expression. Example use case: define functions/subroutines in a file you specify with -f, then call @@ -124,8 +124,7 @@ func transformerPutOrFilterParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - mainRecordWriterOptions *cliutil.TWriterOptions, + mainOptions *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -134,6 +133,7 @@ func transformerPutOrFilterParseCLI( argi++ var dslStrings []string = make([]string, 0) + haveDSLStringsHere := false echoDSLString := false printASTAsTree := false printASTMultiLine := false @@ -146,15 +146,26 @@ func transformerPutOrFilterParseCLI( presets := make([]string, 0) // TODO: make sure this is a full nested-struct copy. - var recordWriterOptions *cliutil.TWriterOptions = nil - if mainRecordWriterOptions != nil { - // TODO: make a .Copy() method - copyThereof := *mainRecordWriterOptions - recordWriterOptions = ©Thereof + var options *cliutil.TOptions = nil + if mainOptions != nil { + copyThereof := *mainOptions // struct copy + options = ©Thereof + } + + // If there was a global --load/--mload, load those DSL strings here (e.g. + // someone's local function library). + for _, filename := range options.DSLPreloadFileNames { + theseDSLStrings, err := lib.LoadStringsFromFileOrDir(filename, ".mlr") + if err != nil { + fmt.Fprintf(os.Stderr, "%s %s: cannot load DSL expression from \"%s\": ", + lib.MlrExeName(), verb, filename) + fmt.Println(err) + return nil + } + dslStrings = append(dslStrings, theseDSLStrings...) } // Parse local flags. - for argi < argc /* variable increment: 1 or 2 depending on flag */ { opt := args[argi] if !strings.HasPrefix(opt, "-") { @@ -168,19 +179,20 @@ func transformerPutOrFilterParseCLI( } else if opt == "-f" { // Get a DSL string from the user-specified filename filename := cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc) - data, err := ioutil.ReadFile(filename) + theseDSLStrings, err := lib.LoadStringsFromFileOrDir(filename, ".mlr") if err != nil { fmt.Fprintf(os.Stderr, "%s %s: cannot load DSL expression from file \"%s\": ", lib.MlrExeName(), verb, filename) fmt.Println(err) return nil } - dslString := string(data) - dslStrings = append(dslStrings, dslString) + dslStrings = append(dslStrings, theseDSLStrings...) + haveDSLStringsHere = true } else if opt == "-e" { dslString := cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc) dslStrings = append(dslStrings, dslString) + haveDSLStringsHere = true } else if opt == "-s" { // E.g. @@ -228,7 +240,7 @@ func transformerPutOrFilterParseCLI( // loop (so individual if-statements don't need to). However, // ParseWriterOptions expects it unadvanced. wargi := argi - 1 - if cliutil.ParseWriterOptions(args, argc, &wargi, recordWriterOptions) { + if cliutil.ParseWriterOptions(args, argc, &wargi, &options.WriterOptions) { // This lets mlr main and mlr put have different output formats. // Nothing else to handle here. argi = wargi @@ -241,7 +253,7 @@ func transformerPutOrFilterParseCLI( // If they've used either of 'mlr put -f {filename}' or 'mlr put -e // {expression}' then that specifies their DSL expression. But if they've // done neither then we expect 'mlr put {expression}'. - if len(dslStrings) == 0 { + if !haveDSLStringsHere { // Get the DSL string from the command line, after the flags if argi >= argc { transformerPutUsage(os.Stderr, true, 1) @@ -265,7 +277,7 @@ func transformerPutOrFilterParseCLI( warningsAreFatal, invertFilter, suppressOutputRecord, - recordWriterOptions, + options, ) if err != nil { // Error message already printed out @@ -299,10 +311,10 @@ func NewTransformerPut( warningsAreFatal bool, invertFilter bool, suppressOutputRecord bool, - recordWriterOptions *cliutil.TWriterOptions, + options *cliutil.TOptions, ) (*TransformerPut, error) { - cstRootNode := cst.NewEmptyRoot(recordWriterOptions) + cstRootNode := cst.NewEmptyRoot(&options.WriterOptions) for _, dslString := range dslStrings { astRootNode, err := BuildASTFromStringWithMessage(dslString) diff --git a/go/src/transformers/regularize.go b/go/src/transformers/regularize.go index 36be888eb..9b66df7d4 100644 --- a/go/src/transformers/regularize.go +++ b/go/src/transformers/regularize.go @@ -40,8 +40,7 @@ func transformerRegularizeParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/remove-empty-columns.go b/go/src/transformers/remove-empty-columns.go index 227fbc082..542374be9 100644 --- a/go/src/transformers/remove-empty-columns.go +++ b/go/src/transformers/remove-empty-columns.go @@ -41,8 +41,7 @@ func transformerRemoveEmptyColumnsParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/rename.go b/go/src/transformers/rename.go index 5fcaae74f..a0abd98db 100644 --- a/go/src/transformers/rename.go +++ b/go/src/transformers/rename.go @@ -63,8 +63,7 @@ func transformerRenameParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/reorder.go b/go/src/transformers/reorder.go index 9b47454be..cccfbb18b 100644 --- a/go/src/transformers/reorder.go +++ b/go/src/transformers/reorder.go @@ -56,8 +56,7 @@ func transformerReorderParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/repeat.go b/go/src/transformers/repeat.go index 60de63320..cf662eedb 100644 --- a/go/src/transformers/repeat.go +++ b/go/src/transformers/repeat.go @@ -70,8 +70,7 @@ func transformerRepeatParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { repeatCountSource := repeatCountSourceUnspecified diff --git a/go/src/transformers/reshape.go b/go/src/transformers/reshape.go index 840c6e247..e0624f91f 100644 --- a/go/src/transformers/reshape.go +++ b/go/src/transformers/reshape.go @@ -123,8 +123,7 @@ func transformerReshapeParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/sample.go b/go/src/transformers/sample.go index c82068d29..291b9b8e0 100644 --- a/go/src/transformers/sample.go +++ b/go/src/transformers/sample.go @@ -45,8 +45,7 @@ func transformerSampleParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/sec2gmt.go b/go/src/transformers/sec2gmt.go index dd192abb3..fc2abe11f 100644 --- a/go/src/transformers/sec2gmt.go +++ b/go/src/transformers/sec2gmt.go @@ -48,8 +48,7 @@ func transformerSec2GMTParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/sec2gmtdate.go b/go/src/transformers/sec2gmtdate.go index 589fb070f..81de24722 100644 --- a/go/src/transformers/sec2gmtdate.go +++ b/go/src/transformers/sec2gmtdate.go @@ -42,8 +42,7 @@ func transformerSec2GMTDateParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/seqgen.go b/go/src/transformers/seqgen.go index f4d1c5b56..43d38f823 100644 --- a/go/src/transformers/seqgen.go +++ b/go/src/transformers/seqgen.go @@ -52,8 +52,7 @@ func transformerSeqgenParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/shuffle.go b/go/src/transformers/shuffle.go index f57b7ae26..7ab2f83ab 100644 --- a/go/src/transformers/shuffle.go +++ b/go/src/transformers/shuffle.go @@ -44,8 +44,7 @@ func transformerShuffleParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/skip-trivial-records.go b/go/src/transformers/skip-trivial-records.go index 35bc16998..cd8bd18b9 100644 --- a/go/src/transformers/skip-trivial-records.go +++ b/go/src/transformers/skip-trivial-records.go @@ -41,8 +41,7 @@ func transformerSkipTrivialRecordsParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/sort-within-records.go b/go/src/transformers/sort-within-records.go index 5c96c8f90..63eede371 100644 --- a/go/src/transformers/sort-within-records.go +++ b/go/src/transformers/sort-within-records.go @@ -41,8 +41,7 @@ func transformerSortWithinRecordsParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/sort.go b/go/src/transformers/sort.go index e29ae6321..06df67cdf 100644 --- a/go/src/transformers/sort.go +++ b/go/src/transformers/sort.go @@ -98,8 +98,7 @@ func transformerSortParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/stats1.go b/go/src/transformers/stats1.go index 7dc718e28..b7cd96fa5 100644 --- a/go/src/transformers/stats1.go +++ b/go/src/transformers/stats1.go @@ -86,8 +86,7 @@ func transformerStats1ParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/stats2.go b/go/src/transformers/stats2.go index 191b6ee9a..217b2d4e9 100644 --- a/go/src/transformers/stats2.go +++ b/go/src/transformers/stats2.go @@ -69,8 +69,7 @@ func transformerStats2ParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/step.go b/go/src/transformers/step.go index 0f6d616b1..b320e5fa8 100644 --- a/go/src/transformers/step.go +++ b/go/src/transformers/step.go @@ -80,8 +80,7 @@ func transformerStepParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/tac.go b/go/src/transformers/tac.go index 706a53b2a..2eb6b3426 100644 --- a/go/src/transformers/tac.go +++ b/go/src/transformers/tac.go @@ -41,8 +41,7 @@ func transformerTacParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/tail.go b/go/src/transformers/tail.go index ea14e0d73..73913a794 100644 --- a/go/src/transformers/tail.go +++ b/go/src/transformers/tail.go @@ -44,8 +44,7 @@ func transformerTailParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/tee.go b/go/src/transformers/tee.go index 22441aa43..d6dce9537 100644 --- a/go/src/transformers/tee.go +++ b/go/src/transformers/tee.go @@ -48,8 +48,7 @@ func transformerTeeParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - mainRecordWriterOptions *cliutil.TWriterOptions, + mainOptions *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -61,8 +60,8 @@ func transformerTeeParseCLI( piping := false // TODO: make sure this is a full nested-struct copy. var recordWriterOptions *cliutil.TWriterOptions = nil - if mainRecordWriterOptions != nil { - copyThereof := *mainRecordWriterOptions + if mainOptions != nil { + copyThereof := mainOptions.WriterOptions // struct copy recordWriterOptions = ©Thereof } diff --git a/go/src/transformers/top.go b/go/src/transformers/top.go index 445855dc7..cabd6f47d 100644 --- a/go/src/transformers/top.go +++ b/go/src/transformers/top.go @@ -53,8 +53,7 @@ func transformerTopParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/unflatten.go b/go/src/transformers/unflatten.go index 0e662624c..a757ca844 100644 --- a/go/src/transformers/unflatten.go +++ b/go/src/transformers/unflatten.go @@ -45,8 +45,7 @@ func transformerUnflattenParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/uniq.go b/go/src/transformers/uniq.go index 59246e012..862df22be 100644 --- a/go/src/transformers/uniq.go +++ b/go/src/transformers/uniq.go @@ -62,8 +62,7 @@ func transformerCountDistinctParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -162,8 +161,7 @@ func transformerUniqParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transformers/unsparsify.go b/go/src/transformers/unsparsify.go index fb3955880..484b14d1d 100644 --- a/go/src/transformers/unsparsify.go +++ b/go/src/transformers/unsparsify.go @@ -56,8 +56,7 @@ func transformerUnsparsifyParseCLI( pargi *int, argc int, args []string, - _ *cliutil.TReaderOptions, - __ *cliutil.TWriterOptions, + _ *cliutil.TOptions, ) transforming.IRecordTransformer { // Skip the verb name from the current spot in the mlr command line diff --git a/go/src/transforming/record_transformer.go b/go/src/transforming/record_transformer.go index 4f8fcdaa3..a675c5117 100644 --- a/go/src/transforming/record_transformer.go +++ b/go/src/transforming/record_transformer.go @@ -29,8 +29,7 @@ type TransformerParseCLIFunc func( pargi *int, argc int, args []string, - readerOptions *cliutil.TReaderOptions, - writerOptions *cliutil.TWriterOptions, + mainOptions *cliutil.TOptions, ) IRecordTransformer type TransformerSetup struct { diff --git a/go/todo.txt b/go/todo.txt index 1c355f264..fe2d60e5d 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -11,7 +11,18 @@ TOP OF LIST: o check JSON-parser x 2 -- is there really a 'restart'? - infinite-loop avoidance for sure * json-triple-quote -- what can be done here? -* -l/-L +* --load/--mload + k load-from-file helper function + k load-from-dir helper function + k move to lib ddir + k put/filter usage func + k mlr main help + k port x all to repl too + k repl usage func + k make :load handle dirs ... :load usage too ... + o regtest cases including mrpl + ! fix all g3b04k3n ${CASEDIR}/input cases :( + ! suppress from .mlrrc with comments as to why, & local-alias suggestion * tools/perf: o https://eng.uber.com/pprof-go-profiler/ @@ -623,3 +634,8 @@ i https://en.wikipedia.org/wiki/Delimiter#Delimiter_collision - needs doc-ent for JSON_NULL -- only for JSON user data only only ... except for this * sec2gmt --millis/--micros/--nanos doc * doc --mfrom +* sort-within-records --recursive doc +* doc re --load/--mload at mlrmain & repl +* doc re put -f dirname will load dirname/*.mlr + mlr --opprint --from foo.dat --load u/stdlib/ put '$f=fibo($i); $g=factorial($i)' + mlr --opprint --from foo.dat put -f u/stdlib/ -e '$f=fibo($i); $g=factorial($i)' diff --git a/go/u/stdlib/factorial.mlr b/go/u/stdlib/factorial.mlr new file mode 100644 index 000000000..713a075a1 --- /dev/null +++ b/go/u/stdlib/factorial.mlr @@ -0,0 +1,7 @@ +func factorial(int n): int { + if (n <= 1) { + return n + } else { + return n * factorial(n-1) + } +} diff --git a/go/u/stdlib/fibo.mlr b/go/u/stdlib/fibo.mlr new file mode 100644 index 000000000..51a9c2ded --- /dev/null +++ b/go/u/stdlib/fibo.mlr @@ -0,0 +1,7 @@ +func fibo(int n): int { + if (n <= 1) { + return n + } else { + return fibo(n-1) + fibo(n-2) + } +}