DSL-library-load feature

go/regtest/cases/cli-help/0001/expout

@@ -138,7 +138,8 @@ Options:
 ================================================================
 Usage: mlr filter [options] {DSL expression}
 Options:
--f {file name} File containing a DSL expression.
+-f {file name} File containing a DSL expression. If the filename is a directory,
+   all *.mlr files in that directory are loaded.
 
 -e {expression} You can use this after -f to add an expression. Example use
    case: define functions/subroutines in a file you specify with -f, then call
@@ -472,7 +473,8 @@ Options:
 ================================================================
 Usage: mlr put [options] {DSL expression}
 Options:
--f {file name} File containing a DSL expression.
+-f {file name} File containing a DSL expression. If the filename is a directory,
+   all *.mlr files in that directory are loaded.
 
 -e {expression} You can use this after -f to add an expression. Example use
    case: define functions/subroutines in a file you specify with -f, then call

go/regtest/cases/repl/0001/cmd

@@ -1 +1 @@
-mlr repl -q -s ./${CASEDIR}/input
+mlr repl -q -s < ./${CASEDIR}/input

go/regtest/cases/repl/0002/cmd

@@ -1 +1 @@
-mlr repl -q -s ./${CASEDIR}/input
+mlr repl -q -s < ./${CASEDIR}/input

go/regtest/cases/repl/0002/expout

@@ -0,0 +1 @@
+3

go/regtest/cases/repl/0003/cmd

@@ -1 +1 @@
-mlr repl -q -s ./${CASEDIR}/input
+mlr repl -q -s < ./${CASEDIR}/input

go/regtest/cases/repl/0003/expout

@@ -0,0 +1,3 @@
+{
+  "x": 3
+}

go/regtest/cases/repl/0003/input

@@ -1,2 +1,2 @@
-\$x=3
-\$*
+$x=3
+$*

go/regtest/cases/repl/0004/cmd

@@ -1 +1 @@
-mlr repl -q -s ./${CASEDIR}/input
+mlr repl -q -s < ./${CASEDIR}/input

go/regtest/cases/repl/0004/expout

@@ -0,0 +1,13 @@
+HOW ARE THINGS?
+3
+#begin 3
+#main  2
+#end   3
+In the beginning:
+HELLO
+WORLD
+In ...
+... the middle!
+At the end:
+GOODBYE
+WORLD

go/regtest/cases/repl/0005/cmd

@@ -1 +1 @@
-mlr repl -q -s ./${CASEDIR}/input
+mlr repl -q -s < ./${CASEDIR}/input

go/regtest/cases/repl/0005/expout

@@ -0,0 +1,53 @@
+FILENAME="(stdin)",FILENUM=0,NR=0,FNR=0
+FILENAME="regtest/input/medium.dkvp",FILENUM=1,NR=10,FNR=10
+{
+  "a": "pan",
+  "b": "wye",
+  "i": 10,
+  "x": 0.5026260055412137,
+  "y": 0.9526183602969864
+}
+a=pan,b=pan,i=11,x=0.7930488423451967,y=0.6505816637259333
+a=zee,b=pan,i=12,x=0.3676141320555616,y=0.23614420670296965
+a=eks,b=pan,i=13,x=0.4915175580479536,y=0.7709126592971468
+a=eks,b=zee,i=14,x=0.5207382318405251,y=0.34141681118811673
+a=eks,b=pan,i=15,x=0.07155556372719507,y=0.3596137145616235
+a=pan,b=pan,i=16,x=0.5736853980681922,y=0.7554169353781729
+a=zee,b=eks,i=17,x=0.29081949506712723,y=0.054478717073354166
+a=hat,b=zee,i=18,x=0.05727869223575699,y=0.13343527626645157
+a=zee,b=pan,i=19,x=0.43144132839222604,y=0.8442204830496998
+a=eks,b=wye,i=20,x=0.38245149780530685,y=0.4730652428100751
+FILENAME="regtest/input/medium.dkvp",FILENUM=1,NR=20,FNR=20
+{
+  "a": "eks",
+  "b": "wye",
+  "i": 20,
+  "x": 0.38245149780530685,
+  "y": 0.4730652428100751
+}
+FILENAME="regtest/input/medium.dkvp",FILENUM=1,NR=30,FNR=30
+{
+  "a": "zee",
+  "b": "pan",
+  "i": 30,
+  "x": 0.7514525581952406,
+  "y": 0.5869901584461873
+}
+a=eks,b=pan,i=31,x=0.5701563454882129,y=0.822178550628332
+a=wye,b=wye,i=32,x=0.5112738997714514,y=0.33107563648490446
+a=hat,b=pan,i=33,x=0.09303747345405,y=0.18404048980291832
+a=hat,b=wye,i=34,x=0.49061881414751796,y=0.4721354893261027
+a=zee,b=zee,i=35,x=0.12565828887119346,y=0.1936186210637818
+a=pan,b=hat,i=36,x=0.9436794292675323,y=0.9111218711707734
+a=wye,b=pan,i=37,x=0.056537252420374995,y=0.5450112991603523
+a=pan,b=zee,i=38,x=0.508250895522605,y=0.12919877119684198
+a=hat,b=pan,i=39,x=0.4591095934684415,y=0.6526252088703994
+a=pan,b=hat,i=40,x=0.30302850095892986,y=0.2884018711352886
+FILENAME="regtest/input/medium.dkvp",FILENUM=1,NR=40,FNR=40
+{
+  "a": "pan",
+  "b": "hat",
+  "i": 40,
+  "x": 0.30302850095892986,
+  "y": 0.2884018711352886
+}

go/regtest/cases/repl/0005/input

@@ -1,19 +1,19 @@
 
-:open $indir/medium.dkvp
+:open regtest/input/medium.dkvp
 :context
 
 :skip 10
 :context
-\$*
+$*
 
 :process 10
 :context
-\$*
+$*
 
 :skip until NR == 30
 :context
-\$*
+$*
 
 :process until NR == 40
 :context
-\$*
+$*

go/regtest/cases/repl/0006/cmd

@@ -1 +1 @@
-mlr repl -q -s --j2x regtest/input/flatten-input-2.json ./${CASEDIR}/input
+mlr repl -q -s --j2x regtest/input/flatten-input-2.json < ./${CASEDIR}/input

go/regtest/cases/repl/0006/expout

@@ -0,0 +1,16 @@
+hostname                    localhost
+pid                         12345
+req.id                      6789
+req.method                  GET
+req.path                    api/check
+req.host                    foo.bar
+req.headers.host            bar.baz
+req.headers.user-agent      browser
+res.status_code             200
+res.header.content-type     text
+res.header.content-encoding plain
+empty1                      {}
+empty2                      []
+wrapper.empty3              {}
+wrapper.emtpy4              []
+End of record stream

go/regtest/cases/repl/0007/cmd

@@ -1 +1 @@
-mlr repl -q -s --x2j regtest/input/unflatten-input.xtab ./${CASEDIR}/input
+mlr repl -q -s --x2j regtest/input/unflatten-input.xtab < ./${CASEDIR}/input

go/regtest/cases/repl/0007/expout

@@ -0,0 +1,28 @@
+{
+  "hostname": "localhost",
+  "pid": 12345,
+  "req": {
+    "id": 6789,
+    "method": "GET",
+    "path": "api/check",
+    "host": "foo.bar",
+    "headers": {
+      "host": "bar.baz",
+      "user-agent": "browser"
+    }
+  },
+  "res": {
+    "status_code": 200,
+    "header": {
+      "content-type": "text",
+      "content-encoding": "plain"
+    }
+  },
+  "empty1": {},
+  "empty2": [],
+  "wrapper": {
+    "empty3": {},
+    "emtpy4": []
+  }
+}
+End of record stream

go/regtest/cases/repl/0008/cmd

@@ -1 +1 @@
-mlr repl -q -s --xtab regtest/input/unflatten-input.xtab ./${CASEDIR}/input
+mlr repl -q -s --xtab regtest/input/unflatten-input.xtab < ./${CASEDIR}/input

go/regtest/cases/repl/0008/expout

@@ -0,0 +1,16 @@
+hostname                    localhost
+pid                         12345
+req.id                      6789
+req.method                  GET
+req.path                    api/check
+req.host                    foo.bar
+req.headers.host            bar.baz
+req.headers.user-agent      browser
+res.status_code             200
+res.header.content-type     text
+res.header.content-encoding plain
+empty1                      {}
+empty2                      []
+wrapper.empty3              {}
+wrapper.emtpy4              []
+End of record stream

go/regtest/cases/repl/0009/cmd

@@ -1 +1 @@
-mlr repl -q -s --json regtest/input/flatten-input-2.json ./${CASEDIR}/input
+mlr repl -q -s --json regtest/input/flatten-input-2.json < ./${CASEDIR}/input

go/regtest/cases/repl/0009/expout

@@ -0,0 +1,28 @@
+{
+  "hostname": "localhost",
+  "pid": 12345,
+  "req": {
+    "id": 6789,
+    "method": "GET",
+    "path": "api/check",
+    "host": "foo.bar",
+    "headers": {
+      "host": "bar.baz",
+      "user-agent": "browser"
+    }
+  },
+  "res": {
+    "status_code": 200,
+    "header": {
+      "content-type": "text",
+      "content-encoding": "plain"
+    }
+  },
+  "empty1": {},
+  "empty2": [],
+  "wrapper": {
+    "empty3": {},
+    "emtpy4": []
+  }
+}
+End of record stream

go/regtest/cases/repl/0010/cmd

@@ -1 +1 @@
-mlr repl -q -s --json regtest/input/flatten-input-2.json ./${CASEDIR}/input
+mlr repl -q -s --json regtest/input/flatten-input-2.json < ./${CASEDIR}/input

go/regtest/cases/repl/0010/expout

@@ -0,0 +1,56 @@
+{
+  "hostname": "localhost",
+  "pid": 12345,
+  "req": {
+    "id": 6789,
+    "method": "GET",
+    "path": "api/check",
+    "host": "foo.bar",
+    "headers": {
+      "host": "bar.baz",
+      "user-agent": "browser"
+    }
+  },
+  "res": {
+    "status_code": 200,
+    "header": {
+      "content-type": "text",
+      "content-encoding": "plain"
+    }
+  },
+  "empty1": {},
+  "empty2": [],
+  "wrapper": {
+    "empty3": {},
+    "emtpy4": []
+  }
+}
+End of record stream
+{
+  "hostname": "localhost",
+  "pid": 12345,
+  "req": {
+    "id": 6789,
+    "method": "GET",
+    "path": "api/check",
+    "host": "foo.bar",
+    "headers": {
+      "host": "bar.baz",
+      "user-agent": "browser"
+    }
+  },
+  "res": {
+    "status_code": 200,
+    "header": {
+      "content-type": "text",
+      "content-encoding": "plain"
+    }
+  },
+  "empty1": {},
+  "empty2": [],
+  "wrapper": {
+    "empty3": {},
+    "emtpy4": []
+  }
+}
+End of record stream

go/regtest/cases/repl/0011/cmd

@@ -1 +1 @@
-mlr repl -q -s -d ./${CASEDIR}/input
+mlr repl -q -s -d < ./${CASEDIR}/input

go/regtest/cases/repl/0011/expout

@@ -0,0 +1,6 @@
+(statement-block
+    (=
+        y
+        (+ x 1)
+    )
+)

go/regtest/cases/repl/0012/cmd

@@ -1 +1 @@
-mlr repl -q -s -D ./${CASEDIR}/input
+mlr repl -q -s -D < ./${CASEDIR}/input

go/regtest/cases/repl/0012/expout

@@ -0,0 +1 @@
+(statement-block (= y (+ x 1)))

go/regtest/cases/repl/0013/cmd

@@ -1 +1 @@
-mlr repl -q -s -v ./${CASEDIR}/input
+mlr repl -q -s -v < ./${CASEDIR}/input

go/regtest/cases/repl/0013/expout

@@ -0,0 +1,6 @@
+* statement block
+    * assignment "="
+        * local variable "y"
+        * operator "+"
+            * local variable "x"
+            * int literal "1"

go/regtest/cases/repl/0014/cmd

@@ -0,0 +1 @@
+mlr repl -q -s --load u/stdlib < ./${CASEDIR}/input

go/regtest/cases/repl/0014/expout

@@ -0,0 +1,2 @@
+55
+3628800

go/regtest/cases/repl/0014/input

@@ -0,0 +1,2 @@
+fibo(10)
+factorial(10)

go/regtest/cases/repl/0015/cmd

@@ -0,0 +1 @@
+mlr repl -q -s --mload u/stdlib/fibo.mlr u/stdlib/factorial.mlr --  < ./${CASEDIR}/input

go/regtest/cases/repl/0015/expout

@@ -0,0 +1,2 @@
+55
+3628800

go/regtest/cases/repl/0015/input

@@ -0,0 +1,2 @@
+fibo(10)
+factorial(10)

go/src/auxents/repl/entry.go

@@ -60,6 +60,13 @@ func ReplUsage(verbName string, o *os.File, exitCode int) {
 
 -s Don't show prompts
 
+--load {DSL script file} Load script file before presenting the prompt.
+   If the name following --load is a directory, load all "*.mlr" files
+   in that directory.
+
+--mload {DSL script files} -- Like --load but works with more than one filename,
+   e.g. '--mload *.mlr --'.
+
 -h|--help Show this message.
 
 Or any --icsv, --ojson, etc. reader/writer options as for the main Miller command line.
@@ -111,6 +118,26 @@ func ReplMain(args []string) int {
 			doWarnings = true
 			argi++
 
+		} else if args[argi] == "--load" {
+			if argc-argi < 2 {
+				ReplUsage(replName, os.Stderr, 1)
+			}
+			options.DSLPreloadFileNames = append(options.DSLPreloadFileNames, args[argi+1])
+			argi += 2
+
+		} else if args[argi] == "--mload" {
+			if argc-argi < 2 {
+				ReplUsage(replName, os.Stderr, 1)
+			}
+			argi += 1
+			for argi < argc && args[argi] != "--" {
+				options.DSLPreloadFileNames = append(options.DSLPreloadFileNames, args[argi])
+				argi += 1
+			}
+			if args[argi] == "--" {
+				argi += 1
+			}
+
 		} else if cliutil.ParseReaderWriterOptions(
 			args, argc, &argi, &options.ReaderOptions, &options.WriterOptions,
 		) {

go/src/auxents/repl/session.go

@@ -29,6 +29,7 @@ import (
 	"miller/src/cliutil"
 	"miller/src/dsl/cst"
 	"miller/src/input"
+	"miller/src/lib"
 	"miller/src/output"
 	"miller/src/runtime"
 	"miller/src/types"
@@ -75,7 +76,24 @@ func NewRepl(
 	signal.Notify(sysToSignalHandlerChannel, os.Interrupt, syscall.SIGTERM)
 	go controlCHandler(sysToSignalHandlerChannel, appSignalNotificationChannel)
 
-	return &Repl{
+	cstRootNode := cst.NewEmptyRoot(&options.WriterOptions).WithRedefinableUDFUDS()
+
+	// TODO
+
+	// If there was a --load/--mload on the command line, load those DSL strings here (e.g.
+	// someone's local function library).
+	dslStrings := make([]string, 0)
+	for _, filename := range options.DSLPreloadFileNames {
+		theseDSLStrings, err := lib.LoadStringsFromFileOrDir(filename, ".mlr")
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s %s: cannot load DSL expression from \"%s\": ",
+				exeName, replName, filename)
+			return nil, err
+		}
+		dslStrings = append(dslStrings, theseDSLStrings...)
+	}
+
+	repl := &Repl{
 		exeName:  exeName,
 		replName: replName,
 
@@ -87,7 +105,7 @@ func NewRepl(
 
 		astPrintMode: astPrintMode,
 		doWarnings:   doWarnings,
-		cstRootNode:  cst.NewEmptyRoot(&options.WriterOptions).WithRedefinableUDFUDS(),
+		cstRootNode:  cstRootNode,
 
 		options:      options,
 		inputChannel: nil,
@@ -99,7 +117,17 @@ func NewRepl(
 		runtimeState:                 runtimeState,
 		sysToSignalHandlerChannel:    sysToSignalHandlerChannel,
 		appSignalNotificationChannel: appSignalNotificationChannel,
-	}, nil
+	}
+
+	for _, dslString := range dslStrings {
+		err := repl.handleDSLStringBulk(dslString, doWarnings)
+		if err != nil {
+			// Error message already printed out
+			return nil, err
+		}
+	}
+
+	return repl, nil
 }
 
 // When the user types control-C, immediately print something to the screen,

go/src/auxents/repl/verbs.go

@@ -6,7 +6,6 @@ package repl
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strings"
 
@@ -100,6 +99,7 @@ func (this *Repl) handleNonDSLLine(trimmedLine string) bool {
 // ----------------------------------------------------------------
 func usageLoad(this *Repl) {
 	fmt.Println(":load {one or more filenames containing Miller DSL statements}")
+	fmt.Println("If a filename is a directory, all \"*.mlr\" files will be loaded from within it.")
 	fmt.Print(
 		`Any 'begin {...}' / 'end{...}' blocks are parsed and saved. (You can then type
 ':begin' or ':end', respectively, to execute them.) User-defined functions and
@@ -115,18 +115,19 @@ func handleLoad(this *Repl, args []string) bool {
 		return false
 	}
 	for _, filename := range args {
-		dslBytes, err := ioutil.ReadFile(filename)
+		dslStrings, err := lib.LoadStringsFromFileOrDir(filename, ".mlr")
 		if err != nil {
-			fmt.Printf("Cannot load DSL expression from file \"%s\": ",
+			fmt.Printf("Cannot load DSL expression file \"%s\": ",
 				filename)
 			fmt.Println(err)
 			return true
 		}
-		dslString := string(dslBytes)
 
-		err = this.handleDSLStringBulk(dslString, this.doWarnings)
-		if err != nil {
-			fmt.Println(err)
+		for _, dslString := range dslStrings {
+			err = this.handleDSLStringBulk(dslString, this.doWarnings)
+			if err != nil {
+				fmt.Println(err)
+			}
 		}
 	}
 	return true

go/src/cli/mlrcli_parse.go

@@ -288,8 +288,7 @@ func parseTransformers(
 			&argi,
 			argc,
 			args,
-			&options.ReaderOptions,
-			&options.WriterOptions,
+			options,
 		)
 		lib.InternalCodingErrorIf(transformer == nil)
 

go/src/cli/mlrcli_usage.go

@@ -496,6 +496,13 @@ func mainUsageOtherOptions(o *os.File, argv0 string) {
 	fmt.Fprintf(o, "                     The list of filename must end with \"--\". This is useful\n")
 	fmt.Fprintf(o, "                     for example since \"--from *.csv\" doesn't do what you might\n")
 	fmt.Fprintf(o, "                     hope but \"--mfrom *.csv --\" does.\n")
+	fmt.Fprintf(o, "  --load {filename}  Load DSL script file for all put/filter operations on the command line.\n")
+	fmt.Fprintf(o, "                     If the name following --load is a directory, load all \"*.mlr\" files\n")
+	fmt.Fprintf(o, "                     in that directory. This is just like \"put -f\" and \"filter -f\"\n")
+	fmt.Fprintf(o, "                     except it's up-front on the command line, so you can do something like\n")
+	fmt.Fprintf(o, "                     alias mlr='mlr --load ~/myscripts' if you like.\n")
+	fmt.Fprintf(o, "  --mload {names} -- Like --load but works with more than one filename,\n")
+	fmt.Fprintf(o, "                     e.g. '--mload *.mlr --'.\n")
 	fmt.Fprintf(o, "  -n                 Process no input files, nor standard input either. Useful\n")
 	fmt.Fprintf(o, "                     for %s put with begin/end statements only. (Same as --from\n", argv0)
 	fmt.Fprintf(o, "                     /dev/null.) Also useful in \"%s -n put -v '...'\" for\n", argv0)

go/src/cliutil/option_parse.go

@@ -871,6 +871,22 @@ func ParseMiscOptions(
 			argi += 1
 		}
 
+	} else if args[argi] == "--load" {
+		CheckArgCount(args, argi, argc, 2)
+		options.DSLPreloadFileNames = append(options.DSLPreloadFileNames, args[argi+1])
+		argi += 2
+
+	} else if args[argi] == "--mload" {
+		CheckArgCount(args, argi, argc, 2)
+		argi += 1
+		for argi < argc && args[argi] != "--" {
+			options.DSLPreloadFileNames = append(options.DSLPreloadFileNames, args[argi])
+			argi += 1
+		}
+		if args[argi] == "--" {
+			argi += 1
+		}
+
 		//	} else if args[argi] == "--ofmt" {
 		//		CheckArgCount(args, argi, argc, 2);
 		//		options.ofmt = args[argi+1];

go/src/cliutil/option_types.go

@@ -95,8 +95,20 @@ type TOptions struct {
 	ReaderOptions TReaderOptions
 	WriterOptions TWriterOptions
 
+	// Data files to be operated on: e.g. given 'mlr cat foo.dat bar.dat', this
+	// is ["foo.dat", "bar.dat"].
 	FileNames []string
 
+	// DSL files to be loaded for every put/filter operation -- like 'put -f'
+	// or 'filter -f' but specified up front on the command line, suitable for
+	// .mlrrc. Use-case is someone has DSL functions they always want to be
+	// defined.
+	//
+	// TODO: risk of CVE?!? :( -- exclude from .mlrrc & only maybe suggest
+	// people use an alias -- ?
+	// TODO: comment re if-dir then load all *.mlr
+	DSLPreloadFileNames []string
+
 	// These are used to construct the transformer list. In particular, for in-place mode
 	// they're reconstructed for each file.  We make copies since each pass through a
 	// CLI-parser operates destructively, principally by running strtok over
@@ -123,8 +135,9 @@ func DefaultOptions() TOptions {
 		ReaderOptions: DefaultReaderOptions(),
 		WriterOptions: DefaultWriterOptions(),
 
-		FileNames: make([]string, 0),
-		NoInput:   false,
+		FileNames:           make([]string, 0),
+		DSLPreloadFileNames: make([]string, 0),
+		NoInput:             false,
 	}
 }
 

go/src/lib/readfiles.go

@@ -0,0 +1,74 @@
+// ================================================================
+// Routines for loading strings from files. Nominally for the put/filter verbs
+// to load DSL strings from .mlr files.
+// ================================================================
+
+package lib
+
+import (
+	"io/ioutil"
+	"os"
+	"strings"
+)
+
+// LoadStringsFromFileOrDir calls LoadStringFromFile if path exists and is a
+// file, or LoadStringsFromDir if path exists and is a directory.  In the
+// former case the extension is ignored; in the latter case it's used as a
+// filter on the directory entries.
+func LoadStringsFromFileOrDir(path string, extension string) ([]string, error) {
+	fileInfo, err := os.Stat(path)
+	if err != nil {
+		return nil, err
+	}
+	if fileInfo.IsDir() {
+		return LoadStringsFromDir(path, extension)
+	} else {
+		dslString, err := LoadStringFromFile(path)
+		if err != nil {
+			return nil, err
+		} else {
+			return []string{dslString}, nil
+		}
+	}
+}
+
+// LoadStringFromFile is just a wrapper around ioutil.ReadFile,
+// with a cast from []byte to string.
+func LoadStringFromFile(filename string) (string, error) {
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}
+
+// LoadStringsFromDir loads all file contents for files in the given directory
+// having the given extension. E.g. LoadStringsFromDir("/u/myfiles", ".mlr")
+// will load /u/myfiles/foo.mlr and /u/myfiles/bar.mlr but will skip over
+// /u/myfiles/data.csv and /u/myfiles/todo.txt.
+func LoadStringsFromDir(dirname string, extension string) ([]string, error) {
+	dslStrings := make([]string, 0)
+
+	entries, err := os.ReadDir(dirname)
+	if err != nil {
+		return nil, err
+	}
+
+	for i := range entries {
+		entry := &entries[i]
+		name := (*entry).Name()
+		if !strings.HasSuffix(name, extension) {
+			continue
+		}
+
+		path := dirname + "/" + name
+		dslString, err := LoadStringFromFile(path)
+		if err != nil {
+			return nil, err
+		}
+
+		dslStrings = append(dslStrings, dslString)
+	}
+
+	return dslStrings, nil
+}

go/src/transformers/altkv.go

@@ -40,8 +40,7 @@ func transformerAltkvParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/bar.go

@@ -61,8 +61,7 @@ func transformerBarParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/bootstrap.go

@@ -48,8 +48,7 @@ func transformerBootstrapParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/cat.go

@@ -43,8 +43,7 @@ func transformerCatParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/check.go

@@ -41,8 +41,7 @@ func transformerCheckParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/clean-whitespace.go

@@ -49,8 +49,7 @@ func transformerCleanWhitespaceParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	doKeys := true

go/src/transformers/count-similar.go

@@ -44,8 +44,7 @@ func transformerCountSimilarParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/count.go

@@ -45,8 +45,7 @@ func transformerCountParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/cut.go

@@ -55,8 +55,7 @@ func transformerCutParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/decimate.go

@@ -44,8 +44,7 @@ func transformerDecimateParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/fill-down.go

@@ -50,8 +50,7 @@ func transformerFillDownParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/fill-empty.go

@@ -41,8 +41,7 @@ func transformerFillEmptyParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/flatten.go

@@ -45,8 +45,7 @@ func transformerFlattenParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/fraction.go

@@ -59,8 +59,7 @@ func transformerFractionParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/gap.go

@@ -46,8 +46,7 @@ func transformerGapParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/grep.go

@@ -57,8 +57,7 @@ func transformerGrepParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/group-by.go

@@ -41,8 +41,7 @@ func transformerGroupByParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/group-like.go

@@ -41,8 +41,7 @@ func transformerGroupLikeParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/having-fields.go

@@ -67,8 +67,7 @@ func transformerHavingFieldsParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	havingFieldsCriterion := havingFieldsCriterionUnspecified

go/src/transformers/head.go

@@ -49,8 +49,7 @@ func transformerHeadParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/histogram.go

@@ -48,8 +48,7 @@ func transformerHistogramParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/join.go

@@ -67,9 +67,6 @@ func newJoinOptions() *tJoinOptions {
 
 		leftFileName: "",
 		prepipe:      "",
-
-		// TODO
-		// readerOptions: readerOptions,
 	}
 }
 
@@ -134,8 +131,7 @@ func transformerJoinParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	mainReaderOptions *cliutil.TReaderOptions, // Options for the right-files
-	__ *cliutil.TWriterOptions,
+	mainOptions *cliutil.TOptions, // Options for the right-files
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line
@@ -146,9 +142,9 @@ func transformerJoinParseCLI(
 	// Parse local flags
 	opts := newJoinOptions()
 
-	if mainReaderOptions != nil { // for 'mlr --usage-all-verbs', it's nil
+	if mainOptions != nil { // for 'mlr --usage-all-verbs', it's nil
 		// TODO: make sure this is a full nested-struct copy.
-		opts.joinReaderOptions = *mainReaderOptions // struct copy
+		opts.joinReaderOptions = mainOptions.ReaderOptions // struct copy
 	}
 
 	for argi < argc /* variable increment: 1 or 2 depending on flag */ {

go/src/transformers/json-parse.go

@@ -44,8 +44,7 @@ func transformerJSONParseParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/json-stringify.go

@@ -45,8 +45,7 @@ func transformerJSONStringifyParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/label.go

@@ -45,8 +45,7 @@ func transformerLabelParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/merge-fields.go

@@ -82,8 +82,7 @@ func transformerMergeFieldsParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/most-or-least-frequent.go

@@ -79,8 +79,7 @@ func transformerMostFrequentParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 	return transformerMostOrLeastFrequentParseCLI(pargi, argc, args, true, transformerMostFrequentUsage)
 }
@@ -89,8 +88,7 @@ func transformerLeastFrequentParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 	return transformerMostOrLeastFrequentParseCLI(pargi, argc, args, false, transformerLeastFrequentUsage)
 }

go/src/transformers/nest.go

@@ -97,8 +97,7 @@ func transformerNestParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/nothing.go

@@ -41,8 +41,7 @@ func transformerNothingParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/put-or-filter.go

@@ -3,7 +3,6 @@ package transformers
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strings"
 
@@ -63,7 +62,8 @@ func transformerPutOrFilterUsage(
 	fmt.Fprintf(o, "Usage: %s %s [options] {DSL expression}\n", lib.MlrExeName(), verb)
 	fmt.Fprintf(o, "Options:\n")
 	fmt.Fprintf(o,
-		`-f {file name} File containing a DSL expression.
+		`-f {file name} File containing a DSL expression. If the filename is a directory,
+   all *.mlr files in that directory are loaded.
 
 -e {expression} You can use this after -f to add an expression. Example use
    case: define functions/subroutines in a file you specify with -f, then call
@@ -124,8 +124,7 @@ func transformerPutOrFilterParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	mainRecordWriterOptions *cliutil.TWriterOptions,
+	mainOptions *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line
@@ -134,6 +133,7 @@ func transformerPutOrFilterParseCLI(
 	argi++
 
 	var dslStrings []string = make([]string, 0)
+	haveDSLStringsHere := false
 	echoDSLString := false
 	printASTAsTree := false
 	printASTMultiLine := false
@@ -146,15 +146,26 @@ func transformerPutOrFilterParseCLI(
 	presets := make([]string, 0)
 
 	// TODO: make sure this is a full nested-struct copy.
-	var recordWriterOptions *cliutil.TWriterOptions = nil
-	if mainRecordWriterOptions != nil {
-		// TODO: make a .Copy() method
-		copyThereof := *mainRecordWriterOptions
-		recordWriterOptions = &copyThereof
+	var options *cliutil.TOptions = nil
+	if mainOptions != nil {
+		copyThereof := *mainOptions // struct copy
+		options = &copyThereof
+	}
+
+	// If there was a global --load/--mload, load those DSL strings here (e.g.
+	// someone's local function library).
+	for _, filename := range options.DSLPreloadFileNames {
+		theseDSLStrings, err := lib.LoadStringsFromFileOrDir(filename, ".mlr")
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s %s: cannot load DSL expression from \"%s\": ",
+				lib.MlrExeName(), verb, filename)
+			fmt.Println(err)
+			return nil
+		}
+		dslStrings = append(dslStrings, theseDSLStrings...)
 	}
 
 	// Parse local flags.
-
 	for argi < argc /* variable increment: 1 or 2 depending on flag */ {
 		opt := args[argi]
 		if !strings.HasPrefix(opt, "-") {
@@ -168,19 +179,20 @@ func transformerPutOrFilterParseCLI(
 		} else if opt == "-f" {
 			// Get a DSL string from the user-specified filename
 			filename := cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc)
-			data, err := ioutil.ReadFile(filename)
+			theseDSLStrings, err := lib.LoadStringsFromFileOrDir(filename, ".mlr")
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "%s %s: cannot load DSL expression from file \"%s\": ",
 					lib.MlrExeName(), verb, filename)
 				fmt.Println(err)
 				return nil
 			}
-			dslString := string(data)
-			dslStrings = append(dslStrings, dslString)
+			dslStrings = append(dslStrings, theseDSLStrings...)
+			haveDSLStringsHere = true
 
 		} else if opt == "-e" {
 			dslString := cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc)
 			dslStrings = append(dslStrings, dslString)
+			haveDSLStringsHere = true
 
 		} else if opt == "-s" {
 			// E.g.
@@ -228,7 +240,7 @@ func transformerPutOrFilterParseCLI(
 			// loop (so individual if-statements don't need to). However,
 			// ParseWriterOptions expects it unadvanced.
 			wargi := argi - 1
-			if cliutil.ParseWriterOptions(args, argc, &wargi, recordWriterOptions) {
+			if cliutil.ParseWriterOptions(args, argc, &wargi, &options.WriterOptions) {
 				// This lets mlr main and mlr put have different output formats.
 				// Nothing else to handle here.
 				argi = wargi
@@ -241,7 +253,7 @@ func transformerPutOrFilterParseCLI(
 	// If they've used either of 'mlr put -f {filename}' or 'mlr put -e
 	// {expression}' then that specifies their DSL expression. But if they've
 	// done neither then we expect 'mlr put {expression}'.
-	if len(dslStrings) == 0 {
+	if !haveDSLStringsHere {
 		// Get the DSL string from the command line, after the flags
 		if argi >= argc {
 			transformerPutUsage(os.Stderr, true, 1)
@@ -265,7 +277,7 @@ func transformerPutOrFilterParseCLI(
 		warningsAreFatal,
 		invertFilter,
 		suppressOutputRecord,
-		recordWriterOptions,
+		options,
 	)
 	if err != nil {
 		// Error message already printed out
@@ -299,10 +311,10 @@ func NewTransformerPut(
 	warningsAreFatal bool,
 	invertFilter bool,
 	suppressOutputRecord bool,
-	recordWriterOptions *cliutil.TWriterOptions,
+	options *cliutil.TOptions,
 ) (*TransformerPut, error) {
 
-	cstRootNode := cst.NewEmptyRoot(recordWriterOptions)
+	cstRootNode := cst.NewEmptyRoot(&options.WriterOptions)
 
 	for _, dslString := range dslStrings {
 		astRootNode, err := BuildASTFromStringWithMessage(dslString)

go/src/transformers/regularize.go

@@ -40,8 +40,7 @@ func transformerRegularizeParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/remove-empty-columns.go

@@ -41,8 +41,7 @@ func transformerRemoveEmptyColumnsParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/rename.go

@@ -63,8 +63,7 @@ func transformerRenameParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/reorder.go

@@ -56,8 +56,7 @@ func transformerReorderParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/repeat.go

@@ -70,8 +70,7 @@ func transformerRepeatParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	repeatCountSource := repeatCountSourceUnspecified

go/src/transformers/reshape.go

@@ -123,8 +123,7 @@ func transformerReshapeParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/sample.go

@@ -45,8 +45,7 @@ func transformerSampleParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/sec2gmt.go

@@ -48,8 +48,7 @@ func transformerSec2GMTParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/sec2gmtdate.go

@@ -42,8 +42,7 @@ func transformerSec2GMTDateParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/seqgen.go

@@ -52,8 +52,7 @@ func transformerSeqgenParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/shuffle.go

@@ -44,8 +44,7 @@ func transformerShuffleParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/skip-trivial-records.go

@@ -41,8 +41,7 @@ func transformerSkipTrivialRecordsParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/sort-within-records.go

@@ -41,8 +41,7 @@ func transformerSortWithinRecordsParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/sort.go

@@ -98,8 +98,7 @@ func transformerSortParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/stats1.go

@@ -86,8 +86,7 @@ func transformerStats1ParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/stats2.go

@@ -69,8 +69,7 @@ func transformerStats2ParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/step.go

@@ -80,8 +80,7 @@ func transformerStepParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/tac.go

@@ -41,8 +41,7 @@ func transformerTacParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/tail.go

@@ -44,8 +44,7 @@ func transformerTailParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/tee.go

@@ -48,8 +48,7 @@ func transformerTeeParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	mainRecordWriterOptions *cliutil.TWriterOptions,
+	mainOptions *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line
@@ -61,8 +60,8 @@ func transformerTeeParseCLI(
 	piping := false
 	// TODO: make sure this is a full nested-struct copy.
 	var recordWriterOptions *cliutil.TWriterOptions = nil
-	if mainRecordWriterOptions != nil {
-		copyThereof := *mainRecordWriterOptions
+	if mainOptions != nil {
+		copyThereof := mainOptions.WriterOptions // struct copy
 		recordWriterOptions = &copyThereof
 	}
 

go/src/transformers/top.go

@@ -53,8 +53,7 @@ func transformerTopParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/unflatten.go

@@ -45,8 +45,7 @@ func transformerUnflattenParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/uniq.go

@@ -62,8 +62,7 @@ func transformerCountDistinctParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line
@@ -162,8 +161,7 @@ func transformerUniqParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transformers/unsparsify.go

@@ -56,8 +56,7 @@ func transformerUnsparsifyParseCLI(
 	pargi *int,
 	argc int,
 	args []string,
-	_ *cliutil.TReaderOptions,
-	__ *cliutil.TWriterOptions,
+	_ *cliutil.TOptions,
 ) transforming.IRecordTransformer {
 
 	// Skip the verb name from the current spot in the mlr command line

go/src/transforming/record_transformer.go

@@ -29,8 +29,7 @@ type TransformerParseCLIFunc func(
 	pargi *int,
 	argc int,
 	args []string,
-	readerOptions *cliutil.TReaderOptions,
-	writerOptions *cliutil.TWriterOptions,
+	mainOptions *cliutil.TOptions,
 ) IRecordTransformer
 
 type TransformerSetup struct {

go/todo.txt

@@ -11,7 +11,18 @@ TOP OF LIST:
   o check JSON-parser x 2 -- is there really a 'restart'?
     - infinite-loop avoidance for sure
 * json-triple-quote -- what can be done here?
-* -l/-L
+* --load/--mload
+  k load-from-file helper function
+  k load-from-dir helper function
+  k move to lib ddir
+  k put/filter usage func
+  k mlr main help
+  k port x all to repl too
+    k repl usage func
+    k make :load handle dirs ... :load usage too ...
+  o regtest cases including mrpl
+  ! fix all g3b04k3n ${CASEDIR}/input cases :(
+  ! suppress from .mlrrc with comments as to why, & local-alias suggestion
 
 * tools/perf:
   o https://eng.uber.com/pprof-go-profiler/
@@ -623,3 +634,8 @@ i https://en.wikipedia.org/wiki/Delimiter#Delimiter_collision
   - needs doc-ent for JSON_NULL -- only for JSON user data only only ... except for this
 * sec2gmt --millis/--micros/--nanos doc
 * doc --mfrom
+* sort-within-records --recursive doc
+* doc re --load/--mload at mlrmain & repl
+* doc re put -f dirname will load dirname/*.mlr
+  mlr --opprint --from foo.dat --load u/stdlib/ put '$f=fibo($i); $g=factorial($i)'
+  mlr --opprint --from foo.dat put -f u/stdlib/ -e '$f=fibo($i); $g=factorial($i)'