flatten/unflatten code-dedupe

This commit is contained in:
John Kerl 2021-02-11 01:00:45 -05:00
parent ee48c884e9
commit eddd8830ad
6 changed files with 317 additions and 167 deletions

View file

@ -66,3 +66,33 @@ run_mlr repl <<EOF
:context
\$*
EOF
# ----------------------------------------------------------------
run_mlr repl --j2x $indir/flatten-input-2.json <<EOF
:rw
:rw
EOF
run_mlr repl --x2j $indir/unflatten-input.xtab <<EOF
:rw
:rw
EOF
run_mlr repl --xtab $indir/unflatten-input.xtab <<EOF
:rw
:rw
EOF
run_mlr repl --json $indir/flatten-input-2.json <<EOF
:rw
:rw
EOF
# ----------------------------------------------------------------
run_mlr repl --json $indir/flatten-input-2.json <<EOF
:rw
:rw
:reopen
:rw
:rw
EOF

View file

@ -78,3 +78,157 @@ FILENAME="./reg-test/input/medium.dkvp",FILENUM=1,NR=40,FNR=40
"y": 0.2884018711352886
}
mlr repl --j2x ./reg-test/input/flatten-input-2.json
hostname localhost
pid 12345
req.id 6789
req.method GET
req.path api/check
req.host foo.bar
req.headers.host bar.baz
req.headers.user-agent browser
res.status_code 200
res.header.content-type text
res.header.content-encoding plain
empty1 {}
empty2 []
wrapper.empty3 {}
wrapper.emtpy4 []
End of record stream
mlr repl --x2j ./reg-test/input/unflatten-input.xtab
{
"hostname": "localhost",
"pid": 12345,
"req": {
"id": 6789,
"method": "GET",
"path": "api/check",
"host": "foo.bar",
"headers": {
"host": "bar.baz",
"user-agent": "browser"
}
},
"res": {
"status_code": 200,
"header": {
"content-type": "text",
"content-encoding": "plain"
}
},
"empty1": {},
"empty2": [],
"wrapper": {
"empty3": {},
"emtpy4": []
}
}
End of record stream
mlr repl --xtab ./reg-test/input/unflatten-input.xtab
hostname localhost
pid 12345
req.id 6789
req.method GET
req.path api/check
req.host foo.bar
req.headers.host bar.baz
req.headers.user-agent browser
res.status_code 200
res.header.content-type text
res.header.content-encoding plain
empty1 {}
empty2 []
wrapper.empty3 {}
wrapper.emtpy4 []
End of record stream
mlr repl --json ./reg-test/input/flatten-input-2.json
{
"hostname": "localhost",
"pid": 12345,
"req": {
"id": 6789,
"method": "GET",
"path": "api/check",
"host": "foo.bar",
"headers": {
"host": "bar.baz",
"user-agent": "browser"
}
},
"res": {
"status_code": 200,
"header": {
"content-type": "text",
"content-encoding": "plain"
}
},
"empty1": {},
"empty2": [],
"wrapper": {
"empty3": {},
"emtpy4": []
}
}
End of record stream
mlr repl --json ./reg-test/input/flatten-input-2.json
{
"hostname": "localhost",
"pid": 12345,
"req": {
"id": 6789,
"method": "GET",
"path": "api/check",
"host": "foo.bar",
"headers": {
"host": "bar.baz",
"user-agent": "browser"
}
},
"res": {
"status_code": 200,
"header": {
"content-type": "text",
"content-encoding": "plain"
}
},
"empty1": {},
"empty2": [],
"wrapper": {
"empty3": {},
"emtpy4": []
}
}
End of record stream
{
"hostname": "localhost",
"pid": 12345,
"req": {
"id": 6789,
"method": "GET",
"path": "api/check",
"host": "foo.bar",
"headers": {
"host": "bar.baz",
"user-agent": "browser"
}
},
"res": {
"status_code": 200,
"header": {
"content-type": "text",
"content-encoding": "plain"
}
},
"empty1": {},
"empty2": [],
"wrapper": {
"empty3": {},
"emtpy4": []
}
}
End of record stream

View file

@ -27,6 +27,7 @@ import (
"path"
"strings"
"miller/cli"
"miller/cliutil"
)
@ -106,6 +107,11 @@ func ReplMain(args []string) int {
}
}
// --auto-flatten is on by default. But if input and output formats are both JSON,
// then we don't need to actually do anything. See also mlrcli_parse.go.
options.WriterOptions.AutoFlatten = cli.DecideFinalFlatten(&options)
options.WriterOptions.AutoUnflatten = cli.DecideFinalUnflatten(&options)
repl, err := NewRepl(
exeName,
replName,

View file

@ -37,6 +37,7 @@ func init() {
handlerLookupTable = []handlerInfo{
{verbNames: []string{":l", ":load"}, handlerFunc: handleLoad, usageFunc: usageLoad},
{verbNames: []string{":o", ":open"}, handlerFunc: handleOpen, usageFunc: usageOpen},
{verbNames: []string{":reopen"}, handlerFunc: handleReopen, usageFunc: usageReopen},
{verbNames: []string{":r", ":read"}, handlerFunc: handleRead, usageFunc: usageRead},
{verbNames: []string{":w", ":write"}, handlerFunc: handleWrite, usageFunc: usageWrite},
{verbNames: []string{":rw"}, handlerFunc: handleReadWrite, usageFunc: usageReadWrite},
@ -186,6 +187,9 @@ func openFilesPreCheck(this *Repl, args []string) bool {
// Also invoked from the main entry-point, hence split out as a separate method.
func (this *Repl) openFiles(filenames []string) {
// Remember for :reopen
this.options.FileNames = filenames
this.inputChannel = make(chan *types.RecordAndContext, 10)
this.errorChannel = make(chan error, 1)
@ -197,6 +201,24 @@ func (this *Repl) openFiles(filenames []string) {
)
}
// ----------------------------------------------------------------
func usageReopen(this *Repl) {
fmt.Println(":reopen with no arguments.")
fmt.Println("Like :open with the same filenames you provided at the time you typed :open.")
}
func handleReopen(this *Repl, args []string) bool {
args = args[1:] // strip off verb
if len(args) != 0 {
return false
}
if openFilesPreCheck(this, this.options.FileNames) {
this.openFiles(this.options.FileNames)
}
return true
}
// ----------------------------------------------------------------
func usageRead(this *Repl) {
fmt.Println(":read with no arguments.")
@ -540,74 +562,17 @@ func handleWrite(this *Repl, args []string) bool {
return true
}
// ================================================================
// Takes care of flattening nested JSON data structures to multiple fields for
// JSON -> non-JSON.
//
// TODO: centralize a function/data between here & mlrcli_parse.go & refer to it.
// TODO: centralize the narrative comments as well.
//
// ----------------------------------------------------------------
// PROBLEM TO BE SOLVED:
//
// JSON has nested structures and CSV et al. do not. For example:
// {
// "req" : {
// "method": "GET",
// "path": "api/check",
// }
// }
//
// For CSV we flatten this down to
//
// {
// "req.method": "GET",
// "req.path": "api/check"
// }
//
// ----------------------------------------------------------------
// APPROACH:
//
// Use the Principle of Least Surprise (POLS).
//
// * If input is JSON and output is JSON:
// o Records can be nested from record-read
// o They remain that way through the Miller record-processing stream
// o They are nested on record-write
// o No action needs to be taken
// * If input is JSON and output is non-JSON:
// o Records can be nested from record-read
// o They remain that way through the Miller record-processing stream
// o On record-write, nested structures will be converted to string (carriage
// returns and all) using json_stringify. People *might* want this but
// (using POLS) we will (by default) AUTO-FLATTEN for them. There is a
// --no-auto-unflatten CLI flag for those who want it.
// * If input is non-JSON and output is non-JSON:
// o Leave records as-is.
// o Example, if there is a "req.method" field, people should be able to do
// 'mlr sort -f req.method' with no surprises. (Again, POLS.)
// o People can insert an unflatten verb into their verb chain if they really
// want unflatten for non-JSON files.
// * If input is non-JSON and output is JSON:
// o Default is to auto-unflatten at output.
// o There is a --no-auto-unflatten for those who want it.
// ================================================================
func writeRecord(this *Repl, outrec *types.Mlrmap) {
ropt := &this.options.ReaderOptions
wopt := &this.options.WriterOptions
ifmt := ropt.InputFileFormat
ofmt := wopt.OutputFileFormat
if wopt.AutoFlatten {
if ifmt == "json" && ofmt != "json" {
outrec.Flatten(wopt.OFLATSEP)
if outrec != nil {
// E.g. '{"req": {"method": "GET", "path": "/api/check"}}' becomes
// req.method=GET,req.path=/api/check.
if this.options.WriterOptions.AutoFlatten {
outrec.Flatten(this.options.WriterOptions.OFLATSEP)
}
}
if wopt.AutoUnflatten {
if ifmt != "json" && ofmt == "json" {
outrec.Unflatten(wopt.OFLATSEP)
// E.g. req.method=GET,req.path=/api/check becomes
// '{"req": {"method": "GET", "path": "/api/check"}}'
if this.options.WriterOptions.AutoUnflatten {
outrec.Unflatten(this.options.WriterOptions.OFLATSEP)
}
}
this.recordWriter.Write(outrec, this.outputStream)

View file

@ -110,82 +110,22 @@ func ParseCommandLine(args []string) (
options.NoInput = true // e.g. then-chain begins with seqgen
}
// ================================================================
// TODO: centralize a function/data between here & repl/verbs.go & refer to it.
// TODO: centralize the narrative comments as well.
//
// ----------------------------------------------------------------
// PROBLEM TO BE SOLVED:
//
// JSON has nested structures and CSV et al. do not. For example:
// {
// "req" : {
// "method": "GET",
// "path": "api/check",
// }
// }
//
// For CSV we flatten this down to
//
// {
// "req.method": "GET",
// "req.path": "api/check"
// }
//
// ----------------------------------------------------------------
// APPROACH:
//
// Use the Principle of Least Surprise (POLS).
//
// * If input is JSON and output is JSON:
// o Records can be nested from record-read
// o They remain that way through the Miller record-processing stream
// o They are nested on record-write
// o No action needs to be taken
//
// * If input is JSON and output is non-JSON:
// o Records can be nested from record-read
// o They remain that way through the Miller record-processing stream
// o On record-write, nested structures will be converted to string (carriage
// returns and all) using json_stringify. People *might* want this but
// (using POLS) we will (by default) AUTO-FLATTEN for them. There is a
// --no-auto-unflatten CLI flag for those who want it.
//
// * If input is non-JSON and output is non-JSON:
// o If there is a "req.method" field, people should be able to do
// 'mlr sort -f req.method' with no surprises. (Again, POLS.) Therefore
// no auto-unflatten on input. People can insert an unflatten verb
// into their verb chain if they really want unflatten for non-JSON
// files.
// o The DSL can make nested data, so AUTO-FLATTEN at output.
//
// * If input is non-JSON and output is JSON:
// o Default is to auto-unflatten at output.
// o There is a --no-auto-unflatten for those who want it.
// ================================================================
ifmt := options.ReaderOptions.InputFileFormat
ofmt := options.WriterOptions.OutputFileFormat
oflatsep := options.WriterOptions.OFLATSEP
if options.WriterOptions.AutoFlatten {
if ofmt != "json" {
transformer, err := transformers.NewTransformerFlatten(oflatsep, nil)
lib.InternalCodingErrorIf(err != nil)
lib.InternalCodingErrorIf(transformer == nil)
recordTransformers = append(recordTransformers, transformer)
}
if DecideFinalFlatten(&options) {
// E.g. '{"req": {"method": "GET", "path": "/api/check"}}' becomes
// req.method=GET,req.path=/api/check.
transformer, err := transformers.NewTransformerFlatten(options.WriterOptions.OFLATSEP, nil)
lib.InternalCodingErrorIf(err != nil)
lib.InternalCodingErrorIf(transformer == nil)
recordTransformers = append(recordTransformers, transformer)
}
if options.WriterOptions.AutoUnflatten {
if ifmt != "json" {
if ofmt == "json" {
transformer, err := transformers.NewTransformerUnflatten(oflatsep, nil)
lib.InternalCodingErrorIf(err != nil)
lib.InternalCodingErrorIf(transformer == nil)
recordTransformers = append(recordTransformers, transformer)
}
}
if DecideFinalUnflatten(&options) {
// E.g. req.method=GET,req.path=/api/check becomes
// '{"req": {"method": "GET", "path": "/api/check"}}'
transformer, err := transformers.NewTransformerUnflatten(options.WriterOptions.OFLATSEP, nil)
lib.InternalCodingErrorIf(err != nil)
lib.InternalCodingErrorIf(transformer == nil)
recordTransformers = append(recordTransformers, transformer)
}
// There may already be one or more because of --from on the command line,
@ -211,6 +151,84 @@ func ParseCommandLine(args []string) (
return options, recordTransformers, nil
}
// ================================================================
// Decide whether to insert a flatten or unflatten verb at the end of the
// chain. See also repl/verbs.go which handles the same issue in the REPL.
//
// ----------------------------------------------------------------
// PROBLEM TO BE SOLVED:
//
// JSON has nested structures and CSV et al. do not. For example:
// {
// "req" : {
// "method": "GET",
// "path": "api/check",
// }
// }
//
// For CSV we flatten this down to
//
// {
// "req.method": "GET",
// "req.path": "api/check"
// }
//
// ----------------------------------------------------------------
// APPROACH:
//
// Use the Principle of Least Surprise (POLS).
//
// * If input is JSON and output is JSON:
// o Records can be nested from record-read
// o They remain that way through the Miller record-processing stream
// o They are nested on record-write
// o No action needs to be taken
//
// * If input is JSON and output is non-JSON:
// o Records can be nested from record-read
// o They remain that way through the Miller record-processing stream
// o On record-write, nested structures will be converted to string (carriage
// returns and all) using json_stringify. People *might* want this but
// (using POLS) we will (by default) AUTO-FLATTEN for them. There is a
// --no-auto-unflatten CLI flag for those who want it.
//
// * If input is non-JSON and output is non-JSON:
// o If there is a "req.method" field, people should be able to do
// 'mlr sort -f req.method' with no surprises. (Again, POLS.) Therefore
// no auto-unflatten on input. People can insert an unflatten verb
// into their verb chain if they really want unflatten for non-JSON
// files.
// o The DSL can make nested data, so AUTO-FLATTEN at output.
//
// * If input is non-JSON and output is JSON:
// o Default is to auto-unflatten at output.
// o There is a --no-auto-unflatten for those who want it.
// ================================================================
func DecideFinalFlatten(options *cliutil.TOptions) bool {
ofmt := options.WriterOptions.OutputFileFormat
if options.WriterOptions.AutoFlatten {
if ofmt != "json" {
return true
}
}
return false
}
func DecideFinalUnflatten(options *cliutil.TOptions) bool {
ifmt := options.ReaderOptions.InputFileFormat
ofmt := options.WriterOptions.OutputFileFormat
if options.WriterOptions.AutoUnflatten {
if ifmt != "json" {
if ofmt == "json" {
return true
}
}
}
return false
}
// ----------------------------------------------------------------
// Returns a list of transformers, from the starting point in args given by *pargi.
// Bumps *pargi to point to remaining post-transformer-setup args, i.e. filenames.

View file

@ -1,4 +1,5 @@
---------------------------------------------------------------- TOP OF LIST:
----------------------------------------------------------------
TOP OF LIST:
! issues !
! rmd ex1 even simpler -- commarect
@ -13,7 +14,6 @@ mlrtut links:
https://guillim.github.io/terminal/2018/06/19/MLR-for-CSV-manipulation.html
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
* help-refactor:
o audit for DEFAULT_FOOs @
@ -28,31 +28,6 @@ mlrtut links:
* note hexfmt for non-ints -- especially floats -- ?!?
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
* revisit flatten/flatten:
! mlr --csv --from x.csv put -q '@a["b"]=1;@a["c"]=2; emitp @a'`
mlr --opprint put '$f=asserting_map($*)' ./reg-test/input/nullvals.dkvp
mlr --opprint put '$f=asserting_map($*)' then flatten ./reg-test/input/nullvals.dkvp
- centralize POLS comment/method from $repl/entry.go
- put into the go/README.md
- schedule for doc6
- refactor/rename args in mlrmain
- avoid if flatten/unflatten verbs are anywhere in the chain?
* repl fu:
* :rw -- doc & UT
* :reopen verb
* auto-unflatten / auto-flatten UT
o tilde-expand for load/open ...
- if '~' is in the string, run it though sh -c echo ...
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
! fix all typemasks; 'int x = 1; x = "abc"'
! strptime/strftime experiments ...
@ -510,6 +485,7 @@ i https://en.wikipedia.org/wiki/Delimiter#Delimiter_collision
o the former is not necessarily in sync with the output record stream
* dev-note on why `int` not `int64` -- processor-arch & those who most need it get it
* document tee -p
* doc auto-flatten/auto-unflatten -- incl narrative from mlrcli_parse.go
* doc6: default flatsep is now "." not ":" in keeping with JSON culture
? allow [[...]] / [[[...]]] at assignment LHS
@ -538,3 +514,4 @@ i https://en.wikipedia.org/wiki/Delimiter#Delimiter_collision
* mlr inp parse error failstring retback?
* https://blog.golang.org/go1.13-errors
* split REPL lines on ';' -- ?
* tilde-expand for REPL load/open: if '~' is at the start of the string, run it though 'sh -c echo'