From 2ba4a364ffe563d8a04c233054efbc31b4ffa1d1 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Wed, 2 Sep 2020 13:14:51 -0400 Subject: [PATCH] neaten --- go/README.md | 78 ++++++++++++++++++++++++++++++++++++++++----- go/ppp | 24 -------------- go/rrr | 50 ----------------------------- go/todo.txt | 12 +++++-- go/u/README.md | 2 ++ go/u/ppp | 24 ++++++++++++++ go/{ => u}/qqq | 4 +-- go/u/rrr | 50 +++++++++++++++++++++++++++++ go/{ => u}/s.csv | 0 go/{ => u}/s.dkvp | 0 go/{ => u}/s.json | 0 go/{ => u}/s.nidx | 0 go/{ => u}/t.csv | 0 go/{ => u}/t.dkvp | 0 go/{ => u}/t.json | 0 go/{ => u}/t.nidx | 0 go/{ => u}/test-dsl | 0 go/{ => u}/uuu | 6 ++-- 18 files changed, 160 insertions(+), 90 deletions(-) delete mode 100755 go/ppp delete mode 100755 go/rrr create mode 100644 go/u/README.md create mode 100755 go/u/ppp rename go/{ => u}/qqq (94%) create mode 100755 go/u/rrr rename go/{ => u}/s.csv (100%) rename go/{ => u}/s.dkvp (100%) rename go/{ => u}/s.json (100%) rename go/{ => u}/s.nidx (100%) rename go/{ => u}/t.csv (100%) rename go/{ => u}/t.dkvp (100%) rename go/{ => u}/t.json (100%) rename go/{ => u}/t.nidx (100%) rename go/{ => u}/test-dsl (100%) rename go/{ => u}/uuu (61%) diff --git a/go/README.md b/go/README.md index f7b82b19f..6e1bb921b 100644 --- a/go/README.md +++ b/go/README.md @@ -1,13 +1,75 @@ # Status of the Go port -* This is not necessarily a full Go port of Miller. At the moment, it's a little spot for some experimentation. Things are very rough and very iterative and very incomplete. +* This is not necessarily a full Go port of Miller. At the moment, it's a little spot for some experimentation. Things are very rough and very iterative and very incomplete. I don't commit to finishing a Go port but I very much hope to. * One reason Miller exists is to be a useful tool for myself and others; another is it's fun to write. At bare minimum, I'll re-teach myself some Go. -* In all likelihood though this will turn into a full port which will someday become Miller 6.0. +* In all likelihood, though, this will turn into a full port which will someday become Miller 6.0. * I hope to retain backward compatibility at the command-line level as much as possible. -* Benefits: - * The lack of a streaming (record-by-record) JSON reader in the C implementation (https://github.com/johnkerl/miller/issues/99) is immediately solved in the Go implementation. - * The quoted-DKVP feature from https://github.com/johnkerl/miller/issues/266 will be easily addressed. - * String/number-formatting issues in https://github.com/johnkerl/miller/issues/211 https://github.com/johnkerl/miller/issues/178 https://github.com/johnkerl/miller/issues/151 https://github.com/johnkerl/miller/issues/259 will be fixed during the Go port. - * I think some DST/timezone issues such as https://github.com/johnkerl/miller/issues/359 will be easier to fix using the Go datetime library than using the C datetime library +* Benefits of porting to Go: + * The lack of a streaming (record-by-record) JSON reader in the C implementation ([issue 99](https://github.com/johnkerl/miller/issues/99)) is immediately solved in the Go implementation. + * The quoted-DKVP feature from [issue 266](https://github.com/johnkerl/miller/issues/266) will be easily addressed. + * String/number-formatting issues in [issue 211](https://github.com/johnkerl/miller/issues/211), [issue 178](https://github.com/johnkerl/miller/issues/178), [issue 151](https://github.com/johnkerl/miller/issues/151), and [issue 259](https://github.com/johnkerl/miller/issues/259) will be fixed during the Go port. + * I think some DST/timezone issues such as [issue 359](https://github.com/johnkerl/miller/issues/359) will be easier to fix using the Go datetime library than using the C datetime library * The code will be easier to read and, I hope, easier for others to contribute to. -* In the meantime I will still keep fixing bugs, doing some features, etc in C on Miller 5.x. +* In the meantime I will still keep fixing bugs, doing some features, etc. in C on Miller 5.x -- in the near term, support for Miller's C implementation continues as before. + +# Directory structure + +Miller is a multi-format record-stream processor, where a *record* is a +sequence of key-value pairs. The basic *stream* operation is: + +* *read* records in some specified file format; +* *map* the input records to output records in some user-specified way, using a *chain* of *verbs* (sort, filter, cut, put, etc.); +* *write* the records in some specified file format. + +## Directory-structure overview + +So, in broad overview, the key packages are: + +``` +src/miller/stream -- connect input -> mapping -> output via Go channels +src/miller/input -- read input records +src/miller/mapping -- map input records to output records +src/miller/output -- write output records +``` + +## Directory-structure details + +### Dependencies + +* Miller dependencies are all in the Go standard library, except a couple local ones: + * Insertion-ordered (order-preserving) maps from [gitlab.com/c0b/go-ordered-json](https://gitlab.com/c0b/go-ordered-json): + * If you have a JSON data record `{"x":3,"y":4,"z":5}` then the keys `x,y,z` should stay that way. This package makes that happen. + * GOCC lexer/parser code-generator from [github.com/goccmack/gocc](https://github.com/goccmack/gocc): + * This package defines the grammar for Miller's domain-specific language (DSL) for the Miller `put` and `filter` verbs. And, GOCC is a joy to use. :) + + +``` +src/localdeps/ordered +src/github.com/goccmack +src/miller +``` + +### More + +``` +mlr.go +src/miller/lib +src/miller/containers + +src/miller/cli +src/miller/clitypes +src/miller/stream +src/miller/input +src/miller/mapping +src/miller/output + +src/miller/mappers +src/miller/parsing +src/miller/parsing/token +src/miller/parsing/util +src/miller/parsing/lexer +src/miller/parsing/parser +src/miller/parsing/errors +src/miller/dsl +src/miller/dsl/cst +``` diff --git a/go/ppp b/go/ppp deleted file mode 100755 index d219ceb4f..000000000 --- a/go/ppp +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -echo; mlr put -v '$y = 1 || 2' /dev/null -echo; mlr null put -v '$y = 1 || 2 || 3' /dev/null -echo; mlr null put -v '$y = 1 || 2 && 3' /dev/null -echo; mlr null put -v '$y = 1 && 2 || 3' /dev/null -echo; mlr null put -v '$y = 1 ? 2 : 3' /dev/null -echo; mlr null put -v '$y = $a + $b * $c' /dev/null -echo; mlr null put -v '$y = $a * $b * $c' /dev/null -echo; mlr null put -v '$y = $a ** $b ** $c' /dev/null -echo; mlr null put -v '$[2] = 3' /dev/null -echo; mlr null put -v '$[$y] = 4' /dev/null -#echo; mlr null put -v '${1} = 4' /dev/null -echo; mlr null put -v '$x = "abc"' /dev/null -echo; mlr null put -v '$["abc"] = "def"' /dev/null -echo; mlr null put -v '$[FILENAME] = FNR' /dev/null -echo; mlr null put -v '$x = $a + $b + $c' /dev/null -echo; mlr null put -v '$x = ($a + $b) + $c; $y = $a + ($b + $c); $z = $a + ($b)+ $c' /dev/null -echo; mlr null put -v '$x = 2 * $a + $b . $c' /dev/null -echo; mlr null put -v '$x = 2 * $a + ($b . $c)' /dev/null -echo; mlr null put -v '$x = (NF + NR) * 7; $y = OFS . $y . "hello"' /dev/null -echo; mlr null put -v '$x = 123. + 1e-2 / .2e3 + 1.e-3' /dev/null diff --git a/go/rrr b/go/rrr deleted file mode 100755 index 59c4cd978..000000000 --- a/go/rrr +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -echo; mlr --idkvp --opprint put '$j = $i + $i' s.dkvp -echo; mlr --idkvp --opprint put '$j = $i + $x' s.dkvp -echo; mlr --idkvp --opprint put '$j = $y + $x' s.dkvp -echo; mlr --idkvp --opprint put '$j = $y + $i' s.dkvp -echo; mlr --idkvp --opprint put '$j = $y + $y' s.dkvp -echo; mlr --idkvp --opprint put '$j = $i+$i' s.dkvp -echo; mlr --idkvp --opprint put '$y=$x*1e6' s.dkvp -echo; mlr --idkvp --opprint put '$y=$x+1e6' s.dkvp -echo; mlr --idkvp --opprint put '$y=$x+1' s.dkvp -echo; mlr --idkvp --opprint put '$y=FILENAME' s.dkvp -echo; mlr --idkvp --opprint put '$y=FILENUM' s.dkvp -echo; mlr --idkvp --opprint put '$y=NF' s.dkvp -echo; mlr --idkvp --opprint put '$y=NR' s.dkvp -echo; mlr --idkvp --opprint put '$y=FNR' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$y=NR' s.dkvp s.dkvp -echo; mlr --icsv --opprint put '$y=FNR' s.csv s.csv -echo; mlr --idkvp --opprint put '$y=FNR+1' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$y=FNR+$i' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$y=FNR+3' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$y=FNR+3+$i' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$y=$i+$y' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$y=$i+$x' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$z=$x+$y' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$z=$x+$i' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$z=NR+$i' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$z=NR-$i' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$z=4-1' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$z=NR' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$z=$i' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$z=100*NR-$i' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$z=100*$i+$x' s.dkvp s.dkvp -echo; mlr --idkvp --opprint put '$z=100*$i+$x' s.dkvp -echo; mlr --idkvp --opprint put '$z=100*$i/$x' s.dkvp -echo; mlr --idkvp --opprint put '$z=NR/$i' s.dkvp -echo; mlr --idkvp --opprint put '$z=100/$i' s.dkvp -echo; mlr --idkvp --opprint put '$z=100//$i' s.dkvp -echo; mlr --idkvp --opprint put '$z=100//$x' s.dkvp -echo; mlr --idkvp --opprint put '$z=100.0//$i' s.dkvp -echo; mlr --idkvp --opprint put '$z=100.0 // $i' s.dkvp -echo; mlr --idkvp --opprint put '$z=100.0 / $i' s.dkvp -echo; mlr --idkvp --opprint put '$z=100.0 ' s.dkvp -echo; mlr --idkvp --opprint put '$z=100 ' s.dkvp -echo; mlr --idkvp --opprint put '$z=100.4 ' s.dkvp -echo; mlr --idkvp --opprint put '$z=1.2 ' s.dkvp -echo; mlr --idkvp --opprint put '$z=100.0 / $i ' s.dkvp -echo; mlr --idkvp --opprint put '$z=100.0 // $i ' s.dkvp diff --git a/go/todo.txt b/go/todo.txt index 0c00f45b0..5ac3d768a 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -1,6 +1,14 @@ ---------------------------------------------------------------- TOP OF LIST: * README.md at various levels +* widen CLI coverage + o --c2x et al. +* widen DSL coverage + o begin/end blocks + o indirect field names, at LHS and RHS + o hex inputs + o bitwise/int operators + o support the filter verb * widen reader coverage o TSV/ASV o XTAB @@ -20,10 +28,8 @@ TOP OF LIST: o repeat o head/tail o sort + o filter * implement mlrrc -* widen DSL coverage - o begin/end blocks - o indirect field names, at LHS and RHS ---------------------------------------------------------------- general plan: diff --git a/go/u/README.md b/go/u/README.md new file mode 100644 index 000000000..1d081ae94 --- /dev/null +++ b/go/u/README.md @@ -0,0 +1,2 @@ +These are manually invoked sanity-check scripts, eventually to be replaced by +duplicating the `../c/reg_test/run` framework. diff --git a/go/u/ppp b/go/u/ppp new file mode 100755 index 000000000..6116a075a --- /dev/null +++ b/go/u/ppp @@ -0,0 +1,24 @@ +#!/bin/bash + +set -euo pipefail + +echo; mlr -n put -v '$y = 1 || 2' +echo; mlr -n put -v '$y = 1 || 2 || 3' +echo; mlr -n put -v '$y = 1 || 2 && 3' +echo; mlr -n put -v '$y = 1 && 2 || 3' +echo; mlr -n put -v '$y = 1 ? 2 : 3' +echo; mlr -n put -v '$y = $a + $b * $c' +echo; mlr -n put -v '$y = $a * $b * $c' +echo; mlr -n put -v '$y = $a ** $b ** $c' +echo; mlr -n put -v '$[2] = 3' +echo; mlr -n put -v '$[$y] = 4' +#echo; mlr -n put -v '${1} = 4' +echo; mlr -n put -v '$x = "abc"' +echo; mlr -n put -v '$["abc"] = "def"' +echo; mlr -n put -v '$[FILENAME] = FNR' +echo; mlr -n put -v '$x = $a + $b + $c' +echo; mlr -n put -v '$x = ($a + $b) + $c; $y = $a + ($b + $c); $z = $a + ($b)+ $c' +echo; mlr -n put -v '$x = 2 * $a + $b . $c' +echo; mlr -n put -v '$x = 2 * $a + ($b . $c)' +echo; mlr -n put -v '$x = (NF + NR) * 7; $y = OFS . $y . "hello"' +echo; mlr -n put -v '$x = 123. + 1e-2 / .2e3 + 1.e-3' diff --git a/go/qqq b/go/u/qqq similarity index 94% rename from go/qqq rename to go/u/qqq index d0e267531..a2dfba332 100755 --- a/go/qqq +++ b/go/u/qqq @@ -25,7 +25,7 @@ for ifmt in csv dkvp nidx json; do $fnr=FNR; $filename=FILENAME; $filenum=FILENUM; - ' s.$ifmt t.$ifmt + ' u/s.$ifmt u/t.$ifmt echo done @@ -42,6 +42,6 @@ for ofmt in pprint csv dkvp nidx json; do $fnr=FNR; $filename=FILENAME; $filenum=FILENUM; - ' s.$ifmt t.$ifmt + ' u/s.$ifmt u/t.$ifmt echo done diff --git a/go/u/rrr b/go/u/rrr new file mode 100755 index 000000000..e22cd5cb3 --- /dev/null +++ b/go/u/rrr @@ -0,0 +1,50 @@ +#!/bin/bash + +set -euo pipefail + +echo; mlr --idkvp --opprint put '$j=$i+$i' u/s.dkvp +echo; mlr --idkvp --opprint put '$j=$i+$x' u/s.dkvp +echo; mlr --idkvp --opprint put '$j=$y+$x' u/s.dkvp +echo; mlr --idkvp --opprint put '$j=$y+$i' u/s.dkvp +echo; mlr --idkvp --opprint put '$j=$y+$y' u/s.dkvp +echo; mlr --idkvp --opprint put '$j=$i+$i' u/s.dkvp +echo; mlr --idkvp --opprint put '$y=$x*1e6' u/s.dkvp +echo; mlr --idkvp --opprint put '$y=$x+1e6' u/s.dkvp +echo; mlr --idkvp --opprint put '$y=$x+1' u/s.dkvp +echo; mlr --idkvp --opprint put '$y=FILENAME' u/s.dkvp +echo; mlr --idkvp --opprint put '$y=FILENUM' u/s.dkvp +echo; mlr --idkvp --opprint put '$y=NF' u/s.dkvp +echo; mlr --idkvp --opprint put '$y=NR' u/s.dkvp +echo; mlr --idkvp --opprint put '$y=FNR' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$y=NR' u/s.dkvp u/t.dkvp +echo; mlr --icsv --opprint put '$y=FNR' u/s.csv u/t.csv +echo; mlr --idkvp --opprint put '$y=FNR+1' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$y=FNR+$i' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$y=FNR+3' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$y=FNR+3+$i' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$y=$i+$y' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$y=$i+$x' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$z=$x+$y' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$z=$x+$i' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$z=NR+$i' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$z=NR-$i' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$z=4-1' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$z=NR' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$z=$i' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$z=100*NR-$i' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$z=100*$i+$x' u/s.dkvp u/t.dkvp +echo; mlr --idkvp --opprint put '$z=100*$i+$x' u/s.dkvp +echo; mlr --idkvp --opprint put '$z=100*$i/$x' u/s.dkvp +echo; mlr --idkvp --opprint put '$z=NR/$i' u/s.dkvp +echo; mlr --idkvp --opprint put '$z=100/$i' u/s.dkvp +echo; mlr --idkvp --opprint put '$z=100//$i' u/s.dkvp +echo; mlr --idkvp --opprint put '$z=100//$x' u/s.dkvp +echo; mlr --idkvp --opprint put '$z=100.0//$i' u/s.dkvp +echo; mlr --idkvp --opprint put '$z=100.0//$i' u/s.dkvp +echo; mlr --idkvp --opprint put '$z=100.0/$i' u/s.dkvp +echo; mlr --idkvp --opprint put '$z=100.0' u/s.dkvp +echo; mlr --idkvp --opprint put '$z=100' u/s.dkvp +echo; mlr --idkvp --opprint put '$z=100.4' u/s.dkvp +echo; mlr --idkvp --opprint put '$z=1.2' u/s.dkvp +echo; mlr --idkvp --opprint put '$z=100.0/$i' u/s.dkvp +echo; mlr --idkvp --opprint put '$z=100.0//$i' u/s.dkvp diff --git a/go/s.csv b/go/u/s.csv similarity index 100% rename from go/s.csv rename to go/u/s.csv diff --git a/go/s.dkvp b/go/u/s.dkvp similarity index 100% rename from go/s.dkvp rename to go/u/s.dkvp diff --git a/go/s.json b/go/u/s.json similarity index 100% rename from go/s.json rename to go/u/s.json diff --git a/go/s.nidx b/go/u/s.nidx similarity index 100% rename from go/s.nidx rename to go/u/s.nidx diff --git a/go/t.csv b/go/u/t.csv similarity index 100% rename from go/t.csv rename to go/u/t.csv diff --git a/go/t.dkvp b/go/u/t.dkvp similarity index 100% rename from go/t.dkvp rename to go/u/t.dkvp diff --git a/go/t.json b/go/u/t.json similarity index 100% rename from go/t.json rename to go/u/t.json diff --git a/go/t.nidx b/go/u/t.nidx similarity index 100% rename from go/t.nidx rename to go/u/t.nidx diff --git a/go/test-dsl b/go/u/test-dsl similarity index 100% rename from go/test-dsl rename to go/u/test-dsl diff --git a/go/uuu b/go/u/uuu similarity index 61% rename from go/uuu rename to go/u/uuu index afe577387..97f81558c 100755 --- a/go/uuu +++ b/go/u/uuu @@ -1,7 +1,7 @@ echo ================================================================ -mlr --idkvp --opprint cat then tac s.dkvp t.dkvp +mlr --idkvp --opprint cat then tac u/s.dkvp u/t.dkvp echo ================================================================ -mlr --idkvp --opprint cat -n then tac s.dkvp t.dkvp +mlr --idkvp --opprint cat -n then tac u/s.dkvp u/t.dkvp echo ================================================================ -mlr --idkvp --opprint cat -n then tac then cat -n s.dkvp t.dkvp +mlr --idkvp --opprint cat -n then tac then cat -n u/s.dkvp u/t.dkvp echo ================================================================