diff --git a/c/draft-release-notes.md b/c/draft-release-notes.md index 0af20a055..c9c7c1097 100644 --- a/c/draft-release-notes.md +++ b/c/draft-release-notes.md @@ -49,3 +49,5 @@ longer-term follow-on [**issue 151**](https://github.com/johnkerl/miller/issues/ * [**Issue 161**](https://github.com/johnkerl/miller/issues/161) fixes a CSV-parse error (with error message "unwrapped double quote at line 0") when a CSV file starts with the UTF-8 bill-of-materials ("BOM") sequence `0xef` `0xbb` `0xbf` and the header line has double-quoted fields. ([Release 5.2.0](https://github.com/johnkerl/miller/releases/tag/v5.2.0) introduced handling for UTF-8 BOMs, but missed the case of double-quoted header line.) * [**Issue 162**](https://github.com/johnkerl/miller/issues/162) fixes a corner case doing multi-emit of aggregate variables when the first variable name is a typo. + +There is no prebuilt Windows executable for this release; my apologies. diff --git a/c/input/lrec_reader_mmap_xtab.c b/c/input/lrec_reader_mmap_xtab.c index 43e9c4c64..b48dbe0f0 100644 --- a/c/input/lrec_reader_mmap_xtab.c +++ b/c/input/lrec_reader_mmap_xtab.c @@ -147,7 +147,6 @@ static lrec_t* lrec_parse_mmap_xtab_single_ifs_single_ips(file_reader_mmap_state char* comment_string = pstate->comment_string; int comment_string_length = pstate->comment_string_length; - // xxx more if (pstate->do_auto_line_term) { // Skip over otherwise empty LF-only or CRLF-only lines. while (phandle->sol < phandle->eof) { @@ -334,8 +333,6 @@ static lrec_t* lrec_parse_mmap_xtab_single_ifs_multi_ips(file_reader_mmap_state_ phandle->sol++; } - // xxx skip comments ... - if (phandle->sol >= phandle->eof) return NULL; @@ -472,7 +469,6 @@ static lrec_t* lrec_parse_mmap_xtab_multi_ifs_single_ips(file_reader_mmap_state_ if (!skipped_anything) break; } - // xxx skip comments ... if (phandle->sol >= phandle->eof) return NULL; diff --git a/c/oo b/c/oo index 882219e7f..12577114c 100755 --- a/c/oo +++ b/c/oo @@ -36,10 +36,77 @@ flags="-a" #vee=-v vee= -# ---------------------------------------------------------------- -announce +## ---------------------------------------------------------------- +#announce XTAB +# +#mention input comments1.xtab +#cat reg_test/input/comments/comments1.xtab +# +#mention skip comments1.xtab +#run_mlr --skip-comments --ixtab --odkvp cat < reg_test/input/comments/comments1.xtab +#run_mlr --skip-comments --ixtab --odkvp cat reg_test/input/comments/comments1.xtab +# +#mention pass comments1.xtab +#run_mlr --pass-comments --ixtab --odkvp cat < reg_test/input/comments/comments1.xtab +#run_mlr --pass-comments --ixtab --odkvp cat reg_test/input/comments/comments1.xtab +# +#mention input comments2.xtab +#cat reg_test/input/comments/comments2.xtab +# +#mention skip comments2.xtab +#run_mlr --skip-comments --ixtab --odkvp cat < reg_test/input/comments/comments2.xtab +#run_mlr --skip-comments --ixtab --odkvp cat reg_test/input/comments/comments2.xtab +# +#mention pass comments2.xtab +#run_mlr --pass-comments --ixtab --odkvp cat < reg_test/input/comments/comments2.xtab +#run_mlr --pass-comments --ixtab --odkvp cat reg_test/input/comments/comments2.xtab -run_mlr --from $indir/abixy --opprint stats1 -a sum --gr '[a-h]' --fr '[i-z]' -run_mlr --from $indir/abixy --opprint stats1 -a sum -g a,b --fr '[i-z]' -run_mlr --from $indir/abixy --opprint stats1 -a sum --gr '[a-h]' -f i,x,y -run_mlr --from $indir/abixy --opprint stats1 -a sum --grfx '[a-h]' +## ---------------------------------------------------------------- +#announce DKVP +# +#mention input comments1.dkvp +#cat reg_test/input/comments/comments1.dkvp +# +#mention skip comments1.dkvp +#run_mlr --skip-comments --idkvp --oxtab cat < reg_test/input/comments/comments1.dkvp +#run_mlr --skip-comments --idkvp --oxtab cat reg_test/input/comments/comments1.dkvp +# +#mention pass comments1.dkvp +#run_mlr --pass-comments --idkvp --oxtab cat < reg_test/input/comments/comments1.dkvp +#run_mlr --pass-comments --idkvp --oxtab cat reg_test/input/comments/comments1.dkvp +# +#mention input comments2.dkvp +#cat reg_test/input/comments/comments2.dkvp +# +#mention skip comments2.dkvp +#run_mlr --skip-comments --idkvp --oxtab cat < reg_test/input/comments/comments2.dkvp +#run_mlr --skip-comments --idkvp --oxtab cat reg_test/input/comments/comments2.dkvp +# +#mention pass comments2.dkvp +#run_mlr --pass-comments --idkvp --oxtab cat < reg_test/input/comments/comments2.dkvp +#run_mlr --pass-comments --idkvp --oxtab cat reg_test/input/comments/comments2.dkvp + +# ---------------------------------------------------------------- +announce NIDX + +mention input comments1.nidx +cat reg_test/input/comments/comments1.nidx + +mention skip comments1.nidx +run_mlr --skip-comments --inidx --oxtab cat < reg_test/input/comments/comments1.nidx +run_mlr --skip-comments --inidx --oxtab cat reg_test/input/comments/comments1.nidx + +mention pass comments1.nidx +run_mlr --pass-comments --inidx --oxtab cat < reg_test/input/comments/comments1.nidx +run_mlr --pass-comments --inidx --oxtab cat reg_test/input/comments/comments1.nidx + +mention input comments2.nidx +cat reg_test/input/comments/comments2.nidx + +mention skip comments2.nidx +run_mlr --skip-comments --inidx --oxtab cat < reg_test/input/comments/comments2.nidx +run_mlr --skip-comments --inidx --oxtab cat reg_test/input/comments/comments2.nidx + +mention pass comments2.nidx +run_mlr --pass-comments --inidx --oxtab cat < reg_test/input/comments/comments2.nidx +run_mlr --pass-comments --inidx --oxtab cat reg_test/input/comments/comments2.nidx diff --git a/c/reg_test/input/Makefile.am b/c/reg_test/input/Makefile.am index 8fd7f7b99..d5b72ab0f 100644 --- a/c/reg_test/input/Makefile.am +++ b/c/reg_test/input/Makefile.am @@ -1,4 +1,6 @@ -SUBDIRS= rfc-csv +SUBDIRS= rfc-csv \ + comments + EXTRA_DIST= \ a.csv \ a.pprint \ @@ -25,6 +27,7 @@ EXTRA_DIST= \ c.pprint \ capture-lengths.dkvp \ capture.dkvp \ + comments \ d.csv \ d.pprint \ date1.csv \ diff --git a/c/reg_test/input/comments/Makefile.am b/c/reg_test/input/comments/Makefile.am new file mode 100644 index 000000000..b2da0b476 --- /dev/null +++ b/c/reg_test/input/comments/Makefile.am @@ -0,0 +1,11 @@ +EXTRA_DIST= \ + comments1.dkvp \ + comments2.dkvp \ + comments3.dkvp \ + \ + comments1.nidx \ + comments2.nidx \ + comments3.nidx \ + \ + comments1.xtab \ + comments2.xtab diff --git a/c/reg_test/input/comments/comments1.dkvp b/c/reg_test/input/comments/comments1.dkvp new file mode 100644 index 000000000..9c3dad9cf --- /dev/null +++ b/c/reg_test/input/comments/comments1.dkvp @@ -0,0 +1,3 @@ +# hello world 1 +a=1,b=2,c=3 +a=4,b=5,c=6 diff --git a/c/reg_test/input/comments/comments1.nidx b/c/reg_test/input/comments/comments1.nidx new file mode 100644 index 000000000..eb2f2f10d --- /dev/null +++ b/c/reg_test/input/comments/comments1.nidx @@ -0,0 +1,3 @@ +# hello world 1 +aX1 bX2 cX3 +aX4 bX5 cX6 diff --git a/c/reg_test/input/comments/comments1.xtab b/c/reg_test/input/comments/comments1.xtab new file mode 100644 index 000000000..4d9f43db8 --- /dev/null +++ b/c/reg_test/input/comments/comments1.xtab @@ -0,0 +1,8 @@ +x 1 +#hello world3 +z 3 + +x 2 +#hello world4 +z 5 + diff --git a/c/reg_test/input/comments/comments2.dkvp b/c/reg_test/input/comments/comments2.dkvp new file mode 100644 index 000000000..b13f182fb --- /dev/null +++ b/c/reg_test/input/comments/comments2.dkvp @@ -0,0 +1,3 @@ +a=1,b=2,c=3 +# hello world 2 +a=4,b=5,c=6 diff --git a/c/reg_test/input/comments/comments2.nidx b/c/reg_test/input/comments/comments2.nidx new file mode 100644 index 000000000..8ee3ae723 --- /dev/null +++ b/c/reg_test/input/comments/comments2.nidx @@ -0,0 +1,3 @@ +aX1 bX2 cX3 +# hello world 2 +aX4 bX5 cX6 diff --git a/c/reg_test/input/comments/comments2.xtab b/c/reg_test/input/comments/comments2.xtab new file mode 100644 index 000000000..043e7736b --- /dev/null +++ b/c/reg_test/input/comments/comments2.xtab @@ -0,0 +1,13 @@ +#hello world1 + +#hello world2 +x 1 +#hello world3 +z 3 + +x 2 +#hello world4 +z 5 +#hello world5 + +#hello world6 diff --git a/c/reg_test/input/comments/comments3.dkvp b/c/reg_test/input/comments/comments3.dkvp new file mode 100644 index 000000000..23c5e97c8 --- /dev/null +++ b/c/reg_test/input/comments/comments3.dkvp @@ -0,0 +1,3 @@ +a=1,b=2,c=3 +a=4,b=5,c=6 +# hello world 3 diff --git a/c/reg_test/input/comments/comments3.nidx b/c/reg_test/input/comments/comments3.nidx new file mode 100644 index 000000000..2d23d575d --- /dev/null +++ b/c/reg_test/input/comments/comments3.nidx @@ -0,0 +1,3 @@ +aX1 bX2 cX3 +aX4 bX5 cX6 +# hello world 3 diff --git a/c/todo.txt b/c/todo.txt index 6ed3da5a2..bc8e0a782 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -11,8 +11,6 @@ BUGFIXES x=9223372036854775802,y=-9223372036854775806 x=9223372036854775805,y=-9223372036854775802 -$ mlr --from s put 'tee > "boo.".$a, mapexcept($*,"a")' - ================================================================ 5.3.0 TO DO: @@ -22,25 +20,35 @@ $ mlr --from s put 'tee > "boo.".$a, mapexcept($*,"a")' o streqn-past-eof scenarios @ page boundaries ... o UT for skip/pass - input/lrec_reader_mmap_csv.c:9 - input/lrec_reader_mmap_csvlite.c:37 - input/lrec_reader_mmap_dkvp.c:25 - input/lrec_reader_mmap_json.c:7 - input/lrec_reader_mmap_nidx.c:10 - input/lrec_reader_mmap_xtab.c:47 + k input/lrec_reader_mmap_dkvp.c:25 + k input/lrec_reader_stdio_dkvp.c:21 - input/lrec_reader_stdio_csv.c:9 - input/lrec_reader_stdio_csvlite.c:13 - input/lrec_reader_stdio_dkvp.c:21 - input/lrec_reader_stdio_json.c:7 - input/lrec_reader_stdio_nidx.c:21 + k input/lrec_reader_mmap_nidx.c:10 + k input/lrec_reader_stdio_nidx.c:21 + + input/lrec_reader_mmap_xtab.c:47 input/lrec_reader_stdio_xtab.c:8 + input/lrec_reader_mmap_csv.c:9 + input/lrec_reader_stdio_csv.c:9 + + input/lrec_reader_mmap_csvlite.c:37 + input/lrec_reader_stdio_csvlite.c:13 + + input/lrec_reader_mmap_json.c:7 + input/lrec_reader_stdio_json.c:7 + o mlh o mld -* sql in/outs -> own page? + * sql-setup doc somewhere ....... +* ./configure + +* valgrinds + +* xxxes + ================================================================ FUNDAM: @@ -160,7 +168,7 @@ MAPVAR CHECKLIST: * clarify ownership semantics in localstack & mlhmmv via function names, & top-of-file comments ================================================================ -5.3.0 ideas: +5.4.0 ideas: ---------------------------------------------------------------- ! multi-field x many verbs: -f/-r field-name-spec opportunities throughout @@ -198,8 +206,6 @@ which verbs: * mt_error/fatal/skip-assign audit -* perf note: DSL stuff performant w/r/t ruby/python (mand bench & maybe others) - ! more than 'syntax error' from lemon-parse failures ... lemon API research ? some sort of debug/single-step/trace ... include the AST in the CST & reverse-generate ? @@ -260,7 +266,7 @@ COMPARES: * https://www.gnu.org/software/datamash/manual/datamash.html ================================================================ -5.3.0 IDEAS: +5.4.0 IDEAS: !! coroutines: line-oriented; operate on field(s); r/w popen