Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-18 00:45:47 +00:00
Code Issues Projects Releases Wiki Activity

skip/pass-comments UT

Browse source
This commit is contained in:
John Kerl 2018-01-03 22:11:31 -05:00
parent c7caa29d7c
commit 749bd458ad
14 changed files with 153 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

2
c/draft-release-notes.md
View file

@ -49,3 +49,5 @@ longer-term follow-on [**issue 151**](https://github.com/johnkerl/miller/issues/
* [**Issue 161**](https://github.com/johnkerl/miller/issues/161) fixes a CSV-parse error (with error message "unwrapped double quote at line 0") when a CSV file starts with the UTF-8 bill-of-materials ("BOM") sequence `0xef` `0xbb` `0xbf` and the header line has double-quoted fields. ([Release 5.2.0](https://github.com/johnkerl/miller/releases/tag/v5.2.0) introduced handling for UTF-8 BOMs, but missed the case of double-quoted header line.)
* [**Issue 162**](https://github.com/johnkerl/miller/issues/162) fixes a corner case doing multi-emit of aggregate variables when the first variable name is a typo.
There is no prebuilt Windows executable for this release; my apologies.

4
c/input/lrec_reader_mmap_xtab.c
View file

@ -147,7 +147,6 @@ static lrec_t* lrec_parse_mmap_xtab_single_ifs_single_ips(file_reader_mmap_state
char* comment_string = pstate->comment_string;
int comment_string_length = pstate->comment_string_length;
// xxx more
if (pstate->do_auto_line_term) {
// Skip over otherwise empty LF-only or CRLF-only lines.
while (phandle->sol < phandle->eof) {
@ -334,8 +333,6 @@ static lrec_t* lrec_parse_mmap_xtab_single_ifs_multi_ips(file_reader_mmap_state_
phandle->sol++;
}
// xxx skip comments ...
if (phandle->sol >= phandle->eof)
return NULL;
@ -472,7 +469,6 @@ static lrec_t* lrec_parse_mmap_xtab_multi_ifs_single_ips(file_reader_mmap_state_
if (!skipped_anything)
break;
}
// xxx skip comments ...
if (phandle->sol >= phandle->eof)
return NULL;

79
c/oo
View file

@ -36,10 +36,77 @@ flags="-a"
#vee=-v
vee=
# ----------------------------------------------------------------
announce
## ----------------------------------------------------------------
#announce XTAB
#
#mention input comments1.xtab
#cat reg_test/input/comments/comments1.xtab
#
#mention skip comments1.xtab
#run_mlr --skip-comments --ixtab --odkvp cat < reg_test/input/comments/comments1.xtab
#run_mlr --skip-comments --ixtab --odkvp cat reg_test/input/comments/comments1.xtab
#
#mention pass comments1.xtab
#run_mlr --pass-comments --ixtab --odkvp cat < reg_test/input/comments/comments1.xtab
#run_mlr --pass-comments --ixtab --odkvp cat reg_test/input/comments/comments1.xtab
#
#mention input comments2.xtab
#cat reg_test/input/comments/comments2.xtab
#
#mention skip comments2.xtab
#run_mlr --skip-comments --ixtab --odkvp cat < reg_test/input/comments/comments2.xtab
#run_mlr --skip-comments --ixtab --odkvp cat reg_test/input/comments/comments2.xtab
#
#mention pass comments2.xtab
#run_mlr --pass-comments --ixtab --odkvp cat < reg_test/input/comments/comments2.xtab
#run_mlr --pass-comments --ixtab --odkvp cat reg_test/input/comments/comments2.xtab
run_mlr --from $indir/abixy --opprint stats1 -a sum --gr '[a-h]' --fr '[i-z]'
run_mlr --from $indir/abixy --opprint stats1 -a sum -g a,b --fr '[i-z]'
run_mlr --from $indir/abixy --opprint stats1 -a sum --gr '[a-h]' -f i,x,y
run_mlr --from $indir/abixy --opprint stats1 -a sum --grfx '[a-h]'
## ----------------------------------------------------------------
#announce DKVP
#
#mention input comments1.dkvp
#cat reg_test/input/comments/comments1.dkvp
#
#mention skip comments1.dkvp
#run_mlr --skip-comments --idkvp --oxtab cat < reg_test/input/comments/comments1.dkvp
#run_mlr --skip-comments --idkvp --oxtab cat reg_test/input/comments/comments1.dkvp
#
#mention pass comments1.dkvp
#run_mlr --pass-comments --idkvp --oxtab cat < reg_test/input/comments/comments1.dkvp
#run_mlr --pass-comments --idkvp --oxtab cat reg_test/input/comments/comments1.dkvp
#
#mention input comments2.dkvp
#cat reg_test/input/comments/comments2.dkvp
#
#mention skip comments2.dkvp
#run_mlr --skip-comments --idkvp --oxtab cat < reg_test/input/comments/comments2.dkvp
#run_mlr --skip-comments --idkvp --oxtab cat reg_test/input/comments/comments2.dkvp
#
#mention pass comments2.dkvp
#run_mlr --pass-comments --idkvp --oxtab cat < reg_test/input/comments/comments2.dkvp
#run_mlr --pass-comments --idkvp --oxtab cat reg_test/input/comments/comments2.dkvp
# ----------------------------------------------------------------
announce NIDX
mention input comments1.nidx
cat reg_test/input/comments/comments1.nidx
mention skip comments1.nidx
run_mlr --skip-comments --inidx --oxtab cat < reg_test/input/comments/comments1.nidx
run_mlr --skip-comments --inidx --oxtab cat reg_test/input/comments/comments1.nidx
mention pass comments1.nidx
run_mlr --pass-comments --inidx --oxtab cat < reg_test/input/comments/comments1.nidx
run_mlr --pass-comments --inidx --oxtab cat reg_test/input/comments/comments1.nidx
mention input comments2.nidx
cat reg_test/input/comments/comments2.nidx
mention skip comments2.nidx
run_mlr --skip-comments --inidx --oxtab cat < reg_test/input/comments/comments2.nidx
run_mlr --skip-comments --inidx --oxtab cat reg_test/input/comments/comments2.nidx
mention pass comments2.nidx
run_mlr --pass-comments --inidx --oxtab cat < reg_test/input/comments/comments2.nidx
run_mlr --pass-comments --inidx --oxtab cat reg_test/input/comments/comments2.nidx

5
c/reg_test/input/Makefile.am
View file

@ -1,4 +1,6 @@
SUBDIRS= rfc-csv
SUBDIRS= rfc-csv \
comments
EXTRA_DIST= \
a.csv \
a.pprint \
@ -25,6 +27,7 @@ EXTRA_DIST= \
c.pprint \
capture-lengths.dkvp \
capture.dkvp \
comments \
d.csv \
d.pprint \
date1.csv \

11
c/reg_test/input/comments/Makefile.am Normal file
View file

@ -0,0 +1,11 @@
EXTRA_DIST= \
comments1.dkvp \
comments2.dkvp \
comments3.dkvp \
\
comments1.nidx \
comments2.nidx \
comments3.nidx \
\
comments1.xtab \
comments2.xtab

3
c/reg_test/input/comments/comments1.dkvp Normal file
View file

@ -0,0 +1,3 @@
# hello world 1
a=1,b=2,c=3
a=4,b=5,c=6

3
c/reg_test/input/comments/comments1.nidx Normal file
View file

@ -0,0 +1,3 @@
# hello world 1
aX1 bX2 cX3
aX4 bX5 cX6

8
c/reg_test/input/comments/comments1.xtab Normal file
View file

@ -0,0 +1,8 @@
x 1
#hello world3
z 3
x 2
#hello world4
z 5

3
c/reg_test/input/comments/comments2.dkvp Normal file
View file

@ -0,0 +1,3 @@
a=1,b=2,c=3
# hello world 2
a=4,b=5,c=6

3
c/reg_test/input/comments/comments2.nidx Normal file
View file

@ -0,0 +1,3 @@
aX1 bX2 cX3
# hello world 2
aX4 bX5 cX6

13
c/reg_test/input/comments/comments2.xtab Normal file
View file

@ -0,0 +1,13 @@
#hello world1
#hello world2
x 1
#hello world3
z 3
x 2
#hello world4
z 5
#hello world5
#hello world6

3
c/reg_test/input/comments/comments3.dkvp Normal file
View file

@ -0,0 +1,3 @@
a=1,b=2,c=3
a=4,b=5,c=6
# hello world 3

3
c/reg_test/input/comments/comments3.nidx Normal file
View file

@ -0,0 +1,3 @@
aX1 bX2 cX3
aX4 bX5 cX6
# hello world 3

42
c/todo.txt
View file

@ -11,8 +11,6 @@ BUGFIXES
x=9223372036854775802,y=-9223372036854775806
x=9223372036854775805,y=-9223372036854775802
$ mlr --from s put 'tee > "boo.".$a, mapexcept($*,"a")'
================================================================
5.3.0 TO DO:
@ -22,25 +20,35 @@ $ mlr --from s put 'tee > "boo.".$a, mapexcept($*,"a")'
o streqn-past-eof scenarios @ page boundaries ...
o UT for skip/pass
input/lrec_reader_mmap_csv.c:9
input/lrec_reader_mmap_csvlite.c:37
input/lrec_reader_mmap_dkvp.c:25
input/lrec_reader_mmap_json.c:7
input/lrec_reader_mmap_nidx.c:10
input/lrec_reader_mmap_xtab.c:47
k input/lrec_reader_mmap_dkvp.c:25
k input/lrec_reader_stdio_dkvp.c:21
input/lrec_reader_stdio_csv.c:9
input/lrec_reader_stdio_csvlite.c:13
input/lrec_reader_stdio_dkvp.c:21
input/lrec_reader_stdio_json.c:7
input/lrec_reader_stdio_nidx.c:21
k input/lrec_reader_mmap_nidx.c:10
k input/lrec_reader_stdio_nidx.c:21
input/lrec_reader_mmap_xtab.c:47
input/lrec_reader_stdio_xtab.c:8
input/lrec_reader_mmap_csv.c:9
input/lrec_reader_stdio_csv.c:9
input/lrec_reader_mmap_csvlite.c:37
input/lrec_reader_stdio_csvlite.c:13
input/lrec_reader_mmap_json.c:7
input/lrec_reader_stdio_json.c:7
o mlh
o mld
* sql in/outs -> own page?
* sql-setup doc somewhere .......
* ./configure
* valgrinds
* xxxes
================================================================
FUNDAM:
@ -160,7 +168,7 @@ MAPVAR CHECKLIST:
* clarify ownership semantics in localstack & mlhmmv via function names, & top-of-file comments
================================================================
5.3.0 ideas:
5.4.0 ideas:
----------------------------------------------------------------
! multi-field x many verbs: -f/-r field-name-spec opportunities throughout
@ -198,8 +206,6 @@ which verbs:
* mt_error/fatal/skip-assign audit
* perf note: DSL stuff performant w/r/t ruby/python (mand bench & maybe others)
! more than 'syntax error' from lemon-parse failures ... lemon API research
? some sort of debug/single-step/trace ... include the AST in the CST & reverse-generate ?
@ -260,7 +266,7 @@ COMPARES:
* https://www.gnu.org/software/datamash/manual/datamash.html
================================================================
5.3.0 IDEAS:
5.4.0 IDEAS:
!! coroutines: line-oriented; operate on field(s); r/w popen