From 094652bc1ddf4e7301d9a977977afafb52bf4824 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Thu, 4 Jan 2018 20:38:55 -0500 Subject: [PATCH] skip/pass-comments UT iterate --- c/input/lrec_reader_mmap_csv.c | 24 -------- c/input/lrec_reader_stdio_csv.c | 20 ------- c/reg_test/expected/out | 79 --------------------------- c/reg_test/input/Makefile.am | 2 - c/reg_test/input/with-comments.csv | 7 --- c/reg_test/input/without-comments.csv | 3 - c/reg_test/run | 29 ---------- 7 files changed, 164 deletions(-) delete mode 100644 c/reg_test/input/with-comments.csv delete mode 100644 c/reg_test/input/without-comments.csv diff --git a/c/input/lrec_reader_mmap_csv.c b/c/input/lrec_reader_mmap_csv.c index 6295839d9..56b32afbe 100644 --- a/c/input/lrec_reader_mmap_csv.c +++ b/c/input/lrec_reader_mmap_csv.c @@ -202,18 +202,6 @@ static lrec_t* lrec_reader_mmap_csv_process(void* pvstate, void* pvhandle, conte return NULL; pstate->ilno++; - if (pstate->comment_string != NULL) { - if (pstate->pfields->phead != NULL) { - if (streqn(pstate->pfields->phead->value, pstate->comment_string, pstate->comment_string_length)) { - if (pstate->comment_handling == PASS_COMMENTS) { - // xxx fix me - } - rslls_reset(pstate->pfields); - continue; - } - } - } - slls_t* pheader_fields = slls_alloc(); int i = 0; for (rsllse_t* pe = pstate->pfields->phead; i < pstate->pfields->length && pe != NULL; pe = pe->pnext, i++) { @@ -250,18 +238,6 @@ static lrec_t* lrec_reader_mmap_csv_process(void* pvstate, void* pvhandle, conte if (rc == FALSE) // EOF return NULL; - if (pstate->comment_string != NULL) { - if (pstate->pfields->phead != NULL) { - if (streqn(pstate->pfields->phead->value, pstate->comment_string, pstate->comment_string_length)) { - if (pstate->comment_handling == PASS_COMMENTS) { - // xxx fix me - } - rslls_reset(pstate->pfields); - continue; - } - } - } - lrec_t* prec = pstate->use_implicit_header ? paste_indices_and_data(pstate, pstate->pfields, pctx) : paste_header_and_data(pstate, pstate->pfields, pctx); diff --git a/c/input/lrec_reader_stdio_csv.c b/c/input/lrec_reader_stdio_csv.c index 4d5ab0ee4..9ce8f9387 100644 --- a/c/input/lrec_reader_stdio_csv.c +++ b/c/input/lrec_reader_stdio_csv.c @@ -231,16 +231,6 @@ static lrec_t* lrec_reader_stdio_csv_process(void* pvstate, void* pvhandle, cont return NULL; pstate->ilno++; - if (pstate->comment_string != NULL) { - if (pstate->pfields->phead != NULL) { - if (streqn(pstate->pfields->phead->value, pstate->comment_string, pstate->comment_string_length)) { - // xxx fix me -- doesn't do well with pass-through, post-parse ... - rslls_reset(pstate->pfields); - continue; - } - } - } - slls_t* pheader_fields = slls_alloc(); int i = 0; for (rsllse_t* pe = pstate->pfields->phead; i < pstate->pfields->length && pe != NULL; pe = pe->pnext) { @@ -277,16 +267,6 @@ static lrec_t* lrec_reader_stdio_csv_process(void* pvstate, void* pvhandle, cont if (rc == FALSE) // EOF return NULL; - if (pstate->comment_string != NULL) { - if (pstate->pfields->phead != NULL) { - // xxx fix me - if (streqn(pstate->pfields->phead->value, pstate->comment_string, pstate->comment_string_length)) { - rslls_reset(pstate->pfields); - continue; - } - } - } - lrec_t* prec = pstate->use_implicit_header ? paste_indices_and_data(pstate, pstate->pfields, pctx) : paste_header_and_data(pstate, pstate->pfields, pctx); diff --git a/c/reg_test/expected/out b/c/reg_test/expected/out index f985f53de..2e9d48e07 100644 --- a/c/reg_test/expected/out +++ b/c/reg_test/expected/out @@ -49952,85 +49952,6 @@ mlr --jlistwrap --json --jvstack cat ./reg_test/input/line-term-crlf-wrap.json ] -================================================================ -SKIP-COMMENTS - - ----------------------------------------------------------------- CSV mmap -mlr --csv --mmap cat ./reg_test/input/without-comments.csv -a,b,c -1,2,3 -4,5,6 - -mlr --csv --skip-comments --mmap cat ./reg_test/input/without-comments.csv -a,b,c -1,2,3 -4,5,6 - -mlr --csv --mmap cat ./reg_test/input/with-comments.csv -mlr: Header/data length mismatch (1 != 3) at file "./reg_test/input/with-comments.csv" line 2. - -mlr --csv --skip-comments --mmap cat ./reg_test/input/with-comments.csv -a,b,c -1,2,3 -4,5,6 - - ----------------------------------------------------------------- CSV stdio -mlr --csv --no-mmap cat ./reg_test/input/without-comments.csv -a,b,c -1,2,3 -4,5,6 - -mlr --csv --skip-comments --no-mmap cat ./reg_test/input/without-comments.csv -a,b,c -1,2,3 -4,5,6 - -mlr --csv --no-mmap cat ./reg_test/input/with-comments.csv -mlr: Header/data length mismatch (1 != 3) at file "./reg_test/input/with-comments.csv" line 2. - -mlr --csv --skip-comments --no-mmap cat ./reg_test/input/with-comments.csv -a,b,c -1,2,3 -4,5,6 - - ----------------------------------------------------------------- CSVLITE mmap -mlr --csvlite --mmap cat ./reg_test/input/without-comments.csv -a,b,c -1,2,3 -4,5,6 - -mlr --csvlite --skip-comments --mmap cat ./reg_test/input/without-comments.csv -a,b,c -1,2,3 -4,5,6 - -mlr --csvlite --mmap cat ./reg_test/input/with-comments.csv -mlr: Header-data length mismatch in file ./reg_test/input/with-comments.csv at line 1. - - ----------------------------------------------------------------- CSVLITE stdio -mlr --csvlite --no-mmap cat ./reg_test/input/without-comments.csv -a,b,c -1,2,3 -4,5,6 - -mlr --csvlite --skip-comments --no-mmap cat ./reg_test/input/without-comments.csv -a,b,c -1,2,3 -4,5,6 - -mlr --csvlite --no-mmap cat ./reg_test/input/with-comments.csv -mlr: Header-data length mismatch in file ./reg_test/input/with-comments.csv at line 2. - -mlr --csvlite --skip-comments --no-mmap cat ./reg_test/input/with-comments.csv -a,b,c -1,2,3 -4,5,6 - - ================================================================ USER-DEFINED FUNCTIONS AND SUBROUTINES diff --git a/c/reg_test/input/Makefile.am b/c/reg_test/input/Makefile.am index d5b72ab0f..501bfb08c 100644 --- a/c/reg_test/input/Makefile.am +++ b/c/reg_test/input/Makefile.am @@ -191,8 +191,6 @@ EXTRA_DIST= \ utf8-align.dkvp \ utf8-align.nidx \ valgrind.txt \ - with-comments.csv \ - without-comments.csv \ x0to10.dat \ xy40.dkvp \ xyz345 \ diff --git a/c/reg_test/input/with-comments.csv b/c/reg_test/input/with-comments.csv deleted file mode 100644 index d8e996a14..000000000 --- a/c/reg_test/input/with-comments.csv +++ /dev/null @@ -1,7 +0,0 @@ -# hello -a,b,c -# hi -1,2,3 -# there -4,5,6 -# haha diff --git a/c/reg_test/input/without-comments.csv b/c/reg_test/input/without-comments.csv deleted file mode 100644 index 88700c714..000000000 --- a/c/reg_test/input/without-comments.csv +++ /dev/null @@ -1,3 +0,0 @@ -a,b,c -1,2,3 -4,5,6 diff --git a/c/reg_test/run b/c/reg_test/run index ccec80f5f..0573a9f7e 100755 --- a/c/reg_test/run +++ b/c/reg_test/run @@ -5631,35 +5631,6 @@ run_mlr --irs auto --ors lf --jlistwrap --json --jvstack cat $indir/line-term-cr run_mlr --jlistwrap --json --jvstack cat $indir/line-term-lf-wrap.json run_mlr --jlistwrap --json --jvstack cat $indir/line-term-crlf-wrap.json -# ---------------------------------------------------------------- -announce SKIP-COMMENTS - -mention CSV mmap -run_mlr --csv --mmap cat $indir/without-comments.csv -run_mlr --csv --skip-comments --mmap cat $indir/without-comments.csv -mlr_expect_fail --csv --mmap cat $indir/with-comments.csv -run_mlr --csv --skip-comments --mmap cat $indir/with-comments.csv - -mention CSV stdio -run_mlr --csv --no-mmap cat $indir/without-comments.csv -run_mlr --csv --skip-comments --no-mmap cat $indir/without-comments.csv -mlr_expect_fail --csv --no-mmap cat $indir/with-comments.csv -run_mlr --csv --skip-comments --no-mmap cat $indir/with-comments.csv - - -mention CSVLITE mmap -run_mlr --csvlite --mmap cat $indir/without-comments.csv -run_mlr --csvlite --skip-comments --mmap cat $indir/without-comments.csv -mlr_expect_fail --csvlite --mmap cat $indir/with-comments.csv -# xxx fix me: -#run_mlr --csvlite --skip-comments --mmap cat $indir/with-comments.csv - -mention CSVLITE stdio -run_mlr --csvlite --no-mmap cat $indir/without-comments.csv -run_mlr --csvlite --skip-comments --no-mmap cat $indir/without-comments.csv -mlr_expect_fail --csvlite --no-mmap cat $indir/with-comments.csv -run_mlr --csvlite --skip-comments --no-mmap cat $indir/with-comments.csv - # ---------------------------------------------------------------- announce USER-DEFINED FUNCTIONS AND SUBROUTINES