From 4f5c95eaf6faee4e8cb9918f49fb68d35e2bf054 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Thu, 4 Jan 2018 22:50:45 -0500 Subject: [PATCH] mmap xtab comment-handling iterate --- c/input/lrec_reader_mmap_xtab.c | 209 ++++++++++++-------------------- c/oo | 34 +++--- c/todo.txt | 1 + 3 files changed, 98 insertions(+), 146 deletions(-) diff --git a/c/input/lrec_reader_mmap_xtab.c b/c/input/lrec_reader_mmap_xtab.c index eb2a5d17b..996261159 100644 --- a/c/input/lrec_reader_mmap_xtab.c +++ b/c/input/lrec_reader_mmap_xtab.c @@ -45,6 +45,15 @@ static lrec_t* lrec_parse_mmap_xtab_multi_ifs_single_ips(file_reader_mmap_state_ static lrec_t* lrec_parse_mmap_xtab_multi_ifs_multi_ips(file_reader_mmap_state_t* phandle, lrec_reader_mmap_xtab_state_t* pstate); +static int skip_comment_line_single_ifs( + file_reader_mmap_state_t* phandle, + lrec_reader_mmap_xtab_state_t* pstate, + char ifs); + +static int skip_comment_line_multi_ifs( + file_reader_mmap_state_t* phandle, + lrec_reader_mmap_xtab_state_t* pstate); + // ---------------------------------------------------------------- lrec_reader_t* lrec_reader_mmap_xtab_alloc(char* ifs, char* ips, int allow_repeat_ips, comment_handling_t comment_handling, char* comment_string) @@ -145,7 +154,6 @@ static lrec_t* lrec_parse_mmap_xtab_single_ifs_single_ips(file_reader_mmap_state lrec_reader_mmap_xtab_state_t* pstate, context_t* pctx) { char* comment_string = pstate->comment_string; - int comment_string_length = pstate->comment_string_length; if (pstate->do_auto_line_term) { // Skip over otherwise empty LF-only or CRLF-only lines. @@ -167,33 +175,17 @@ static lrec_t* lrec_parse_mmap_xtab_single_ifs_single_ips(file_reader_mmap_state } } else { // Skip over otherwise empty IFS-only lines, or comment lines + // xxx auto-line-term in the functions as well! while (TRUE) { - // xxx funcify int skipped_anything = FALSE; if (phandle->sol < phandle->eof && *phandle->sol == ifs) { skipped_anything = TRUE; phandle->sol++; - continue; } if (comment_string != NULL) { - if (phandle->sol < phandle->eof && streqn(phandle->sol, comment_string, comment_string_length)) { + if (skip_comment_line_single_ifs(phandle, pstate, ifs)) skipped_anything = TRUE; - if (pstate->comment_handling == PASS_COMMENTS) - for (int i = 0; i < pstate->comment_string_length; i++) - fputc(phandle->sol[i], stdout); - phandle->sol += comment_string_length; - while (*phandle->sol != ifs) { - if (pstate->comment_handling == PASS_COMMENTS) - fputc(*phandle->sol, stdout); - phandle->sol++; - } - if (*phandle->sol == ifs) { - if (pstate->comment_handling == PASS_COMMENTS) - fputc(*phandle->sol, stdout); - phandle->sol++; - } - } - continue; + } if (!skipped_anything) break; @@ -210,25 +202,9 @@ static lrec_t* lrec_parse_mmap_xtab_single_ifs_single_ips(file_reader_mmap_state if (phandle->sol >= phandle->eof) break; - // Skip comment lines if (comment_string != NULL) { - if (streqn(phandle->sol, comment_string, comment_string_length)) { - if (pstate->comment_handling == PASS_COMMENTS) - for (int i = 0; i < pstate->comment_string_length; i++) - fputc(phandle->sol[i], stdout); - phandle->sol += comment_string_length; - while (phandle->sol < phandle->eof && *phandle->sol != ifs) { - if (pstate->comment_handling == PASS_COMMENTS) - fputc(*phandle->sol, stdout); - phandle->sol++; - } - if (phandle->sol < phandle->eof && *phandle->sol == ifs) { - if (pstate->comment_handling == PASS_COMMENTS) - fputc(*phandle->sol, stdout); - phandle->sol++; - } + if (skip_comment_line_single_ifs(phandle, pstate, ifs)) continue; - } } char* line = phandle->sol; @@ -311,7 +287,6 @@ static lrec_t* lrec_parse_mmap_xtab_single_ifs_multi_ips(file_reader_mmap_state_ lrec_reader_mmap_xtab_state_t* pstate, context_t* pctx) { char* comment_string = pstate->comment_string; - int comment_string_length = pstate->comment_string_length; if (pstate->do_auto_line_term) { // Skip over otherwise empty LF-only or CRLF-only lines. while (phandle->sol < phandle->eof) { @@ -347,25 +322,9 @@ static lrec_t* lrec_parse_mmap_xtab_single_ifs_multi_ips(file_reader_mmap_state_ // Loop over fields, one per line while (TRUE) { - // Skip comment lines if (comment_string != NULL) { - if (streqn(phandle->sol, comment_string, comment_string_length)) { - if (pstate->comment_handling == PASS_COMMENTS) - for (int i = 0; i < pstate->comment_string_length; i++) - fputc(phandle->sol[i], stdout); - phandle->sol += comment_string_length; - while (phandle->sol < phandle->eof && *phandle->sol != ifs) { - if (pstate->comment_handling == PASS_COMMENTS) - fputc(*phandle->sol, stdout); - phandle->sol++; - } - if (phandle->sol < phandle->eof && *phandle->sol == ifs) { - if (pstate->comment_handling == PASS_COMMENTS) - fputc(*phandle->sol, stdout); - phandle->sol++; - } + if (skip_comment_line_single_ifs(phandle, pstate, ifs)) continue; - } } char* line = phandle->sol; @@ -438,7 +397,6 @@ static lrec_t* lrec_parse_mmap_xtab_multi_ifs_single_ips(file_reader_mmap_state_ char* ifs = pstate->ifs; int ifslen = pstate->ifslen; char* comment_string = pstate->comment_string; - int comment_string_length = pstate->comment_string_length; // Skip blank lines and comment lines while (TRUE) { @@ -450,24 +408,7 @@ static lrec_t* lrec_parse_mmap_xtab_multi_ifs_single_ips(file_reader_mmap_state_ } // Skip comment lines if (comment_string != NULL) { - if (phandle->sol < phandle->eof && streqn(phandle->sol, comment_string, comment_string_length)) { - skipped_anything = TRUE; - if (pstate->comment_handling == PASS_COMMENTS) - for (int i = 0; i < pstate->comment_string_length; i++) - fputc(phandle->sol[i], stdout); - phandle->sol += comment_string_length; - while (phandle->sol < phandle->eof && !streqn(phandle->sol, ifs, ifslen)) { - if (pstate->comment_handling == PASS_COMMENTS) - fputc(*phandle->sol, stdout); - phandle->sol++; - } - if (phandle->sol < phandle->eof && streqn(phandle->sol, ifs, ifslen)) { - if (pstate->comment_handling == PASS_COMMENTS) - for (int i = 0; i < ifslen; i++) - fputc(phandle->sol[i], stdout); - phandle->sol += ifslen; - } - } + skipped_anything |= skip_comment_line_multi_ifs(phandle, pstate); } if (!skipped_anything) break; @@ -481,26 +422,9 @@ static lrec_t* lrec_parse_mmap_xtab_multi_ifs_single_ips(file_reader_mmap_state_ // Loop over fields, one per line while (TRUE) { - // Skip comment lines if (comment_string != NULL) { - if (streqn(phandle->sol, comment_string, comment_string_length)) { - if (pstate->comment_handling == PASS_COMMENTS) - for (int i = 0; i < pstate->comment_string_length; i++) - fputc(phandle->sol[i], stdout); - phandle->sol += comment_string_length; - while (phandle->sol < phandle->eof && !streqn(phandle->sol, comment_string, comment_string_length)) { - if (pstate->comment_handling == PASS_COMMENTS) - fputc(*phandle->sol, stdout); - phandle->sol++; - } - if (phandle->sol < phandle->eof && streqn(phandle->sol, ifs, ifslen)) { - if (pstate->comment_handling == PASS_COMMENTS) - for (int i = 0; i < ifslen; i++) - fputc(phandle->sol[i], stdout); - phandle->sol += ifslen; - } + if (skip_comment_line_multi_ifs(phandle, pstate)) continue; - } } char* line = phandle->sol; @@ -565,7 +489,6 @@ static lrec_t* lrec_parse_mmap_xtab_multi_ifs_multi_ips(file_reader_mmap_state_t char* ifs = pstate->ifs; int ifslen = pstate->ifslen; char* comment_string = pstate->comment_string; - int comment_string_length = pstate->comment_string_length; // Skip blank lines and comment lines while (TRUE) { @@ -575,26 +498,8 @@ static lrec_t* lrec_parse_mmap_xtab_multi_ifs_multi_ips(file_reader_mmap_state_t skipped_anything = TRUE; phandle->sol += ifslen; } - // Skip comment lines if (comment_string != NULL) { - if (phandle->sol < phandle->eof && streqn(phandle->sol, comment_string, comment_string_length)) { - skipped_anything = TRUE; - if (pstate->comment_handling == PASS_COMMENTS) - for (int i = 0; i < pstate->comment_string_length; i++) - fputc(phandle->sol[i], stdout); - phandle->sol += comment_string_length; - while (phandle->sol < phandle->eof && !streqn(phandle->sol, ifs, ifslen)) { - if (pstate->comment_handling == PASS_COMMENTS) - fputc(*phandle->sol, stdout); - phandle->sol++; - } - if (phandle->sol < phandle->eof && streqn(phandle->sol, ifs, ifslen)) { - if (pstate->comment_handling == PASS_COMMENTS) - for (int i = 0; i < ifslen; i++) - fputc(phandle->sol[i], stdout); - phandle->sol += ifslen; - } - } + skipped_anything |= skip_comment_line_multi_ifs(phandle, pstate); } if (!skipped_anything) break; @@ -610,24 +515,8 @@ static lrec_t* lrec_parse_mmap_xtab_multi_ifs_multi_ips(file_reader_mmap_state_t // Skip comment lines if (comment_string != NULL) { - if (streqn(phandle->sol, comment_string, comment_string_length)) { - if (pstate->comment_handling == PASS_COMMENTS) - for (int i = 0; i < pstate->comment_string_length; i++) - fputc(phandle->sol[i], stdout); - phandle->sol += comment_string_length; - while (phandle->sol < phandle->eof && !streqn(phandle->sol, comment_string, comment_string_length)) { - if (pstate->comment_handling == PASS_COMMENTS) - fputc(*phandle->sol, stdout); - phandle->sol++; - } - if (phandle->sol < phandle->eof && streqn(phandle->sol, ifs, ifslen)) { - if (pstate->comment_handling == PASS_COMMENTS) - for (int i = 0; i < ifslen; i++) - fputc(phandle->sol[i], stdout); - phandle->sol += ifslen; - } + if (skip_comment_line_multi_ifs(phandle, pstate)) continue; - } } char* line = phandle->sol; @@ -683,3 +572,65 @@ static lrec_t* lrec_parse_mmap_xtab_multi_ifs_multi_ips(file_reader_mmap_state_t return prec; } } + +// ---------------------------------------------------------------- +static int skip_comment_line_single_ifs( + file_reader_mmap_state_t* phandle, + lrec_reader_mmap_xtab_state_t* pstate, + char ifs) +{ + char* cs = pstate->comment_string; + int csl = pstate->comment_string_length; + comment_handling_t ch = pstate->comment_handling; + + if (phandle->eof - phandle->sol >= csl && streqn(phandle->sol, cs, csl)) { + if (ch == PASS_COMMENTS) + for (int i = 0; i < csl; i++) + fputc(phandle->sol[i], stdout); + phandle->sol += csl; + while (phandle->sol < phandle->eof && *phandle->sol != ifs) { + if (ch == PASS_COMMENTS) + fputc(*phandle->sol, stdout); + phandle->sol++; + } + if (phandle->sol < phandle->eof && *phandle->sol == ifs) { + if (ch == PASS_COMMENTS) + fputc(*phandle->sol, stdout); + phandle->sol++; + } + return TRUE; + } + return FALSE; +} + +// ---------------------------------------------------------------- +static int skip_comment_line_multi_ifs( + file_reader_mmap_state_t* phandle, + lrec_reader_mmap_xtab_state_t* pstate) +{ + char* ifs = pstate->ifs; + int ifslen = pstate->ifslen; + char* cs = pstate->comment_string; + int csl = pstate->comment_string_length; + comment_handling_t ch = pstate->comment_handling; + + if (phandle->eof - phandle->sol >= csl && streqn(phandle->sol, cs, csl)) { + if (ch == PASS_COMMENTS) + for (int i = 0; i < csl; i++) + fputc(phandle->sol[i], stdout); + phandle->sol += csl; + while (phandle->sol - phandle->eof >= ifslen && !streqn(phandle->sol, ifs, ifslen)) { + if (ch == PASS_COMMENTS) + fputc(*phandle->sol, stdout); + phandle->sol++; + } + if (phandle->sol - phandle->eof >= ifslen && streqn(phandle->sol, ifs, ifslen)) { + if (ch == PASS_COMMENTS) + for (int i = 0; i < ifslen; i++) + fputc(phandle->sol[i], stdout); + phandle->sol += ifslen; + } + return TRUE; + } + return FALSE; +} diff --git a/c/oo b/c/oo index fa0c5005c..33b840b95 100755 --- a/c/oo +++ b/c/oo @@ -37,26 +37,26 @@ flags="-a" vee= # ---------------------------------------------------------------- -announce CSV +announce XTAB -mention input comments1.csv -cat reg_test/input/comments/comments1.csv +mention input comments1.xtab +cat reg_test/input/comments/comments1.xtab -mention skip comments1.csv -run_mlr --skip-comments --icsv --odkvp cat < reg_test/input/comments/comments1.csv -run_mlr --skip-comments --icsv --odkvp cat reg_test/input/comments/comments1.csv +mention skip comments1.xtab +run_mlr --skip-comments --ixtab --odkvp cat < reg_test/input/comments/comments1.xtab +run_mlr --skip-comments --ixtab --odkvp cat reg_test/input/comments/comments1.xtab -mention pass comments1.csv -run_mlr --pass-comments --icsv --odkvp cat < reg_test/input/comments/comments1.csv -run_mlr --pass-comments --icsv --odkvp cat reg_test/input/comments/comments1.csv +mention pass comments1.xtab +run_mlr --pass-comments --ixtab --odkvp cat < reg_test/input/comments/comments1.xtab +run_mlr --pass-comments --ixtab --odkvp cat reg_test/input/comments/comments1.xtab -mention input comments2.csv -cat reg_test/input/comments/comments2.csv +mention input comments2.xtab +cat reg_test/input/comments/comments2.xtab -mention skip comments2.csv -run_mlr --skip-comments --icsv --odkvp cat < reg_test/input/comments/comments2.csv -run_mlr --skip-comments --icsv --odkvp cat reg_test/input/comments/comments2.csv +mention skip comments2.xtab +run_mlr --skip-comments --ixtab --odkvp cat < reg_test/input/comments/comments2.xtab +run_mlr --skip-comments --ixtab --odkvp cat reg_test/input/comments/comments2.xtab -mention pass comments2.csv -run_mlr --pass-comments --icsv --odkvp cat < reg_test/input/comments/comments2.csv -run_mlr --pass-comments --icsv --odkvp cat reg_test/input/comments/comments2.csv +mention pass comments2.xtab +run_mlr --pass-comments --ixtab --odkvp cat < reg_test/input/comments/comments2.xtab +run_mlr --pass-comments --ixtab --odkvp cat reg_test/input/comments/comments2.xtab diff --git a/c/todo.txt b/c/todo.txt index 7311837d1..0215cfa24 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -22,6 +22,7 @@ BUGFIXES o xxxes o streqn-past-eof scenarios @ page boundaries ... o UT for skip/pass + o UT for CRLFs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ! single/multi irs/ifs test cases