diff --git a/.gitignore b/.gitignore index 83f4559f8..337e7187f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ mlr-unit-tester test-mlrutil test-string-builder test-byte-readers +test-peek-file-reader test-parse-trie test-lrec test-join-bucket-keeper diff --git a/c/Makefile.am b/c/Makefile.am index d4fa8e542..e33f53174 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -21,7 +21,6 @@ lib/mlrutil.c \ lib/mlr_globals.c \ lib/string_builder.c \ input/file_reader_mmap.c \ -input/old_peek_file_reader.c \ experimental/getlines.c # ================================================================ @@ -63,9 +62,10 @@ mlrp: .always dsls tests: unit-test reg-test -unit-test: test-mlrutil test-byte-readers test-parse-trie test-lrec test-string-builder test-join-bucket-keeper +unit-test: test-mlrutil test-byte-readers test-peek-file-reader test-parse-trie test-lrec test-string-builder test-join-bucket-keeper ./test-mlrutil ./test-byte-readers + ./test-peek-file-reader ./test-parse-trie ./test-lrec ./test-string-builder @@ -83,10 +83,11 @@ dev-tests: dev-unit-test reg-test # Unfortunately --error-exitcode=1 doesn't work well since there are # valgrind-detected errors in stdlibs. :( -dev-unit-test: test-mlrutil test-byte-readers test-parse-trie test-lrec test-string-builder test-join-bucket-keeper +dev-unit-test: test-mlrutil test-byte-readers test-peek-file-reader test-parse-trie test-lrec test-string-builder test-join-bucket-keeper # valgrind --leak-check=full --error-exitcode=1 ./a.out valgrind --leak-check=full ./test-mlrutil valgrind --leak-check=full ./test-byte-readers + valgrind --leak-check=full ./test-Peek-file-reader valgrind --leak-check=full ./test-parse-trie valgrind --leak-check=full ./test-lrec valgrind --leak-check=full ./test-string-builder diff --git a/c/cli/mlrcli.c b/c/cli/mlrcli.c index 8a2976334..21716a863 100644 --- a/c/cli/mlrcli.c +++ b/c/cli/mlrcli.c @@ -289,24 +289,14 @@ cli_opts_t* parse_command_line(int argc, char** argv) { argi++; } -// ready for cutover! :) -// just leaving it off for a while to compare performance. -#if 0 - else if (streq(argv[argi], "--csv")) { popts->ifmt = "csvex"; ofmt = "csv"; } - else if (streq(argv[argi], "--icsv")) { popts->ifmt = "csvex"; } - else if (streq(argv[argi], "--ocsv")) { ofmt = "csv"; } -#else else if (streq(argv[argi], "--csv")) { popts->ifmt = ofmt = "csv"; } else if (streq(argv[argi], "--icsv")) { popts->ifmt = "csv"; } else if (streq(argv[argi], "--ocsv")) { ofmt = "csv"; } -#endif + else if (streq(argv[argi], "--csvlite")) { popts->ifmt = ofmt = "csvlite"; } else if (streq(argv[argi], "--icsvlite")) { popts->ifmt = "csvlite"; } else if (streq(argv[argi], "--ocsvlite")) { ofmt = "csvlite"; } - else if (streq(argv[argi], "--csvex")) { popts->ifmt = "csvex"; ofmt = "csv"; } - else if (streq(argv[argi], "--icsvex")) { popts->ifmt = "csvex"; } - else if (streq(argv[argi], "--dkvp")) { popts->ifmt = ofmt = "dkvp"; } else if (streq(argv[argi], "--idkvp")) { popts->ifmt = "dkvp"; } else if (streq(argv[argi], "--odkvp")) { ofmt = "dkvp"; } diff --git a/c/containers/parse_trie.c b/c/containers/parse_trie.c index b2032a45b..70cafaf6c 100644 --- a/c/containers/parse_trie.c +++ b/c/containers/parse_trie.c @@ -80,39 +80,3 @@ static void parse_trie_add_string_aux(parse_trie_node_t* pnode, char* string, in parse_trie_add_string_aux(pnext, &string[1], stridx, len); } } - -// ---------------------------------------------------------------- -// Example input: -// * string 0 is "a" -// * string 1 is "aa" -// * buf is "aaabc" -// Output: -// * return value is TRUE -// * stridx is 1 since longest match is "aa" -// * matchlen is 2 since "aa" has length 2 - -// xxx cmt longest-prefix match - -int parse_trie_match(parse_trie_t* ptrie, char* buf, int buflen, int* pstridx, int* pmatchlen) { - parse_trie_node_t* pnode = ptrie->proot; - parse_trie_node_t* pnext; - parse_trie_node_t* pterm = NULL; - int i; - for (i = 0; i < buflen; i++) { - char c = buf[i]; - pnext = pnode->pnexts[(unsigned char) c]; - if (pnext == NULL) - break; - if (pnext->strlen > 0) { - pterm = pnext; - } - pnode = pnext; - } - if (pterm == NULL) { - return FALSE; - } else { - *pstridx = pterm->stridx; - *pmatchlen = pterm->strlen; - return TRUE; - } -} diff --git a/c/containers/parse_trie.h b/c/containers/parse_trie.h index cd7eba67d..e7fbf8958 100644 --- a/c/containers/parse_trie.h +++ b/c/containers/parse_trie.h @@ -21,6 +21,47 @@ parse_trie_t* parse_trie_alloc(); void parse_trie_free(parse_trie_t* ptrie); void parse_trie_print(parse_trie_t* ptrie); void parse_trie_add_string(parse_trie_t* ptrie, char* string, int stridx); -int parse_trie_match(parse_trie_t* ptrie, char* buf, int buflen, int* pstridx, int* pmatchlen); + +// ---------------------------------------------------------------- +// Example input: +// * string 0 is "a" +// * string 1 is "aa" +// * buf is "aaabc" +// Output: +// * return value is TRUE +// * stridx is 1 since longest match is "aa" +// * matchlen is 2 since "aa" has length 2 + +// xxx cmt longest-prefix match + +// xxx cmt assumption that enough has been peeked -- there is no check here. +// (recall that this is called on every single byte of each data file, e.g. +// for a 1GB data file we'll be in this function a billion times ... it needs +// to be tight.) + +// xxx cmt re sob/mask semantics (ring buffer) + +static inline int parse_trie_match(parse_trie_t* ptrie, char* buf, int sob, int buflen, int mask, int* pstridx, int* pmatchlen) { + parse_trie_node_t* pnode = ptrie->proot; + parse_trie_node_t* pnext; + parse_trie_node_t* pterm = NULL; + for (int i = 0; i < buflen; i++) { + char c = buf[(sob+i)&mask]; + pnext = pnode->pnexts[(unsigned char) c]; + if (pnext == NULL) + break; + if (pnext->strlen > 0) { + pterm = pnext; + } + pnode = pnext; + } + if (pterm == NULL) { + return FALSE; + } else { + *pstridx = pterm->stridx; + *pmatchlen = pterm->strlen; + return TRUE; + } +} #endif // PARSE_TRIE_H diff --git a/c/containers/test_parse_trie.c b/c/containers/test_parse_trie.c index d5a447c89..ac3dec759 100644 --- a/c/containers/test_parse_trie.c +++ b/c/containers/test_parse_trie.c @@ -36,7 +36,7 @@ static void test_case( stridx = -2; matchlen = -2; - rc = parse_trie_match(ptrie, buf, strlen(buf), &stridx, &matchlen); + rc = parse_trie_match(ptrie, buf, 0, strlen(buf), 0xff, &stridx, &matchlen); parse_trie_free(ptrie); @@ -135,61 +135,61 @@ static char* test_dkvp() { } parse_trie_print(ptrie); - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == TRUE); mu_assert_lf(stridx == PS_TOKEN); mu_assert_lf(matchlen == strlen(strings[PS_TOKEN])); p += matchlen; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == TRUE); mu_assert_lf(stridx == FS_TOKEN); mu_assert_lf(matchlen == strlen(strings[FS_TOKEN])); p += matchlen; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == TRUE); mu_assert_lf(stridx == PS_TOKEN); mu_assert_lf(matchlen == strlen(strings[PS_TOKEN])); p += matchlen; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == TRUE); mu_assert_lf(stridx == RS_TOKEN); mu_assert_lf(matchlen == strlen(strings[RS_TOKEN])); p += matchlen; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == TRUE); mu_assert_lf(stridx == PS_TOKEN); mu_assert_lf(matchlen == strlen(strings[PS_TOKEN])); p += matchlen; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == TRUE); mu_assert_lf(stridx == EOF_TOKEN); mu_assert_lf(matchlen == strlen(strings[EOF_TOKEN])); @@ -221,7 +221,7 @@ static char* show_it() { parse_trie_print(ptrie); while (TRUE) { - rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); + rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); if (rc) { printf("match token %d (%s)\n", stridx, strings[stridx]); p += matchlen; diff --git a/c/experimental/csv0.c b/c/experimental/csv0.c deleted file mode 100644 index f656a0a3e..000000000 --- a/c/experimental/csv0.c +++ /dev/null @@ -1,140 +0,0 @@ -#include -#include -#include -#include "lib/mlrutil.h" -#include "containers/slls.h" -#include "lib/string_builder.h" -#include "input/old_peek_file_reader.h" - -#define TERMIND_RS 0x1111 -#define TERMIND_FS 0x2222 -#define TERMIND_EOF 0x3333 - -typedef struct _field_wrapper_t { - char* contents; - int termind; -} field_wrapper_t; - -typedef struct _record_wrapper_t { - slls_t* contents; - int at_eof; -} record_wrapper_t; - -static field_wrapper_t get_csv_field_not_dquoted(old_peek_file_reader_t* pfr, string_builder_t* psb) { - // Note that "\"," etc. will be encoded in the rfc_csv_reader_t ctor -- this is just sketch - printf("\n"); - printf("ENTER\n"); - while (TRUE) { - if (old_pfr_at_eof(pfr)) { - printf("--case 1\n"); - printf("EXIT\n"); - return (field_wrapper_t) { .contents = sb_is_empty(psb) ? NULL: sb_finish(psb), .termind = TERMIND_EOF }; - } else if (old_pfr_next_is(pfr, ",\xff", 2)) { - printf("--case 2\n"); - old_pfr_advance_by(pfr, 2); - printf("EXIT\n"); - return (field_wrapper_t) { .contents = sb_finish(psb), .termind = TERMIND_EOF }; - } else if (old_pfr_next_is(pfr, ",", 1)) { - printf("--case 3\n"); - old_pfr_advance_by(pfr, 1); - printf("EXIT\n"); - return (field_wrapper_t) { .contents = sb_finish(psb), .termind = TERMIND_FS }; - } else if (old_pfr_next_is(pfr, "\r\n", 2)) { - printf("--case 4\n"); - old_pfr_advance_by(pfr, 2); - printf("EXIT\n"); - return (field_wrapper_t) { .contents = sb_finish(psb), .termind = TERMIND_RS }; - } else { - //old_pfr_dump(pfr); - char c = old_pfr_read_char(pfr); - printf("--case 5 %c [%02x]\n", isprint(c) ? c : '?', c); - //old_pfr_dump(pfr); - sb_append_char(psb, c); - //sb_append_char(psb, old_pfr_read_char(pfr)); - } - } -} - -static field_wrapper_t get_csv_field_dquoted(old_peek_file_reader_t* pfr, string_builder_t* psb) { - old_pfr_advance_by(pfr, 1); - while (TRUE) { - if (old_pfr_at_eof(pfr)) { - // xxx imbalanced-dquote error - fprintf(stderr, "xxx k0d3 me up b04k3n b04k3n b04ken %d\n", __LINE__); - exit(1); - } else if (old_pfr_next_is(pfr, "\"\xff", 2)) { - old_pfr_advance_by(pfr, 2); - return (field_wrapper_t) { .contents = sb_finish(psb), .termind = TERMIND_EOF }; - } else if (old_pfr_next_is(pfr, "\",", 2)) { - old_pfr_advance_by(pfr, 2); - return (field_wrapper_t) { .contents = sb_finish(psb), .termind = TERMIND_FS }; - } else if (old_pfr_next_is(pfr, "\"\r\n", 3)) { - old_pfr_advance_by(pfr, 3); - return (field_wrapper_t) { .contents = sb_finish(psb), .termind = TERMIND_RS }; - } else { - sb_append_char(psb, old_pfr_read_char(pfr)); - } - } -} - -field_wrapper_t get_csv_field(old_peek_file_reader_t* pfr, string_builder_t* psb) { - field_wrapper_t wrapper; - if (old_pfr_at_eof(pfr)) { - wrapper.contents = NULL; - wrapper.termind = TERMIND_EOF; - return wrapper; - } else if (old_pfr_next_is(pfr, "\"", 1)) { - return get_csv_field_dquoted(pfr, psb); - } else { - return get_csv_field_not_dquoted(pfr, psb); - } -} - -record_wrapper_t get_csv_record(old_peek_file_reader_t* pfr, string_builder_t* psb) { - slls_t* fields = slls_alloc(); - record_wrapper_t rwrapper; - rwrapper.contents = fields; - rwrapper.at_eof = FALSE; - while (TRUE) { - field_wrapper_t fwrapper = get_csv_field(pfr, psb); - if (fwrapper.termind == TERMIND_EOF) { - rwrapper.at_eof = TRUE; - } - if (fwrapper.contents != NULL) { - printf("CONT=>>%s<<[%d]\n", fwrapper.contents, (int)strlen(fwrapper.contents)); - slls_add_with_free(fields, fwrapper.contents); - } - if (fwrapper.termind != TERMIND_FS) - break; - } - printf("FLEN=%d\n", fields->length); - printf("FEOF=%d\n", rwrapper.at_eof); - if (fields->length == 0 && rwrapper.at_eof) { - slls_free(fields); - rwrapper.contents = NULL; - } - return rwrapper; -} - -int main() { - FILE* fp = stdin; - old_peek_file_reader_t* pfr = old_pfr_alloc(fp, 32); - string_builder_t sb; - string_builder_t* psb = &sb; - sb_init(psb, 1024); - - while (TRUE) { - record_wrapper_t rwrapper = get_csv_record(pfr, psb); - if (rwrapper.contents != NULL) { - printf("++++ [NF=%d]\n", rwrapper.contents->length); - for (sllse_t* pe = rwrapper.contents->phead; pe != NULL; pe = pe->pnext) { - printf(" [%s]\n", pe->value); - } - slls_free(rwrapper.contents); - } - if (rwrapper.at_eof) - break; - } - - return 0; -} diff --git a/c/experimental/pfr.c b/c/experimental/pfr.c deleted file mode 100644 index 67453648a..000000000 --- a/c/experimental/pfr.c +++ /dev/null @@ -1,25 +0,0 @@ -#include -#include -#include "input/old_peek_file_reader.h" - -int main(int argc, char** argv) { - FILE* fp = stdin; - old_peek_file_reader_t* pfr = old_pfr_alloc(fp, 32); - - printf("@eof = %d\n", old_pfr_at_eof(pfr)); - old_pfr_dump(pfr); - printf("read 0x%02x\n", (unsigned)old_pfr_read_char(pfr)); - old_pfr_dump(pfr); - char* s = "//"; - old_pfr_dump(pfr); - printf("next is %s = %d\n", s, old_pfr_next_is(pfr, s, strlen(s))); - old_pfr_dump(pfr); - char c = old_pfr_read_char(pfr); - printf("read %c [0x%02x]\n", c, (unsigned)c); - old_pfr_dump(pfr); - - printf("@eof = %d\n", old_pfr_at_eof(pfr)); - - old_pfr_free(pfr); - return 0; -} diff --git a/c/input/lrec_reader_csvex.c b/c/input/lrec_reader_csv.c similarity index 83% rename from c/input/lrec_reader_csvex.c rename to c/input/lrec_reader_csv.c index ff64d966c..db558f2e6 100644 --- a/c/input/lrec_reader_csvex.c +++ b/c/input/lrec_reader_csv.c @@ -1,5 +1,6 @@ #include #include +#include #include "lib/mlr_globals.h" #include "lib/mlrutil.h" #include "lib/string_builder.h" @@ -11,14 +12,6 @@ #include "containers/lhmslv.h" #include "containers/parse_trie.h" -// ================================================================ -// xxx to do: -// * avoid the separate paster: just inline lrec_put as in the csvlite reader. -// * profile .............. -// * ring buffer in pfr? -// -> split out a separate pfr ut per se -// ================================================================ - // Idea of pheader_keepers: each header_keeper object retains the input-line backing // and the slls_t for a CSV header line which is used by one or more CSV data // lines. Meanwhile some mappers retain input records from the entire data @@ -44,7 +37,7 @@ #define DQUOTE_DQUOTE_STRIDX 0x2008 // ---------------------------------------------------------------- -typedef struct _lrec_reader_csvex_state_t { +typedef struct _lrec_reader_csv_state_t { // Input line number is not the same as the record-counter in context_t, // which counts records. long long ilno; @@ -83,19 +76,19 @@ typedef struct _lrec_reader_csvex_state_t { header_keeper_t* pheader_keeper; lhmslv_t* pheader_keepers; -} lrec_reader_csvex_state_t; +} lrec_reader_csv_state_t; -static slls_t* lrec_reader_csvex_get_fields(lrec_reader_csvex_state_t* pstate); -static lrec_t* paste_header_and_data(lrec_reader_csvex_state_t* pstate, slls_t* pdata_fields); +static slls_t* lrec_reader_csv_get_fields(lrec_reader_csv_state_t* pstate); +static lrec_t* paste_header_and_data(lrec_reader_csv_state_t* pstate, slls_t* pdata_fields); // ---------------------------------------------------------------- // xxx needs abend on null lhs. etc. -static lrec_t* lrec_reader_csvex_process(void* pvstate, void* pvhandle, context_t* pctx) { - lrec_reader_csvex_state_t* pstate = pvstate; +static lrec_t* lrec_reader_csv_process(void* pvstate, void* pvhandle, context_t* pctx) { + lrec_reader_csv_state_t* pstate = pvstate; if (pstate->expect_header_line_next) { - slls_t* pheader_fields = lrec_reader_csvex_get_fields(pstate); + slls_t* pheader_fields = lrec_reader_csv_get_fields(pstate); if (pheader_fields == NULL) return NULL; pstate->ilno++; @@ -111,14 +104,14 @@ static lrec_t* lrec_reader_csvex_process(void* pvstate, void* pvhandle, context_ } pstate->ilno++; - slls_t* pdata_fields = lrec_reader_csvex_get_fields(pstate); + slls_t* pdata_fields = lrec_reader_csv_get_fields(pstate); if (pdata_fields == NULL) // EOF return NULL; else return paste_header_and_data(pstate, pdata_fields); } -static slls_t* lrec_reader_csvex_get_fields(lrec_reader_csvex_state_t* pstate) { +static slls_t* lrec_reader_csv_get_fields(lrec_reader_csv_state_t* pstate) { int rc, stridx, matchlen, record_done, field_done; peek_file_reader_t* pfr = pstate->pfr; string_builder_t* psb = pstate->psb; @@ -137,8 +130,17 @@ static slls_t* lrec_reader_csvex_get_fields(lrec_reader_csvex_state_t* pstate) { field_done = FALSE; while (!field_done) { pfr_buffer_by(pfr, pstate->pno_dquote_parse_trie->maxlen); - rc = parse_trie_match(pstate->pno_dquote_parse_trie, pfr->peekbuf, pfr->npeeked, &stridx, &matchlen); + + rc = parse_trie_match(pstate->pno_dquote_parse_trie, + pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask, + &stridx, &matchlen); +#ifdef DEBUG_PARSER + pfr_dump(pfr); +#endif if (rc) { +#ifdef DEBUG_PARSER + printf("RC=%d stridx=0x%04x matchlen=%d\n", rc, stridx, matchlen); +#endif switch(stridx) { case EOF_STRIDX: // end of record slls_add_with_free(pfields, sb_finish(psb)); @@ -172,7 +174,13 @@ static slls_t* lrec_reader_csvex_get_fields(lrec_reader_csvex_state_t* pstate) { } pfr_advance_by(pfr, matchlen); } else { +#ifdef DEBUG_PARSER + char c = pfr_read_char(pfr); + printf("CHAR=%c [%02x]\n", isprint(c) ? c : ' ', (unsigned)c); + sb_append_char(psb, c); +#else sb_append_char(psb, pfr_read_char(pfr)); +#endif } } @@ -183,7 +191,11 @@ static slls_t* lrec_reader_csvex_get_fields(lrec_reader_csvex_state_t* pstate) { field_done = FALSE; while (!field_done) { pfr_buffer_by(pfr, pstate->pdquote_parse_trie->maxlen); - rc = parse_trie_match(pstate->pdquote_parse_trie, pfr->peekbuf, pfr->npeeked, &stridx, &matchlen); + + rc = parse_trie_match(pstate->pdquote_parse_trie, + pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask, + &stridx, &matchlen); + if (rc) { switch(stridx) { case EOF_STRIDX: // end of record @@ -227,7 +239,7 @@ static slls_t* lrec_reader_csvex_get_fields(lrec_reader_csvex_state_t* pstate) { } // ---------------------------------------------------------------- -static lrec_t* paste_header_and_data(lrec_reader_csvex_state_t* pstate, slls_t* pdata_fields) { +static lrec_t* paste_header_and_data(lrec_reader_csv_state_t* pstate, slls_t* pdata_fields) { if (pstate->pheader_keeper->pkeys->length != pdata_fields->length) { fprintf(stderr, "%s: Header/data length mismatch: %d != %d at line %lld.\n", MLR_GLOBALS.argv0, pstate->pheader_keeper->pkeys->length, pdata_fields->length, pstate->ilno); @@ -238,34 +250,35 @@ static lrec_t* paste_header_and_data(lrec_reader_csvex_state_t* pstate, slls_t* sllse_t* pd = pdata_fields->phead; for ( ; ph != NULL && pd != NULL; ph = ph->pnext, pd = pd->pnext) { lrec_put_no_free(prec, ph->value, pd->value); + //lrec_put(prec, ph->value, pd->value, LREC_FREE_ENTRY_VALUE); } return prec; } // ---------------------------------------------------------------- -void* lrec_reader_csvex_open(void* pvstate, char* filename) { - lrec_reader_csvex_state_t* pstate = pvstate; +void* lrec_reader_csv_open(void* pvstate, char* filename) { + lrec_reader_csv_state_t* pstate = pvstate; pstate->pfr->pbr->popen_func(pstate->pfr->pbr, filename); pfr_reset(pstate->pfr); return NULL; // xxx modify the API after the functional refactor is complete } -void lrec_reader_csvex_close(void* pvstate, void* pvhandle) { - lrec_reader_csvex_state_t* pstate = pvstate; +void lrec_reader_csv_close(void* pvstate, void* pvhandle) { + lrec_reader_csv_state_t* pstate = pvstate; pstate->pfr->pbr->pclose_func(pstate->pfr->pbr); } // ---------------------------------------------------------------- // xxx after the pfr/pbr refactor is complete, vsof and vopen may be redundant. -static void lrec_reader_csvex_sof(void* pvstate) { - lrec_reader_csvex_state_t* pstate = pvstate; +static void lrec_reader_csv_sof(void* pvstate) { + lrec_reader_csv_state_t* pstate = pvstate; pstate->ilno = 0LL; pstate->expect_header_line_next = TRUE; } // ---------------------------------------------------------------- -static void lrec_reader_csvex_free(void* pvstate) { - lrec_reader_csvex_state_t* pstate = pvstate; +static void lrec_reader_csv_free(void* pvstate) { + lrec_reader_csv_state_t* pstate = pvstate; for (lhmslve_t* pe = pstate->pheader_keepers->phead; pe != NULL; pe = pe->pnext) { header_keeper_t* pheader_keeper = pe->pvvalue; header_keeper_free(pheader_keeper); @@ -274,10 +287,10 @@ static void lrec_reader_csvex_free(void* pvstate) { } // ---------------------------------------------------------------- -lrec_reader_t* lrec_reader_csvex_alloc(byte_reader_t* pbr, char irs, char ifs) { +lrec_reader_t* lrec_reader_csv_alloc(byte_reader_t* pbr, char irs, char ifs) { lrec_reader_t* plrec_reader = mlr_malloc_or_die(sizeof(lrec_reader_t)); - lrec_reader_csvex_state_t* pstate = mlr_malloc_or_die(sizeof(lrec_reader_csvex_state_t)); + lrec_reader_csv_state_t* pstate = mlr_malloc_or_die(sizeof(lrec_reader_csv_state_t)); pstate->ilno = 0LL; pstate->eof = "\xff"; @@ -337,11 +350,11 @@ lrec_reader_t* lrec_reader_csvex_alloc(byte_reader_t* pbr, char irs, char ifs) { pstate->pheader_keepers = lhmslv_alloc(); plrec_reader->pvstate = (void*)pstate; - plrec_reader->popen_func = &lrec_reader_csvex_open; - plrec_reader->pclose_func = &lrec_reader_csvex_close; - plrec_reader->pprocess_func = &lrec_reader_csvex_process; - plrec_reader->psof_func = &lrec_reader_csvex_sof; - plrec_reader->pfree_func = &lrec_reader_csvex_free; + plrec_reader->popen_func = &lrec_reader_csv_open; + plrec_reader->pclose_func = &lrec_reader_csv_close; + plrec_reader->pprocess_func = &lrec_reader_csv_process; + plrec_reader->psof_func = &lrec_reader_csv_sof; + plrec_reader->pfree_func = &lrec_reader_csv_free; return plrec_reader; } diff --git a/c/input/lrec_reader_stdio_csv.c b/c/input/lrec_reader_stdio_csv.c deleted file mode 100644 index 456b3fc0b..000000000 --- a/c/input/lrec_reader_stdio_csv.c +++ /dev/null @@ -1,288 +0,0 @@ -#include -#include -#include "lib/mlr_globals.h" -#include "lib/mlrutil.h" -#include "containers/slls.h" -#include "containers/lhmslv.h" -#include "input/file_reader_stdio.h" -#include "input/lrec_readers.h" -#include "lib/string_builder.h" -#include "input/old_peek_file_reader.h" - -// Idea of pheader_keepers: each header_keeper object retains the input-line backing -// and the slls_t for a CSV header line which is used by one or more CSV data -// lines. Meanwhile some mappers retain input records from the entire data -// stream, including header-schema changes in the input stream. This means we -// need to keep headers intact as long as any lrecs are pointing to them. One -// option is reference-counting which I experimented with; it was messy and -// error-prone. The approach used here is to keep a hash map from header-schema -// to header_keeper object. The current pheader_keeper is a pointer into one of -// those. Then when the reader is freed, all the header-keepers are freed. - -// ---------------------------------------------------------------- -#define STRING_BUILDER_INIT_SIZE 1024 -#define TERMIND_RS 0x1111 -#define TERMIND_FS 0x2222 -#define TERMIND_EOF 0x3333 - -typedef struct _field_wrapper_t { - char* contents; - int termind; -} field_wrapper_t; - -typedef struct _record_wrapper_t { - slls_t* contents; - int at_eof; -} record_wrapper_t; - -// ---------------------------------------------------------------- -typedef struct _lrec_reader_stdio_csv_state_t { - // Input line number is not the same as the record-counter in context_t, - // which counts records. - long long ilno; - - char* irs; - char* ifs; - char* dquote_irs; - char* dquote_ifs; - char* dquote_eof; - char* dquote; - char* dquote_dquote; - char* ifs_eof; - - int irs_len; - int ifs_len; - int dquote_irs_len; - int dquote_ifs_len; - int dquote_eof_len; - int dquote_len; - int dquote_dquote_len; - int ifs_eof_len; - - int peek_buf_len; - - string_builder_t sb; - string_builder_t* psb; - old_peek_file_reader_t* pfr; - - int expect_header_line_next; - header_keeper_t* pheader_keeper; - lhmslv_t* pheader_keepers; - -} lrec_reader_stdio_csv_state_t; - -// ---------------------------------------------------------------- -static record_wrapper_t lrec_reader_stdio_csv_get_record(lrec_reader_stdio_csv_state_t* pstate); - -static field_wrapper_t get_csv_field(lrec_reader_stdio_csv_state_t* pstate); -static field_wrapper_t get_csv_field_not_dquoted(lrec_reader_stdio_csv_state_t* pstate); -static field_wrapper_t get_csv_field_dquoted(lrec_reader_stdio_csv_state_t* pstate); -static lrec_t* paste_header_and_data(lrec_reader_stdio_csv_state_t* pstate, slls_t* pdata_fields); - -// ---------------------------------------------------------------- -// xxx needs abend on null lhs. etc. - -static lrec_t* lrec_reader_stdio_csv_process(void* pvstate, void* pvhandle, context_t* pctx) { - lrec_reader_stdio_csv_state_t* pstate = pvstate; - if (pstate->pfr == NULL) { - pstate->pfr = old_pfr_alloc((FILE*)pvhandle, pstate->peek_buf_len); - } - - record_wrapper_t rwrapper; - - if (pstate->expect_header_line_next) { - rwrapper = lrec_reader_stdio_csv_get_record(pstate); - - if (rwrapper.contents == NULL && rwrapper.at_eof) - return NULL; - pstate->ilno++; - - pstate->expect_header_line_next = FALSE; - - pstate->pheader_keeper = lhmslv_get(pstate->pheader_keepers, rwrapper.contents); - if (pstate->pheader_keeper == NULL) { - pstate->pheader_keeper = header_keeper_alloc(NULL, rwrapper.contents); - lhmslv_put(pstate->pheader_keepers, rwrapper.contents, pstate->pheader_keeper); - } else { // Re-use the header-keeper in the header cache - slls_free(rwrapper.contents); - } - } - - rwrapper = lrec_reader_stdio_csv_get_record(pstate); - if (rwrapper.contents == NULL && rwrapper.at_eof) - return NULL; - - pstate->ilno++; - return paste_header_and_data(pstate, rwrapper.contents); -} - -static record_wrapper_t lrec_reader_stdio_csv_get_record(lrec_reader_stdio_csv_state_t* pstate) { - slls_t* pfields = slls_alloc(); - record_wrapper_t rwrapper; - rwrapper.contents = pfields; - rwrapper.at_eof = FALSE; - while (TRUE) { - field_wrapper_t fwrapper = get_csv_field(pstate); - if (fwrapper.termind == TERMIND_EOF) - rwrapper.at_eof = TRUE; - if (fwrapper.contents != NULL) - slls_add_with_free(pfields, fwrapper.contents); - if (fwrapper.termind != TERMIND_FS) - break; - } - if (pfields->length == 0 && rwrapper.at_eof) { - slls_free(pfields); - rwrapper.contents = NULL; - } - return rwrapper; -} - -static field_wrapper_t get_csv_field(lrec_reader_stdio_csv_state_t* pstate) { - field_wrapper_t wrapper; - if (old_pfr_at_eof(pstate->pfr)) { - wrapper.contents = NULL; - wrapper.termind = TERMIND_EOF; - return wrapper; - } else if (old_pfr_next_is(pstate->pfr, pstate->dquote, pstate->dquote_len)) { - old_pfr_advance_by(pstate->pfr, pstate->dquote_len); - return get_csv_field_dquoted(pstate); - } else { - return get_csv_field_not_dquoted(pstate); - } -} - -static field_wrapper_t get_csv_field_not_dquoted(lrec_reader_stdio_csv_state_t* pstate) { - while (TRUE) { - if (old_pfr_at_eof(pstate->pfr)) { - return (field_wrapper_t) { - .contents = sb_is_empty(pstate->psb) ? NULL: sb_finish(pstate->psb), - .termind = TERMIND_EOF - }; - } else if (old_pfr_next_is(pstate->pfr, pstate->ifs_eof, pstate->ifs_eof_len)) { - old_pfr_advance_by(pstate->pfr, pstate->ifs_eof_len); - return (field_wrapper_t) { .contents = sb_finish(pstate->psb), .termind = TERMIND_EOF }; - } else if (old_pfr_next_is(pstate->pfr, pstate->ifs, pstate->ifs_len)) { - old_pfr_advance_by(pstate->pfr, pstate->ifs_len); - return (field_wrapper_t) { .contents = sb_finish(pstate->psb), .termind = TERMIND_FS }; - } else if (old_pfr_next_is(pstate->pfr, pstate->irs, pstate->irs_len)) { - old_pfr_advance_by(pstate->pfr, pstate->irs_len); - return (field_wrapper_t) { .contents = sb_finish(pstate->psb), .termind = TERMIND_RS }; - } else if (old_pfr_next_is(pstate->pfr, pstate->dquote, pstate->dquote_len)) { - fprintf(stderr, "%s: non-compliant field-internal double-quote at line %lld.\n", - MLR_GLOBALS.argv0, pstate->ilno); - exit(1); - } else { - sb_append_char(pstate->psb, old_pfr_read_char(pstate->pfr)); - } - } -} - -static field_wrapper_t get_csv_field_dquoted(lrec_reader_stdio_csv_state_t* pstate) { - while (TRUE) { - if (old_pfr_at_eof(pstate->pfr)) { - fprintf(stderr, "%s: imbalanced double-quote at line %lld.\n", MLR_GLOBALS.argv0, pstate->ilno); - exit(1); - } else if (old_pfr_next_is(pstate->pfr, pstate->dquote_eof, pstate->dquote_eof_len)) { - old_pfr_advance_by(pstate->pfr, pstate->dquote_eof_len); - return (field_wrapper_t) { .contents = sb_finish(pstate->psb), .termind = TERMIND_EOF }; - } else if (old_pfr_next_is(pstate->pfr, pstate->dquote_ifs, pstate->dquote_ifs_len)) { - old_pfr_advance_by(pstate->pfr, pstate->dquote_ifs_len); - return (field_wrapper_t) { .contents = sb_finish(pstate->psb), .termind = TERMIND_FS }; - } else if (old_pfr_next_is(pstate->pfr, pstate->dquote_irs, pstate->dquote_irs_len)) { - old_pfr_advance_by(pstate->pfr, pstate->dquote_irs_len); - return (field_wrapper_t) { .contents = sb_finish(pstate->psb), .termind = TERMIND_RS }; - } else if (old_pfr_next_is(pstate->pfr, pstate->dquote_dquote, pstate->dquote_dquote_len)) { - // "" inside a dquoted field is an escape for " - old_pfr_advance_by(pstate->pfr, pstate->dquote_dquote_len); - sb_append_string(pstate->psb, pstate->dquote); - } else { - sb_append_char(pstate->psb, old_pfr_read_char(pstate->pfr)); - } - } -} - -static lrec_t* paste_header_and_data(lrec_reader_stdio_csv_state_t* pstate, slls_t* pdata_fields) { - if (pstate->pheader_keeper->pkeys->length != pdata_fields->length) { - fprintf(stderr, "%s: Header/data length mismatch: %d != %d at line %lld.\n", - MLR_GLOBALS.argv0, pstate->pheader_keeper->pkeys->length, pdata_fields->length, pstate->ilno); - exit(1); - } - lrec_t* prec = lrec_unbacked_alloc(); - sllse_t* ph = pstate->pheader_keeper->pkeys->phead; - sllse_t* pd = pdata_fields->phead; - for ( ; ph != NULL && pd != NULL; ph = ph->pnext, pd = pd->pnext) { - lrec_put_no_free(prec, ph->value, pd->value); - } - return prec; -} - -// ---------------------------------------------------------------- -static void lrec_reader_stdio_csv_sof(void* pvstate) { - lrec_reader_stdio_csv_state_t* pstate = pvstate; - pstate->ilno = 0LL; - pstate->expect_header_line_next = TRUE; - pstate->pfr = NULL; -} - -// ---------------------------------------------------------------- -static void lrec_reader_stdio_csv_free(void* pvstate) { - lrec_reader_stdio_csv_state_t* pstate = pvstate; - for (lhmslve_t* pe = pstate->pheader_keepers->phead; pe != NULL; pe = pe->pnext) { - header_keeper_t* pheader_keeper = pe->pvvalue; - header_keeper_free(pheader_keeper); - } - old_pfr_free(pstate->pfr); -} - -// ---------------------------------------------------------------- -lrec_reader_t* lrec_reader_stdio_csv_alloc(char irs, char ifs) { - lrec_reader_t* plrec_reader = mlr_malloc_or_die(sizeof(lrec_reader_t)); - - lrec_reader_stdio_csv_state_t* pstate = mlr_malloc_or_die(sizeof(lrec_reader_stdio_csv_state_t)); - pstate->ilno = 0LL; - pstate->irs = "\r\n"; // xxx multi-byte the cli irs/ifs/etc, and integrate here - pstate->ifs = ","; // xxx multi-byte the cli irs/ifs/etc, and integrate here - - pstate->dquote_irs = mlr_paste_2_strings("\"", pstate->irs); - pstate->dquote_ifs = mlr_paste_2_strings("\"", pstate->ifs); - pstate->dquote_eof = "\"\xff"; - pstate->dquote = "\""; - pstate->dquote_dquote = "\"\""; - pstate->ifs_eof = mlr_paste_2_strings(pstate->ifs, "\xff"); - - pstate->irs_len = strlen(pstate->irs); - pstate->ifs_len = strlen(pstate->ifs); - pstate->dquote_irs_len = strlen(pstate->dquote_irs); - pstate->dquote_ifs_len = strlen(pstate->dquote_ifs); - pstate->dquote_eof_len = strlen(pstate->dquote_eof); - pstate->dquote_len = strlen(pstate->dquote); - pstate->dquote_dquote_len = strlen(pstate->dquote_dquote); - pstate->ifs_eof_len = strlen(pstate->ifs_eof); - - pstate->peek_buf_len = pstate->irs_len; - pstate->peek_buf_len = mlr_imax2(pstate->peek_buf_len, pstate->ifs_len); - pstate->peek_buf_len = mlr_imax2(pstate->peek_buf_len, pstate->dquote_irs_len); - pstate->peek_buf_len = mlr_imax2(pstate->peek_buf_len, pstate->dquote_ifs_len); - pstate->peek_buf_len = mlr_imax2(pstate->peek_buf_len, pstate->dquote_eof_len); - pstate->peek_buf_len = mlr_imax2(pstate->peek_buf_len, pstate->dquote_len); - pstate->peek_buf_len = mlr_imax2(pstate->peek_buf_len, pstate->dquote_dquote_len); - pstate->peek_buf_len = mlr_imax2(pstate->peek_buf_len, pstate->ifs_eof_len); - pstate->peek_buf_len += 2; - - sb_init(&pstate->sb, STRING_BUILDER_INIT_SIZE); - pstate->psb = &pstate->sb; - pstate->pfr = NULL; - - pstate->expect_header_line_next = TRUE; - pstate->pheader_keeper = NULL; - pstate->pheader_keepers = lhmslv_alloc(); - - plrec_reader->pvstate = (void*)pstate; - plrec_reader->popen_func = &file_reader_stdio_vopen; - plrec_reader->pclose_func = &file_reader_stdio_vclose; - plrec_reader->pprocess_func = &lrec_reader_stdio_csv_process; - plrec_reader->psof_func = &lrec_reader_stdio_csv_sof; - plrec_reader->pfree_func = &lrec_reader_stdio_csv_free; - - return plrec_reader; -} diff --git a/c/input/lrec_readers.c b/c/input/lrec_readers.c index 3efffedb1..5d72f47f5 100644 --- a/c/input/lrec_readers.c +++ b/c/input/lrec_readers.c @@ -14,14 +14,12 @@ lrec_reader_t* lrec_reader_alloc(char* fmtdesc, int use_mmap, char irs, char if else return lrec_reader_stdio_dkvp_alloc(irs, ifs, ips, allow_repeat_ifs); } else if (streq(fmtdesc, "csv")) { - return lrec_reader_stdio_csv_alloc(irs, ifs); + return lrec_reader_csv_alloc(pbr, irs, ifs); } else if (streq(fmtdesc, "csvlite")) { if (use_mmap) return lrec_reader_mmap_csvlite_alloc(irs, ifs, allow_repeat_ifs); else return lrec_reader_stdio_csvlite_alloc(irs, ifs, allow_repeat_ifs); - } else if (streq(fmtdesc, "csvex")) { - return lrec_reader_csvex_alloc(pbr, irs, ifs); } else if (streq(fmtdesc, "nidx")) { if (use_mmap) return lrec_reader_mmap_nidx_alloc(irs, ifs, allow_repeat_ifs); diff --git a/c/input/lrec_readers.h b/c/input/lrec_readers.h index ea04e6318..c793492aa 100644 --- a/c/input/lrec_readers.h +++ b/c/input/lrec_readers.h @@ -10,15 +10,14 @@ lrec_reader_t* lrec_reader_alloc(char* fmtdesc, int use_mmap, char irs, char ifs, int allow_repeat_ifs, char ips, int allow_repeat_ips); -lrec_reader_t* lrec_reader_stdio_csv_alloc(char irs, char ifs); lrec_reader_t* lrec_reader_stdio_csvlite_alloc(char irs, char ifs, int allow_repeat_ifs); -lrec_reader_t* lrec_reader_csvex_alloc(byte_reader_t* pbr, char irs, char ifs); +lrec_reader_t* lrec_reader_csv_alloc(byte_reader_t* pbr, char irs, char ifs); lrec_reader_t* lrec_reader_stdio_dkvp_alloc(char irs, char ifs, char ips, int allow_repeat_ifs); lrec_reader_t* lrec_reader_stdio_nidx_alloc(char irs, char ifs, int allow_repeat_ifs); lrec_reader_t* lrec_reader_stdio_xtab_alloc(char ips, int allow_repeat_ips); -lrec_reader_t* lrec_reader_mmap_csv_alloc(char irs, char ifs, int allow_repeat_ifs); -lrec_reader_t* lrec_reader_mmap_csvlite_alloc(char irs, char ifs, int allow_repeat_ifs); +lrec_reader_t* lrec_reader_mmap_csv_alloc(char irs, char ifs, int allow_repeat_ifs); +lrec_reader_t* lrec_reader_mmap_csvlite_alloc(char irs, char ifs, int allow_repeat_ifs); lrec_reader_t* lrec_reader_mmap_dkvp_alloc(char irs, char ifs, char ips, int allow_repeat_ifs); lrec_reader_t* lrec_reader_mmap_nidx_alloc(char irs, char ifs, int allow_repeat_ifs); lrec_reader_t* lrec_reader_mmap_xtab_alloc(char irs, char ips, int allow_repeat_ips); @@ -33,8 +32,6 @@ slls_t* split_csv_header_line(char* line, char ifs, int allow_repeat_ifs); slls_t* split_csvlite_header_line(char* line, char ifs, int allow_repeat_ifs); lrec_t* lrec_parse_stdio_csvlite_data_line(header_keeper_t* pheader_keeper, char* data_line, char ifs, int allow_repeat_ifs); -lrec_t* lrec_parse_stdio_csv_data_line(header_keeper_t* pheader_keeper, char* data_line, char ifs, - int allow_repeat_ifs); lrec_t* lrec_parse_stdio_xtab(slls_t* pxtab_lines, char ips, int allow_repeat_ips); lrec_t* lrec_parse_mmap_nidx(file_reader_mmap_state_t* phandle, char irs, char ifs, int allow_repeat_ifs); diff --git a/c/input/old_peek_file_reader.c b/c/input/old_peek_file_reader.c deleted file mode 100644 index f8c6371a7..000000000 --- a/c/input/old_peek_file_reader.c +++ /dev/null @@ -1,89 +0,0 @@ -#include -#include -#include "lib/mlrutil.h" -#include "lib/mlr_globals.h" -#include "old_peek_file_reader.h" - -// xxx comment about efficiency here: enough to deliver rfc-csv feature with performance tuning still tbd. - -// tripartite ascii art w/ chars-to-caller; the peekbuf; pending data in the fp. -// label in particular eof handling. - -// ---------------------------------------------------------------- -old_peek_file_reader_t* old_pfr_alloc(FILE* fp, int maxnpeek) { - old_peek_file_reader_t* pfr = mlr_malloc_or_die(sizeof(old_peek_file_reader_t)); - pfr->fp = fp; - pfr->peekbuflen = maxnpeek + 1; - pfr->peekbuf = mlr_malloc_or_die(pfr->peekbuflen); - memset(pfr->peekbuf, 0, pfr->peekbuflen); - pfr->npeeked = 0; - - // Pre-read one char into the peekbuf so that we can say old_pfr_at_eof - // right away on the first call on an empty file. - // getc_unlocked() is appropriate since Miller is single-threaded. - pfr->peekbuf[pfr->npeeked++] = getc_unlocked(pfr->fp); // maybe EOF - - return pfr; -} - -// ---------------------------------------------------------------- -int old_pfr_at_eof(old_peek_file_reader_t* pfr) { - return pfr->npeeked >= 1 && pfr->peekbuf[0] == EOF; -} - -// ---------------------------------------------------------------- -// xxx inline this for perf. -int old_pfr_next_is(old_peek_file_reader_t* pfr, char* string, int len) { - // xxx abend on len > peekbuflen - while (pfr->npeeked < len) { - char c = getc_unlocked(pfr->fp); // maybe EOF - pfr->peekbuf[pfr->npeeked++] = c; - } - // xxx make a memeq, inlined. - return memcmp(string, pfr->peekbuf, len) == 0; -} - -// ---------------------------------------------------------------- -char old_pfr_read_char(old_peek_file_reader_t* pfr) { - if (pfr->npeeked == 1 && pfr->peekbuf[0] == EOF) { - return EOF; - } else if (pfr->npeeked == 0) { - pfr->peekbuf[0] = getc_unlocked(pfr->fp); // maybe EOF - pfr->npeeked = 1; - return pfr->peekbuf[0]; - } else { - char c = pfr->peekbuf[0]; - pfr->npeeked--; - for (int i = 0; i < pfr->npeeked; i++) - pfr->peekbuf[i] = pfr->peekbuf[i+1]; - return c; - } -} - -// ---------------------------------------------------------------- -void old_pfr_advance_by(old_peek_file_reader_t* pfr, int len) { - for (int i = 0; i < len; i++) - old_pfr_read_char(pfr); -} - -// ---------------------------------------------------------------- -void old_pfr_free(old_peek_file_reader_t* pfr) { - if (pfr == NULL) - return; - free(pfr->peekbuf); - pfr->fp = NULL; - pfr->peekbuf = NULL; - free(pfr); -} - -// ---------------------------------------------------------------- -void old_pfr_dump(old_peek_file_reader_t* pfr) { - printf("======================== pfr at %p\n", pfr); - printf(" peekbuflen = %d\n", pfr->peekbuflen); - printf(" npeeked = %d\n", pfr->npeeked); - for (int i = 0; i < pfr->npeeked; i++) { - char c = pfr->peekbuf[i]; - printf(" i=%d c=%c [%02x]\n", i, isprint((unsigned char)c) ? c : ' ', c); - } - printf("------------------------\n"); -} diff --git a/c/input/old_peek_file_reader.h b/c/input/old_peek_file_reader.h deleted file mode 100644 index 85fbb0797..000000000 --- a/c/input/old_peek_file_reader.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef OLD_PEEK_FILE_READER_H -#define OLD_PEEK_FILE_READER_H - -#include - -typedef struct _old_peek_file_reader_t { - FILE* fp; - int peekbuflen; - char* peekbuf; - int npeeked; -} old_peek_file_reader_t; - -// The caller should fclose the fp, since presumably it will have opened it. We -// could have our constructor do the fopen (taking not fp but filename as -// argument) and the destructor do the fclose but that would break reading from -// stdin. - -old_peek_file_reader_t* old_pfr_alloc(FILE* fp, int maxnpeek); -// xxx needing contextual comments here. -int old_pfr_at_eof(old_peek_file_reader_t* pfr); -int old_pfr_next_is(old_peek_file_reader_t* pfr, char* string, int len); -char old_pfr_read_char(old_peek_file_reader_t* pfr); -void old_pfr_advance_by(old_peek_file_reader_t* pfr, int len); -void old_pfr_free(old_peek_file_reader_t* pfr); - -void old_pfr_dump(old_peek_file_reader_t* pfr); - -#endif // OLD_PEEK_FILE_READER_H diff --git a/c/input/peek_file_reader.c b/c/input/peek_file_reader.c new file mode 100644 index 000000000..03dd154bf --- /dev/null +++ b/c/input/peek_file_reader.c @@ -0,0 +1,28 @@ +#include +#include +#include "input/peek_file_reader.h" + +// ---------------------------------------------------------------- +void pfr_dump(peek_file_reader_t* pfr) { + printf("======================== pfr at %p\n", pfr); + printf(" peekbuflen = %d\n", pfr->peekbuflen); + printf(" npeeked = %d\n", pfr->npeeked); + int sob = pfr->sob; + int eob = (pfr->sob + pfr->npeeked) & pfr->peekbuflenmask; + for (int i = 0; i < pfr->peekbuflen; i++) { + char c = pfr->peekbuf[i]; + char* sdesc = (i == sob) ? "START" : ""; + char* edesc = (i == eob) ? "END" : ""; + char* occdesc = ""; + if (sob <= eob) { + if (sob <= i && i < eob) + occdesc = "OCC"; + } else { + if (i < eob || sob <= i) + occdesc = "OCC"; + } + printf(" %-5s %-5s %-3s i=%2d c=%c [%02x]\n", + sdesc, edesc, occdesc, i, isprint((unsigned char)c) ? c : ' ', c); + } + printf("------------------------\n"); +} diff --git a/c/input/peek_file_reader.h b/c/input/peek_file_reader.h index a43b9bb24..54ef3ba20 100644 --- a/c/input/peek_file_reader.h +++ b/c/input/peek_file_reader.h @@ -2,28 +2,38 @@ #define PEEK_FILE_READER_H #include +#include "lib/mlrutil.h" +#include "lib/mlrmath.h" +#include "lib/mlr_globals.h" #include "input/byte_reader.h" +// This is a ring-buffered peekahead file/string reader. + +// Note: Throughout Miller as a general rule I treat struct attributes as if +// there were private attributes. However, for performance, parse_trie_match +// accesses this ring buffer directly. typedef struct _peek_file_reader_t { byte_reader_t* pbr; int peekbuflen; + int peekbuflenmask; char* peekbuf; + int sob; // start of ring-buffer int npeeked; } peek_file_reader_t; -// xxx needing contextual comments here. - -// xxx to do: try using a ring buffer (power-of-two length >= buflen) instead -// of the current slipback buffer, for performance - // ---------------------------------------------------------------- static inline peek_file_reader_t* pfr_alloc(byte_reader_t* pbr, int maxnpeek) { peek_file_reader_t* pfr = mlr_malloc_or_die(sizeof(peek_file_reader_t)); - pfr->pbr = pbr; - pfr->peekbuflen = maxnpeek + 1; - pfr->peekbuf = mlr_malloc_or_die(pfr->peekbuflen); - memset(pfr->peekbuf, 0, pfr->peekbuflen); - pfr->npeeked = 0; + pfr->pbr = pbr; + pfr->peekbuflen = power_of_two_ceil(maxnpeek); + pfr->peekbuflenmask = pfr->peekbuflen - 1; + // The peek-buffer doesn't contain null-terminated C strings, but we + // nonetheless null-terminate the buffer with an extra never-touched byte + // so that print statements in the debugger, etc. will be nice. + pfr->peekbuf = mlr_malloc_or_die(pfr->peekbuflen + 1); + memset(pfr->peekbuf, 0, pfr->peekbuflen + 1); + pfr->sob = 0; + pfr->npeeked = 0; return pfr; } @@ -39,15 +49,17 @@ static inline void pfr_free(peek_file_reader_t* pfr) { // ---------------------------------------------------------------- static inline void pfr_reset(peek_file_reader_t* pfr) { memset(pfr->peekbuf, 0, pfr->peekbuflen); + pfr->sob = 0; pfr->npeeked = 0; } // ---------------------------------------------------------------- static inline char pfr_peek_char(peek_file_reader_t* pfr) { if (pfr->npeeked < 1) { - pfr->peekbuf[pfr->npeeked++] = pfr->pbr->pread_func(pfr->pbr); + pfr->peekbuf[pfr->sob] = pfr->pbr->pread_func(pfr->pbr); + pfr->npeeked++; } - return pfr->peekbuf[0]; + return pfr->peekbuf[pfr->sob]; } // ---------------------------------------------------------------- @@ -55,10 +67,8 @@ static inline char pfr_read_char(peek_file_reader_t* pfr) { if (pfr->npeeked < 1) { return pfr->pbr->pread_func(pfr->pbr); } else { - // xxx to do: make this a ring buffer to avoid the shifts. - char c = pfr->peekbuf[0]; - for (int i = 1; i < pfr->npeeked; i++) - pfr->peekbuf[i-1] = pfr->peekbuf[i]; + char c = pfr->peekbuf[pfr->sob]; + pfr->sob = (pfr->sob + 1) & pfr->peekbuflenmask; pfr->npeeked--; return c; } @@ -67,7 +77,7 @@ static inline char pfr_read_char(peek_file_reader_t* pfr) { // ---------------------------------------------------------------- static inline void pfr_buffer_by(peek_file_reader_t* pfr, int len) { while (pfr->npeeked < len) { - pfr->peekbuf[pfr->npeeked++] = pfr->pbr->pread_func(pfr->pbr); + pfr->peekbuf[(pfr->sob + pfr->npeeked++) & pfr->peekbuflenmask] = pfr->pbr->pread_func(pfr->pbr); } } @@ -78,9 +88,11 @@ static inline void pfr_advance_by(peek_file_reader_t* pfr, int len) { MLR_GLOBALS.argv0, len, pfr->npeeked); exit(1); } - for (int i = len; i < pfr->npeeked; i++) - pfr->peekbuf[i-len] = pfr->peekbuf[i]; + pfr->sob = (pfr->sob + len) & pfr->peekbuflenmask; pfr->npeeked -= len; } +// ---------------------------------------------------------------- +void pfr_dump(peek_file_reader_t* pfr); + #endif // PEEK_FILE_READER_H diff --git a/c/input/stdio_byte_reader.c b/c/input/stdio_byte_reader.c index ce33d0cf4..097b4a13c 100644 --- a/c/input/stdio_byte_reader.c +++ b/c/input/stdio_byte_reader.c @@ -13,11 +13,15 @@ typedef struct _stdio_byte_reader_state_t { static int stdio_byte_reader_open_func(struct _byte_reader_t* pbr, char* filename) { stdio_byte_reader_state_t* pstate = mlr_malloc_or_die(sizeof(stdio_byte_reader_state_t)); pstate->filename = strdup(filename); - pstate->fp = fopen(filename, "r"); - if (pstate->fp == NULL) { - perror("fopen"); - fprintf(stderr, "%s: Couldn't fopen \"%s\" for read.\n", MLR_GLOBALS.argv0, filename); - exit(1); + if (streq(pstate->filename, "-")) { + pstate->fp = stdin; + } else { + pstate->fp = fopen(filename, "r"); + if (pstate->fp == NULL) { + perror("fopen"); + fprintf(stderr, "%s: Couldn't fopen \"%s\" for read.\n", MLR_GLOBALS.argv0, filename); + exit(1); + } } pbr->pvstate = pstate; return TRUE; @@ -36,7 +40,8 @@ static int stdio_byte_reader_read_func(struct _byte_reader_t* pbr) { static void stdio_byte_reader_close_func(struct _byte_reader_t* pbr) { stdio_byte_reader_state_t* pstate = pbr->pvstate; - fclose(pstate->fp); + if (pstate->fp != stdin) + fclose(pstate->fp); free(pstate); pbr->pvstate = NULL; } diff --git a/c/input/test_peek_file_reader.c b/c/input/test_peek_file_reader.c new file mode 100644 index 000000000..9957ed3c3 --- /dev/null +++ b/c/input/test_peek_file_reader.c @@ -0,0 +1,90 @@ +#include +#include +#include +#include "lib/mlrutil.h" +#include "lib/minunit.h" +#include "lib/mlr_test_util.h" +#include "input/byte_readers.h" +#include "input/peek_file_reader.h" + +#ifdef __TEST_PEEK_FILE_READER_MAIN__ +int tests_run = 0; +int tests_failed = 0; +int assertions_run = 0; +int assertions_failed = 0; + +// ---------------------------------------------------------------- +static char* test_empty() { + byte_reader_t* pbr = string_byte_reader_alloc(); + int ok = pbr->popen_func(pbr, ""); + mu_assert_lf(ok == TRUE); + + peek_file_reader_t* pfr = pfr_alloc(pbr, 7); + + mu_assert_lf(pfr_peek_char(pfr) == EOF); + mu_assert_lf(pfr_read_char(pfr) == EOF); + + pbr->pclose_func(pbr); + pfr_free(pfr); + + return NULL; +} + +// ---------------------------------------------------------------- +static char* test_non_empty() { + byte_reader_t* pbr = string_byte_reader_alloc(); + int ok = pbr->popen_func(pbr, + "ab,cde\n" + "123,4567\n" + ); + mu_assert_lf(ok == TRUE); + + peek_file_reader_t* pfr = pfr_alloc(pbr, 7); + + pfr_dump(pfr); mu_assert_lf(pfr_peek_char(pfr) == 'a'); + pfr_dump(pfr); mu_assert_lf(pfr_read_char(pfr) == 'a'); + pfr_dump(pfr); mu_assert_lf(pfr_peek_char(pfr) == 'b'); + pfr_dump(pfr); mu_assert_lf(pfr_read_char(pfr) == 'b'); + + pfr_dump(pfr); mu_assert_lf(pfr_peek_char(pfr) == ','); + pfr_dump(pfr); mu_assert_lf(pfr_peek_char(pfr) == ','); + pfr_dump(pfr); mu_assert_lf(pfr_read_char(pfr) == ','); + pfr_dump(pfr); pfr_buffer_by(pfr, 5); + pfr_dump(pfr); pfr_advance_by(pfr, 5); + pfr_dump(pfr); mu_assert_lf(pfr_read_char(pfr) == '2'); + + pfr_dump(pfr); mu_assert_lf(pfr_peek_char(pfr) == '3'); + pfr_dump(pfr); mu_assert_lf(pfr_peek_char(pfr) == '3'); + pfr_dump(pfr); mu_assert_lf(pfr_read_char(pfr) == '3'); + pfr_dump(pfr); pfr_buffer_by(pfr, 5); + pfr_dump(pfr); pfr_advance_by(pfr, 5); + pfr_dump(pfr); mu_assert_lf(pfr_read_char(pfr) == '\n'); + + pbr->pclose_func(pbr); + pfr_free(pfr); + + return NULL; +} + +// ================================================================ +static char * run_all_tests() { + mu_run_test(test_empty); + mu_run_test(test_non_empty); + return 0; +} + +int main(int argc, char **argv) { + char *result = run_all_tests(); + printf("\n"); + if (result != 0) { + printf("Not all unit tests passed\n"); + } + else { + printf("TEST_PEEK_FILE_READER: ALL UNIT TESTS PASSED\n"); + } + printf("Tests passed: %d of %d\n", tests_run - tests_failed, tests_run); + printf("Assertions passed: %d of %d\n", assertions_run - assertions_failed, assertions_run); + + return result != 0; +} +#endif // __TEST_PEEK_FILE_READER_MAIN__ diff --git a/c/lib/mlrutil.c b/c/lib/mlrutil.c index 497b0421a..bfd8081e0 100644 --- a/c/lib/mlrutil.c +++ b/c/lib/mlrutil.c @@ -286,3 +286,12 @@ int mlr_imax2(int a, int b) { else return b; } + +// ---------------------------------------------------------------- +// This is inefficient. It's quite fine for call-once, small-n use. + +int power_of_two_ceil(int n) { + while (n&(n-1)) + n++; + return n; +} diff --git a/c/lib/mlrutil.h b/c/lib/mlrutil.h index 6cee34461..b166e31f3 100644 --- a/c/lib/mlrutil.h +++ b/c/lib/mlrutil.h @@ -66,5 +66,6 @@ time_t mlr_timegm (struct tm *tm); int strlen_for_utf8_display(char* str); int mlr_imax2(int a, int b); +int power_of_two_ceil(int n); #endif // MLRUTIL_H diff --git a/c/todo.txt b/c/todo.txt index 7f14e7388..32be97d2e 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -1,7 +1,12 @@ ================================================================ BUGFIXES -:D +$ sh foo|termcvt --lf2crlf|mlr --no-mmap --csv check +fopen: No such file or directory +mlr: Couldn't fopen "-" for read. +-> needs UT case! + +! memory leak in csv reader! careful about slls data, and do not use lrec_put_no_free ================================================================ TOP OF LIST