From e1b5a7e305f3edc572f8a4ebbd749b33713f4e38 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 7 Dec 2015 23:23:36 -0500 Subject: [PATCH] csv-read performance iterate --- c/containers/parse_trie.h | 2 +- c/experimental/getlines.c | 2 +- c/input/lrec_reader_csv.c | 4 +-- c/input/lrec_reader_mmap_csv.c | 45 +++++++++++++++++++++++++++-- c/input/peek_file_reader.h | 2 +- c/unit_test/test_parse_trie.c | 52 +++++++++++++++++----------------- 6 files changed, 73 insertions(+), 34 deletions(-) diff --git a/c/containers/parse_trie.h b/c/containers/parse_trie.h index dfc303b4b..72a6c10c9 100644 --- a/c/containers/parse_trie.h +++ b/c/containers/parse_trie.h @@ -48,7 +48,7 @@ void parse_trie_add_string(parse_trie_t* ptrie, char* string, int stridx); // The start of buffer (sob), buflen, and mask attributes are nominally // presented from a ring_buffer object. -static inline int parse_trie_match(parse_trie_t* ptrie, char* buf, int sob, int buflen, int mask, +static inline int parse_trie_ring_match(parse_trie_t* ptrie, char* buf, int sob, int buflen, int mask, int* pstridx, int* pmatchlen) { parse_trie_node_t* pnode = ptrie->proot; diff --git a/c/experimental/getlines.c b/c/experimental/getlines.c index 99463966a..6b3133bd3 100644 --- a/c/experimental/getlines.c +++ b/c/experimental/getlines.c @@ -303,7 +303,7 @@ static char* read_line_pfr_psb(peek_file_reader_t* pfr, string_builder_t* psb, p int rc, stridx, matchlen; while (TRUE) { pfr_buffer_by(pfr, ptrie->maxlen); - rc = parse_trie_match(ptrie, pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask, + rc = parse_trie_ring_match(ptrie, pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask, &stridx, &matchlen); if (rc) { pfr_advance_by(pfr, matchlen); diff --git a/c/input/lrec_reader_csv.c b/c/input/lrec_reader_csv.c index 6c66e600c..bb0986b5c 100644 --- a/c/input/lrec_reader_csv.c +++ b/c/input/lrec_reader_csv.c @@ -214,7 +214,7 @@ static slls_t* lrec_reader_csv_get_fields(lrec_reader_csv_state_t* pstate) { while (!field_done) { pfr_buffer_by(pfr, pstate->pno_dquote_parse_trie->maxlen); - rc = parse_trie_match(pstate->pno_dquote_parse_trie, + rc = parse_trie_ring_match(pstate->pno_dquote_parse_trie, pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask, &stridx, &matchlen); #ifdef DEBUG_PARSER @@ -275,7 +275,7 @@ static slls_t* lrec_reader_csv_get_fields(lrec_reader_csv_state_t* pstate) { while (!field_done) { pfr_buffer_by(pfr, pstate->pdquote_parse_trie->maxlen); - rc = parse_trie_match(pstate->pdquote_parse_trie, + rc = parse_trie_ring_match(pstate->pdquote_parse_trie, pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask, &stridx, &matchlen); diff --git a/c/input/lrec_reader_mmap_csv.c b/c/input/lrec_reader_mmap_csv.c index 24e2c75c5..47d497eeb 100644 --- a/c/input/lrec_reader_mmap_csv.c +++ b/c/input/lrec_reader_mmap_csv.c @@ -4,7 +4,7 @@ #include "lib/mlr_globals.h" #include "lib/mlrutil.h" #include "lib/string_builder.h" -#include "input/file_reader_stdio.h" +#include "input/file_reader_mmap.h" #include "input/byte_reader.h" #include "input/lrec_readers.h" #include "input/peek_file_reader.h" @@ -216,7 +216,7 @@ static slls_t* lrec_reader_mmap_csv_get_fields(lrec_reader_mmap_csv_state_t* pst while (!field_done) { pfr_buffer_by(pfr, pstate->pno_dquote_parse_trie->maxlen); - rc = parse_trie_match(pstate->pno_dquote_parse_trie, + rc = parse_trie_ring_match(pstate->pno_dquote_parse_trie, pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask, &stridx, &matchlen); #ifdef DEBUG_PARSER @@ -277,7 +277,7 @@ static slls_t* lrec_reader_mmap_csv_get_fields(lrec_reader_mmap_csv_state_t* pst while (!field_done) { pfr_buffer_by(pfr, pstate->pdquote_parse_trie->maxlen); - rc = parse_trie_match(pstate->pdquote_parse_trie, + rc = parse_trie_ring_match(pstate->pdquote_parse_trie, pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask, &stridx, &matchlen); @@ -372,3 +372,42 @@ static void lrec_reader_mmap_csv_close(void* pvstate, void* pvhandle) { lrec_reader_mmap_csv_state_t* pstate = pvstate; pstate->pfr->pbr->pclose_func(pstate->pfr->pbr); } + +//// ---------------------------------------------------------------- +//lrec_reader_t* lrec_reader_mmap_csvlite_alloc(char* irs, char* ifs, int allow_repeat_ifs, int use_implicit_header) { +// lrec_reader_t* plrec_reader = mlr_malloc_or_die(sizeof(lrec_reader_t)); +// ... +// plrec_reader->popen_func = file_reader_mmap_vopen; +// plrec_reader->pclose_func = file_reader_mmap_vclose; +// ... +//} + +//// ---------------------------------------------------------------- +//static void lrec_reader_mmap_csvlite_sof(void* pvstate) { +// lrec_reader_mmap_csvlite_state_t* pstate = pvstate; +// pstate->ifnr = 0LL; +// pstate->ilno = 0LL; +// pstate->expect_header_line_next = pstate->use_implicit_header ? FALSE : TRUE; +//} + +//static slls_t* lrec_reader_mmap_csvlite_get_header_multi_seps(file_reader_mmap_state_t* phandle, +// lrec_reader_mmap_csvlite_state_t* pstate) +//{ +// ... +// while ((phandle->eof - phandle->sol) >= irslen && streqn(phandle->sol, irs, irslen)) { +// phandle->sol += irslen; +// pstate->ilno++; +// } +// ... +//} + +//static lrec_t* lrec_reader_mmap_csvlite_get_record_multi_seps(file_reader_mmap_state_t* phandle, +// lrec_reader_mmap_csvlite_state_t* pstate, context_t* pctx, header_keeper_t* pheader_keeper, int* pend_of_stanza) +//{ +// if (phandle->sol >= phandle->eof) +// return NULL; +// +// ... +// phandle->sol = p + irslen; +// ... +//} diff --git a/c/input/peek_file_reader.h b/c/input/peek_file_reader.h index 21313ecab..848df9822 100644 --- a/c/input/peek_file_reader.h +++ b/c/input/peek_file_reader.h @@ -10,7 +10,7 @@ // This is a ring-buffered peekahead file/string reader. // Note: Throughout Miller as a general rule I treat struct attributes as if -// they were private attributes. However, for performance, parse_trie_match +// they were private attributes. However, for performance, parse_trie_ring_match // accesses this ring buffer directly. typedef struct _peek_file_reader_t { byte_reader_t* pbr; diff --git a/c/unit_test/test_parse_trie.c b/c/unit_test/test_parse_trie.c index 7380962fc..e4271cd22 100644 --- a/c/unit_test/test_parse_trie.c +++ b/c/unit_test/test_parse_trie.c @@ -45,7 +45,7 @@ static void test_case( stridx = -2; matchlen = -2; - rc = parse_trie_match(ptrie, buf, 0, strlen(buf), 0xff, &stridx, &matchlen); + rc = parse_trie_ring_match(ptrie, buf, 0, strlen(buf), 0xff, &stridx, &matchlen); parse_trie_free(ptrie); @@ -146,61 +146,61 @@ static char* test_dkvp() { } parse_trie_print(ptrie); - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == TRUE); mu_assert_lf(stridx == PS_STRIDX); mu_assert_lf(matchlen == strlen(strings[PS_STRIDX])); p += matchlen; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == TRUE); mu_assert_lf(stridx == FS_STRIDX); mu_assert_lf(matchlen == strlen(strings[FS_STRIDX])); p += matchlen; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == TRUE); mu_assert_lf(stridx == PS_STRIDX); mu_assert_lf(matchlen == strlen(strings[PS_STRIDX])); p += matchlen; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == TRUE); mu_assert_lf(stridx == RS_STRIDX); mu_assert_lf(matchlen == strlen(strings[RS_STRIDX])); p += matchlen; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == TRUE); mu_assert_lf(stridx == PS_STRIDX); mu_assert_lf(matchlen == strlen(strings[PS_STRIDX])); p += matchlen; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++; - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == TRUE); mu_assert_lf(stridx == EOF_STRIDX); mu_assert_lf(matchlen == strlen(strings[EOF_STRIDX])); @@ -234,7 +234,7 @@ static char* show_it() { parse_trie_print(ptrie); while (TRUE) { - rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); + rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); if (rc) { printf("match token %d (", stridx); print_with_unprintables(strings[stridx]);