Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-22 07:30:43 +00:00
Code Issues Projects Releases Wiki Activity

csv-read performance iterate

Browse source
This commit is contained in:
John Kerl 2015-12-07 23:23:36 -05:00
parent 03ca872e45
commit e1b5a7e305
6 changed files with 73 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

2
c/containers/parse_trie.h
View file

@ -48,7 +48,7 @@ void parse_trie_add_string(parse_trie_t* ptrie, char* string, int stridx);
// The start of buffer (sob), buflen, and mask attributes are nominally
// presented from a ring_buffer object.
static inline int parse_trie_match(parse_trie_t* ptrie, char* buf, int sob, int buflen, int mask,
static inline int parse_trie_ring_match(parse_trie_t* ptrie, char* buf, int sob, int buflen, int mask,
int* pstridx, int* pmatchlen)
{
parse_trie_node_t* pnode = ptrie->proot;

2
c/experimental/getlines.c
View file

@ -303,7 +303,7 @@ static char* read_line_pfr_psb(peek_file_reader_t* pfr, string_builder_t* psb, p
int rc, stridx, matchlen;
while (TRUE) {
pfr_buffer_by(pfr, ptrie->maxlen);
rc = parse_trie_match(ptrie, pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask,
rc = parse_trie_ring_match(ptrie, pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask,
&stridx, &matchlen);
if (rc) {
pfr_advance_by(pfr, matchlen);

4
c/input/lrec_reader_csv.c
View file

@ -214,7 +214,7 @@ static slls_t* lrec_reader_csv_get_fields(lrec_reader_csv_state_t* pstate) {
while (!field_done) {
pfr_buffer_by(pfr, pstate->pno_dquote_parse_trie->maxlen);
rc = parse_trie_match(pstate->pno_dquote_parse_trie,
rc = parse_trie_ring_match(pstate->pno_dquote_parse_trie,
pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask,
&stridx, &matchlen);
#ifdef DEBUG_PARSER
@ -275,7 +275,7 @@ static slls_t* lrec_reader_csv_get_fields(lrec_reader_csv_state_t* pstate) {
while (!field_done) {
pfr_buffer_by(pfr, pstate->pdquote_parse_trie->maxlen);
rc = parse_trie_match(pstate->pdquote_parse_trie,
rc = parse_trie_ring_match(pstate->pdquote_parse_trie,
pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask,
&stridx, &matchlen);

45
c/input/lrec_reader_mmap_csv.c
View file

@ -4,7 +4,7 @@
#include "lib/mlr_globals.h"
#include "lib/mlrutil.h"
#include "lib/string_builder.h"
#include "input/file_reader_stdio.h"
#include "input/file_reader_mmap.h"
#include "input/byte_reader.h"
#include "input/lrec_readers.h"
#include "input/peek_file_reader.h"
@ -216,7 +216,7 @@ static slls_t* lrec_reader_mmap_csv_get_fields(lrec_reader_mmap_csv_state_t* pst
while (!field_done) {
pfr_buffer_by(pfr, pstate->pno_dquote_parse_trie->maxlen);
rc = parse_trie_match(pstate->pno_dquote_parse_trie,
rc = parse_trie_ring_match(pstate->pno_dquote_parse_trie,
pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask,
&stridx, &matchlen);
#ifdef DEBUG_PARSER
@ -277,7 +277,7 @@ static slls_t* lrec_reader_mmap_csv_get_fields(lrec_reader_mmap_csv_state_t* pst
while (!field_done) {
pfr_buffer_by(pfr, pstate->pdquote_parse_trie->maxlen);
rc = parse_trie_match(pstate->pdquote_parse_trie,
rc = parse_trie_ring_match(pstate->pdquote_parse_trie,
pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask,
&stridx, &matchlen);
@ -372,3 +372,42 @@ static void lrec_reader_mmap_csv_close(void* pvstate, void* pvhandle) {
lrec_reader_mmap_csv_state_t* pstate = pvstate;
pstate->pfr->pbr->pclose_func(pstate->pfr->pbr);
}
//// ----------------------------------------------------------------
//lrec_reader_t* lrec_reader_mmap_csvlite_alloc(char* irs, char* ifs, int allow_repeat_ifs, int use_implicit_header) {
// lrec_reader_t* plrec_reader = mlr_malloc_or_die(sizeof(lrec_reader_t));
// ...
// plrec_reader->popen_func = file_reader_mmap_vopen;
// plrec_reader->pclose_func = file_reader_mmap_vclose;
// ...
//}
//// ----------------------------------------------------------------
//static void lrec_reader_mmap_csvlite_sof(void* pvstate) {
// lrec_reader_mmap_csvlite_state_t* pstate = pvstate;
// pstate->ifnr = 0LL;
// pstate->ilno = 0LL;
// pstate->expect_header_line_next = pstate->use_implicit_header ? FALSE : TRUE;
//}
//static slls_t* lrec_reader_mmap_csvlite_get_header_multi_seps(file_reader_mmap_state_t* phandle,
// lrec_reader_mmap_csvlite_state_t* pstate)
//{
// ...
// while ((phandle->eof - phandle->sol) >= irslen && streqn(phandle->sol, irs, irslen)) {
// phandle->sol += irslen;
// pstate->ilno++;
// }
// ...
//}
//static lrec_t* lrec_reader_mmap_csvlite_get_record_multi_seps(file_reader_mmap_state_t* phandle,
// lrec_reader_mmap_csvlite_state_t* pstate, context_t* pctx, header_keeper_t* pheader_keeper, int* pend_of_stanza)
//{
// if (phandle->sol >= phandle->eof)
// return NULL;
//
// ...
// phandle->sol = p + irslen;
// ...
//}

2
c/input/peek_file_reader.h
View file

@ -10,7 +10,7 @@
// This is a ring-buffered peekahead file/string reader.
// Note: Throughout Miller as a general rule I treat struct attributes as if
// they were private attributes. However, for performance, parse_trie_match
// they were private attributes. However, for performance, parse_trie_ring_match
// accesses this ring buffer directly.
typedef struct _peek_file_reader_t {
byte_reader_t* pbr;

52
c/unit_test/test_parse_trie.c
View file

@ -45,7 +45,7 @@ static void test_case(
stridx = -2;
matchlen = -2;
rc = parse_trie_match(ptrie, buf, 0, strlen(buf), 0xff, &stridx, &matchlen);
rc = parse_trie_ring_match(ptrie, buf, 0, strlen(buf), 0xff, &stridx, &matchlen);
parse_trie_free(ptrie);
@ -146,61 +146,61 @@ static char* test_dkvp() {
}
parse_trie_print(ptrie);
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
mu_assert_lf(rc == TRUE);
mu_assert_lf(stridx == PS_STRIDX);
mu_assert_lf(matchlen == strlen(strings[PS_STRIDX]));
p += matchlen;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
mu_assert_lf(rc == TRUE);
mu_assert_lf(stridx == FS_STRIDX);
mu_assert_lf(matchlen == strlen(strings[FS_STRIDX]));
p += matchlen;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
mu_assert_lf(rc == TRUE);
mu_assert_lf(stridx == PS_STRIDX);
mu_assert_lf(matchlen == strlen(strings[PS_STRIDX]));
p += matchlen;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
mu_assert_lf(rc == TRUE);
mu_assert_lf(stridx == RS_STRIDX);
mu_assert_lf(matchlen == strlen(strings[RS_STRIDX]));
p += matchlen;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
mu_assert_lf(rc == TRUE);
mu_assert_lf(stridx == PS_STRIDX);
mu_assert_lf(matchlen == strlen(strings[PS_STRIDX]));
p += matchlen;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
mu_assert_lf(rc == TRUE);
mu_assert_lf(stridx == EOF_STRIDX);
mu_assert_lf(matchlen == strlen(strings[EOF_STRIDX]));
@ -234,7 +234,7 @@ static char* show_it() {
parse_trie_print(ptrie);
while (TRUE) {
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
rc = parse_trie_ring_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
if (rc) {
printf("match token %d (", stridx);
print_with_unprintables(strings[stridx]);