From 101b7abfcf58a5019fc621223656b6e893cab0a9 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Tue, 8 Dec 2015 21:11:43 -0500 Subject: [PATCH] csv-read performance iterate --- c/input/lrec_reader_mmap_csv.c | 19 ++++++++++++++++++- c/todo.txt | 1 + 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/c/input/lrec_reader_mmap_csv.c b/c/input/lrec_reader_mmap_csv.c index cf9ad8b1f..68dd3d299 100644 --- a/c/input/lrec_reader_mmap_csv.c +++ b/c/input/lrec_reader_mmap_csv.c @@ -68,6 +68,7 @@ typedef struct _lrec_reader_mmap_csv_state_t { static void lrec_reader_mmap_csv_free(void* pvstate); static void lrec_reader_mmap_csv_sof(void* pvstate); +static void* lrec_reader_mmap_csv_vopen(void* pvstate, char* file_name); static lrec_t* lrec_reader_mmap_csv_process(void* pvstate, void* pvhandle, context_t* pctx); static slls_t* lrec_reader_mmap_csv_get_fields(lrec_reader_mmap_csv_state_t* pstate, file_reader_mmap_state_t* phandle); @@ -116,7 +117,7 @@ lrec_reader_t* lrec_reader_mmap_csv_alloc(byte_reader_t* pbr, char* irs, char* i pstate->pheader_keepers = lhmslv_alloc(); plrec_reader->pvstate = (void*)pstate; - plrec_reader->popen_func = file_reader_mmap_vopen; + plrec_reader->popen_func = lrec_reader_mmap_csv_vopen; plrec_reader->pclose_func = file_reader_mmap_vclose; plrec_reader->pprocess_func = lrec_reader_mmap_csv_process; plrec_reader->psof_func = lrec_reader_mmap_csv_sof; @@ -144,6 +145,14 @@ static void lrec_reader_mmap_csv_sof(void* pvstate) { pstate->expect_header_line_next = pstate->use_implicit_header ? FALSE : TRUE; } +// ---------------------------------------------------------------- +static void* lrec_reader_mmap_csv_vopen(void* pvstate, char* file_name) { + void* pvhandle = file_reader_mmap_open(file_name); + file_reader_mmap_state_t* phandle = pvhandle; + *phandle->eof = EOF; + return pvhandle; +} + // ---------------------------------------------------------------- static lrec_t* lrec_reader_mmap_csv_process(void* pvstate, void* pvhandle, context_t* pctx) { lrec_reader_mmap_csv_state_t* pstate = pvstate; @@ -253,6 +262,14 @@ static slls_t* lrec_reader_mmap_csv_get_fields(lrec_reader_mmap_csv_state_t* pst break; } e += matchlen; + } else if (e >= phandle->eof) { + // xxx this is awkward w/r/t EOF_STRIDX + *e = 0; + slls_add_no_free(pfields, p); + p = e + matchlen; + field_done = TRUE; + record_done = TRUE; + break; } else { ////printf("CHAR %d %c\n", *e, *e); e++; diff --git a/c/todo.txt b/c/todo.txt index 36a83b30f..cdcc1a3ed 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -27,6 +27,7 @@ MAJOR: csv mem-leak/read-perf ! mmap vopen/vclose -> api, for unit-testability. & x all readers. ! eliminate string-builder except where necessary ! reusable_slls: x both csv and both csvlite readers; xtab too? + ! eofpoke in wrapper vopen o memory leak in csv reader! careful about slls data, and do not use lrec_put_no_free o redo inline-pasting but this time correctly weight the fragging effect o power-of-two