Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-29 10:40:22 +00:00
Code Issues Projects Releases Wiki Activity

csv-read performance iterate

Browse source
This commit is contained in:
John Kerl 2015-12-08 21:11:43 -05:00
parent 6d1ff3530b
commit 101b7abfcf
2 changed files with 19 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

19
c/input/lrec_reader_mmap_csv.c
View file

@ -68,6 +68,7 @@ typedef struct _lrec_reader_mmap_csv_state_t {
static void lrec_reader_mmap_csv_free(void* pvstate);
static void lrec_reader_mmap_csv_sof(void* pvstate);
static void* lrec_reader_mmap_csv_vopen(void* pvstate, char* file_name);
static lrec_t* lrec_reader_mmap_csv_process(void* pvstate, void* pvhandle, context_t* pctx);
static slls_t* lrec_reader_mmap_csv_get_fields(lrec_reader_mmap_csv_state_t* pstate,
file_reader_mmap_state_t* phandle);
@ -116,7 +117,7 @@ lrec_reader_t* lrec_reader_mmap_csv_alloc(byte_reader_t* pbr, char* irs, char* i
pstate->pheader_keepers = lhmslv_alloc();
plrec_reader->pvstate = (void*)pstate;
plrec_reader->popen_func = file_reader_mmap_vopen;
plrec_reader->popen_func = lrec_reader_mmap_csv_vopen;
plrec_reader->pclose_func = file_reader_mmap_vclose;
plrec_reader->pprocess_func = lrec_reader_mmap_csv_process;
plrec_reader->psof_func = lrec_reader_mmap_csv_sof;
@ -144,6 +145,14 @@ static void lrec_reader_mmap_csv_sof(void* pvstate) {
pstate->expect_header_line_next = pstate->use_implicit_header ? FALSE : TRUE;
}
// ----------------------------------------------------------------
static void* lrec_reader_mmap_csv_vopen(void* pvstate, char* file_name) {
void* pvhandle = file_reader_mmap_open(file_name);
file_reader_mmap_state_t* phandle = pvhandle;
*phandle->eof = EOF;
return pvhandle;
}
// ----------------------------------------------------------------
static lrec_t* lrec_reader_mmap_csv_process(void* pvstate, void* pvhandle, context_t* pctx) {
lrec_reader_mmap_csv_state_t* pstate = pvstate;
@ -253,6 +262,14 @@ static slls_t* lrec_reader_mmap_csv_get_fields(lrec_reader_mmap_csv_state_t* pst
break;
}
e += matchlen;
} else if (e >= phandle->eof) {
// xxx this is awkward w/r/t EOF_STRIDX
*e = 0;
slls_add_no_free(pfields, p);
p = e + matchlen;
field_done = TRUE;
record_done = TRUE;
break;
} else {
////printf("CHAR %d %c\n", *e, *e);
e++;

1
c/todo.txt
View file

@ -27,6 +27,7 @@ MAJOR: csv mem-leak/read-perf
! mmap vopen/vclose -> api, for unit-testability. & x all readers.
! eliminate string-builder except where necessary
! reusable_slls: x both csv and both csvlite readers; xtab too?
! eofpoke in wrapper vopen
o memory leak in csv reader! careful about slls data, and do not use lrec_put_no_free
o redo inline-pasting but this time correctly weight the fragging effect
o power-of-two