Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-22 15:37:59 +00:00
Code Issues Projects Releases Wiki Activity

csv-read performance iterate

Browse source
This commit is contained in:
John Kerl 2015-12-09 23:08:04 -05:00
parent 6a1e819233
commit 253b1a8e1a
4 changed files with 13 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

3
c/input/lrec_reader_mmap_csv.c
View file

@ -370,7 +370,8 @@ static lrec_t* paste_header_and_data(lrec_reader_mmap_csv_state_t* pstate, rslls
sllse_t* ph = pstate->pheader_keeper->pkeys->phead;
rsllse_t* pd = pdata_fields->phead;
for ( ; ph != NULL && pd != NULL; ph = ph->pnext, pd = pd->pnext) {
lrec_put(prec, ph->value, pd->value, pd->free_flag);
//lrec_put(prec, ph->value, pd->value, pd->free_flag);
lrec_put_no_free(prec, ph->value, pd->value);
pd->free_flag = 0;
}
return prec;

3
c/lib/mlrutil.c
View file

@ -62,7 +62,7 @@ void* mlr_malloc_or_die(size_t size) {
fprintf(stderr, "malloc(%lu) failed.\n", (unsigned long)size);
exit(1);
}
//fprintf(stderr, "size=%d,p=%p\n", (int)size, p);
//fprintf(stderr, "MALLOC size=%d,p=%p\n", (int)size, p);
return p;
}
@ -73,6 +73,7 @@ void* mlr_realloc_or_die(void *optr, size_t size) {
fprintf(stderr, "realloc(%lu) failed.\n", (unsigned long)size);
exit(1);
}
//fprintf(stderr, "REALLOC size=%d,p=%p\n", (int)size, nptr);
return nptr;
}

1
c/lib/mlrutil.h
View file

@ -76,6 +76,7 @@ static inline char * mlr_strdup_or_die(const char *s1) {
fprintf(stderr, "malloc/strdup failed\n");
exit(1);
}
//fprintf(stderr, "STRDUP size=%d,p=%p\n", (int)strlen(s2), s2);
return s2;
}

15
c/todo.txt
View file

@ -25,14 +25,15 @@ TOP OF LIST
MAJOR: csv mem-leak/read-perf
* current option runs faster w/o free, apparently due to heap-fragging
! malloc-trace ... verify heap-frag hypothesis for fp-csv reader.
! mmap vopen/vclose -> api, for unit-testability. & x all readers.
! eliminate string-builder except where necessary
! reusable_slls: x both csv and both csvlite readers; xtab too?
! eofpoke in wrapper vopen
o memory leak in csv reader! careful about slls data, and do not use lrec_put_no_free
~ malloc-trace ... verify heap-frag hypothesis for fp-csv reader.
? mmap vopen/vclose -> api, for unit-testability. & x all readers.
k eliminate string-builder except where necessary
* in mmap-csv, try harder to not use the psb even in the dquoted case
k reusable_slls: x both csv and both csvlite readers; xtab too.
k eofpoke in wrapper vopen
! rethink lrec_put/lrec_put_no_free entirely. 1st get timings for all filetypes.
o redo inline-pasting but this time correctly weight the fragging effect
o power-of-two
o power-of-two ...
* for stdio, needs some thought ...
* ... but for mmap, it's almost always not necessary to strdup at all:
only on escaped-double-quote case.