From 267c899ee3f24006b21311ef93e7119a82c93cfd Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sun, 6 Sep 2015 23:36:28 -0400 Subject: [PATCH] [read performance iterate] CSV performance tuning --- c/Makefile | 1 - c/input/lrec_reader_csvex.c | 7 ++++ c/input/peek_file_reader.c | 76 ------------------------------------- c/input/peek_file_reader.h | 72 +++++++++++++++++++++++++++++++---- 4 files changed, 71 insertions(+), 85 deletions(-) delete mode 100644 c/input/peek_file_reader.c diff --git a/c/Makefile b/c/Makefile index cb1ba64c5..2a192e538 100644 --- a/c/Makefile +++ b/c/Makefile @@ -70,7 +70,6 @@ input/lrec_reader_mmap_nidx.c input/lrec_reader_stdio_nidx.c \ input/lrec_reader_mmap_xtab.c input/lrec_reader_stdio_xtab.c \ input/file_reader_mmap.c input/file_reader_stdio.c \ input/old_peek_file_reader.c \ -input/peek_file_reader.c \ containers/test_join_bucket_keeper.c EXPERIMENTAL_READER_SRCS = \ diff --git a/c/input/lrec_reader_csvex.c b/c/input/lrec_reader_csvex.c index 6884f6418..8f0b05d2f 100644 --- a/c/input/lrec_reader_csvex.c +++ b/c/input/lrec_reader_csvex.c @@ -11,6 +11,13 @@ #include "containers/lhmslv.h" #include "containers/parse_trie.h" +// ================================================================ +// xxx to do: +// * avoid the separate paster: just inline lrec_put as in the csvlite reader. +// * profile .............. +// * ring buffer in pfr? +// ================================================================ + // Idea of pheader_keepers: each header_keeper object retains the input-line backing // and the slls_t for a CSV header line which is used by one or more CSV data // lines. Meanwhile some mappers retain input records from the entire data diff --git a/c/input/peek_file_reader.c b/c/input/peek_file_reader.c deleted file mode 100644 index 0f1680458..000000000 --- a/c/input/peek_file_reader.c +++ /dev/null @@ -1,76 +0,0 @@ -#include "lib/mlr_globals.h" -#include "lib/mlrutil.h" -#include "input/peek_file_reader.h" - -// ---------------------------------------------------------------- -peek_file_reader_t* pfr_alloc(byte_reader_t* pbr, int maxnpeek) { - peek_file_reader_t* pfr = mlr_malloc_or_die(sizeof(peek_file_reader_t)); - pfr->pbr = pbr; - pfr->peekbuflen = maxnpeek + 1; - pfr->peekbuf = mlr_malloc_or_die(pfr->peekbuflen); - memset(pfr->peekbuf, 0, pfr->peekbuflen); - pfr->npeeked = 0; - - return pfr; -} - -// ---------------------------------------------------------------- -void pfr_free(peek_file_reader_t* pfr) { - if (pfr == NULL) - return; - free(pfr->peekbuf); - free(pfr); -} - -// ---------------------------------------------------------------- -void pfr_reset(peek_file_reader_t* pfr) { - memset(pfr->peekbuf, 0, pfr->peekbuflen); - pfr->npeeked = 0; -} - -// ---------------------------------------------------------------- -char pfr_peek_char(peek_file_reader_t* pfr) { - if (pfr->npeeked < 1) { - pfr->peekbuf[pfr->npeeked++] = pfr->pbr->pread_func(pfr->pbr); - } - return pfr->peekbuf[0]; -} - -// ---------------------------------------------------------------- -char pfr_read_char(peek_file_reader_t* pfr) { - if (pfr->npeeked < 1) { - return pfr->pbr->pread_func(pfr->pbr); - } else { - // xxx to do: make this a ring buffer to avoid the shifts. - char c = pfr->peekbuf[0]; - for (int i = 1; i < pfr->npeeked; i++) - pfr->peekbuf[i-1] = pfr->peekbuf[i]; - pfr->npeeked--; - return c; - } -} - -// ---------------------------------------------------------------- -void pfr_buffer_by(peek_file_reader_t* pfr, int len) { - while (pfr->npeeked < len) { - pfr->peekbuf[pfr->npeeked++] = pfr->pbr->pread_func(pfr->pbr); - } -} - -// ---------------------------------------------------------------- -void pfr_advance_by(peek_file_reader_t* pfr, int len) { - if (len > pfr->npeeked) { - fprintf(stderr, "%s: internal coding error: advance-by %d exceeds buffer depth %d.\n", - MLR_GLOBALS.argv0, len, pfr->npeeked); - exit(1); - } - for (int i = len; i < pfr->npeeked; i++) - pfr->peekbuf[i-len] = pfr->peekbuf[i]; - pfr->npeeked -= len; -} - -// ---------------------------------------------------------------- -void pfr_dump(peek_file_reader_t* pfr) { - // xxx stub - printf("PFR DUMP STUB. MAYBE REMOVE THIS ROUTINE.\n"); -} diff --git a/c/input/peek_file_reader.h b/c/input/peek_file_reader.h index 81736ae85..2138af489 100644 --- a/c/input/peek_file_reader.h +++ b/c/input/peek_file_reader.h @@ -16,15 +16,71 @@ typedef struct _peek_file_reader_t { // xxx to do: try using a ring buffer (power-of-two length >= buflen) instead // of the current slipback buffer, for performance -peek_file_reader_t* pfr_alloc(byte_reader_t* pbr, int maxnpeek); -void pfr_free(peek_file_reader_t* pfr); +// ---------------------------------------------------------------- +static inline peek_file_reader_t* pfr_alloc(byte_reader_t* pbr, int maxnpeek) { + peek_file_reader_t* pfr = mlr_malloc_or_die(sizeof(peek_file_reader_t)); + pfr->pbr = pbr; + pfr->peekbuflen = maxnpeek + 1; + pfr->peekbuf = mlr_malloc_or_die(pfr->peekbuflen); + memset(pfr->peekbuf, 0, pfr->peekbuflen); + pfr->npeeked = 0; -char pfr_peek_char(peek_file_reader_t* pfr); -char pfr_read_char(peek_file_reader_t* pfr); -void pfr_buffer_by(peek_file_reader_t* pfr, int len); -void pfr_advance_by(peek_file_reader_t* pfr, int len); -void pfr_reset(peek_file_reader_t* pfr); + return pfr; +} -void pfr_dump(peek_file_reader_t* pfr); +// ---------------------------------------------------------------- +static inline void pfr_free(peek_file_reader_t* pfr) { + if (pfr == NULL) + return; + free(pfr->peekbuf); + free(pfr); +} + +// ---------------------------------------------------------------- +static inline void pfr_reset(peek_file_reader_t* pfr) { + memset(pfr->peekbuf, 0, pfr->peekbuflen); + pfr->npeeked = 0; +} + +// ---------------------------------------------------------------- +static inline char pfr_peek_char(peek_file_reader_t* pfr) { + if (pfr->npeeked < 1) { + pfr->peekbuf[pfr->npeeked++] = pfr->pbr->pread_func(pfr->pbr); + } + return pfr->peekbuf[0]; +} + +// ---------------------------------------------------------------- +static inline char pfr_read_char(peek_file_reader_t* pfr) { + if (pfr->npeeked < 1) { + return pfr->pbr->pread_func(pfr->pbr); + } else { + // xxx to do: make this a ring buffer to avoid the shifts. + char c = pfr->peekbuf[0]; + for (int i = 1; i < pfr->npeeked; i++) + pfr->peekbuf[i-1] = pfr->peekbuf[i]; + pfr->npeeked--; + return c; + } +} + +// ---------------------------------------------------------------- +static inline void pfr_buffer_by(peek_file_reader_t* pfr, int len) { + while (pfr->npeeked < len) { + pfr->peekbuf[pfr->npeeked++] = pfr->pbr->pread_func(pfr->pbr); + } +} + +// ---------------------------------------------------------------- +static inline void pfr_advance_by(peek_file_reader_t* pfr, int len) { + if (len > pfr->npeeked) { + fprintf(stderr, "%s: internal coding error: advance-by %d exceeds buffer depth %d.\n", + MLR_GLOBALS.argv0, len, pfr->npeeked); + exit(1); + } + for (int i = len; i < pfr->npeeked; i++) + pfr->peekbuf[i-len] = pfr->peekbuf[i]; + pfr->npeeked -= len; +} #endif // PEEK_FILE_READER_H