From d045b863e4de6654a2a6affe7e9e7dce5046dcaa Mon Sep 17 00:00:00 2001 From: John Kerl Date: Fri, 4 Sep 2015 21:53:19 -0400 Subject: [PATCH] [read performance iterate] byte-reader iterate --- c/Makefile | 6 ++-- c/experimental/csv0.c | 12 ++++---- c/experimental/getlines.c | 8 +++--- c/experimental/pfr.c | 4 +-- c/input/lrec_reader_stdio_csv.c | 4 +-- ...k_file_reader.c => old_peek_file_reader.c} | 18 ++++++------ c/input/old_peek_file_reader.h | 28 +++++++++++++++++++ c/input/peek_file_reader.h | 27 ------------------ 8 files changed, 54 insertions(+), 53 deletions(-) rename c/input/{peek_file_reader.c => old_peek_file_reader.c} (83%) create mode 100644 c/input/old_peek_file_reader.h delete mode 100644 c/input/peek_file_reader.h diff --git a/c/Makefile b/c/Makefile index 5eee01bbc..78f7f1a18 100644 --- a/c/Makefile +++ b/c/Makefile @@ -42,7 +42,7 @@ containers/slls.c containers/lhmslv.c \ input/file_reader_mmap.c input/file_reader_stdio.c \ input/lrec_reader_stdio_csv.c \ input/lrec_reader_mmap_csvlite.c input/lrec_reader_stdio_csvlite.c \ -input/peek_file_reader.c \ +input/old_peek_file_reader.c \ input/lrec_reader_mmap_dkvp.c input/lrec_reader_stdio_dkvp.c \ input/lrec_reader_mmap_nidx.c input/lrec_reader_stdio_nidx.c \ input/lrec_reader_mmap_xtab.c input/lrec_reader_stdio_xtab.c \ @@ -62,7 +62,7 @@ input/lrec_reader_mmap_dkvp.c input/lrec_reader_stdio_dkvp.c \ input/lrec_reader_mmap_nidx.c input/lrec_reader_stdio_nidx.c \ input/lrec_reader_mmap_xtab.c input/lrec_reader_stdio_xtab.c \ input/file_reader_mmap.c input/file_reader_stdio.c \ -input/peek_file_reader.c \ +input/old_peek_file_reader.c \ containers/test_join_bucket_keeper.c EXPERIMENTAL_READER_SRCS = \ @@ -70,7 +70,7 @@ lib/mlrutil.c \ lib/mlr_globals.c \ lib/string_builder.c \ input/file_reader_mmap.c \ -input/peek_file_reader.c \ +input/old_peek_file_reader.c \ experimental/getlines.c # ================================================================ diff --git a/c/experimental/csv0.c b/c/experimental/csv0.c index 7f85f6fd7..b3d6ba77e 100644 --- a/c/experimental/csv0.c +++ b/c/experimental/csv0.c @@ -4,7 +4,7 @@ #include "lib/mlrutil.h" #include "containers/slls.h" #include "lib/string_builder.h" -#include "input/peek_file_reader.h" +#include "input/old_peek_file_reader.h" #define TERMIND_RS 0x1111 #define TERMIND_FS 0x2222 @@ -20,7 +20,7 @@ typedef struct _record_wrapper_t { int at_eof; } record_wrapper_t; -static field_wrapper_t get_csv_field_not_dquoted(peek_file_reader_t* pfr, string_builder_t* psb) { +static field_wrapper_t get_csv_field_not_dquoted(old_peek_file_reader_t* pfr, string_builder_t* psb) { // Note that "\"," etc. will be encoded in the rfc_csv_reader_t ctor -- this is just sketch printf("\n"); printf("ENTER\n"); @@ -55,7 +55,7 @@ static field_wrapper_t get_csv_field_not_dquoted(peek_file_reader_t* pfr, string } } -static field_wrapper_t get_csv_field_dquoted(peek_file_reader_t* pfr, string_builder_t* psb) { +static field_wrapper_t get_csv_field_dquoted(old_peek_file_reader_t* pfr, string_builder_t* psb) { pfr_advance_by(pfr, 1); while (TRUE) { if (pfr_at_eof(pfr)) { @@ -77,7 +77,7 @@ static field_wrapper_t get_csv_field_dquoted(peek_file_reader_t* pfr, string_bui } } -field_wrapper_t get_csv_field(peek_file_reader_t* pfr, string_builder_t* psb) { +field_wrapper_t get_csv_field(old_peek_file_reader_t* pfr, string_builder_t* psb) { field_wrapper_t wrapper; if (pfr_at_eof(pfr)) { wrapper.contents = NULL; @@ -90,7 +90,7 @@ field_wrapper_t get_csv_field(peek_file_reader_t* pfr, string_builder_t* psb) { } } -record_wrapper_t get_csv_record(peek_file_reader_t* pfr, string_builder_t* psb) { +record_wrapper_t get_csv_record(old_peek_file_reader_t* pfr, string_builder_t* psb) { slls_t* fields = slls_alloc(); record_wrapper_t rwrapper; rwrapper.contents = fields; @@ -118,7 +118,7 @@ record_wrapper_t get_csv_record(peek_file_reader_t* pfr, string_builder_t* psb) int main() { FILE* fp = stdin; - peek_file_reader_t* pfr = pfr_alloc(fp, 32); + old_peek_file_reader_t* pfr = pfr_alloc(fp, 32); string_builder_t sb; string_builder_t* psb = &sb; sb_init(psb, 1024); diff --git a/c/experimental/getlines.c b/c/experimental/getlines.c index 6fea63b31..5c6c581b1 100644 --- a/c/experimental/getlines.c +++ b/c/experimental/getlines.c @@ -6,7 +6,7 @@ #include "input/file_reader_mmap.h" #include "input/lrec_readers.h" #include "lib/string_builder.h" -#include "input/peek_file_reader.h" +#include "input/old_peek_file_reader.h" #define PEEK_BUF_LEN 32 #define STRING_BUILDER_INIT_SIZE 1024 @@ -263,7 +263,7 @@ static int read_file_mmap_psb(char* filename, int do_write) { } // ================================================================ -static char* read_line_pfr_psb(peek_file_reader_t* pfr, string_builder_t* psb, char* irs, int irs_len) { +static char* read_line_pfr_psb(old_peek_file_reader_t* pfr, string_builder_t* psb, char* irs, int irs_len) { while (TRUE) { if (pfr_at_eof(pfr)) { if (sb_is_empty(psb)) @@ -284,7 +284,7 @@ static int read_file_pfr_psb(char* filename, int do_write) { char* irs = "\n"; int irs_len = strlen(irs); - peek_file_reader_t* pfr = pfr_alloc(fp, PEEK_BUF_LEN); + old_peek_file_reader_t* pfr = pfr_alloc(fp, PEEK_BUF_LEN); string_builder_t sb; string_builder_t* psb = &sb; sb_init(&sb, STRING_BUILDER_INIT_SIZE); @@ -441,4 +441,4 @@ int main(int argc, char** argv) { // * getc_unlocked vs. fgetc, no-brainer for this single-threaded code. // * string-builder is a little than fixed-length malloc, as expected // -- it's adding value. -// ! peek_file_reader is where the optimization opportunities are +// ! old_peek_file_reader is where the optimization opportunities are diff --git a/c/experimental/pfr.c b/c/experimental/pfr.c index 0f4e84de7..a802b6d30 100644 --- a/c/experimental/pfr.c +++ b/c/experimental/pfr.c @@ -1,10 +1,10 @@ #include #include -#include "input/peek_file_reader.h" +#include "input/old_peek_file_reader.h" int main(int argc, char** argv) { FILE* fp = stdin; - peek_file_reader_t* pfr = pfr_alloc(fp, 32); + old_peek_file_reader_t* pfr = pfr_alloc(fp, 32); printf("@eof = %d\n", pfr_at_eof(pfr)); pfr_dump(pfr); diff --git a/c/input/lrec_reader_stdio_csv.c b/c/input/lrec_reader_stdio_csv.c index 4b3c12f99..33bae3fe1 100644 --- a/c/input/lrec_reader_stdio_csv.c +++ b/c/input/lrec_reader_stdio_csv.c @@ -7,7 +7,7 @@ #include "input/file_reader_stdio.h" #include "input/lrec_readers.h" #include "lib/string_builder.h" -#include "input/peek_file_reader.h" +#include "input/old_peek_file_reader.h" // Idea of pheader_keepers: each header_keeper object retains the input-line backing // and the slls_t for a CSV header line which is used by one or more CSV data @@ -64,7 +64,7 @@ typedef struct _lrec_reader_stdio_csv_state_t { string_builder_t sb; string_builder_t* psb; - peek_file_reader_t* pfr; + old_peek_file_reader_t* pfr; int expect_header_line_next; header_keeper_t* pheader_keeper; diff --git a/c/input/peek_file_reader.c b/c/input/old_peek_file_reader.c similarity index 83% rename from c/input/peek_file_reader.c rename to c/input/old_peek_file_reader.c index 387fd1efb..36426cf12 100644 --- a/c/input/peek_file_reader.c +++ b/c/input/old_peek_file_reader.c @@ -2,7 +2,7 @@ #include #include "lib/mlrutil.h" #include "lib/mlr_globals.h" -#include "peek_file_reader.h" +#include "old_peek_file_reader.h" // xxx comment about efficiency here: enough to deliver rfc-csv feature with performance tuning still tbd. @@ -10,8 +10,8 @@ // label in particular eof handling. // ---------------------------------------------------------------- -peek_file_reader_t* pfr_alloc(FILE* fp, int maxnpeek) { - peek_file_reader_t* pfr = mlr_malloc_or_die(sizeof(peek_file_reader_t)); +old_peek_file_reader_t* pfr_alloc(FILE* fp, int maxnpeek) { + old_peek_file_reader_t* pfr = mlr_malloc_or_die(sizeof(old_peek_file_reader_t)); pfr->fp = fp; pfr->peekbuflen = maxnpeek + 1; pfr->peekbuf = mlr_malloc_or_die(pfr->peekbuflen); @@ -27,13 +27,13 @@ peek_file_reader_t* pfr_alloc(FILE* fp, int maxnpeek) { } // ---------------------------------------------------------------- -int pfr_at_eof(peek_file_reader_t* pfr) { +int pfr_at_eof(old_peek_file_reader_t* pfr) { return pfr->npeeked >= 1 && pfr->peekbuf[0] == EOF; } // ---------------------------------------------------------------- // xxx inline this for perf. -int pfr_next_is(peek_file_reader_t* pfr, char* string, int len) { +int pfr_next_is(old_peek_file_reader_t* pfr, char* string, int len) { // xxx abend on len > peekbuflen while (pfr->npeeked < len) { char c = getc_unlocked(pfr->fp); // maybe EOF @@ -44,7 +44,7 @@ int pfr_next_is(peek_file_reader_t* pfr, char* string, int len) { } // ---------------------------------------------------------------- -char pfr_read_char(peek_file_reader_t* pfr) { +char pfr_read_char(old_peek_file_reader_t* pfr) { if (pfr->npeeked == 1 && pfr->peekbuf[0] == EOF) { return EOF; } else if (pfr->npeeked == 0) { @@ -61,13 +61,13 @@ char pfr_read_char(peek_file_reader_t* pfr) { } // ---------------------------------------------------------------- -void pfr_advance_by(peek_file_reader_t* pfr, int len) { +void pfr_advance_by(old_peek_file_reader_t* pfr, int len) { for (int i = 0; i < len; i++) pfr_read_char(pfr); } // ---------------------------------------------------------------- -void pfr_free(peek_file_reader_t* pfr) { +void pfr_free(old_peek_file_reader_t* pfr) { if (pfr == NULL) return; free(pfr->peekbuf); @@ -77,7 +77,7 @@ void pfr_free(peek_file_reader_t* pfr) { } // ---------------------------------------------------------------- -void pfr_dump(peek_file_reader_t* pfr) { +void pfr_dump(old_peek_file_reader_t* pfr) { printf("======================== pfr at %p\n", pfr); printf(" peekbuflen = %d\n", pfr->peekbuflen); printf(" npeeked = %d\n", pfr->npeeked); diff --git a/c/input/old_peek_file_reader.h b/c/input/old_peek_file_reader.h new file mode 100644 index 000000000..6ea844bf0 --- /dev/null +++ b/c/input/old_peek_file_reader.h @@ -0,0 +1,28 @@ +#ifndef OLD_PEEK_FILE_READER_H +#define OLD_PEEK_FILE_READER_H + +#include + +typedef struct _old_peek_file_reader_t { + FILE* fp; + int peekbuflen; + char* peekbuf; + int npeeked; +} old_peek_file_reader_t; + +// The caller should fclose the fp, since presumably it will have opened it. We +// could have our constructor do the fopen (taking not fp but filename as +// argument) and the destructor do the fclose but that would break reading from +// stdin. + +old_peek_file_reader_t* pfr_alloc(FILE* fp, int maxnpeek); +// xxx needing contextual comments here. +int pfr_at_eof(old_peek_file_reader_t* pfr); +int pfr_next_is(old_peek_file_reader_t* pfr, char* string, int len); +char pfr_read_char(old_peek_file_reader_t* pfr); +void pfr_advance_by(old_peek_file_reader_t* pfr, int len); +void pfr_free(old_peek_file_reader_t* pfr); + +void pfr_dump(old_peek_file_reader_t* pfr); + +#endif // OLD_PEEK_FILE_READER_H diff --git a/c/input/peek_file_reader.h b/c/input/peek_file_reader.h deleted file mode 100644 index 1df2e2341..000000000 --- a/c/input/peek_file_reader.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef PEEK_FILE_READER_H -#define PEEK_FILE_READER_H - -#include - -typedef struct _peek_file_reader_t { - FILE* fp; - int peekbuflen; - char* peekbuf; - int npeeked; -} peek_file_reader_t; - -// The caller should fclose the fp, since presumably it will have opened it. We -// could have our constructor fopen (taking not fp but filename as argument) -// and the destructor fclose but that would break reading from stdin. - -peek_file_reader_t* pfr_alloc(FILE* fp, int maxnpeek); -// xxx needing contextual comments here. -int pfr_at_eof(peek_file_reader_t* pfr); -int pfr_next_is(peek_file_reader_t* pfr, char* string, int len); -char pfr_read_char(peek_file_reader_t* pfr); -void pfr_advance_by(peek_file_reader_t* pfr, int len); -void pfr_free(peek_file_reader_t* pfr); - -void pfr_dump(peek_file_reader_t* pfr); - -#endif // PEEK_FILE_READER_H