Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-23 07:59:37 +00:00
Code Issues Projects Releases Wiki Activity

[read performance iterate] CSV performance tuning

Browse source
This commit is contained in:
John Kerl 2015-09-06 23:36:28 -04:00
parent 0180482e6b
commit 267c899ee3
4 changed files with 71 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

1
c/Makefile
View file

@ -70,7 +70,6 @@ input/lrec_reader_mmap_nidx.c input/lrec_reader_stdio_nidx.c \
input/lrec_reader_mmap_xtab.c input/lrec_reader_stdio_xtab.c \
input/file_reader_mmap.c input/file_reader_stdio.c \
input/old_peek_file_reader.c \
input/peek_file_reader.c \
containers/test_join_bucket_keeper.c
EXPERIMENTAL_READER_SRCS = \

7
c/input/lrec_reader_csvex.c
View file

@ -11,6 +11,13 @@
#include "containers/lhmslv.h"
#include "containers/parse_trie.h"
// ================================================================
// xxx to do:
// * avoid the separate paster: just inline lrec_put as in the csvlite reader.
// * profile ..............
// * ring buffer in pfr?
// ================================================================
// Idea of pheader_keepers: each header_keeper object retains the input-line backing
// and the slls_t for a CSV header line which is used by one or more CSV data
// lines. Meanwhile some mappers retain input records from the entire data

76
c/input/peek_file_reader.c
View file

@ -1,76 +0,0 @@
#include "lib/mlr_globals.h"
#include "lib/mlrutil.h"
#include "input/peek_file_reader.h"
// ----------------------------------------------------------------
peek_file_reader_t* pfr_alloc(byte_reader_t* pbr, int maxnpeek) {
peek_file_reader_t* pfr = mlr_malloc_or_die(sizeof(peek_file_reader_t));
pfr->pbr = pbr;
pfr->peekbuflen = maxnpeek + 1;
pfr->peekbuf = mlr_malloc_or_die(pfr->peekbuflen);
memset(pfr->peekbuf, 0, pfr->peekbuflen);
pfr->npeeked = 0;
return pfr;
}
// ----------------------------------------------------------------
void pfr_free(peek_file_reader_t* pfr) {
if (pfr == NULL)
return;
free(pfr->peekbuf);
free(pfr);
}
// ----------------------------------------------------------------
void pfr_reset(peek_file_reader_t* pfr) {
memset(pfr->peekbuf, 0, pfr->peekbuflen);
pfr->npeeked = 0;
}
// ----------------------------------------------------------------
char pfr_peek_char(peek_file_reader_t* pfr) {
if (pfr->npeeked < 1) {
pfr->peekbuf[pfr->npeeked++] = pfr->pbr->pread_func(pfr->pbr);
}
return pfr->peekbuf[0];
}
// ----------------------------------------------------------------
char pfr_read_char(peek_file_reader_t* pfr) {
if (pfr->npeeked < 1) {
return pfr->pbr->pread_func(pfr->pbr);
} else {
// xxx to do: make this a ring buffer to avoid the shifts.
char c = pfr->peekbuf[0];
for (int i = 1; i < pfr->npeeked; i++)
pfr->peekbuf[i-1] = pfr->peekbuf[i];
pfr->npeeked--;
return c;
}
}
// ----------------------------------------------------------------
void pfr_buffer_by(peek_file_reader_t* pfr, int len) {
while (pfr->npeeked < len) {
pfr->peekbuf[pfr->npeeked++] = pfr->pbr->pread_func(pfr->pbr);
}
}
// ----------------------------------------------------------------
void pfr_advance_by(peek_file_reader_t* pfr, int len) {
if (len > pfr->npeeked) {
fprintf(stderr, "%s: internal coding error: advance-by %d exceeds buffer depth %d.\n",
MLR_GLOBALS.argv0, len, pfr->npeeked);
exit(1);
}
for (int i = len; i < pfr->npeeked; i++)
pfr->peekbuf[i-len] = pfr->peekbuf[i];
pfr->npeeked -= len;
}
// ----------------------------------------------------------------
void pfr_dump(peek_file_reader_t* pfr) {
// xxx stub
printf("PFR DUMP STUB. MAYBE REMOVE THIS ROUTINE.\n");
}

72
c/input/peek_file_reader.h
View file

@ -16,15 +16,71 @@ typedef struct _peek_file_reader_t {
// xxx to do: try using a ring buffer (power-of-two length >= buflen) instead
// of the current slipback buffer, for performance
peek_file_reader_t* pfr_alloc(byte_reader_t* pbr, int maxnpeek);
void pfr_free(peek_file_reader_t* pfr);
// ----------------------------------------------------------------
static inline peek_file_reader_t* pfr_alloc(byte_reader_t* pbr, int maxnpeek) {
peek_file_reader_t* pfr = mlr_malloc_or_die(sizeof(peek_file_reader_t));
pfr->pbr = pbr;
pfr->peekbuflen = maxnpeek + 1;
pfr->peekbuf = mlr_malloc_or_die(pfr->peekbuflen);
memset(pfr->peekbuf, 0, pfr->peekbuflen);
pfr->npeeked = 0;
char pfr_peek_char(peek_file_reader_t* pfr);
char pfr_read_char(peek_file_reader_t* pfr);
void pfr_buffer_by(peek_file_reader_t* pfr, int len);
void pfr_advance_by(peek_file_reader_t* pfr, int len);
void pfr_reset(peek_file_reader_t* pfr);
return pfr;
}
void pfr_dump(peek_file_reader_t* pfr);
// ----------------------------------------------------------------
static inline void pfr_free(peek_file_reader_t* pfr) {
if (pfr == NULL)
return;
free(pfr->peekbuf);
free(pfr);
}
// ----------------------------------------------------------------
static inline void pfr_reset(peek_file_reader_t* pfr) {
memset(pfr->peekbuf, 0, pfr->peekbuflen);
pfr->npeeked = 0;
}
// ----------------------------------------------------------------
static inline char pfr_peek_char(peek_file_reader_t* pfr) {
if (pfr->npeeked < 1) {
pfr->peekbuf[pfr->npeeked++] = pfr->pbr->pread_func(pfr->pbr);
}
return pfr->peekbuf[0];
}
// ----------------------------------------------------------------
static inline char pfr_read_char(peek_file_reader_t* pfr) {
if (pfr->npeeked < 1) {
return pfr->pbr->pread_func(pfr->pbr);
} else {
// xxx to do: make this a ring buffer to avoid the shifts.
char c = pfr->peekbuf[0];
for (int i = 1; i < pfr->npeeked; i++)
pfr->peekbuf[i-1] = pfr->peekbuf[i];
pfr->npeeked--;
return c;
}
}
// ----------------------------------------------------------------
static inline void pfr_buffer_by(peek_file_reader_t* pfr, int len) {
while (pfr->npeeked < len) {
pfr->peekbuf[pfr->npeeked++] = pfr->pbr->pread_func(pfr->pbr);
}
}
// ----------------------------------------------------------------
static inline void pfr_advance_by(peek_file_reader_t* pfr, int len) {
if (len > pfr->npeeked) {
fprintf(stderr, "%s: internal coding error: advance-by %d exceeds buffer depth %d.\n",
MLR_GLOBALS.argv0, len, pfr->npeeked);
exit(1);
}
for (int i = len; i < pfr->npeeked; i++)
pfr->peekbuf[i-len] = pfr->peekbuf[i];
pfr->npeeked -= len;
}
#endif // PEEK_FILE_READER_H