Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-22 23:48:30 +00:00
Code Issues Projects Releases Wiki Activity

[read performance iterate] byte-reader iterate

Browse source
This commit is contained in:
John Kerl 2015-09-04 21:53:19 -04:00
parent e6c39a8856
commit d045b863e4
8 changed files with 54 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

6
c/Makefile
View file

@ -42,7 +42,7 @@ containers/slls.c containers/lhmslv.c \
input/file_reader_mmap.c input/file_reader_stdio.c \
input/lrec_reader_stdio_csv.c \
input/lrec_reader_mmap_csvlite.c input/lrec_reader_stdio_csvlite.c \
input/peek_file_reader.c \
input/old_peek_file_reader.c \
input/lrec_reader_mmap_dkvp.c input/lrec_reader_stdio_dkvp.c \
input/lrec_reader_mmap_nidx.c input/lrec_reader_stdio_nidx.c \
input/lrec_reader_mmap_xtab.c input/lrec_reader_stdio_xtab.c \
@ -62,7 +62,7 @@ input/lrec_reader_mmap_dkvp.c input/lrec_reader_stdio_dkvp.c \
input/lrec_reader_mmap_nidx.c input/lrec_reader_stdio_nidx.c \
input/lrec_reader_mmap_xtab.c input/lrec_reader_stdio_xtab.c \
input/file_reader_mmap.c input/file_reader_stdio.c \
input/peek_file_reader.c \
input/old_peek_file_reader.c \
containers/test_join_bucket_keeper.c
EXPERIMENTAL_READER_SRCS = \
@ -70,7 +70,7 @@ lib/mlrutil.c \
lib/mlr_globals.c \
lib/string_builder.c \
input/file_reader_mmap.c \
input/peek_file_reader.c \
input/old_peek_file_reader.c \
experimental/getlines.c
# ================================================================

12
c/experimental/csv0.c
View file

@ -4,7 +4,7 @@
#include "lib/mlrutil.h"
#include "containers/slls.h"
#include "lib/string_builder.h"
#include "input/peek_file_reader.h"
#include "input/old_peek_file_reader.h"
#define TERMIND_RS 0x1111
#define TERMIND_FS 0x2222
@ -20,7 +20,7 @@ typedef struct _record_wrapper_t {
int at_eof;
} record_wrapper_t;
static field_wrapper_t get_csv_field_not_dquoted(peek_file_reader_t* pfr, string_builder_t* psb) {
static field_wrapper_t get_csv_field_not_dquoted(old_peek_file_reader_t* pfr, string_builder_t* psb) {
// Note that "\"," etc. will be encoded in the rfc_csv_reader_t ctor -- this is just sketch
printf("\n");
printf("ENTER\n");
@ -55,7 +55,7 @@ static field_wrapper_t get_csv_field_not_dquoted(peek_file_reader_t* pfr, string
}
}
static field_wrapper_t get_csv_field_dquoted(peek_file_reader_t* pfr, string_builder_t* psb) {
static field_wrapper_t get_csv_field_dquoted(old_peek_file_reader_t* pfr, string_builder_t* psb) {
pfr_advance_by(pfr, 1);
while (TRUE) {
if (pfr_at_eof(pfr)) {
@ -77,7 +77,7 @@ static field_wrapper_t get_csv_field_dquoted(peek_file_reader_t* pfr, string_bui
}
}
field_wrapper_t get_csv_field(peek_file_reader_t* pfr, string_builder_t* psb) {
field_wrapper_t get_csv_field(old_peek_file_reader_t* pfr, string_builder_t* psb) {
field_wrapper_t wrapper;
if (pfr_at_eof(pfr)) {
wrapper.contents = NULL;
@ -90,7 +90,7 @@ field_wrapper_t get_csv_field(peek_file_reader_t* pfr, string_builder_t* psb) {
}
}
record_wrapper_t get_csv_record(peek_file_reader_t* pfr, string_builder_t* psb) {
record_wrapper_t get_csv_record(old_peek_file_reader_t* pfr, string_builder_t* psb) {
slls_t* fields = slls_alloc();
record_wrapper_t rwrapper;
rwrapper.contents = fields;
@ -118,7 +118,7 @@ record_wrapper_t get_csv_record(peek_file_reader_t* pfr, string_builder_t* psb)
int main() {
FILE* fp = stdin;
peek_file_reader_t* pfr = pfr_alloc(fp, 32);
old_peek_file_reader_t* pfr = pfr_alloc(fp, 32);
string_builder_t sb;
string_builder_t* psb = &sb;
sb_init(psb, 1024);

8
c/experimental/getlines.c
View file

@ -6,7 +6,7 @@
#include "input/file_reader_mmap.h"
#include "input/lrec_readers.h"
#include "lib/string_builder.h"
#include "input/peek_file_reader.h"
#include "input/old_peek_file_reader.h"
#define PEEK_BUF_LEN 32
#define STRING_BUILDER_INIT_SIZE 1024
@ -263,7 +263,7 @@ static int read_file_mmap_psb(char* filename, int do_write) {
}
// ================================================================
static char* read_line_pfr_psb(peek_file_reader_t* pfr, string_builder_t* psb, char* irs, int irs_len) {
static char* read_line_pfr_psb(old_peek_file_reader_t* pfr, string_builder_t* psb, char* irs, int irs_len) {
while (TRUE) {
if (pfr_at_eof(pfr)) {
if (sb_is_empty(psb))
@ -284,7 +284,7 @@ static int read_file_pfr_psb(char* filename, int do_write) {
char* irs = "\n";
int irs_len = strlen(irs);
peek_file_reader_t* pfr = pfr_alloc(fp, PEEK_BUF_LEN);
old_peek_file_reader_t* pfr = pfr_alloc(fp, PEEK_BUF_LEN);
string_builder_t sb;
string_builder_t* psb = &sb;
sb_init(&sb, STRING_BUILDER_INIT_SIZE);
@ -441,4 +441,4 @@ int main(int argc, char** argv) {
// * getc_unlocked vs. fgetc, no-brainer for this single-threaded code.
// * string-builder is a little than fixed-length malloc, as expected
// -- it's adding value.
// ! peek_file_reader is where the optimization opportunities are
// ! old_peek_file_reader is where the optimization opportunities are

4
c/experimental/pfr.c
View file

@ -1,10 +1,10 @@
#include <stdio.h>
#include <string.h>
#include "input/peek_file_reader.h"
#include "input/old_peek_file_reader.h"
int main(int argc, char** argv) {
FILE* fp = stdin;
peek_file_reader_t* pfr = pfr_alloc(fp, 32);
old_peek_file_reader_t* pfr = pfr_alloc(fp, 32);
printf("@eof = %d\n", pfr_at_eof(pfr));
pfr_dump(pfr);

4
c/input/lrec_reader_stdio_csv.c
View file

@ -7,7 +7,7 @@
#include "input/file_reader_stdio.h"
#include "input/lrec_readers.h"
#include "lib/string_builder.h"
#include "input/peek_file_reader.h"
#include "input/old_peek_file_reader.h"
// Idea of pheader_keepers: each header_keeper object retains the input-line backing
// and the slls_t for a CSV header line which is used by one or more CSV data
@ -64,7 +64,7 @@ typedef struct _lrec_reader_stdio_csv_state_t {
string_builder_t sb;
string_builder_t* psb;
peek_file_reader_t* pfr;
old_peek_file_reader_t* pfr;
int expect_header_line_next;
header_keeper_t* pheader_keeper;

18
c/input/peek_file_reader.c → c/input/old_peek_file_reader.c
View file

@ -2,7 +2,7 @@
#include <ctype.h>
#include "lib/mlrutil.h"
#include "lib/mlr_globals.h"
#include "peek_file_reader.h"
#include "old_peek_file_reader.h"
// xxx comment about efficiency here: enough to deliver rfc-csv feature with performance tuning still tbd.
@ -10,8 +10,8 @@
// label in particular eof handling.
// ----------------------------------------------------------------
peek_file_reader_t* pfr_alloc(FILE* fp, int maxnpeek) {
peek_file_reader_t* pfr = mlr_malloc_or_die(sizeof(peek_file_reader_t));
old_peek_file_reader_t* pfr_alloc(FILE* fp, int maxnpeek) {
old_peek_file_reader_t* pfr = mlr_malloc_or_die(sizeof(old_peek_file_reader_t));
pfr->fp = fp;
pfr->peekbuflen = maxnpeek + 1;
pfr->peekbuf = mlr_malloc_or_die(pfr->peekbuflen);
@ -27,13 +27,13 @@ peek_file_reader_t* pfr_alloc(FILE* fp, int maxnpeek) {
}
// ----------------------------------------------------------------
int pfr_at_eof(peek_file_reader_t* pfr) {
int pfr_at_eof(old_peek_file_reader_t* pfr) {
return pfr->npeeked >= 1 && pfr->peekbuf[0] == EOF;
}
// ----------------------------------------------------------------
// xxx inline this for perf.
int pfr_next_is(peek_file_reader_t* pfr, char* string, int len) {
int pfr_next_is(old_peek_file_reader_t* pfr, char* string, int len) {
// xxx abend on len > peekbuflen
while (pfr->npeeked < len) {
char c = getc_unlocked(pfr->fp); // maybe EOF
@ -44,7 +44,7 @@ int pfr_next_is(peek_file_reader_t* pfr, char* string, int len) {
}
// ----------------------------------------------------------------
char pfr_read_char(peek_file_reader_t* pfr) {
char pfr_read_char(old_peek_file_reader_t* pfr) {
if (pfr->npeeked == 1 && pfr->peekbuf[0] == EOF) {
return EOF;
} else if (pfr->npeeked == 0) {
@ -61,13 +61,13 @@ char pfr_read_char(peek_file_reader_t* pfr) {
}
// ----------------------------------------------------------------
void pfr_advance_by(peek_file_reader_t* pfr, int len) {
void pfr_advance_by(old_peek_file_reader_t* pfr, int len) {
for (int i = 0; i < len; i++)
pfr_read_char(pfr);
}
// ----------------------------------------------------------------
void pfr_free(peek_file_reader_t* pfr) {
void pfr_free(old_peek_file_reader_t* pfr) {
if (pfr == NULL)
return;
free(pfr->peekbuf);
@ -77,7 +77,7 @@ void pfr_free(peek_file_reader_t* pfr) {
}
// ----------------------------------------------------------------
void pfr_dump(peek_file_reader_t* pfr) {
void pfr_dump(old_peek_file_reader_t* pfr) {
printf("======================== pfr at %p\n", pfr);
printf(" peekbuflen = %d\n", pfr->peekbuflen);
printf(" npeeked = %d\n", pfr->npeeked);

28
c/input/old_peek_file_reader.h Normal file
View file

@ -0,0 +1,28 @@
#ifndef OLD_PEEK_FILE_READER_H
#define OLD_PEEK_FILE_READER_H
#include <stdio.h>
typedef struct _old_peek_file_reader_t {
FILE* fp;
int peekbuflen;
char* peekbuf;
int npeeked;
} old_peek_file_reader_t;
// The caller should fclose the fp, since presumably it will have opened it. We
// could have our constructor do the fopen (taking not fp but filename as
// argument) and the destructor do the fclose but that would break reading from
// stdin.
old_peek_file_reader_t* pfr_alloc(FILE* fp, int maxnpeek);
// xxx needing contextual comments here.
int pfr_at_eof(old_peek_file_reader_t* pfr);
int pfr_next_is(old_peek_file_reader_t* pfr, char* string, int len);
char pfr_read_char(old_peek_file_reader_t* pfr);
void pfr_advance_by(old_peek_file_reader_t* pfr, int len);
void pfr_free(old_peek_file_reader_t* pfr);
void pfr_dump(old_peek_file_reader_t* pfr);
#endif // OLD_PEEK_FILE_READER_H

27
c/input/peek_file_reader.h
View file

@ -1,27 +0,0 @@
#ifndef PEEK_FILE_READER_H
#define PEEK_FILE_READER_H
#include <stdio.h>
typedef struct _peek_file_reader_t {
FILE* fp;
int peekbuflen;
char* peekbuf;
int npeeked;
} peek_file_reader_t;
// The caller should fclose the fp, since presumably it will have opened it. We
// could have our constructor fopen (taking not fp but filename as argument)
// and the destructor fclose but that would break reading from stdin.
peek_file_reader_t* pfr_alloc(FILE* fp, int maxnpeek);
// xxx needing contextual comments here.
int pfr_at_eof(peek_file_reader_t* pfr);
int pfr_next_is(peek_file_reader_t* pfr, char* string, int len);
char pfr_read_char(peek_file_reader_t* pfr);
void pfr_advance_by(peek_file_reader_t* pfr, int len);
void pfr_free(peek_file_reader_t* pfr);
void pfr_dump(peek_file_reader_t* pfr);
#endif // PEEK_FILE_READER_H