Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-18 00:45:47 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/johnkerl/miller

Browse source
This commit is contained in:
Thomas Klausner 2015-09-08 11:40:31 +02:00
commit 4459dfeafc
21 changed files with 302 additions and 717 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -6,6 +6,7 @@ mlr-unit-tester
test-mlrutil
test-string-builder
test-byte-readers
test-peek-file-reader
test-parse-trie
test-lrec
test-join-bucket-keeper

7
c/Makefile.am
View file

@ -21,7 +21,6 @@ lib/mlrutil.c \
lib/mlr_globals.c \
lib/string_builder.c \
input/file_reader_mmap.c \
input/old_peek_file_reader.c \
experimental/getlines.c
# ================================================================
@ -63,9 +62,10 @@ mlrp: .always dsls
tests: unit-test reg-test
unit-test: test-mlrutil test-byte-readers test-parse-trie test-lrec test-string-builder test-join-bucket-keeper
unit-test: test-mlrutil test-byte-readers test-peek-file-reader test-parse-trie test-lrec test-string-builder test-join-bucket-keeper
./test-mlrutil
./test-byte-readers
./test-peek-file-reader
./test-parse-trie
./test-lrec
./test-string-builder
@ -83,10 +83,11 @@ dev-tests: dev-unit-test reg-test
# Unfortunately --error-exitcode=1 doesn't work well since there are
# valgrind-detected errors in stdlibs. :(
dev-unit-test: test-mlrutil test-byte-readers test-parse-trie test-lrec test-string-builder test-join-bucket-keeper
dev-unit-test: test-mlrutil test-byte-readers test-peek-file-reader test-parse-trie test-lrec test-string-builder test-join-bucket-keeper
# valgrind --leak-check=full --error-exitcode=1 ./a.out
valgrind --leak-check=full ./test-mlrutil
valgrind --leak-check=full ./test-byte-readers
valgrind --leak-check=full ./test-Peek-file-reader
valgrind --leak-check=full ./test-parse-trie
valgrind --leak-check=full ./test-lrec
valgrind --leak-check=full ./test-string-builder

12
c/cli/mlrcli.c
View file

@ -289,24 +289,14 @@ cli_opts_t* parse_command_line(int argc, char** argv) {
argi++;
}
// ready for cutover! :)
// just leaving it off for a while to compare performance.
#if 0
else if (streq(argv[argi], "--csv")) { popts->ifmt = "csvex"; ofmt = "csv"; }
else if (streq(argv[argi], "--icsv")) { popts->ifmt = "csvex"; }
else if (streq(argv[argi], "--ocsv")) { ofmt = "csv"; }
#else
else if (streq(argv[argi], "--csv")) { popts->ifmt = ofmt = "csv"; }
else if (streq(argv[argi], "--icsv")) { popts->ifmt = "csv"; }
else if (streq(argv[argi], "--ocsv")) { ofmt = "csv"; }
#endif
else if (streq(argv[argi], "--csvlite")) { popts->ifmt = ofmt = "csvlite"; }
else if (streq(argv[argi], "--icsvlite")) { popts->ifmt = "csvlite"; }
else if (streq(argv[argi], "--ocsvlite")) { ofmt = "csvlite"; }
else if (streq(argv[argi], "--csvex")) { popts->ifmt = "csvex"; ofmt = "csv"; }
else if (streq(argv[argi], "--icsvex")) { popts->ifmt = "csvex"; }
else if (streq(argv[argi], "--dkvp")) { popts->ifmt = ofmt = "dkvp"; }
else if (streq(argv[argi], "--idkvp")) { popts->ifmt = "dkvp"; }
else if (streq(argv[argi], "--odkvp")) { ofmt = "dkvp"; }

36
c/containers/parse_trie.c
View file

@ -80,39 +80,3 @@ static void parse_trie_add_string_aux(parse_trie_node_t* pnode, char* string, in
parse_trie_add_string_aux(pnext, &string[1], stridx, len);
}
}
// ----------------------------------------------------------------
// Example input:
// * string 0 is "a"
// * string 1 is "aa"
// * buf is "aaabc"
// Output:
// * return value is TRUE
// * stridx is 1 since longest match is "aa"
// * matchlen is 2 since "aa" has length 2
// xxx cmt longest-prefix match
int parse_trie_match(parse_trie_t* ptrie, char* buf, int buflen, int* pstridx, int* pmatchlen) {
parse_trie_node_t* pnode = ptrie->proot;
parse_trie_node_t* pnext;
parse_trie_node_t* pterm = NULL;
int i;
for (i = 0; i < buflen; i++) {
char c = buf[i];
pnext = pnode->pnexts[(unsigned char) c];
if (pnext == NULL)
break;
if (pnext->strlen > 0) {
pterm = pnext;
}
pnode = pnext;
}
if (pterm == NULL) {
return FALSE;
} else {
*pstridx = pterm->stridx;
*pmatchlen = pterm->strlen;
return TRUE;
}
}

43
c/containers/parse_trie.h
View file

@ -21,6 +21,47 @@ parse_trie_t* parse_trie_alloc();
void parse_trie_free(parse_trie_t* ptrie);
void parse_trie_print(parse_trie_t* ptrie);
void parse_trie_add_string(parse_trie_t* ptrie, char* string, int stridx);
int parse_trie_match(parse_trie_t* ptrie, char* buf, int buflen, int* pstridx, int* pmatchlen);
// ----------------------------------------------------------------
// Example input:
// * string 0 is "a"
// * string 1 is "aa"
// * buf is "aaabc"
// Output:
// * return value is TRUE
// * stridx is 1 since longest match is "aa"
// * matchlen is 2 since "aa" has length 2
// xxx cmt longest-prefix match
// xxx cmt assumption that enough has been peeked -- there is no check here.
// (recall that this is called on every single byte of each data file, e.g.
// for a 1GB data file we'll be in this function a billion times ... it needs
// to be tight.)
// xxx cmt re sob/mask semantics (ring buffer)
static inline int parse_trie_match(parse_trie_t* ptrie, char* buf, int sob, int buflen, int mask, int* pstridx, int* pmatchlen) {
parse_trie_node_t* pnode = ptrie->proot;
parse_trie_node_t* pnext;
parse_trie_node_t* pterm = NULL;
for (int i = 0; i < buflen; i++) {
char c = buf[(sob+i)&mask];
pnext = pnode->pnexts[(unsigned char) c];
if (pnext == NULL)
break;
if (pnext->strlen > 0) {
pterm = pnext;
}
pnode = pnext;
}
if (pterm == NULL) {
return FALSE;
} else {
*pstridx = pterm->stridx;
*pmatchlen = pterm->strlen;
return TRUE;
}
}
#endif // PARSE_TRIE_H

52
c/containers/test_parse_trie.c
View file

@ -36,7 +36,7 @@ static void test_case(
stridx = -2;
matchlen = -2;
rc = parse_trie_match(ptrie, buf, strlen(buf), &stridx, &matchlen);
rc = parse_trie_match(ptrie, buf, 0, strlen(buf), 0xff, &stridx, &matchlen);
parse_trie_free(ptrie);
@ -135,61 +135,61 @@ static char* test_dkvp() {
}
parse_trie_print(ptrie);
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen);
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
mu_assert_lf(rc == TRUE);
mu_assert_lf(stridx == PS_TOKEN);
mu_assert_lf(matchlen == strlen(strings[PS_TOKEN]));
p += matchlen;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen);
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
mu_assert_lf(rc == TRUE);
mu_assert_lf(stridx == FS_TOKEN);
mu_assert_lf(matchlen == strlen(strings[FS_TOKEN]));
p += matchlen;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen);
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
mu_assert_lf(rc == TRUE);
mu_assert_lf(stridx == PS_TOKEN);
mu_assert_lf(matchlen == strlen(strings[PS_TOKEN]));
p += matchlen;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen);
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
mu_assert_lf(rc == TRUE);
mu_assert_lf(stridx == RS_TOKEN);
mu_assert_lf(matchlen == strlen(strings[RS_TOKEN]));
p += matchlen;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen);
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
mu_assert_lf(rc == TRUE);
mu_assert_lf(stridx == PS_TOKEN);
mu_assert_lf(matchlen == strlen(strings[PS_TOKEN]));
p += matchlen;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen); mu_assert_lf(rc == FALSE); p++;
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen);
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
mu_assert_lf(rc == TRUE);
mu_assert_lf(stridx == EOF_TOKEN);
mu_assert_lf(matchlen == strlen(strings[EOF_TOKEN]));
@ -221,7 +221,7 @@ static char* show_it() {
parse_trie_print(ptrie);
while (TRUE) {
rc = parse_trie_match(ptrie, p, strlen(p), &stridx, &matchlen);
rc = parse_trie_match(ptrie, p, 0, strlen(p), 0xff, &stridx, &matchlen);
if (rc) {
printf("match token %d (%s)\n", stridx, strings[stridx]);
p += matchlen;

140
c/experimental/csv0.c
View file

@ -1,140 +0,0 @@
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include "lib/mlrutil.h"
#include "containers/slls.h"
#include "lib/string_builder.h"
#include "input/old_peek_file_reader.h"
#define TERMIND_RS 0x1111
#define TERMIND_FS 0x2222
#define TERMIND_EOF 0x3333
typedef struct _field_wrapper_t {
char* contents;
int termind;
} field_wrapper_t;
typedef struct _record_wrapper_t {
slls_t* contents;
int at_eof;
} record_wrapper_t;
static field_wrapper_t get_csv_field_not_dquoted(old_peek_file_reader_t* pfr, string_builder_t* psb) {
// Note that "\"," etc. will be encoded in the rfc_csv_reader_t ctor -- this is just sketch
printf("\n");
printf("ENTER\n");
while (TRUE) {
if (old_pfr_at_eof(pfr)) {
printf("--case 1\n");
printf("EXIT\n");
return (field_wrapper_t) { .contents = sb_is_empty(psb) ? NULL: sb_finish(psb), .termind = TERMIND_EOF };
} else if (old_pfr_next_is(pfr, ",\xff", 2)) {
printf("--case 2\n");
old_pfr_advance_by(pfr, 2);
printf("EXIT\n");
return (field_wrapper_t) { .contents = sb_finish(psb), .termind = TERMIND_EOF };
} else if (old_pfr_next_is(pfr, ",", 1)) {
printf("--case 3\n");
old_pfr_advance_by(pfr, 1);
printf("EXIT\n");
return (field_wrapper_t) { .contents = sb_finish(psb), .termind = TERMIND_FS };
} else if (old_pfr_next_is(pfr, "\r\n", 2)) {
printf("--case 4\n");
old_pfr_advance_by(pfr, 2);
printf("EXIT\n");
return (field_wrapper_t) { .contents = sb_finish(psb), .termind = TERMIND_RS };
} else {
//old_pfr_dump(pfr);
char c = old_pfr_read_char(pfr);
printf("--case 5 %c [%02x]\n", isprint(c) ? c : '?', c);
//old_pfr_dump(pfr);
sb_append_char(psb, c);
//sb_append_char(psb, old_pfr_read_char(pfr));
}
}
}
static field_wrapper_t get_csv_field_dquoted(old_peek_file_reader_t* pfr, string_builder_t* psb) {
old_pfr_advance_by(pfr, 1);
while (TRUE) {
if (old_pfr_at_eof(pfr)) {
// xxx imbalanced-dquote error
fprintf(stderr, "xxx k0d3 me up b04k3n b04k3n b04ken %d\n", __LINE__);
exit(1);
} else if (old_pfr_next_is(pfr, "\"\xff", 2)) {
old_pfr_advance_by(pfr, 2);
return (field_wrapper_t) { .contents = sb_finish(psb), .termind = TERMIND_EOF };
} else if (old_pfr_next_is(pfr, "\",", 2)) {
old_pfr_advance_by(pfr, 2);
return (field_wrapper_t) { .contents = sb_finish(psb), .termind = TERMIND_FS };
} else if (old_pfr_next_is(pfr, "\"\r\n", 3)) {
old_pfr_advance_by(pfr, 3);
return (field_wrapper_t) { .contents = sb_finish(psb), .termind = TERMIND_RS };
} else {
sb_append_char(psb, old_pfr_read_char(pfr));
}
}
}
field_wrapper_t get_csv_field(old_peek_file_reader_t* pfr, string_builder_t* psb) {
field_wrapper_t wrapper;
if (old_pfr_at_eof(pfr)) {
wrapper.contents = NULL;
wrapper.termind = TERMIND_EOF;
return wrapper;
} else if (old_pfr_next_is(pfr, "\"", 1)) {
return get_csv_field_dquoted(pfr, psb);
} else {
return get_csv_field_not_dquoted(pfr, psb);
}
}
record_wrapper_t get_csv_record(old_peek_file_reader_t* pfr, string_builder_t* psb) {
slls_t* fields = slls_alloc();
record_wrapper_t rwrapper;
rwrapper.contents = fields;
rwrapper.at_eof = FALSE;
while (TRUE) {
field_wrapper_t fwrapper = get_csv_field(pfr, psb);
if (fwrapper.termind == TERMIND_EOF) {
rwrapper.at_eof = TRUE;
}
if (fwrapper.contents != NULL) {
printf("CONT=>>%s<<[%d]\n", fwrapper.contents, (int)strlen(fwrapper.contents));
slls_add_with_free(fields, fwrapper.contents);
}
if (fwrapper.termind != TERMIND_FS)
break;
}
printf("FLEN=%d\n", fields->length);
printf("FEOF=%d\n", rwrapper.at_eof);
if (fields->length == 0 && rwrapper.at_eof) {
slls_free(fields);
rwrapper.contents = NULL;
}
return rwrapper;
}
int main() {
FILE* fp = stdin;
old_peek_file_reader_t* pfr = old_pfr_alloc(fp, 32);
string_builder_t sb;
string_builder_t* psb = &sb;
sb_init(psb, 1024);
while (TRUE) {
record_wrapper_t rwrapper = get_csv_record(pfr, psb);
if (rwrapper.contents != NULL) {
printf("++++ [NF=%d]\n", rwrapper.contents->length);
for (sllse_t* pe = rwrapper.contents->phead; pe != NULL; pe = pe->pnext) {
printf(" [%s]\n", pe->value);
}
slls_free(rwrapper.contents);
}
if (rwrapper.at_eof)
break;
}
return 0;
}

25
c/experimental/pfr.c
View file

@ -1,25 +0,0 @@
#include <stdio.h>
#include <string.h>
#include "input/old_peek_file_reader.h"
int main(int argc, char** argv) {
FILE* fp = stdin;
old_peek_file_reader_t* pfr = old_pfr_alloc(fp, 32);
printf("@eof = %d\n", old_pfr_at_eof(pfr));
old_pfr_dump(pfr);
printf("read 0x%02x\n", (unsigned)old_pfr_read_char(pfr));
old_pfr_dump(pfr);
char* s = "//";
old_pfr_dump(pfr);
printf("next is %s = %d\n", s, old_pfr_next_is(pfr, s, strlen(s)));
old_pfr_dump(pfr);
char c = old_pfr_read_char(pfr);
printf("read %c [0x%02x]\n", c, (unsigned)c);
old_pfr_dump(pfr);
printf("@eof = %d\n", old_pfr_at_eof(pfr));
old_pfr_free(pfr);
return 0;
}

83
c/input/lrec_reader_csvex.c → c/input/lrec_reader_csv.c
View file

@ -1,5 +1,6 @@
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include "lib/mlr_globals.h"
#include "lib/mlrutil.h"
#include "lib/string_builder.h"
@ -11,14 +12,6 @@
#include "containers/lhmslv.h"
#include "containers/parse_trie.h"
// ================================================================
// xxx to do:
// * avoid the separate paster: just inline lrec_put as in the csvlite reader.
// * profile ..............
// * ring buffer in pfr?
// -> split out a separate pfr ut per se
// ================================================================
// Idea of pheader_keepers: each header_keeper object retains the input-line backing
// and the slls_t for a CSV header line which is used by one or more CSV data
// lines. Meanwhile some mappers retain input records from the entire data
@ -44,7 +37,7 @@
#define DQUOTE_DQUOTE_STRIDX 0x2008
// ----------------------------------------------------------------
typedef struct _lrec_reader_csvex_state_t {
typedef struct _lrec_reader_csv_state_t {
// Input line number is not the same as the record-counter in context_t,
// which counts records.
long long ilno;
@ -83,19 +76,19 @@ typedef struct _lrec_reader_csvex_state_t {
header_keeper_t* pheader_keeper;
lhmslv_t* pheader_keepers;
} lrec_reader_csvex_state_t;
} lrec_reader_csv_state_t;
static slls_t* lrec_reader_csvex_get_fields(lrec_reader_csvex_state_t* pstate);
static lrec_t* paste_header_and_data(lrec_reader_csvex_state_t* pstate, slls_t* pdata_fields);
static slls_t* lrec_reader_csv_get_fields(lrec_reader_csv_state_t* pstate);
static lrec_t* paste_header_and_data(lrec_reader_csv_state_t* pstate, slls_t* pdata_fields);
// ----------------------------------------------------------------
// xxx needs abend on null lhs. etc.
static lrec_t* lrec_reader_csvex_process(void* pvstate, void* pvhandle, context_t* pctx) {
lrec_reader_csvex_state_t* pstate = pvstate;
static lrec_t* lrec_reader_csv_process(void* pvstate, void* pvhandle, context_t* pctx) {
lrec_reader_csv_state_t* pstate = pvstate;
if (pstate->expect_header_line_next) {
slls_t* pheader_fields = lrec_reader_csvex_get_fields(pstate);
slls_t* pheader_fields = lrec_reader_csv_get_fields(pstate);
if (pheader_fields == NULL)
return NULL;
pstate->ilno++;
@ -111,14 +104,14 @@ static lrec_t* lrec_reader_csvex_process(void* pvstate, void* pvhandle, context_
}
pstate->ilno++;
slls_t* pdata_fields = lrec_reader_csvex_get_fields(pstate);
slls_t* pdata_fields = lrec_reader_csv_get_fields(pstate);
if (pdata_fields == NULL) // EOF
return NULL;
else
return paste_header_and_data(pstate, pdata_fields);
}
static slls_t* lrec_reader_csvex_get_fields(lrec_reader_csvex_state_t* pstate) {
static slls_t* lrec_reader_csv_get_fields(lrec_reader_csv_state_t* pstate) {
int rc, stridx, matchlen, record_done, field_done;
peek_file_reader_t* pfr = pstate->pfr;
string_builder_t* psb = pstate->psb;
@ -137,8 +130,17 @@ static slls_t* lrec_reader_csvex_get_fields(lrec_reader_csvex_state_t* pstate) {
field_done = FALSE;
while (!field_done) {
pfr_buffer_by(pfr, pstate->pno_dquote_parse_trie->maxlen);
rc = parse_trie_match(pstate->pno_dquote_parse_trie, pfr->peekbuf, pfr->npeeked, &stridx, &matchlen);
rc = parse_trie_match(pstate->pno_dquote_parse_trie,
pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask,
&stridx, &matchlen);
#ifdef DEBUG_PARSER
pfr_dump(pfr);
#endif
if (rc) {
#ifdef DEBUG_PARSER
printf("RC=%d stridx=0x%04x matchlen=%d\n", rc, stridx, matchlen);
#endif
switch(stridx) {
case EOF_STRIDX: // end of record
slls_add_with_free(pfields, sb_finish(psb));
@ -172,7 +174,13 @@ static slls_t* lrec_reader_csvex_get_fields(lrec_reader_csvex_state_t* pstate) {
}
pfr_advance_by(pfr, matchlen);
} else {
#ifdef DEBUG_PARSER
char c = pfr_read_char(pfr);
printf("CHAR=%c [%02x]\n", isprint(c) ? c : ' ', (unsigned)c);
sb_append_char(psb, c);
#else
sb_append_char(psb, pfr_read_char(pfr));
#endif
}
}
@ -183,7 +191,11 @@ static slls_t* lrec_reader_csvex_get_fields(lrec_reader_csvex_state_t* pstate) {
field_done = FALSE;
while (!field_done) {
pfr_buffer_by(pfr, pstate->pdquote_parse_trie->maxlen);
rc = parse_trie_match(pstate->pdquote_parse_trie, pfr->peekbuf, pfr->npeeked, &stridx, &matchlen);
rc = parse_trie_match(pstate->pdquote_parse_trie,
pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask,
&stridx, &matchlen);
if (rc) {
switch(stridx) {
case EOF_STRIDX: // end of record
@ -227,7 +239,7 @@ static slls_t* lrec_reader_csvex_get_fields(lrec_reader_csvex_state_t* pstate) {
}
// ----------------------------------------------------------------
static lrec_t* paste_header_and_data(lrec_reader_csvex_state_t* pstate, slls_t* pdata_fields) {
static lrec_t* paste_header_and_data(lrec_reader_csv_state_t* pstate, slls_t* pdata_fields) {
if (pstate->pheader_keeper->pkeys->length != pdata_fields->length) {
fprintf(stderr, "%s: Header/data length mismatch: %d != %d at line %lld.\n",
MLR_GLOBALS.argv0, pstate->pheader_keeper->pkeys->length, pdata_fields->length, pstate->ilno);
@ -238,34 +250,35 @@ static lrec_t* paste_header_and_data(lrec_reader_csvex_state_t* pstate, slls_t*
sllse_t* pd = pdata_fields->phead;
for ( ; ph != NULL && pd != NULL; ph = ph->pnext, pd = pd->pnext) {
lrec_put_no_free(prec, ph->value, pd->value);
//lrec_put(prec, ph->value, pd->value, LREC_FREE_ENTRY_VALUE);
}
return prec;
}
// ----------------------------------------------------------------
void* lrec_reader_csvex_open(void* pvstate, char* filename) {
lrec_reader_csvex_state_t* pstate = pvstate;
void* lrec_reader_csv_open(void* pvstate, char* filename) {
lrec_reader_csv_state_t* pstate = pvstate;
pstate->pfr->pbr->popen_func(pstate->pfr->pbr, filename);
pfr_reset(pstate->pfr);
return NULL; // xxx modify the API after the functional refactor is complete
}
void lrec_reader_csvex_close(void* pvstate, void* pvhandle) {
lrec_reader_csvex_state_t* pstate = pvstate;
void lrec_reader_csv_close(void* pvstate, void* pvhandle) {
lrec_reader_csv_state_t* pstate = pvstate;
pstate->pfr->pbr->pclose_func(pstate->pfr->pbr);
}
// ----------------------------------------------------------------
// xxx after the pfr/pbr refactor is complete, vsof and vopen may be redundant.
static void lrec_reader_csvex_sof(void* pvstate) {
lrec_reader_csvex_state_t* pstate = pvstate;
static void lrec_reader_csv_sof(void* pvstate) {
lrec_reader_csv_state_t* pstate = pvstate;
pstate->ilno = 0LL;
pstate->expect_header_line_next = TRUE;
}
// ----------------------------------------------------------------
static void lrec_reader_csvex_free(void* pvstate) {
lrec_reader_csvex_state_t* pstate = pvstate;
static void lrec_reader_csv_free(void* pvstate) {
lrec_reader_csv_state_t* pstate = pvstate;
for (lhmslve_t* pe = pstate->pheader_keepers->phead; pe != NULL; pe = pe->pnext) {
header_keeper_t* pheader_keeper = pe->pvvalue;
header_keeper_free(pheader_keeper);
@ -274,10 +287,10 @@ static void lrec_reader_csvex_free(void* pvstate) {
}
// ----------------------------------------------------------------
lrec_reader_t* lrec_reader_csvex_alloc(byte_reader_t* pbr, char irs, char ifs) {
lrec_reader_t* lrec_reader_csv_alloc(byte_reader_t* pbr, char irs, char ifs) {
lrec_reader_t* plrec_reader = mlr_malloc_or_die(sizeof(lrec_reader_t));
lrec_reader_csvex_state_t* pstate = mlr_malloc_or_die(sizeof(lrec_reader_csvex_state_t));
lrec_reader_csv_state_t* pstate = mlr_malloc_or_die(sizeof(lrec_reader_csv_state_t));
pstate->ilno = 0LL;
pstate->eof = "\xff";
@ -337,11 +350,11 @@ lrec_reader_t* lrec_reader_csvex_alloc(byte_reader_t* pbr, char irs, char ifs) {
pstate->pheader_keepers = lhmslv_alloc();
plrec_reader->pvstate = (void*)pstate;
plrec_reader->popen_func = &lrec_reader_csvex_open;
plrec_reader->pclose_func = &lrec_reader_csvex_close;
plrec_reader->pprocess_func = &lrec_reader_csvex_process;
plrec_reader->psof_func = &lrec_reader_csvex_sof;
plrec_reader->pfree_func = &lrec_reader_csvex_free;
plrec_reader->popen_func = &lrec_reader_csv_open;
plrec_reader->pclose_func = &lrec_reader_csv_close;
plrec_reader->pprocess_func = &lrec_reader_csv_process;
plrec_reader->psof_func = &lrec_reader_csv_sof;
plrec_reader->pfree_func = &lrec_reader_csv_free;
return plrec_reader;
}

288
c/input/lrec_reader_stdio_csv.c
View file

@ -1,288 +0,0 @@
#include <stdio.h>
#include <stdlib.h>
#include "lib/mlr_globals.h"
#include "lib/mlrutil.h"
#include "containers/slls.h"
#include "containers/lhmslv.h"
#include "input/file_reader_stdio.h"
#include "input/lrec_readers.h"
#include "lib/string_builder.h"
#include "input/old_peek_file_reader.h"
// Idea of pheader_keepers: each header_keeper object retains the input-line backing
// and the slls_t for a CSV header line which is used by one or more CSV data
// lines. Meanwhile some mappers retain input records from the entire data
// stream, including header-schema changes in the input stream. This means we
// need to keep headers intact as long as any lrecs are pointing to them. One
// option is reference-counting which I experimented with; it was messy and
// error-prone. The approach used here is to keep a hash map from header-schema
// to header_keeper object. The current pheader_keeper is a pointer into one of
// those. Then when the reader is freed, all the header-keepers are freed.
// ----------------------------------------------------------------
#define STRING_BUILDER_INIT_SIZE 1024
#define TERMIND_RS 0x1111
#define TERMIND_FS 0x2222
#define TERMIND_EOF 0x3333
typedef struct _field_wrapper_t {
char* contents;
int termind;
} field_wrapper_t;
typedef struct _record_wrapper_t {
slls_t* contents;
int at_eof;
} record_wrapper_t;
// ----------------------------------------------------------------
typedef struct _lrec_reader_stdio_csv_state_t {
// Input line number is not the same as the record-counter in context_t,
// which counts records.
long long ilno;
char* irs;
char* ifs;
char* dquote_irs;
char* dquote_ifs;
char* dquote_eof;
char* dquote;
char* dquote_dquote;
char* ifs_eof;
int irs_len;
int ifs_len;
int dquote_irs_len;
int dquote_ifs_len;
int dquote_eof_len;
int dquote_len;
int dquote_dquote_len;
int ifs_eof_len;
int peek_buf_len;
string_builder_t sb;
string_builder_t* psb;
old_peek_file_reader_t* pfr;
int expect_header_line_next;
header_keeper_t* pheader_keeper;
lhmslv_t* pheader_keepers;
} lrec_reader_stdio_csv_state_t;
// ----------------------------------------------------------------
static record_wrapper_t lrec_reader_stdio_csv_get_record(lrec_reader_stdio_csv_state_t* pstate);
static field_wrapper_t get_csv_field(lrec_reader_stdio_csv_state_t* pstate);
static field_wrapper_t get_csv_field_not_dquoted(lrec_reader_stdio_csv_state_t* pstate);
static field_wrapper_t get_csv_field_dquoted(lrec_reader_stdio_csv_state_t* pstate);
static lrec_t* paste_header_and_data(lrec_reader_stdio_csv_state_t* pstate, slls_t* pdata_fields);
// ----------------------------------------------------------------
// xxx needs abend on null lhs. etc.
static lrec_t* lrec_reader_stdio_csv_process(void* pvstate, void* pvhandle, context_t* pctx) {
lrec_reader_stdio_csv_state_t* pstate = pvstate;
if (pstate->pfr == NULL) {
pstate->pfr = old_pfr_alloc((FILE*)pvhandle, pstate->peek_buf_len);
}
record_wrapper_t rwrapper;
if (pstate->expect_header_line_next) {
rwrapper = lrec_reader_stdio_csv_get_record(pstate);
if (rwrapper.contents == NULL && rwrapper.at_eof)
return NULL;
pstate->ilno++;
pstate->expect_header_line_next = FALSE;
pstate->pheader_keeper = lhmslv_get(pstate->pheader_keepers, rwrapper.contents);
if (pstate->pheader_keeper == NULL) {
pstate->pheader_keeper = header_keeper_alloc(NULL, rwrapper.contents);
lhmslv_put(pstate->pheader_keepers, rwrapper.contents, pstate->pheader_keeper);
} else { // Re-use the header-keeper in the header cache
slls_free(rwrapper.contents);
}
}
rwrapper = lrec_reader_stdio_csv_get_record(pstate);
if (rwrapper.contents == NULL && rwrapper.at_eof)
return NULL;
pstate->ilno++;
return paste_header_and_data(pstate, rwrapper.contents);
}
static record_wrapper_t lrec_reader_stdio_csv_get_record(lrec_reader_stdio_csv_state_t* pstate) {
slls_t* pfields = slls_alloc();
record_wrapper_t rwrapper;
rwrapper.contents = pfields;
rwrapper.at_eof = FALSE;
while (TRUE) {
field_wrapper_t fwrapper = get_csv_field(pstate);
if (fwrapper.termind == TERMIND_EOF)
rwrapper.at_eof = TRUE;
if (fwrapper.contents != NULL)
slls_add_with_free(pfields, fwrapper.contents);
if (fwrapper.termind != TERMIND_FS)
break;
}
if (pfields->length == 0 && rwrapper.at_eof) {
slls_free(pfields);
rwrapper.contents = NULL;
}
return rwrapper;
}
static field_wrapper_t get_csv_field(lrec_reader_stdio_csv_state_t* pstate) {
field_wrapper_t wrapper;
if (old_pfr_at_eof(pstate->pfr)) {
wrapper.contents = NULL;
wrapper.termind = TERMIND_EOF;
return wrapper;
} else if (old_pfr_next_is(pstate->pfr, pstate->dquote, pstate->dquote_len)) {
old_pfr_advance_by(pstate->pfr, pstate->dquote_len);
return get_csv_field_dquoted(pstate);
} else {
return get_csv_field_not_dquoted(pstate);
}
}
static field_wrapper_t get_csv_field_not_dquoted(lrec_reader_stdio_csv_state_t* pstate) {
while (TRUE) {
if (old_pfr_at_eof(pstate->pfr)) {
return (field_wrapper_t) {
.contents = sb_is_empty(pstate->psb) ? NULL: sb_finish(pstate->psb),
.termind = TERMIND_EOF
};
} else if (old_pfr_next_is(pstate->pfr, pstate->ifs_eof, pstate->ifs_eof_len)) {
old_pfr_advance_by(pstate->pfr, pstate->ifs_eof_len);
return (field_wrapper_t) { .contents = sb_finish(pstate->psb), .termind = TERMIND_EOF };
} else if (old_pfr_next_is(pstate->pfr, pstate->ifs, pstate->ifs_len)) {
old_pfr_advance_by(pstate->pfr, pstate->ifs_len);
return (field_wrapper_t) { .contents = sb_finish(pstate->psb), .termind = TERMIND_FS };
} else if (old_pfr_next_is(pstate->pfr, pstate->irs, pstate->irs_len)) {
old_pfr_advance_by(pstate->pfr, pstate->irs_len);
return (field_wrapper_t) { .contents = sb_finish(pstate->psb), .termind = TERMIND_RS };
} else if (old_pfr_next_is(pstate->pfr, pstate->dquote, pstate->dquote_len)) {
fprintf(stderr, "%s: non-compliant field-internal double-quote at line %lld.\n",
MLR_GLOBALS.argv0, pstate->ilno);
exit(1);
} else {
sb_append_char(pstate->psb, old_pfr_read_char(pstate->pfr));
}
}
}
static field_wrapper_t get_csv_field_dquoted(lrec_reader_stdio_csv_state_t* pstate) {
while (TRUE) {
if (old_pfr_at_eof(pstate->pfr)) {
fprintf(stderr, "%s: imbalanced double-quote at line %lld.\n", MLR_GLOBALS.argv0, pstate->ilno);
exit(1);
} else if (old_pfr_next_is(pstate->pfr, pstate->dquote_eof, pstate->dquote_eof_len)) {
old_pfr_advance_by(pstate->pfr, pstate->dquote_eof_len);
return (field_wrapper_t) { .contents = sb_finish(pstate->psb), .termind = TERMIND_EOF };
} else if (old_pfr_next_is(pstate->pfr, pstate->dquote_ifs, pstate->dquote_ifs_len)) {
old_pfr_advance_by(pstate->pfr, pstate->dquote_ifs_len);
return (field_wrapper_t) { .contents = sb_finish(pstate->psb), .termind = TERMIND_FS };
} else if (old_pfr_next_is(pstate->pfr, pstate->dquote_irs, pstate->dquote_irs_len)) {
old_pfr_advance_by(pstate->pfr, pstate->dquote_irs_len);
return (field_wrapper_t) { .contents = sb_finish(pstate->psb), .termind = TERMIND_RS };
} else if (old_pfr_next_is(pstate->pfr, pstate->dquote_dquote, pstate->dquote_dquote_len)) {
// "" inside a dquoted field is an escape for "
old_pfr_advance_by(pstate->pfr, pstate->dquote_dquote_len);
sb_append_string(pstate->psb, pstate->dquote);
} else {
sb_append_char(pstate->psb, old_pfr_read_char(pstate->pfr));
}
}
}
static lrec_t* paste_header_and_data(lrec_reader_stdio_csv_state_t* pstate, slls_t* pdata_fields) {
if (pstate->pheader_keeper->pkeys->length != pdata_fields->length) {
fprintf(stderr, "%s: Header/data length mismatch: %d != %d at line %lld.\n",
MLR_GLOBALS.argv0, pstate->pheader_keeper->pkeys->length, pdata_fields->length, pstate->ilno);
exit(1);
}
lrec_t* prec = lrec_unbacked_alloc();
sllse_t* ph = pstate->pheader_keeper->pkeys->phead;
sllse_t* pd = pdata_fields->phead;
for ( ; ph != NULL && pd != NULL; ph = ph->pnext, pd = pd->pnext) {
lrec_put_no_free(prec, ph->value, pd->value);
}
return prec;
}
// ----------------------------------------------------------------
static void lrec_reader_stdio_csv_sof(void* pvstate) {
lrec_reader_stdio_csv_state_t* pstate = pvstate;
pstate->ilno = 0LL;
pstate->expect_header_line_next = TRUE;
pstate->pfr = NULL;
}
// ----------------------------------------------------------------
static void lrec_reader_stdio_csv_free(void* pvstate) {
lrec_reader_stdio_csv_state_t* pstate = pvstate;
for (lhmslve_t* pe = pstate->pheader_keepers->phead; pe != NULL; pe = pe->pnext) {
header_keeper_t* pheader_keeper = pe->pvvalue;
header_keeper_free(pheader_keeper);
}
old_pfr_free(pstate->pfr);
}
// ----------------------------------------------------------------
lrec_reader_t* lrec_reader_stdio_csv_alloc(char irs, char ifs) {
lrec_reader_t* plrec_reader = mlr_malloc_or_die(sizeof(lrec_reader_t));
lrec_reader_stdio_csv_state_t* pstate = mlr_malloc_or_die(sizeof(lrec_reader_stdio_csv_state_t));
pstate->ilno = 0LL;
pstate->irs = "\r\n"; // xxx multi-byte the cli irs/ifs/etc, and integrate here
pstate->ifs = ","; // xxx multi-byte the cli irs/ifs/etc, and integrate here
pstate->dquote_irs = mlr_paste_2_strings("\"", pstate->irs);
pstate->dquote_ifs = mlr_paste_2_strings("\"", pstate->ifs);
pstate->dquote_eof = "\"\xff";
pstate->dquote = "\"";
pstate->dquote_dquote = "\"\"";
pstate->ifs_eof = mlr_paste_2_strings(pstate->ifs, "\xff");
pstate->irs_len = strlen(pstate->irs);
pstate->ifs_len = strlen(pstate->ifs);
pstate->dquote_irs_len = strlen(pstate->dquote_irs);
pstate->dquote_ifs_len = strlen(pstate->dquote_ifs);
pstate->dquote_eof_len = strlen(pstate->dquote_eof);
pstate->dquote_len = strlen(pstate->dquote);
pstate->dquote_dquote_len = strlen(pstate->dquote_dquote);
pstate->ifs_eof_len = strlen(pstate->ifs_eof);
pstate->peek_buf_len = pstate->irs_len;
pstate->peek_buf_len = mlr_imax2(pstate->peek_buf_len, pstate->ifs_len);
pstate->peek_buf_len = mlr_imax2(pstate->peek_buf_len, pstate->dquote_irs_len);
pstate->peek_buf_len = mlr_imax2(pstate->peek_buf_len, pstate->dquote_ifs_len);
pstate->peek_buf_len = mlr_imax2(pstate->peek_buf_len, pstate->dquote_eof_len);
pstate->peek_buf_len = mlr_imax2(pstate->peek_buf_len, pstate->dquote_len);
pstate->peek_buf_len = mlr_imax2(pstate->peek_buf_len, pstate->dquote_dquote_len);
pstate->peek_buf_len = mlr_imax2(pstate->peek_buf_len, pstate->ifs_eof_len);
pstate->peek_buf_len += 2;
sb_init(&pstate->sb, STRING_BUILDER_INIT_SIZE);
pstate->psb = &pstate->sb;
pstate->pfr = NULL;
pstate->expect_header_line_next = TRUE;
pstate->pheader_keeper = NULL;
pstate->pheader_keepers = lhmslv_alloc();
plrec_reader->pvstate = (void*)pstate;
plrec_reader->popen_func = &file_reader_stdio_vopen;
plrec_reader->pclose_func = &file_reader_stdio_vclose;
plrec_reader->pprocess_func = &lrec_reader_stdio_csv_process;
plrec_reader->psof_func = &lrec_reader_stdio_csv_sof;
plrec_reader->pfree_func = &lrec_reader_stdio_csv_free;
return plrec_reader;
}

4
c/input/lrec_readers.c
View file

@ -14,14 +14,12 @@ lrec_reader_t* lrec_reader_alloc(char* fmtdesc, int use_mmap, char irs, char if
else
return lrec_reader_stdio_dkvp_alloc(irs, ifs, ips, allow_repeat_ifs);
} else if (streq(fmtdesc, "csv")) {
return lrec_reader_stdio_csv_alloc(irs, ifs);
return lrec_reader_csv_alloc(pbr, irs, ifs);
} else if (streq(fmtdesc, "csvlite")) {
if (use_mmap)
return lrec_reader_mmap_csvlite_alloc(irs, ifs, allow_repeat_ifs);
else
return lrec_reader_stdio_csvlite_alloc(irs, ifs, allow_repeat_ifs);
} else if (streq(fmtdesc, "csvex")) {
return lrec_reader_csvex_alloc(pbr, irs, ifs);
} else if (streq(fmtdesc, "nidx")) {
if (use_mmap)
return lrec_reader_mmap_nidx_alloc(irs, ifs, allow_repeat_ifs);

9
c/input/lrec_readers.h
View file

@ -10,15 +10,14 @@
lrec_reader_t* lrec_reader_alloc(char* fmtdesc, int use_mmap, char irs, char ifs, int allow_repeat_ifs,
char ips, int allow_repeat_ips);
lrec_reader_t* lrec_reader_stdio_csv_alloc(char irs, char ifs);
lrec_reader_t* lrec_reader_stdio_csvlite_alloc(char irs, char ifs, int allow_repeat_ifs);
lrec_reader_t* lrec_reader_csvex_alloc(byte_reader_t* pbr, char irs, char ifs);
lrec_reader_t* lrec_reader_csv_alloc(byte_reader_t* pbr, char irs, char ifs);
lrec_reader_t* lrec_reader_stdio_dkvp_alloc(char irs, char ifs, char ips, int allow_repeat_ifs);
lrec_reader_t* lrec_reader_stdio_nidx_alloc(char irs, char ifs, int allow_repeat_ifs);
lrec_reader_t* lrec_reader_stdio_xtab_alloc(char ips, int allow_repeat_ips);
lrec_reader_t* lrec_reader_mmap_csv_alloc(char irs, char ifs, int allow_repeat_ifs);
lrec_reader_t* lrec_reader_mmap_csvlite_alloc(char irs, char ifs, int allow_repeat_ifs);
lrec_reader_t* lrec_reader_mmap_csv_alloc(char irs, char ifs, int allow_repeat_ifs);
lrec_reader_t* lrec_reader_mmap_csvlite_alloc(char irs, char ifs, int allow_repeat_ifs);
lrec_reader_t* lrec_reader_mmap_dkvp_alloc(char irs, char ifs, char ips, int allow_repeat_ifs);
lrec_reader_t* lrec_reader_mmap_nidx_alloc(char irs, char ifs, int allow_repeat_ifs);
lrec_reader_t* lrec_reader_mmap_xtab_alloc(char irs, char ips, int allow_repeat_ips);
@ -33,8 +32,6 @@ slls_t* split_csv_header_line(char* line, char ifs, int allow_repeat_ifs);
slls_t* split_csvlite_header_line(char* line, char ifs, int allow_repeat_ifs);
lrec_t* lrec_parse_stdio_csvlite_data_line(header_keeper_t* pheader_keeper, char* data_line, char ifs,
int allow_repeat_ifs);
lrec_t* lrec_parse_stdio_csv_data_line(header_keeper_t* pheader_keeper, char* data_line, char ifs,
int allow_repeat_ifs);
lrec_t* lrec_parse_stdio_xtab(slls_t* pxtab_lines, char ips, int allow_repeat_ips);
lrec_t* lrec_parse_mmap_nidx(file_reader_mmap_state_t* phandle, char irs, char ifs, int allow_repeat_ifs);

89
c/input/old_peek_file_reader.c
View file

@ -1,89 +0,0 @@
#include <stdio.h>
#include <ctype.h>
#include "lib/mlrutil.h"
#include "lib/mlr_globals.h"
#include "old_peek_file_reader.h"
// xxx comment about efficiency here: enough to deliver rfc-csv feature with performance tuning still tbd.
// tripartite ascii art w/ chars-to-caller; the peekbuf; pending data in the fp.
// label in particular eof handling.
// ----------------------------------------------------------------
old_peek_file_reader_t* old_pfr_alloc(FILE* fp, int maxnpeek) {
old_peek_file_reader_t* pfr = mlr_malloc_or_die(sizeof(old_peek_file_reader_t));
pfr->fp = fp;
pfr->peekbuflen = maxnpeek + 1;
pfr->peekbuf = mlr_malloc_or_die(pfr->peekbuflen);
memset(pfr->peekbuf, 0, pfr->peekbuflen);
pfr->npeeked = 0;
// Pre-read one char into the peekbuf so that we can say old_pfr_at_eof
// right away on the first call on an empty file.
// getc_unlocked() is appropriate since Miller is single-threaded.
pfr->peekbuf[pfr->npeeked++] = getc_unlocked(pfr->fp); // maybe EOF
return pfr;
}
// ----------------------------------------------------------------
int old_pfr_at_eof(old_peek_file_reader_t* pfr) {
return pfr->npeeked >= 1 && pfr->peekbuf[0] == EOF;
}
// ----------------------------------------------------------------
// xxx inline this for perf.
int old_pfr_next_is(old_peek_file_reader_t* pfr, char* string, int len) {
// xxx abend on len > peekbuflen
while (pfr->npeeked < len) {
char c = getc_unlocked(pfr->fp); // maybe EOF
pfr->peekbuf[pfr->npeeked++] = c;
}
// xxx make a memeq, inlined.
return memcmp(string, pfr->peekbuf, len) == 0;
}
// ----------------------------------------------------------------
char old_pfr_read_char(old_peek_file_reader_t* pfr) {
if (pfr->npeeked == 1 && pfr->peekbuf[0] == EOF) {
return EOF;
} else if (pfr->npeeked == 0) {
pfr->peekbuf[0] = getc_unlocked(pfr->fp); // maybe EOF
pfr->npeeked = 1;
return pfr->peekbuf[0];
} else {
char c = pfr->peekbuf[0];
pfr->npeeked--;
for (int i = 0; i < pfr->npeeked; i++)
pfr->peekbuf[i] = pfr->peekbuf[i+1];
return c;
}
}
// ----------------------------------------------------------------
void old_pfr_advance_by(old_peek_file_reader_t* pfr, int len) {
for (int i = 0; i < len; i++)
old_pfr_read_char(pfr);
}
// ----------------------------------------------------------------
void old_pfr_free(old_peek_file_reader_t* pfr) {
if (pfr == NULL)
return;
free(pfr->peekbuf);
pfr->fp = NULL;
pfr->peekbuf = NULL;
free(pfr);
}
// ----------------------------------------------------------------
void old_pfr_dump(old_peek_file_reader_t* pfr) {
printf("======================== pfr at %p\n", pfr);
printf(" peekbuflen = %d\n", pfr->peekbuflen);
printf(" npeeked = %d\n", pfr->npeeked);
for (int i = 0; i < pfr->npeeked; i++) {
char c = pfr->peekbuf[i];
printf(" i=%d c=%c [%02x]\n", i, isprint((unsigned char)c) ? c : ' ', c);
}
printf("------------------------\n");
}

28
c/input/old_peek_file_reader.h
View file

@ -1,28 +0,0 @@
#ifndef OLD_PEEK_FILE_READER_H
#define OLD_PEEK_FILE_READER_H
#include <stdio.h>
typedef struct _old_peek_file_reader_t {
FILE* fp;
int peekbuflen;
char* peekbuf;
int npeeked;
} old_peek_file_reader_t;
// The caller should fclose the fp, since presumably it will have opened it. We
// could have our constructor do the fopen (taking not fp but filename as
// argument) and the destructor do the fclose but that would break reading from
// stdin.
old_peek_file_reader_t* old_pfr_alloc(FILE* fp, int maxnpeek);
// xxx needing contextual comments here.
int old_pfr_at_eof(old_peek_file_reader_t* pfr);
int old_pfr_next_is(old_peek_file_reader_t* pfr, char* string, int len);
char old_pfr_read_char(old_peek_file_reader_t* pfr);
void old_pfr_advance_by(old_peek_file_reader_t* pfr, int len);
void old_pfr_free(old_peek_file_reader_t* pfr);
void old_pfr_dump(old_peek_file_reader_t* pfr);
#endif // OLD_PEEK_FILE_READER_H

28
c/input/peek_file_reader.c Normal file
View file

@ -0,0 +1,28 @@
#include <stdio.h>
#include <ctype.h>
#include "input/peek_file_reader.h"
// ----------------------------------------------------------------
void pfr_dump(peek_file_reader_t* pfr) {
printf("======================== pfr at %p\n", pfr);
printf(" peekbuflen = %d\n", pfr->peekbuflen);
printf(" npeeked = %d\n", pfr->npeeked);
int sob = pfr->sob;
int eob = (pfr->sob + pfr->npeeked) & pfr->peekbuflenmask;
for (int i = 0; i < pfr->peekbuflen; i++) {
char c = pfr->peekbuf[i];
char* sdesc = (i == sob) ? "START" : "";
char* edesc = (i == eob) ? "END" : "";
char* occdesc = "";
if (sob <= eob) {
if (sob <= i && i < eob)
occdesc = "OCC";
} else {
if (i < eob || sob <= i)
occdesc = "OCC";
}
printf(" %-5s %-5s %-3s i=%2d c=%c [%02x]\n",
sdesc, edesc, occdesc, i, isprint((unsigned char)c) ? c : ' ', c);
}
printf("------------------------\n");
}

50
c/input/peek_file_reader.h
View file

@ -2,28 +2,38 @@
#define PEEK_FILE_READER_H
#include <stdio.h>
#include "lib/mlrutil.h"
#include "lib/mlrmath.h"
#include "lib/mlr_globals.h"
#include "input/byte_reader.h"
// This is a ring-buffered peekahead file/string reader.
// Note: Throughout Miller as a general rule I treat struct attributes as if
// there were private attributes. However, for performance, parse_trie_match
// accesses this ring buffer directly.
typedef struct _peek_file_reader_t {
byte_reader_t* pbr;
int peekbuflen;
int peekbuflenmask;
char* peekbuf;
int sob; // start of ring-buffer
int npeeked;
} peek_file_reader_t;
// xxx needing contextual comments here.
// xxx to do: try using a ring buffer (power-of-two length >= buflen) instead
// of the current slipback buffer, for performance
// ----------------------------------------------------------------
static inline peek_file_reader_t* pfr_alloc(byte_reader_t* pbr, int maxnpeek) {
peek_file_reader_t* pfr = mlr_malloc_or_die(sizeof(peek_file_reader_t));
pfr->pbr = pbr;
pfr->peekbuflen = maxnpeek + 1;
pfr->peekbuf = mlr_malloc_or_die(pfr->peekbuflen);
memset(pfr->peekbuf, 0, pfr->peekbuflen);
pfr->npeeked = 0;
pfr->pbr = pbr;
pfr->peekbuflen = power_of_two_ceil(maxnpeek);
pfr->peekbuflenmask = pfr->peekbuflen - 1;
// The peek-buffer doesn't contain null-terminated C strings, but we
// nonetheless null-terminate the buffer with an extra never-touched byte
// so that print statements in the debugger, etc. will be nice.
pfr->peekbuf = mlr_malloc_or_die(pfr->peekbuflen + 1);
memset(pfr->peekbuf, 0, pfr->peekbuflen + 1);
pfr->sob = 0;
pfr->npeeked = 0;
return pfr;
}
@ -39,15 +49,17 @@ static inline void pfr_free(peek_file_reader_t* pfr) {
// ----------------------------------------------------------------
static inline void pfr_reset(peek_file_reader_t* pfr) {
memset(pfr->peekbuf, 0, pfr->peekbuflen);
pfr->sob = 0;
pfr->npeeked = 0;
}
// ----------------------------------------------------------------
static inline char pfr_peek_char(peek_file_reader_t* pfr) {
if (pfr->npeeked < 1) {
pfr->peekbuf[pfr->npeeked++] = pfr->pbr->pread_func(pfr->pbr);
pfr->peekbuf[pfr->sob] = pfr->pbr->pread_func(pfr->pbr);
pfr->npeeked++;
}
return pfr->peekbuf[0];
return pfr->peekbuf[pfr->sob];
}
// ----------------------------------------------------------------
@ -55,10 +67,8 @@ static inline char pfr_read_char(peek_file_reader_t* pfr) {
if (pfr->npeeked < 1) {
return pfr->pbr->pread_func(pfr->pbr);
} else {
// xxx to do: make this a ring buffer to avoid the shifts.
char c = pfr->peekbuf[0];
for (int i = 1; i < pfr->npeeked; i++)
pfr->peekbuf[i-1] = pfr->peekbuf[i];
char c = pfr->peekbuf[pfr->sob];
pfr->sob = (pfr->sob + 1) & pfr->peekbuflenmask;
pfr->npeeked--;
return c;
}
@ -67,7 +77,7 @@ static inline char pfr_read_char(peek_file_reader_t* pfr) {
// ----------------------------------------------------------------
static inline void pfr_buffer_by(peek_file_reader_t* pfr, int len) {
while (pfr->npeeked < len) {
pfr->peekbuf[pfr->npeeked++] = pfr->pbr->pread_func(pfr->pbr);
pfr->peekbuf[(pfr->sob + pfr->npeeked++) & pfr->peekbuflenmask] = pfr->pbr->pread_func(pfr->pbr);
}
}
@ -78,9 +88,11 @@ static inline void pfr_advance_by(peek_file_reader_t* pfr, int len) {
MLR_GLOBALS.argv0, len, pfr->npeeked);
exit(1);
}
for (int i = len; i < pfr->npeeked; i++)
pfr->peekbuf[i-len] = pfr->peekbuf[i];
pfr->sob = (pfr->sob + len) & pfr->peekbuflenmask;
pfr->npeeked -= len;
}
// ----------------------------------------------------------------
void pfr_dump(peek_file_reader_t* pfr);
#endif // PEEK_FILE_READER_H

17
c/input/stdio_byte_reader.c
View file

@ -13,11 +13,15 @@ typedef struct _stdio_byte_reader_state_t {
static int stdio_byte_reader_open_func(struct _byte_reader_t* pbr, char* filename) {
stdio_byte_reader_state_t* pstate = mlr_malloc_or_die(sizeof(stdio_byte_reader_state_t));
pstate->filename = strdup(filename);
pstate->fp = fopen(filename, "r");
if (pstate->fp == NULL) {
perror("fopen");
fprintf(stderr, "%s: Couldn't fopen \"%s\" for read.\n", MLR_GLOBALS.argv0, filename);
exit(1);
if (streq(pstate->filename, "-")) {
pstate->fp = stdin;
} else {
pstate->fp = fopen(filename, "r");
if (pstate->fp == NULL) {
perror("fopen");
fprintf(stderr, "%s: Couldn't fopen \"%s\" for read.\n", MLR_GLOBALS.argv0, filename);
exit(1);
}
}
pbr->pvstate = pstate;
return TRUE;
@ -36,7 +40,8 @@ static int stdio_byte_reader_read_func(struct _byte_reader_t* pbr) {
static void stdio_byte_reader_close_func(struct _byte_reader_t* pbr) {
stdio_byte_reader_state_t* pstate = pbr->pvstate;
fclose(pstate->fp);
if (pstate->fp != stdin)
fclose(pstate->fp);
free(pstate);
pbr->pvstate = NULL;
}

90
c/input/test_peek_file_reader.c Normal file
View file

@ -0,0 +1,90 @@
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include "lib/mlrutil.h"
#include "lib/minunit.h"
#include "lib/mlr_test_util.h"
#include "input/byte_readers.h"
#include "input/peek_file_reader.h"
#ifdef __TEST_PEEK_FILE_READER_MAIN__
int tests_run = 0;
int tests_failed = 0;
int assertions_run = 0;
int assertions_failed = 0;
// ----------------------------------------------------------------
static char* test_empty() {
byte_reader_t* pbr = string_byte_reader_alloc();
int ok = pbr->popen_func(pbr, "");
mu_assert_lf(ok == TRUE);
peek_file_reader_t* pfr = pfr_alloc(pbr, 7);
mu_assert_lf(pfr_peek_char(pfr) == EOF);
mu_assert_lf(pfr_read_char(pfr) == EOF);
pbr->pclose_func(pbr);
pfr_free(pfr);
return NULL;
}
// ----------------------------------------------------------------
static char* test_non_empty() {
byte_reader_t* pbr = string_byte_reader_alloc();
int ok = pbr->popen_func(pbr,
"ab,cde\n"
"123,4567\n"
);
mu_assert_lf(ok == TRUE);
peek_file_reader_t* pfr = pfr_alloc(pbr, 7);
pfr_dump(pfr); mu_assert_lf(pfr_peek_char(pfr) == 'a');
pfr_dump(pfr); mu_assert_lf(pfr_read_char(pfr) == 'a');
pfr_dump(pfr); mu_assert_lf(pfr_peek_char(pfr) == 'b');
pfr_dump(pfr); mu_assert_lf(pfr_read_char(pfr) == 'b');
pfr_dump(pfr); mu_assert_lf(pfr_peek_char(pfr) == ',');
pfr_dump(pfr); mu_assert_lf(pfr_peek_char(pfr) == ',');
pfr_dump(pfr); mu_assert_lf(pfr_read_char(pfr) == ',');
pfr_dump(pfr); pfr_buffer_by(pfr, 5);
pfr_dump(pfr); pfr_advance_by(pfr, 5);
pfr_dump(pfr); mu_assert_lf(pfr_read_char(pfr) == '2');
pfr_dump(pfr); mu_assert_lf(pfr_peek_char(pfr) == '3');
pfr_dump(pfr); mu_assert_lf(pfr_peek_char(pfr) == '3');
pfr_dump(pfr); mu_assert_lf(pfr_read_char(pfr) == '3');
pfr_dump(pfr); pfr_buffer_by(pfr, 5);
pfr_dump(pfr); pfr_advance_by(pfr, 5);
pfr_dump(pfr); mu_assert_lf(pfr_read_char(pfr) == '\n');
pbr->pclose_func(pbr);
pfr_free(pfr);
return NULL;
}
// ================================================================
static char * run_all_tests() {
mu_run_test(test_empty);
mu_run_test(test_non_empty);
return 0;
}
int main(int argc, char **argv) {
char *result = run_all_tests();
printf("\n");
if (result != 0) {
printf("Not all unit tests passed\n");
}
else {
printf("TEST_PEEK_FILE_READER: ALL UNIT TESTS PASSED\n");
}
printf("Tests passed: %d of %d\n", tests_run - tests_failed, tests_run);
printf("Assertions passed: %d of %d\n", assertions_run - assertions_failed, assertions_run);
return result != 0;
}
#endif // __TEST_PEEK_FILE_READER_MAIN__

9
c/lib/mlrutil.c
View file

@ -286,3 +286,12 @@ int mlr_imax2(int a, int b) {
else
return b;
}
// ----------------------------------------------------------------
// This is inefficient. It's quite fine for call-once, small-n use.
int power_of_two_ceil(int n) {
while (n&(n-1))
n++;
return n;
}

1
c/lib/mlrutil.h
View file

@ -66,5 +66,6 @@ time_t mlr_timegm (struct tm *tm);
int strlen_for_utf8_display(char* str);
int mlr_imax2(int a, int b);
int power_of_two_ceil(int n);
#endif // MLRUTIL_H

7
c/todo.txt
View file

@ -1,7 +1,12 @@
================================================================
BUGFIXES
:D
$ sh foo|termcvt --lf2crlf|mlr --no-mmap --csv check
fopen: No such file or directory
mlr: Couldn't fopen "-" for read.
-> needs UT case!
! memory leak in csv reader! careful about slls data, and do not use lrec_put_no_free
================================================================
TOP OF LIST