From 0f65ff759ab3d2f5423064064ce64fe3db47cbf4 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sat, 21 Sep 2019 19:48:23 -0400 Subject: [PATCH] neaten CSV parser-debug --- c/input/lrec_reader_stdio_csv.c | 34 +++++++++++++++++++++++++++------ doc/manpage.html | 2 +- doc/manpage.txt | 2 +- doc/mlr.1 | 4 ++-- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/c/input/lrec_reader_stdio_csv.c b/c/input/lrec_reader_stdio_csv.c index c6213492b..aa2ccef66 100644 --- a/c/input/lrec_reader_stdio_csv.c +++ b/c/input/lrec_reader_stdio_csv.c @@ -50,7 +50,7 @@ #define UTF8_BOM "\xef\xbb\xbf" #define UTF8_BOM_LENGTH 3 -#define DEBUG_PARSER +//#define DEBUG_PARSER // ---------------------------------------------------------------- typedef struct _lrec_reader_stdio_csv_state_t { @@ -337,6 +337,9 @@ static int lrec_reader_stdio_csv_get_fields(lrec_reader_stdio_csv_state_t* pstat context_t* pctx, int is_header) { int rc, token = 0, matchlen = 0, record_done = FALSE, field_done = FALSE; +#ifdef DEBUG_PARSER + char c = 0; +#endif peek_file_reader_t* pfr = pstate->pfr; string_builder_t* psb = pstate->psb; char* field = NULL; @@ -356,7 +359,9 @@ static int lrec_reader_stdio_csv_get_fields(lrec_reader_stdio_csv_state_t* pstat pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask, &token, &matchlen); #ifdef DEBUG_PARSER - printf("RC=%d token=0x%04x matchlen=%d\n", rc, token, matchlen); + if (rc) { + printf("BOMRC=%d token=0x%04x matchlen=%d\n", rc, token, matchlen); + } #endif if (rc == TRUE && token == UTF8_BOM_TOKEN) { pfr_advance_by(pfr, matchlen); @@ -382,7 +387,7 @@ static int lrec_reader_stdio_csv_get_fields(lrec_reader_stdio_csv_state_t* pstat #endif if (rc) { #ifdef DEBUG_PARSER - printf("RC=%d token=0x%04x matchlen=%d\n", rc, token, matchlen); + printf("DQ=0 RC=%d token=0x%04x matchlen=%d\n", rc, token, matchlen); #endif switch(token) { case EOF_TOKEN: // end of record @@ -403,7 +408,7 @@ static int lrec_reader_stdio_csv_get_fields(lrec_reader_stdio_csv_state_t* pstat case IRS_TOKEN: // end of record field = sb_finish_with_length(psb, &field_length); - // The line-ending '\n' won't be included in the field buffer. + // The line-ending '\n' wonlt be included in the field buffer. if (pstate->do_auto_line_term) { if (field_length > 0 && field[field_length-1] == '\r') { field[field_length-1] = 0; @@ -431,7 +436,7 @@ static int lrec_reader_stdio_csv_get_fields(lrec_reader_stdio_csv_state_t* pstat pfr_advance_by(pfr, matchlen); } else { #ifdef DEBUG_PARSER - char c = pfr_read_char(pfr); + c = pfr_read_char(pfr); printf("CHAR=%c [%02x]\n", isprint((unsigned char)c) ? c : ' ', (unsigned)c); sb_append_char(psb, c); #else @@ -453,8 +458,13 @@ static int lrec_reader_stdio_csv_get_fields(lrec_reader_stdio_csv_state_t* pstat rc = parse_trie_ring_match(pstate->pdquote_parse_trie, pfr->peekbuf, pfr->sob, pfr->npeeked, pfr->peekbuflenmask, &token, &matchlen); - +#ifdef DEBUG_PARSER + pfr_print(pfr); +#endif if (rc) { +#ifdef DEBUG_PARSER + printf("DQ=1 RC=%d token=0x%04x matchlen=%d\n", rc, token, matchlen); +#endif switch(token) { case EOF_TOKEN: // end of record fprintf(stderr, "%s: unmatched double quote at line %lld.\n", @@ -490,7 +500,13 @@ static int lrec_reader_stdio_csv_get_fields(lrec_reader_stdio_csv_state_t* pstat record_done = TRUE; break; case DQUOTE_DQUOTE_TOKEN: // RFC-4180 CSV: "" inside a dquoted field is an escape for " +#ifdef DEBUG_PARSER + c = pstate->dquote[0]; + printf("CHAR=%c [%02x]\n", isprint((unsigned char)c) ? c : ' ', (unsigned)c); + sb_append_char(psb, c); +#else sb_append_char(psb, pstate->dquote[0]); +#endif break; default: fprintf(stderr, "%s: internal coding error: unexpected token %d at line %lld.\n", @@ -500,7 +516,13 @@ static int lrec_reader_stdio_csv_get_fields(lrec_reader_stdio_csv_state_t* pstat } pfr_advance_by(pfr, matchlen); } else { +#ifdef DEBUG_PARSER + c = pfr_read_char(pfr); + printf("CHAR=%c [%02x]\n", isprint((unsigned char)c) ? c : ' ', (unsigned)c); + sb_append_char(psb, c); +#else sb_append_char(psb, pfr_read_char(pfr)); +#endif } } diff --git a/doc/manpage.html b/doc/manpage.html index a28e8c3d9..516bfdad4 100644 --- a/doc/manpage.html +++ b/doc/manpage.html @@ -2498,7 +2498,7 @@ SEE ALSO - 2019-09-17 MILLER(1) + 2019-09-21 MILLER(1)

diff --git a/doc/manpage.txt b/doc/manpage.txt index e07ec88f4..296fba9e8 100644 --- a/doc/manpage.txt +++ b/doc/manpage.txt @@ -2298,4 +2298,4 @@ SEE ALSO - 2019-09-17 MILLER(1) + 2019-09-21 MILLER(1) diff --git a/doc/mlr.1 b/doc/mlr.1 index 08ab00f8e..d890d190f 100644 --- a/doc/mlr.1 +++ b/doc/mlr.1 @@ -2,12 +2,12 @@ .\" Title: mlr .\" Author: [see the "AUTHOR" section] .\" Generator: ./mkman.rb -.\" Date: 2019-09-17 +.\" Date: 2019-09-21 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "MILLER" "1" "2019-09-17" "\ \&" "\ \&" +.TH "MILLER" "1" "2019-09-21" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Portability definitions .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~