From 49f60a40c0c1957cfd11c47fc698babb1b7a6df5 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sat, 28 Jan 2017 21:01:31 -0500 Subject: [PATCH] autodetect iterate: lrec readers --- c/input/lrec_reader_mmap_dkvp.c | 30 ++++++++++---- c/input/lrec_reader_mmap_nidx.c | 32 +++++++++++---- c/input/lrec_reader_stdio_dkvp.c | 45 +++++++++++--------- c/input/lrec_reader_stdio_nidx.c | 70 +++++++++++++++++++++++++++++++- c/todo.txt | 2 + 5 files changed, 142 insertions(+), 37 deletions(-) diff --git a/c/input/lrec_reader_mmap_dkvp.c b/c/input/lrec_reader_mmap_dkvp.c index 85834cd36..e2fa9fc4e 100644 --- a/c/input/lrec_reader_mmap_dkvp.c +++ b/c/input/lrec_reader_mmap_dkvp.c @@ -148,15 +148,22 @@ lrec_t* lrec_parse_mmap_dkvp_single_irs_single_others(file_reader_mmap_state_t * for ( ; p < phandle->eof && *p; ) { if (*p == irs) { *p = 0; - if (do_auto_irs && !pctx->auto_irs_detected) { + + if (do_auto_irs) { if (p > line && p[-1] == '\r') { p[-1] = 0; - pctx->auto_irs = "\r\n"; + if (!pctx->auto_irs_detected) { + pctx->auto_irs = "\r\n"; + pctx->auto_irs_detected = TRUE; + } } else { - pctx->auto_irs = "\n"; + if (!pctx->auto_irs_detected) { + pctx->auto_irs = "\n"; + pctx->auto_irs_detected = TRUE; + } } - pctx->auto_irs_detected = TRUE; } + phandle->sol = p+1; saw_rs = TRUE; break; @@ -373,15 +380,22 @@ lrec_t* lrec_parse_mmap_dkvp_single_irs_multi_others(file_reader_mmap_state_t *p for ( ; p < phandle->eof && *p; ) { if (*p == irs) { *p = 0; - if (do_auto_irs && !pctx->auto_irs_detected) { + + if (do_auto_irs) { if (p > line && p[-1] == '\r') { p[-1] = 0; - pctx->auto_irs = "\r\n"; + if (!pctx->auto_irs_detected) { + pctx->auto_irs = "\r\n"; + pctx->auto_irs_detected = TRUE; + } } else { - pctx->auto_irs = "\n"; + if (!pctx->auto_irs_detected) { + pctx->auto_irs = "\n"; + pctx->auto_irs_detected = TRUE; + } } - pctx->auto_irs_detected = TRUE; } + phandle->sol = p+1; saw_rs = TRUE; break; diff --git a/c/input/lrec_reader_mmap_nidx.c b/c/input/lrec_reader_mmap_nidx.c index 5be0a21d3..8c0d529a3 100644 --- a/c/input/lrec_reader_mmap_nidx.c +++ b/c/input/lrec_reader_mmap_nidx.c @@ -142,15 +142,22 @@ lrec_t* lrec_parse_mmap_nidx_single_irs_single_ifs(file_reader_mmap_state_t *pha for ( ; p < phandle->eof && *p; ) { if (*p == irs) { *p = 0; - if (do_auto_irs && !pctx->auto_irs_detected) { - if (p > line && p[1] == '\r') { + + if (do_auto_irs) { + if (p > line && p[-1] == '\r') { p[-1] = 0; - pctx->auto_irs = "\r\n"; + if (!pctx->auto_irs_detected) { + pctx->auto_irs = "\r\n"; + pctx->auto_irs_detected = TRUE; + } } else { - pctx->auto_irs = "\n"; + if (!pctx->auto_irs_detected) { + pctx->auto_irs = "\n"; + pctx->auto_irs_detected = TRUE; + } } - pctx->auto_irs_detected = TRUE; } + phandle->sol = p+1; saw_rs = TRUE; break; @@ -216,15 +223,22 @@ lrec_t* lrec_parse_mmap_nidx_single_irs_multi_ifs(file_reader_mmap_state_t *phan for ( ; p < phandle->eof && *p; ) { if (*p == irs) { *p = 0; - if (do_auto_irs && !pctx->auto_irs_detected) { + + if (do_auto_irs) { if (p > line && p[-1] == '\r') { p[-1] = 0; - pctx->auto_irs = "\r\n"; + if (!pctx->auto_irs_detected) { + pctx->auto_irs = "\r\n"; + pctx->auto_irs_detected = TRUE; + } } else { - pctx->auto_irs = "\n"; + if (!pctx->auto_irs_detected) { + pctx->auto_irs = "\n"; + pctx->auto_irs_detected = TRUE; + } } - pctx->auto_irs_detected = TRUE; } + phandle->sol = p+1; saw_rs = TRUE; break; diff --git a/c/input/lrec_reader_stdio_dkvp.c b/c/input/lrec_reader_stdio_dkvp.c index d5f954989..173ed709d 100644 --- a/c/input/lrec_reader_stdio_dkvp.c +++ b/c/input/lrec_reader_stdio_dkvp.c @@ -23,7 +23,6 @@ typedef struct _lrec_reader_stdio_dkvp_state_t { int ifslen; int ipslen; int allow_repeat_ifs; - int do_auto_irs; } lrec_reader_stdio_dkvp_state_t; static void lrec_reader_stdio_dkvp_free(lrec_reader_t* preader); @@ -53,7 +52,6 @@ lrec_reader_t* lrec_reader_stdio_dkvp_alloc(char* irs, char* ifs, char* ips, int pstate->ifslen = strlen(ifs); pstate->ipslen = strlen(ips); pstate->allow_repeat_ifs = allow_repeat_ifs; - pstate->do_auto_irs = FALSE; plrec_reader->pvstate = (void*)pstate; plrec_reader->popen_func = file_reader_stdio_vopen; @@ -63,18 +61,17 @@ lrec_reader_t* lrec_reader_stdio_dkvp_alloc(char* irs, char* ifs, char* ips, int // either case the final character is "\n". Then for autodetect we // simply check if there's a character in the line before the '\n', and // if that is '\r'. - pstate->do_auto_irs = TRUE; pstate->irs = "\n"; pstate->irslen = 1; plrec_reader->pprocess_func = (pstate->ifslen == 1 && pstate->ipslen == 1) ? lrec_reader_stdio_dkvp_process_single_irs_single_others_auto_irs : lrec_reader_stdio_dkvp_process_single_irs_multi_others_auto_irs; } else if (pstate->irslen == 1) { - plrec_reader->pprocess_func = (pstate->ifslen == 1 && pstate->ipslen == 1) + plrec_reader->pprocess_func = (pstate->ifslen == 1) ? &lrec_reader_stdio_dkvp_process_single_irs_single_others : &lrec_reader_stdio_dkvp_process_single_irs_multi_others; } else { - plrec_reader->pprocess_func = (pstate->ifslen == 1 && pstate->ipslen == 1) + plrec_reader->pprocess_func = (pstate->ifslen == 1) ? &lrec_reader_stdio_dkvp_process_multi_irs_single_others : &lrec_reader_stdio_dkvp_process_multi_irs_multi_others; } @@ -104,17 +101,22 @@ static lrec_t* lrec_reader_stdio_dkvp_process_single_irs_single_others_auto_irs( if (line == NULL) { return NULL; } else { - if (!pctx->auto_irs_detected) { - // mlr_get_cline_with_length will have already chomped the trailing '\n', - // and it won't be included in the line length. - if (line_length > 0 && line[line_length-1] == '\r') { - line[line_length-1] = 0; + + // mlr_get_cline_with_length will have already chomped the trailing '\n', + // and it won't be included in the line length. + if (line_length > 0 && line[line_length-1] == '\r') { + line[line_length-1] = 0; + if (!pctx->auto_irs_detected) { + pctx->auto_irs_detected = TRUE; pctx->auto_irs = "\r\n"; - } else { + } + } else { + if (!pctx->auto_irs_detected) { + pctx->auto_irs_detected = TRUE; pctx->auto_irs = "\n"; } - pctx->auto_irs_detected = TRUE; } + return lrec_parse_stdio_dkvp_single_sep(line, pstate->ifs[0], pstate->ips[0], pstate->allow_repeat_ifs); } } @@ -129,17 +131,22 @@ static lrec_t* lrec_reader_stdio_dkvp_process_single_irs_multi_others_auto_irs( if (line == NULL) { return NULL; } else { - if (!pctx->auto_irs_detected) { - // mlr_get_cline_with_length will have already chomped the trailing '\n', - // and it won't be included in the line length. - if (line_length > 0 && line[line_length-1] == '\r') { - line[line_length-1] = 0; + + // mlr_get_cline_with_length will have already chomped the trailing '\n', + // and it won't be included in the line length. + if (line_length > 0 && line[line_length-1] == '\r') { + line[line_length-1] = 0; + if (!pctx->auto_irs_detected) { + pctx->auto_irs_detected = TRUE; pctx->auto_irs = "\r\n"; - } else { + } + } else { + if (!pctx->auto_irs_detected) { + pctx->auto_irs_detected = TRUE; pctx->auto_irs = "\n"; } - pctx->auto_irs_detected = TRUE; } + return lrec_parse_stdio_dkvp_multi_sep(line, pstate->ifs, pstate->ips, pstate->ifslen, pstate->ipslen, pstate->allow_repeat_ifs); } diff --git a/c/input/lrec_reader_stdio_nidx.c b/c/input/lrec_reader_stdio_nidx.c index 7f5d9d314..358315b0b 100644 --- a/c/input/lrec_reader_stdio_nidx.c +++ b/c/input/lrec_reader_stdio_nidx.c @@ -23,6 +23,8 @@ typedef struct _lrec_reader_stdio_nidx_state_t { static void lrec_reader_stdio_nidx_free(lrec_reader_t* preader); static void lrec_reader_stdio_nidx_sof(void* pvstate, void* pvhandle); +static lrec_t* lrec_reader_stdio_nidx_process_single_irs_single_ifs_auto_irs(void* pvstate, void* pvhandle, context_t* pctx); +static lrec_t* lrec_reader_stdio_nidx_process_single_irs_multi_ifs_auto_irs(void* pvstate, void* pvhandle, context_t* pctx); static lrec_t* lrec_reader_stdio_nidx_process_single_irs_single_ifs(void* pvstate, void* pvhandle, context_t* pctx); static lrec_t* lrec_reader_stdio_nidx_process_single_irs_multi_ifs(void* pvstate, void* pvhandle, context_t* pctx); static lrec_t* lrec_reader_stdio_nidx_process_multi_irs_single_ifs(void* pvstate, void* pvhandle, context_t* pctx); @@ -42,7 +44,17 @@ lrec_reader_t* lrec_reader_stdio_nidx_alloc(char* irs, char* ifs, int allow_repe plrec_reader->pvstate = (void*)pstate; plrec_reader->popen_func = file_reader_stdio_vopen; plrec_reader->pclose_func = file_reader_stdio_vclose; - if (pstate->irslen == 1) { + if (streq(irs, "auto")) { + // Auto means either lines end in "\n" or "\r\n" (LF or CRLF). In + // either case the final character is "\n". Then for autodetect we + // simply check if there's a character in the line before the '\n', and + // if that is '\r'. + pstate->irs = "\n"; + pstate->irslen = 1; + plrec_reader->pprocess_func = (pstate->ifslen == 1) + ? lrec_reader_stdio_nidx_process_single_irs_single_ifs_auto_irs + : lrec_reader_stdio_nidx_process_single_irs_multi_ifs_auto_irs; + } else if (pstate->irslen == 1) { plrec_reader->pprocess_func = (pstate->ifslen == 1) ? &lrec_reader_stdio_nidx_process_single_irs_single_ifs : &lrec_reader_stdio_nidx_process_single_irs_multi_ifs; @@ -67,6 +79,62 @@ static void lrec_reader_stdio_nidx_sof(void* pvstate, void* pvhandle) { } // ---------------------------------------------------------------- +static lrec_t* lrec_reader_stdio_nidx_process_single_irs_single_ifs_auto_irs(void* pvstate, void* pvhandle, context_t* pctx) { + FILE* input_stream = pvhandle; + lrec_reader_stdio_nidx_state_t* pstate = pvstate; + int line_length; + char* line = mlr_get_cline_with_length(input_stream, pstate->irs[0], &line_length); + if (line == NULL) { + return NULL; + } else { + + // mlr_get_cline_with_length will have already chomped the trailing '\n', + // and it won't be included in the line length. + if (line_length > 0 && line[line_length-1] == '\r') { + line[line_length-1] = 0; + if (!pctx->auto_irs_detected) { + pctx->auto_irs_detected = TRUE; + pctx->auto_irs = "\r\n"; + } + } else { + if (!pctx->auto_irs_detected) { + pctx->auto_irs_detected = TRUE; + pctx->auto_irs = "\n"; + } + } + + return lrec_parse_stdio_nidx_single_sep(line, pstate->ifs[0], pstate->allow_repeat_ifs); + } +} + +static lrec_t* lrec_reader_stdio_nidx_process_single_irs_multi_ifs_auto_irs(void* pvstate, void* pvhandle, context_t* pctx) { + FILE* input_stream = pvhandle; + lrec_reader_stdio_nidx_state_t* pstate = pvstate; + int line_length; + char* line = mlr_get_cline_with_length(input_stream, pstate->irs[0], &line_length); + if (line == NULL) { + return NULL; + } else { + + // mlr_get_cline_with_length will have already chomped the trailing '\n', + // and it won't be included in the line length. + if (line_length > 0 && line[line_length-1] == '\r') { + line[line_length-1] = 0; + if (!pctx->auto_irs_detected) { + pctx->auto_irs_detected = TRUE; + pctx->auto_irs = "\r\n"; + } + } else { + if (!pctx->auto_irs_detected) { + pctx->auto_irs_detected = TRUE; + pctx->auto_irs = "\n"; + } + } + + return lrec_parse_stdio_nidx_multi_sep(line, pstate->ifs, pstate->ifslen, pstate->allow_repeat_ifs); + } +} + static lrec_t* lrec_reader_stdio_nidx_process_single_irs_single_ifs(void* pvstate, void* pvhandle, context_t* pctx) { FILE* input_stream = pvhandle; lrec_reader_stdio_nidx_state_t* pstate = pvstate; diff --git a/c/todo.txt b/c/todo.txt index 6bf124d19..de9163fb5 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -81,6 +81,8 @@ AUTOTERM * test w/ join as well +* doc: emph w/ 1st line's ending used for all + ================================================================ UT FOR 5.0.0: