Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-21 10:29:02 +00:00
Code Issues Projects Releases Wiki Activity

autodetect iterate: lrec readers

Browse source
This commit is contained in:
John Kerl 2017-01-28 21:01:31 -05:00
parent a717716801
commit 49f60a40c0
5 changed files with 142 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

30
c/input/lrec_reader_mmap_dkvp.c
View file

@ -148,15 +148,22 @@ lrec_t* lrec_parse_mmap_dkvp_single_irs_single_others(file_reader_mmap_state_t *
for ( ; p < phandle->eof && *p; ) {
if (*p == irs) {
*p = 0;
if (do_auto_irs && !pctx->auto_irs_detected) {
if (do_auto_irs) {
if (p > line && p[-1] == '\r') {
p[-1] = 0;
pctx->auto_irs = "\r\n";
if (!pctx->auto_irs_detected) {
pctx->auto_irs = "\r\n";
pctx->auto_irs_detected = TRUE;
}
} else {
pctx->auto_irs = "\n";
if (!pctx->auto_irs_detected) {
pctx->auto_irs = "\n";
pctx->auto_irs_detected = TRUE;
}
}
pctx->auto_irs_detected = TRUE;
}
phandle->sol = p+1;
saw_rs = TRUE;
break;
@ -373,15 +380,22 @@ lrec_t* lrec_parse_mmap_dkvp_single_irs_multi_others(file_reader_mmap_state_t *p
for ( ; p < phandle->eof && *p; ) {
if (*p == irs) {
*p = 0;
if (do_auto_irs && !pctx->auto_irs_detected) {
if (do_auto_irs) {
if (p > line && p[-1] == '\r') {
p[-1] = 0;
pctx->auto_irs = "\r\n";
if (!pctx->auto_irs_detected) {
pctx->auto_irs = "\r\n";
pctx->auto_irs_detected = TRUE;
}
} else {
pctx->auto_irs = "\n";
if (!pctx->auto_irs_detected) {
pctx->auto_irs = "\n";
pctx->auto_irs_detected = TRUE;
}
}
pctx->auto_irs_detected = TRUE;
}
phandle->sol = p+1;
saw_rs = TRUE;
break;

32
c/input/lrec_reader_mmap_nidx.c
View file

@ -142,15 +142,22 @@ lrec_t* lrec_parse_mmap_nidx_single_irs_single_ifs(file_reader_mmap_state_t *pha
for ( ; p < phandle->eof && *p; ) {
if (*p == irs) {
*p = 0;
if (do_auto_irs && !pctx->auto_irs_detected) {
if (p > line && p[1] == '\r') {
if (do_auto_irs) {
if (p > line && p[-1] == '\r') {
p[-1] = 0;
pctx->auto_irs = "\r\n";
if (!pctx->auto_irs_detected) {
pctx->auto_irs = "\r\n";
pctx->auto_irs_detected = TRUE;
}
} else {
pctx->auto_irs = "\n";
if (!pctx->auto_irs_detected) {
pctx->auto_irs = "\n";
pctx->auto_irs_detected = TRUE;
}
}
pctx->auto_irs_detected = TRUE;
}
phandle->sol = p+1;
saw_rs = TRUE;
break;
@ -216,15 +223,22 @@ lrec_t* lrec_parse_mmap_nidx_single_irs_multi_ifs(file_reader_mmap_state_t *phan
for ( ; p < phandle->eof && *p; ) {
if (*p == irs) {
*p = 0;
if (do_auto_irs && !pctx->auto_irs_detected) {
if (do_auto_irs) {
if (p > line && p[-1] == '\r') {
p[-1] = 0;
pctx->auto_irs = "\r\n";
if (!pctx->auto_irs_detected) {
pctx->auto_irs = "\r\n";
pctx->auto_irs_detected = TRUE;
}
} else {
pctx->auto_irs = "\n";
if (!pctx->auto_irs_detected) {
pctx->auto_irs = "\n";
pctx->auto_irs_detected = TRUE;
}
}
pctx->auto_irs_detected = TRUE;
}
phandle->sol = p+1;
saw_rs = TRUE;
break;

45
c/input/lrec_reader_stdio_dkvp.c
View file

@ -23,7 +23,6 @@ typedef struct _lrec_reader_stdio_dkvp_state_t {
int ifslen;
int ipslen;
int allow_repeat_ifs;
int do_auto_irs;
} lrec_reader_stdio_dkvp_state_t;
static void lrec_reader_stdio_dkvp_free(lrec_reader_t* preader);
@ -53,7 +52,6 @@ lrec_reader_t* lrec_reader_stdio_dkvp_alloc(char* irs, char* ifs, char* ips, int
pstate->ifslen = strlen(ifs);
pstate->ipslen = strlen(ips);
pstate->allow_repeat_ifs = allow_repeat_ifs;
pstate->do_auto_irs = FALSE;
plrec_reader->pvstate = (void*)pstate;
plrec_reader->popen_func = file_reader_stdio_vopen;
@ -63,18 +61,17 @@ lrec_reader_t* lrec_reader_stdio_dkvp_alloc(char* irs, char* ifs, char* ips, int
// either case the final character is "\n". Then for autodetect we
// simply check if there's a character in the line before the '\n', and
// if that is '\r'.
pstate->do_auto_irs = TRUE;
pstate->irs = "\n";
pstate->irslen = 1;
plrec_reader->pprocess_func = (pstate->ifslen == 1 && pstate->ipslen == 1)
? lrec_reader_stdio_dkvp_process_single_irs_single_others_auto_irs
: lrec_reader_stdio_dkvp_process_single_irs_multi_others_auto_irs;
} else if (pstate->irslen == 1) {
plrec_reader->pprocess_func = (pstate->ifslen == 1 && pstate->ipslen == 1)
plrec_reader->pprocess_func = (pstate->ifslen == 1)
? &lrec_reader_stdio_dkvp_process_single_irs_single_others
: &lrec_reader_stdio_dkvp_process_single_irs_multi_others;
} else {
plrec_reader->pprocess_func = (pstate->ifslen == 1 && pstate->ipslen == 1)
plrec_reader->pprocess_func = (pstate->ifslen == 1)
? &lrec_reader_stdio_dkvp_process_multi_irs_single_others
: &lrec_reader_stdio_dkvp_process_multi_irs_multi_others;
}
@ -104,17 +101,22 @@ static lrec_t* lrec_reader_stdio_dkvp_process_single_irs_single_others_auto_irs(
if (line == NULL) {
return NULL;
} else {
if (!pctx->auto_irs_detected) {
// mlr_get_cline_with_length will have already chomped the trailing '\n',
// and it won't be included in the line length.
if (line_length > 0 && line[line_length-1] == '\r') {
line[line_length-1] = 0;
// mlr_get_cline_with_length will have already chomped the trailing '\n',
// and it won't be included in the line length.
if (line_length > 0 && line[line_length-1] == '\r') {
line[line_length-1] = 0;
if (!pctx->auto_irs_detected) {
pctx->auto_irs_detected = TRUE;
pctx->auto_irs = "\r\n";
} else {
}
} else {
if (!pctx->auto_irs_detected) {
pctx->auto_irs_detected = TRUE;
pctx->auto_irs = "\n";
}
pctx->auto_irs_detected = TRUE;
}
return lrec_parse_stdio_dkvp_single_sep(line, pstate->ifs[0], pstate->ips[0], pstate->allow_repeat_ifs);
}
}
@ -129,17 +131,22 @@ static lrec_t* lrec_reader_stdio_dkvp_process_single_irs_multi_others_auto_irs(
if (line == NULL) {
return NULL;
} else {
if (!pctx->auto_irs_detected) {
// mlr_get_cline_with_length will have already chomped the trailing '\n',
// and it won't be included in the line length.
if (line_length > 0 && line[line_length-1] == '\r') {
line[line_length-1] = 0;
// mlr_get_cline_with_length will have already chomped the trailing '\n',
// and it won't be included in the line length.
if (line_length > 0 && line[line_length-1] == '\r') {
line[line_length-1] = 0;
if (!pctx->auto_irs_detected) {
pctx->auto_irs_detected = TRUE;
pctx->auto_irs = "\r\n";
} else {
}
} else {
if (!pctx->auto_irs_detected) {
pctx->auto_irs_detected = TRUE;
pctx->auto_irs = "\n";
}
pctx->auto_irs_detected = TRUE;
}
return lrec_parse_stdio_dkvp_multi_sep(line, pstate->ifs, pstate->ips, pstate->ifslen, pstate->ipslen,
pstate->allow_repeat_ifs);
}

70
c/input/lrec_reader_stdio_nidx.c
View file

@ -23,6 +23,8 @@ typedef struct _lrec_reader_stdio_nidx_state_t {
static void lrec_reader_stdio_nidx_free(lrec_reader_t* preader);
static void lrec_reader_stdio_nidx_sof(void* pvstate, void* pvhandle);
static lrec_t* lrec_reader_stdio_nidx_process_single_irs_single_ifs_auto_irs(void* pvstate, void* pvhandle, context_t* pctx);
static lrec_t* lrec_reader_stdio_nidx_process_single_irs_multi_ifs_auto_irs(void* pvstate, void* pvhandle, context_t* pctx);
static lrec_t* lrec_reader_stdio_nidx_process_single_irs_single_ifs(void* pvstate, void* pvhandle, context_t* pctx);
static lrec_t* lrec_reader_stdio_nidx_process_single_irs_multi_ifs(void* pvstate, void* pvhandle, context_t* pctx);
static lrec_t* lrec_reader_stdio_nidx_process_multi_irs_single_ifs(void* pvstate, void* pvhandle, context_t* pctx);
@ -42,7 +44,17 @@ lrec_reader_t* lrec_reader_stdio_nidx_alloc(char* irs, char* ifs, int allow_repe
plrec_reader->pvstate = (void*)pstate;
plrec_reader->popen_func = file_reader_stdio_vopen;
plrec_reader->pclose_func = file_reader_stdio_vclose;
if (pstate->irslen == 1) {
if (streq(irs, "auto")) {
// Auto means either lines end in "\n" or "\r\n" (LF or CRLF). In
// either case the final character is "\n". Then for autodetect we
// simply check if there's a character in the line before the '\n', and
// if that is '\r'.
pstate->irs = "\n";
pstate->irslen = 1;
plrec_reader->pprocess_func = (pstate->ifslen == 1)
? lrec_reader_stdio_nidx_process_single_irs_single_ifs_auto_irs
: lrec_reader_stdio_nidx_process_single_irs_multi_ifs_auto_irs;
} else if (pstate->irslen == 1) {
plrec_reader->pprocess_func = (pstate->ifslen == 1)
? &lrec_reader_stdio_nidx_process_single_irs_single_ifs
: &lrec_reader_stdio_nidx_process_single_irs_multi_ifs;
@ -67,6 +79,62 @@ static void lrec_reader_stdio_nidx_sof(void* pvstate, void* pvhandle) {
}
// ----------------------------------------------------------------
static lrec_t* lrec_reader_stdio_nidx_process_single_irs_single_ifs_auto_irs(void* pvstate, void* pvhandle, context_t* pctx) {
FILE* input_stream = pvhandle;
lrec_reader_stdio_nidx_state_t* pstate = pvstate;
int line_length;
char* line = mlr_get_cline_with_length(input_stream, pstate->irs[0], &line_length);
if (line == NULL) {
return NULL;
} else {
// mlr_get_cline_with_length will have already chomped the trailing '\n',
// and it won't be included in the line length.
if (line_length > 0 && line[line_length-1] == '\r') {
line[line_length-1] = 0;
if (!pctx->auto_irs_detected) {
pctx->auto_irs_detected = TRUE;
pctx->auto_irs = "\r\n";
}
} else {
if (!pctx->auto_irs_detected) {
pctx->auto_irs_detected = TRUE;
pctx->auto_irs = "\n";
}
}
return lrec_parse_stdio_nidx_single_sep(line, pstate->ifs[0], pstate->allow_repeat_ifs);
}
}
static lrec_t* lrec_reader_stdio_nidx_process_single_irs_multi_ifs_auto_irs(void* pvstate, void* pvhandle, context_t* pctx) {
FILE* input_stream = pvhandle;
lrec_reader_stdio_nidx_state_t* pstate = pvstate;
int line_length;
char* line = mlr_get_cline_with_length(input_stream, pstate->irs[0], &line_length);
if (line == NULL) {
return NULL;
} else {
// mlr_get_cline_with_length will have already chomped the trailing '\n',
// and it won't be included in the line length.
if (line_length > 0 && line[line_length-1] == '\r') {
line[line_length-1] = 0;
if (!pctx->auto_irs_detected) {
pctx->auto_irs_detected = TRUE;
pctx->auto_irs = "\r\n";
}
} else {
if (!pctx->auto_irs_detected) {
pctx->auto_irs_detected = TRUE;
pctx->auto_irs = "\n";
}
}
return lrec_parse_stdio_nidx_multi_sep(line, pstate->ifs, pstate->ifslen, pstate->allow_repeat_ifs);
}
}
static lrec_t* lrec_reader_stdio_nidx_process_single_irs_single_ifs(void* pvstate, void* pvhandle, context_t* pctx) {
FILE* input_stream = pvhandle;
lrec_reader_stdio_nidx_state_t* pstate = pvstate;

2
c/todo.txt
View file

@ -81,6 +81,8 @@ AUTOTERM
* test w/ join as well
* doc: emph w/ 1st line's ending used for all
================================================================
UT FOR 5.0.0: