Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-22 07:30:43 +00:00
Code Issues Projects Releases Wiki Activity

autodetect iterate: lrec writers

Browse source
This commit is contained in:
John Kerl 2017-01-28 13:51:21 -05:00
parent e1feac7abf
commit 565a9126fc
3 changed files with 66 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

48
c/input/lrec_reader_mmap_xtab.c
View file

@ -145,14 +145,14 @@ lrec_t* lrec_parse_mmap_xtab_single_ifs_single_ips(file_reader_mmap_state_t* pha
phandle->sol = p+1;
if (saw_eol) {
// Easy and simple case: we read until end of line. We zero-poked the irs to a null character to terminate the
// C string so it's OK to retain a pointer to that.
// Easy and simple case: we read until end of line. We zero-poked the irs to a null character to terminate
// the C string so it's OK to retain a pointer to that.
lrec_put(prec, key, value, NO_FREE);
} else {
// Messier case: we read to end of file without seeing end of line. We can't always zero-poke a null character
// to terminate the C string: if the file size is not a multiple of the OS page size it'll work (it's our
// copy-on-write memory). But if the file size is a multiple of the page size, then zero-poking at EOF is one
// byte past the page and that will segv us.
// Messier case: we read to end of file without seeing end of line. We can't always zero-poke a null
// character to terminate the C string: if the file size is not a multiple of the OS page size it'll work
// (it's our copy-on-write memory). But if the file size is a multiple of the page size, then zero-poking at
// EOF is one byte past the page and that will segv us.
char* copy = mlr_alloc_string_from_char_range(value, phandle->eof - value);
lrec_put(prec, key, copy, FREE_ENTRY_VALUE);
}
@ -212,14 +212,14 @@ lrec_t* lrec_parse_mmap_xtab_single_ifs_multi_ips(file_reader_mmap_state_t* phan
phandle->sol = p+1;
if (saw_eol) {
// Easy and simple case: we read until end of line. We zero-poked the irs to a null character to terminate the
// C string so it's OK to retain a pointer to that.
// Easy and simple case: we read until end of line. We zero-poked the irs to a null character to terminate
// the C string so it's OK to retain a pointer to that.
lrec_put(prec, key, value, NO_FREE);
} else {
// Messier case: we read to end of file without seeing end of line. We can't always zero-poke a null character
// to terminate the C string: if the file size is not a multiple of the OS page size it'll work (it's our
// copy-on-write memory). But if the file size is a multiple of the page size, then zero-poking at EOF is one
// byte past the page and that will segv us.
// Messier case: we read to end of file without seeing end of line. We can't always zero-poke a null
// character to terminate the C string: if the file size is not a multiple of the OS page size it'll work
// (it's our copy-on-write memory). But if the file size is a multiple of the page size, then zero-poking at
// EOF is one byte past the page and that will segv us.
char* copy = mlr_alloc_string_from_char_range(value, phandle->eof - value);
lrec_put(prec, key, copy, FREE_ENTRY_VALUE);
}
@ -279,14 +279,14 @@ lrec_t* lrec_parse_mmap_xtab_multi_ifs_single_ips(file_reader_mmap_state_t* phan
phandle->sol = p+1;
if (saw_eol) {
// Easy and simple case: we read until end of line. We zero-poked the irs to a null character to terminate the
// C string so it's OK to retain a pointer to that.
// Easy and simple case: we read until end of line. We zero-poked the irs to a null character to terminate
// the C string so it's OK to retain a pointer to that.
lrec_put(prec, key, value, NO_FREE);
} else {
// Messier case: we read to end of file without seeing end of line. We can't always zero-poke a null character
// to terminate the C string: if the file size is not a multiple of the OS page size it'll work (it's our
// copy-on-write memory). But if the file size is a multiple of the page size, then zero-poking at EOF is one
// byte past the page and that will segv us.
// Messier case: we read to end of file without seeing end of line. We can't always zero-poke a null
// character to terminate the C string: if the file size is not a multiple of the OS page size it'll work
// (it's our copy-on-write memory). But if the file size is a multiple of the page size, then zero-poking at
// EOF is one byte past the page and that will segv us.
char* copy = mlr_alloc_string_from_char_range(value, phandle->eof - value);
lrec_put(prec, key, copy, FREE_ENTRY_VALUE);
}
@ -346,14 +346,14 @@ lrec_t* lrec_parse_mmap_xtab_multi_ifs_multi_ips(file_reader_mmap_state_t* phand
phandle->sol = p+1;
if (saw_eol) {
// Easy and simple case: we read until end of line. We zero-poked the irs to a null character to terminate the
// C string so it's OK to retain a pointer to that.
// Easy and simple case: we read until end of line. We zero-poked the irs to a null character to terminate
// the C string so it's OK to retain a pointer to that.
lrec_put(prec, key, value, NO_FREE);
} else {
// Messier case: we read to end of file without seeing end of line. We can't always zero-poke a null character
// to terminate the C string: if the file size is not a multiple of the OS page size it'll work (it's our
// copy-on-write memory). But if the file size is a multiple of the page size, then zero-poking at EOF is one
// byte past the page and that will segv us.
// Messier case: we read to end of file without seeing end of line. We can't always zero-poke a null
// character to terminate the C string: if the file size is not a multiple of the OS page size it'll work
// (it's our copy-on-write memory). But if the file size is a multiple of the page size, then zero-poking at
// EOF is one byte past the page and that will segv us.
char* copy = mlr_alloc_string_from_char_range(value, phandle->eof - value);
lrec_put(prec, key, copy, FREE_ENTRY_VALUE);
}

53
c/output/lrec_writer_xtab.c
View file

@ -28,8 +28,12 @@ typedef struct _lrec_writer_xtab_state_t {
} lrec_writer_xtab_state_t;
static void lrec_writer_xtab_free(lrec_writer_t* pwriter, context_t* pctx);
static void lrec_writer_xtab_process_aligned(void* pvstate, FILE* output_stream, lrec_t* prec, context_t* pctx);
static void lrec_writer_xtab_process_unaligned(void* pvstate, FILE* output_stream, lrec_t* prec, context_t* pctx);
static void lrec_writer_xtab_process_aligned(void* pvstate, FILE* output_stream, lrec_t* prec, char* ofs);
static void lrec_writer_xtab_process_aligned_auto_ofs(void* pvstate, FILE* output_stream, lrec_t* prec, context_t* pctx);
static void lrec_writer_xtab_process_aligned_nonauto_ofs(void* pvstate, FILE* output_stream, lrec_t* prec, context_t* pctx);
static void lrec_writer_xtab_process_unaligned(void* pvstate, FILE* output_stream, lrec_t* prec, char* ofs);
static void lrec_writer_xtab_process_unaligned_auto_ofs(void* pvstate, FILE* output_stream, lrec_t* prec, context_t* pctx);
static void lrec_writer_xtab_process_unaligned_nonauto_ofs(void* pvstate, FILE* output_stream, lrec_t* prec, context_t* pctx);
// ----------------------------------------------------------------
lrec_writer_t* lrec_writer_xtab_alloc(char* ofs, char* ops, int right_justify_value) {
@ -42,11 +46,17 @@ lrec_writer_t* lrec_writer_xtab_alloc(char* ofs, char* ops, int right_justify_va
pstate->record_count = 0LL;
pstate->right_justify_value = right_justify_value;
plrec_writer->pvstate = pstate;
plrec_writer->pprocess_func = (pstate->opslen == 1)
? lrec_writer_xtab_process_aligned
: lrec_writer_xtab_process_unaligned;
plrec_writer->pfree_func = lrec_writer_xtab_free;
plrec_writer->pvstate = pstate;
if (pstate->opslen == 1) {
plrec_writer->pprocess_func = streq(ofs, "auto")
? lrec_writer_xtab_process_aligned_auto_ofs
: lrec_writer_xtab_process_aligned_nonauto_ofs;
} else {
plrec_writer->pprocess_func = streq(ofs, "auto")
? lrec_writer_xtab_process_unaligned_auto_ofs
: lrec_writer_xtab_process_unaligned_nonauto_ofs;
}
plrec_writer->pfree_func = lrec_writer_xtab_free;
return plrec_writer;
}
@ -57,12 +67,21 @@ static void lrec_writer_xtab_free(lrec_writer_t* pwriter, context_t* pctx) {
}
// ----------------------------------------------------------------
static void lrec_writer_xtab_process_aligned(void* pvstate, FILE* output_stream, lrec_t* prec, context_t* pctx) {
static void lrec_writer_xtab_process_aligned_auto_ofs(void* pvstate, FILE* output_stream, lrec_t* prec, context_t* pctx) {
lrec_writer_xtab_process_aligned(pvstate, output_stream, prec, pctx->auto_irs);
}
static void lrec_writer_xtab_process_aligned_nonauto_ofs(void* pvstate, FILE* output_stream, lrec_t* prec, context_t* pctx) {
lrec_writer_xtab_state_t* pstate = pvstate;
lrec_writer_xtab_process_aligned(pvstate, output_stream, prec, pstate->ofs);
}
static void lrec_writer_xtab_process_aligned(void* pvstate, FILE* output_stream, lrec_t* prec, char* ofs) {
if (prec == NULL)
return;
lrec_writer_xtab_state_t* pstate = pvstate;
if (pstate->record_count > 0LL)
fputs(pstate->ofs, output_stream);
fputs(ofs, output_stream);
pstate->record_count++;
int max_key_width = 1;
@ -88,12 +107,22 @@ static void lrec_writer_xtab_process_aligned(void* pvstate, FILE* output_stream,
for (int i = 0; i < d; i++)
fputs(pstate->ops, output_stream);
}
fprintf(output_stream, "%s%s%s", pstate->ops, pe->value, pstate->ofs);
fprintf(output_stream, "%s%s%s", pstate->ops, pe->value, ofs);
}
lrec_free(prec); // end of baton-pass
}
static void lrec_writer_xtab_process_unaligned(void* pvstate, FILE* output_stream, lrec_t* prec, context_t* pctx) {
// ----------------------------------------------------------------
static void lrec_writer_xtab_process_unaligned_auto_ofs(void* pvstate, FILE* output_stream, lrec_t* prec, context_t* pctx) {
lrec_writer_xtab_process_unaligned(pvstate, output_stream, prec, pctx->auto_irs);
}
static void lrec_writer_xtab_process_unaligned_nonauto_ofs(void* pvstate, FILE* output_stream, lrec_t* prec, context_t* pctx) {
lrec_writer_xtab_state_t* pstate = pvstate;
lrec_writer_xtab_process_unaligned(pvstate, output_stream, prec, pstate->ofs);
}
static void lrec_writer_xtab_process_unaligned(void* pvstate, FILE* output_stream, lrec_t* prec, char* ofs) {
if (prec == NULL)
return;
lrec_writer_xtab_state_t* pstate = pvstate;
@ -103,7 +132,7 @@ static void lrec_writer_xtab_process_unaligned(void* pvstate, FILE* output_strea
for (lrece_t* pe = prec->phead; pe != NULL; pe = pe->pnext) {
// "%-*s" fprintf format isn't correct for non-ASCII UTF-8
fprintf(output_stream, "%s%s%s%s", pe->key, pstate->ops, pe->value, pstate->ofs);
fprintf(output_stream, "%s%s%s%s", pe->key, pstate->ops, pe->value, ofs);
}
lrec_free(prec); // end of baton-pass
}

2
c/todo.txt
View file

@ -63,12 +63,12 @@ AUTOTERM
? output/lrec_writer_csv.c
? output/lrec_writer_json.c
? output/lrec_writer_xtab.c
k output/lrec_writer_csvlite.c
k output/lrec_writer_dkvp.c
k output/lrec_writer_markdown.c
k output/lrec_writer_nidx.c
k output/lrec_writer_pprint.c
~ output/lrec_writer_xtab.c -- needs ors=auto -> ofs=auto transmutate @ caller
================================================================
UT FOR 5.0.0: