From d46c68f1dfc06e1c2948604c548e65d98a0ecdef Mon Sep 17 00:00:00 2001 From: John Kerl Date: Wed, 17 Jun 2015 08:57:06 -0400 Subject: [PATCH] join iterate --- c/cli/mlrcli.c | 58 ++++++++++++++++++------------------- c/cli/mlrcli.h | 12 ++++---- c/lib/mlr_globals.c | 9 +++--- c/lib/mlr_globals.h | 11 +++++-- c/mapping/lrec_evaluators.c | 2 +- c/mapping/mapper_join.c | 21 ++++++++++---- c/mlrmain.c | 4 +-- c/todo.txt | 3 +- 8 files changed, 67 insertions(+), 53 deletions(-) diff --git a/c/cli/mlrcli.c b/c/cli/mlrcli.c index cd95a0234..9cbc5278c 100644 --- a/c/cli/mlrcli.c +++ b/c/cli/mlrcli.c @@ -167,10 +167,9 @@ cli_opts_t* parse_command_line(int argc, char** argv) { popts->plrec_writer = NULL; popts->filenames = NULL; - int use_file_reader_mmap = TRUE; - - char* rdesc = "dkvp"; - char* wdesc = "dkvp"; + popts->use_mmap_for_read = TRUE; + popts->ifmt = "dkvp"; + char* ofmt = "dkvp"; int left_align_pprint = TRUE; int have_rand_seed = FALSE; @@ -249,36 +248,36 @@ cli_opts_t* parse_command_line(int argc, char** argv) { argi++; } - else if (streq(argv[argi], "--dkvp")) { rdesc = wdesc = "dkvp"; } - else if (streq(argv[argi], "--idkvp")) { rdesc = "dkvp"; } - else if (streq(argv[argi], "--odkvp")) { wdesc = "dkvp"; } + else if (streq(argv[argi], "--dkvp")) { popts->ifmt = ofmt = "dkvp"; } + else if (streq(argv[argi], "--idkvp")) { popts->ifmt = "dkvp"; } + else if (streq(argv[argi], "--odkvp")) { ofmt = "dkvp"; } - else if (streq(argv[argi], "--csv")) { rdesc = wdesc = "csv"; } - else if (streq(argv[argi], "--icsv")) { rdesc = "csv"; } - else if (streq(argv[argi], "--ocsv")) { wdesc = "csv"; } + else if (streq(argv[argi], "--csv")) { popts->ifmt = ofmt = "csv"; } + else if (streq(argv[argi], "--icsv")) { popts->ifmt = "csv"; } + else if (streq(argv[argi], "--ocsv")) { ofmt = "csv"; } - else if (streq(argv[argi], "--nidx")) { rdesc = wdesc = "nidx"; } - else if (streq(argv[argi], "--inidx")) { rdesc = "nidx"; } - else if (streq(argv[argi], "--onidx")) { wdesc = "nidx"; } + else if (streq(argv[argi], "--nidx")) { popts->ifmt = ofmt = "nidx"; } + else if (streq(argv[argi], "--inidx")) { popts->ifmt = "nidx"; } + else if (streq(argv[argi], "--onidx")) { ofmt = "nidx"; } - else if (streq(argv[argi], "--xtab")) { rdesc = wdesc = "xtab"; } - else if (streq(argv[argi], "--ixtab")) { rdesc = "xtab"; } - else if (streq(argv[argi], "--oxtab")) { wdesc = "xtab"; } + else if (streq(argv[argi], "--xtab")) { popts->ifmt = ofmt = "xtab"; } + else if (streq(argv[argi], "--ixtab")) { popts->ifmt = "xtab"; } + else if (streq(argv[argi], "--oxtab")) { ofmt = "xtab"; } else if (streq(argv[argi], "--ipprint")) { - rdesc = "csv"; + popts->ifmt = "csv"; popts->ifs = ' '; popts->allow_repeat_ifs = TRUE; } else if (streq(argv[argi], "--opprint")) { - wdesc = "pprint"; + ofmt = "pprint"; } else if (streq(argv[argi], "--pprint")) { - rdesc = "csv"; + popts->ifmt = "csv"; popts->ifs = ' '; popts->allow_repeat_ifs = TRUE; - wdesc = "pprint"; + ofmt = "pprint"; } else if (streq(argv[argi], "--right")) { left_align_pprint = FALSE; @@ -290,13 +289,12 @@ cli_opts_t* parse_command_line(int argc, char** argv) { argi++; } - // xxx negate the default once this is working. // xxx put into online help. else if (streq(argv[argi], "--mmap")) { - use_file_reader_mmap = TRUE; + popts->use_mmap_for_read = TRUE; } else if (streq(argv[argi], "--no-mmap")) { - use_file_reader_mmap = FALSE; + popts->use_mmap_for_read = FALSE; } else if (streq(argv[argi], "--seed")) { check_arg_count(argv, argi, argc, 2); @@ -314,11 +312,11 @@ cli_opts_t* parse_command_line(int argc, char** argv) { nusage(argv[0], argv[argi]); } - if (streq(wdesc, "dkvp")) popts->plrec_writer = lrec_writer_dkvp_alloc(popts->ors, popts->ofs, popts->ops); - else if (streq(wdesc, "csv")) popts->plrec_writer = lrec_writer_csv_alloc(popts->ors, popts->ofs); - else if (streq(wdesc, "nidx")) popts->plrec_writer = lrec_writer_nidx_alloc(popts->ors, popts->ofs); - else if (streq(wdesc, "xtab")) popts->plrec_writer = lrec_writer_xtab_alloc(); - else if (streq(wdesc, "pprint")) popts->plrec_writer = lrec_writer_pprint_alloc(left_align_pprint); + if (streq(ofmt, "dkvp")) popts->plrec_writer = lrec_writer_dkvp_alloc(popts->ors, popts->ofs, popts->ops); + else if (streq(ofmt, "csv")) popts->plrec_writer = lrec_writer_csv_alloc(popts->ors, popts->ofs); + else if (streq(ofmt, "nidx")) popts->plrec_writer = lrec_writer_nidx_alloc(popts->ors, popts->ofs); + else if (streq(ofmt, "xtab")) popts->plrec_writer = lrec_writer_xtab_alloc(); + else if (streq(ofmt, "pprint")) popts->plrec_writer = lrec_writer_pprint_alloc(left_align_pprint); else { main_usage(argv[0], 1); } @@ -369,9 +367,9 @@ cli_opts_t* parse_command_line(int argc, char** argv) { // No filenames means read from standard input, and standard input cannot be mmaped. if (argi == argc) - use_file_reader_mmap = FALSE; + popts->use_mmap_for_read = FALSE; - popts->plrec_reader = lrec_reader_alloc(rdesc, use_file_reader_mmap, + popts->plrec_reader = lrec_reader_alloc(popts->ifmt, popts->use_mmap_for_read, popts->irs, popts->ifs, popts->allow_repeat_ifs, popts->ips, popts->allow_repeat_ips); if (popts->plrec_reader == NULL) main_usage(argv[0], 1); diff --git a/c/cli/mlrcli.h b/c/cli/mlrcli.h index 2ba2a6f29..65c095864 100644 --- a/c/cli/mlrcli.h +++ b/c/cli/mlrcli.h @@ -7,11 +7,13 @@ #include "output/lrec_writer.h" typedef struct _cli_opts_t { - char irs; - char ifs; - char ips; - int allow_repeat_ifs; - int allow_repeat_ips; + char irs; + char ifs; + char ips; + int allow_repeat_ifs; + int allow_repeat_ips; + int use_mmap_for_read; + char* ifmt; char ors; char ofs; diff --git a/c/lib/mlr_globals.c b/c/lib/mlr_globals.c index c5355eb63..282cbba99 100644 --- a/c/lib/mlr_globals.c +++ b/c/lib/mlr_globals.c @@ -1,8 +1,9 @@ #include #include "lib/mlr_globals.h" -mlr_globals_t MLR_GLOBALS = { .argv0 = NULL, .ofmt = NULL }; -void mlr_global_init(char* argv0, char* ofmt) { - MLR_GLOBALS.argv0 = argv0; - MLR_GLOBALS.ofmt = ofmt; +mlr_globals_t MLR_GLOBALS = { .argv0 = NULL, .ofmt = NULL, .popts = NULL }; +void mlr_global_init(char* argv0, char* ofmt, cli_opts_t* popts) { + MLR_GLOBALS.argv0 = argv0; + MLR_GLOBALS.ofmt = ofmt; + MLR_GLOBALS.popts = popts; } diff --git a/c/lib/mlr_globals.h b/c/lib/mlr_globals.h index 6c3803bb4..4de6727c5 100644 --- a/c/lib/mlr_globals.h +++ b/c/lib/mlr_globals.h @@ -1,11 +1,16 @@ #ifndef MLR_GLOBALS_H #define MLR_GLOBALS_H +#include "cli/mlrcli.h" typedef struct _mlr_globals_t { - char* argv0; - char* ofmt; + char* argv0; + char* ofmt; + // These are shared by mlrcli.c and mlrmain.c. The only reason for their + // exposure anywhere else is to communicate format and separator flags to + // mapper_join, which (unlike other mappers) needs to do its own file I/O. + cli_opts_t* popts; } mlr_globals_t; extern mlr_globals_t MLR_GLOBALS; -void mlr_global_init(char* argv0, char* ofmt); +void mlr_global_init(char* argv0, char* ofmt, cli_opts_t* popts); #endif // MLR_GLOBALS_H diff --git a/c/mapping/lrec_evaluators.c b/c/mapping/lrec_evaluators.c index 2cfdb68d3..aac06fa8b 100644 --- a/c/mapping/lrec_evaluators.c +++ b/c/mapping/lrec_evaluators.c @@ -144,7 +144,7 @@ mv_t lrec_evaluator_f_ff_func(lrec_t* prec, context_t* pctx, void* pvstate) { return MV_ERROR; mv_t val2 = pstate->parg2->pevaluator_func(prec, pctx, pstate->parg2->pvstate); -NULL_OR_ERROR_OUT(val2); + NULL_OR_ERROR_OUT(val2); mt_get_double_strict(&val2); if (val2.type != MT_DOUBLE) return MV_ERROR; diff --git a/c/mapping/mapper_join.c b/c/mapping/mapper_join.c index 8a892c0c0..adc2f114d 100644 --- a/c/mapping/mapper_join.c +++ b/c/mapping/mapper_join.c @@ -133,14 +133,18 @@ static void mapper_join_free(void* pvstate) { } // ---------------------------------------------------------------- -// xxx void-abstract the stdio/mmap readers. this is a pita. also neaten up stream.c. -// xxx have a reader factory which can be called here. -// xxx for the moment, just dev with hard-coded separator & format parameters. +// Format and separator flags are passed to mapper_join in MLR_GLOBALS rather +// than on the stack, since the latter would require complicating the interface +// for all the other mappers which don't do their own file I/O. (Also, while +// some of the information needed to construct an lrec_reader is available on +// the command line before the mapper-allocators are called, some is not +// available until after. Hence our obtaining these flags after mapper-alloc.) static void ingest_left_file(mapper_join_state_t* pstate) { - // xxx temp - lrec_reader_t* plrec_reader = lrec_reader_alloc("dkvp", TRUE, '\n', ',', FALSE, '=', FALSE); + cli_opts_t* popts = MLR_GLOBALS.popts; + lrec_reader_t* plrec_reader = lrec_reader_alloc(popts->ifmt, popts->use_mmap_for_read, + popts->irs, popts->ifs, popts->allow_repeat_ifs, popts->ips, popts->allow_repeat_ips); void* pvhandle = plrec_reader->popen_func(pstate->left_file_name); plrec_reader->psof_func(plrec_reader->pvstate); @@ -262,7 +266,12 @@ static mapper_t* mapper_join_parse_cli(int* pargi, int argc, char** argv) { return NULL; } - // xxx check not all emit-flags are off -- else no output. + if (!emit_pairables && !emit_left_unpairables && !emit_right_unpairables) { + fprintf(stderr, "%s %s: all emit flags are unset; no output is possible.\n", + MLR_GLOBALS.argv0, verb); + mapper_join_usage(argv[0], verb); + return NULL; + } if (pright_field_names == NULL) pright_field_names = slls_copy(pleft_field_names); diff --git a/c/mlrmain.c b/c/mlrmain.c index 7645704d4..ce76c76b2 100644 --- a/c/mlrmain.c +++ b/c/mlrmain.c @@ -23,9 +23,9 @@ int main(int argc, char** argv) { } fprintf(stderr, "Set up mcheck\n"); #endif - mlr_global_init(argv[0], NULL); + mlr_global_init(argv[0], NULL, NULL); cli_opts_t* popts = parse_command_line(argc, argv); - mlr_global_init(argv[0], popts->ofmt); + mlr_global_init(argv[0], popts->ofmt, popts); lrec_reader_t* plrec_reader = popts->plrec_reader; sllv_t* pmapper_list = popts->pmapper_list; diff --git a/c/todo.txt b/c/todo.txt index f63d04538..89a73b493 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -10,11 +10,10 @@ ================================================================ TOP OF LIST -* mapper_list -> lrec_mapper_list etc. - !! should be able to do: mlr put '$ok = $x < 10' !! join +-> parametrized reader; global cliopts; overrides for join per se -> UTs !! mmap cleanup, especially csv