Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-27 17:53:57 +00:00
Code Issues Projects Releases Wiki Activity

join iterate

Browse source
This commit is contained in:
John Kerl 2015-06-17 08:57:06 -04:00
parent 3d1a851efb
commit d46c68f1df
8 changed files with 67 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

58
c/cli/mlrcli.c
View file

@ -167,10 +167,9 @@ cli_opts_t* parse_command_line(int argc, char** argv) {
popts->plrec_writer = NULL;
popts->filenames = NULL;
int use_file_reader_mmap = TRUE;
char* rdesc = "dkvp";
char* wdesc = "dkvp";
popts->use_mmap_for_read = TRUE;
popts->ifmt = "dkvp";
char* ofmt = "dkvp";
int left_align_pprint = TRUE;
int have_rand_seed = FALSE;
@ -249,36 +248,36 @@ cli_opts_t* parse_command_line(int argc, char** argv) {
argi++;
}
else if (streq(argv[argi], "--dkvp")) { rdesc = wdesc = "dkvp"; }
else if (streq(argv[argi], "--idkvp")) { rdesc = "dkvp"; }
else if (streq(argv[argi], "--odkvp")) { wdesc = "dkvp"; }
else if (streq(argv[argi], "--dkvp")) { popts->ifmt = ofmt = "dkvp"; }
else if (streq(argv[argi], "--idkvp")) { popts->ifmt = "dkvp"; }
else if (streq(argv[argi], "--odkvp")) { ofmt = "dkvp"; }
else if (streq(argv[argi], "--csv")) { rdesc = wdesc = "csv"; }
else if (streq(argv[argi], "--icsv")) { rdesc = "csv"; }
else if (streq(argv[argi], "--ocsv")) { wdesc = "csv"; }
else if (streq(argv[argi], "--csv")) { popts->ifmt = ofmt = "csv"; }
else if (streq(argv[argi], "--icsv")) { popts->ifmt = "csv"; }
else if (streq(argv[argi], "--ocsv")) { ofmt = "csv"; }
else if (streq(argv[argi], "--nidx")) { rdesc = wdesc = "nidx"; }
else if (streq(argv[argi], "--inidx")) { rdesc = "nidx"; }
else if (streq(argv[argi], "--onidx")) { wdesc = "nidx"; }
else if (streq(argv[argi], "--nidx")) { popts->ifmt = ofmt = "nidx"; }
else if (streq(argv[argi], "--inidx")) { popts->ifmt = "nidx"; }
else if (streq(argv[argi], "--onidx")) { ofmt = "nidx"; }
else if (streq(argv[argi], "--xtab")) { rdesc = wdesc = "xtab"; }
else if (streq(argv[argi], "--ixtab")) { rdesc = "xtab"; }
else if (streq(argv[argi], "--oxtab")) { wdesc = "xtab"; }
else if (streq(argv[argi], "--xtab")) { popts->ifmt = ofmt = "xtab"; }
else if (streq(argv[argi], "--ixtab")) { popts->ifmt = "xtab"; }
else if (streq(argv[argi], "--oxtab")) { ofmt = "xtab"; }
else if (streq(argv[argi], "--ipprint")) {
rdesc = "csv";
popts->ifmt = "csv";
popts->ifs = ' ';
popts->allow_repeat_ifs = TRUE;
}
else if (streq(argv[argi], "--opprint")) {
wdesc = "pprint";
ofmt = "pprint";
}
else if (streq(argv[argi], "--pprint")) {
rdesc = "csv";
popts->ifmt = "csv";
popts->ifs = ' ';
popts->allow_repeat_ifs = TRUE;
wdesc = "pprint";
ofmt = "pprint";
}
else if (streq(argv[argi], "--right")) {
left_align_pprint = FALSE;
@ -290,13 +289,12 @@ cli_opts_t* parse_command_line(int argc, char** argv) {
argi++;
}
// xxx negate the default once this is working.
// xxx put into online help.
else if (streq(argv[argi], "--mmap")) {
use_file_reader_mmap = TRUE;
popts->use_mmap_for_read = TRUE;
}
else if (streq(argv[argi], "--no-mmap")) {
use_file_reader_mmap = FALSE;
popts->use_mmap_for_read = FALSE;
}
else if (streq(argv[argi], "--seed")) {
check_arg_count(argv, argi, argc, 2);
@ -314,11 +312,11 @@ cli_opts_t* parse_command_line(int argc, char** argv) {
nusage(argv[0], argv[argi]);
}
if (streq(wdesc, "dkvp")) popts->plrec_writer = lrec_writer_dkvp_alloc(popts->ors, popts->ofs, popts->ops);
else if (streq(wdesc, "csv")) popts->plrec_writer = lrec_writer_csv_alloc(popts->ors, popts->ofs);
else if (streq(wdesc, "nidx")) popts->plrec_writer = lrec_writer_nidx_alloc(popts->ors, popts->ofs);
else if (streq(wdesc, "xtab")) popts->plrec_writer = lrec_writer_xtab_alloc();
else if (streq(wdesc, "pprint")) popts->plrec_writer = lrec_writer_pprint_alloc(left_align_pprint);
if (streq(ofmt, "dkvp")) popts->plrec_writer = lrec_writer_dkvp_alloc(popts->ors, popts->ofs, popts->ops);
else if (streq(ofmt, "csv")) popts->plrec_writer = lrec_writer_csv_alloc(popts->ors, popts->ofs);
else if (streq(ofmt, "nidx")) popts->plrec_writer = lrec_writer_nidx_alloc(popts->ors, popts->ofs);
else if (streq(ofmt, "xtab")) popts->plrec_writer = lrec_writer_xtab_alloc();
else if (streq(ofmt, "pprint")) popts->plrec_writer = lrec_writer_pprint_alloc(left_align_pprint);
else {
main_usage(argv[0], 1);
}
@ -369,9 +367,9 @@ cli_opts_t* parse_command_line(int argc, char** argv) {
// No filenames means read from standard input, and standard input cannot be mmaped.
if (argi == argc)
use_file_reader_mmap = FALSE;
popts->use_mmap_for_read = FALSE;
popts->plrec_reader = lrec_reader_alloc(rdesc, use_file_reader_mmap,
popts->plrec_reader = lrec_reader_alloc(popts->ifmt, popts->use_mmap_for_read,
popts->irs, popts->ifs, popts->allow_repeat_ifs, popts->ips, popts->allow_repeat_ips);
if (popts->plrec_reader == NULL)
main_usage(argv[0], 1);

12
c/cli/mlrcli.h
View file

@ -7,11 +7,13 @@
#include "output/lrec_writer.h"
typedef struct _cli_opts_t {
char irs;
char ifs;
char ips;
int allow_repeat_ifs;
int allow_repeat_ips;
char irs;
char ifs;
char ips;
int allow_repeat_ifs;
int allow_repeat_ips;
int use_mmap_for_read;
char* ifmt;
char ors;
char ofs;

9
c/lib/mlr_globals.c
View file

@ -1,8 +1,9 @@
#include <stdlib.h>
#include "lib/mlr_globals.h"
mlr_globals_t MLR_GLOBALS = { .argv0 = NULL, .ofmt = NULL };
void mlr_global_init(char* argv0, char* ofmt) {
MLR_GLOBALS.argv0 = argv0;
MLR_GLOBALS.ofmt = ofmt;
mlr_globals_t MLR_GLOBALS = { .argv0 = NULL, .ofmt = NULL, .popts = NULL };
void mlr_global_init(char* argv0, char* ofmt, cli_opts_t* popts) {
MLR_GLOBALS.argv0 = argv0;
MLR_GLOBALS.ofmt = ofmt;
MLR_GLOBALS.popts = popts;
}

11
c/lib/mlr_globals.h
View file

@ -1,11 +1,16 @@
#ifndef MLR_GLOBALS_H
#define MLR_GLOBALS_H
#include "cli/mlrcli.h"
typedef struct _mlr_globals_t {
char* argv0;
char* ofmt;
char* argv0;
char* ofmt;
// These are shared by mlrcli.c and mlrmain.c. The only reason for their
// exposure anywhere else is to communicate format and separator flags to
// mapper_join, which (unlike other mappers) needs to do its own file I/O.
cli_opts_t* popts;
} mlr_globals_t;
extern mlr_globals_t MLR_GLOBALS;
void mlr_global_init(char* argv0, char* ofmt);
void mlr_global_init(char* argv0, char* ofmt, cli_opts_t* popts);
#endif // MLR_GLOBALS_H

2
c/mapping/lrec_evaluators.c
View file

@ -144,7 +144,7 @@ mv_t lrec_evaluator_f_ff_func(lrec_t* prec, context_t* pctx, void* pvstate) {
return MV_ERROR;
mv_t val2 = pstate->parg2->pevaluator_func(prec, pctx, pstate->parg2->pvstate);
NULL_OR_ERROR_OUT(val2);
NULL_OR_ERROR_OUT(val2);
mt_get_double_strict(&val2);
if (val2.type != MT_DOUBLE)
return MV_ERROR;

21
c/mapping/mapper_join.c
View file

@ -133,14 +133,18 @@ static void mapper_join_free(void* pvstate) {
}
// ----------------------------------------------------------------
// xxx void-abstract the stdio/mmap readers. this is a pita. also neaten up stream.c.
// xxx have a reader factory which can be called here.
// xxx for the moment, just dev with hard-coded separator & format parameters.
// Format and separator flags are passed to mapper_join in MLR_GLOBALS rather
// than on the stack, since the latter would require complicating the interface
// for all the other mappers which don't do their own file I/O. (Also, while
// some of the information needed to construct an lrec_reader is available on
// the command line before the mapper-allocators are called, some is not
// available until after. Hence our obtaining these flags after mapper-alloc.)
static void ingest_left_file(mapper_join_state_t* pstate) {
// xxx temp
lrec_reader_t* plrec_reader = lrec_reader_alloc("dkvp", TRUE, '\n', ',', FALSE, '=', FALSE);
cli_opts_t* popts = MLR_GLOBALS.popts;
lrec_reader_t* plrec_reader = lrec_reader_alloc(popts->ifmt, popts->use_mmap_for_read,
popts->irs, popts->ifs, popts->allow_repeat_ifs, popts->ips, popts->allow_repeat_ips);
void* pvhandle = plrec_reader->popen_func(pstate->left_file_name);
plrec_reader->psof_func(plrec_reader->pvstate);
@ -262,7 +266,12 @@ static mapper_t* mapper_join_parse_cli(int* pargi, int argc, char** argv) {
return NULL;
}
// xxx check not all emit-flags are off -- else no output.
if (!emit_pairables && !emit_left_unpairables && !emit_right_unpairables) {
fprintf(stderr, "%s %s: all emit flags are unset; no output is possible.\n",
MLR_GLOBALS.argv0, verb);
mapper_join_usage(argv[0], verb);
return NULL;
}
if (pright_field_names == NULL)
pright_field_names = slls_copy(pleft_field_names);

4
c/mlrmain.c
View file

@ -23,9 +23,9 @@ int main(int argc, char** argv) {
}
fprintf(stderr, "Set up mcheck\n");
#endif
mlr_global_init(argv[0], NULL);
mlr_global_init(argv[0], NULL, NULL);
cli_opts_t* popts = parse_command_line(argc, argv);
mlr_global_init(argv[0], popts->ofmt);
mlr_global_init(argv[0], popts->ofmt, popts);
lrec_reader_t* plrec_reader = popts->plrec_reader;
sllv_t* pmapper_list = popts->pmapper_list;

3
c/todo.txt
View file

@ -10,11 +10,10 @@
================================================================
TOP OF LIST
* mapper_list -> lrec_mapper_list etc.
!! should be able to do: mlr put '$ok = $x < 10'
!! join
-> parametrized reader; global cliopts; overrides for join per se
-> UTs
!! mmap cleanup, especially csv