Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-18 00:45:47 +00:00
Code Issues Projects Releases Wiki Activity

zcat iterate

Browse source
This commit is contained in:
John Kerl 2015-12-13 17:25:50 -07:00
parent 6fa14775d1
commit 2dd9bb07b4
18 changed files with 62 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

9
c/cli/mlrcli.c
View file

@ -506,10 +506,12 @@ cli_opts_t* parse_command_line(int argc, char** argv) {
popts->plrec_reader = NULL;
popts->plrec_writer = NULL;
popts->prepipe = NULL;
popts->filenames = NULL;
popts->ifile_fmt = "dkvp";
popts->ofile_fmt = "dkvp";
popts->ofile_fmt = "dkvp";
popts->use_mmap_for_read = TRUE;
int left_align_pprint = TRUE;
@ -720,6 +722,11 @@ cli_opts_t* parse_command_line(int argc, char** argv) {
}
argi++;
} else if (streq(argv[argi], "--prepipe")) {
check_arg_count(argv, argi, argc, 2);
popts->prepipe = argv[argi+1];
argi++;
} else {
usage_unrecognized_verb(argv[0], argv[argi]);
}

1
c/cli/mlrcli.h
View file

@ -35,6 +35,7 @@ typedef struct _cli_opts_t {
sllv_t* pmapper_list;
lrec_writer_t* plrec_writer;
char* prepipe; // Command for popen on input, e.g. "zcat -cf <". Can be null.
char** filenames; // null-terminated
} cli_opts_t;

6
c/containers/join_bucket_keeper.c
View file

@ -140,6 +140,7 @@ static char* describe_state(int state);
// ----------------------------------------------------------------
join_bucket_keeper_t* join_bucket_keeper_alloc(
char* prepipe,
char* left_file_name,
char* input_file_format,
int use_mmap_for_read,
@ -154,18 +155,19 @@ join_bucket_keeper_t* join_bucket_keeper_alloc(
lrec_reader_t* plrec_reader = lrec_reader_alloc(input_file_format, use_mmap_for_read,
irs, ifs, allow_repeat_ifs, ips, allow_repeat_ips, use_implicit_csv_header);
return join_bucket_keeper_alloc_from_reader(plrec_reader, left_file_name, pleft_field_names);
return join_bucket_keeper_alloc_from_reader(plrec_reader, prepipe, left_file_name, pleft_field_names);
}
// ----------------------------------------------------------------
join_bucket_keeper_t* join_bucket_keeper_alloc_from_reader(
lrec_reader_t* plrec_reader,
char* prepipe,
char* left_file_name,
slls_t* pleft_field_names)
{
join_bucket_keeper_t* pkeeper = mlr_malloc_or_die(sizeof(join_bucket_keeper_t));
void* pvhandle = plrec_reader->popen_func(plrec_reader->pvstate, left_file_name);
void* pvhandle = plrec_reader->popen_func(plrec_reader->pvstate, prepipe, left_file_name);
plrec_reader->psof_func(plrec_reader->pvstate);
context_t* pctx = mlr_malloc_or_die(sizeof(context_t));

2
c/containers/join_bucket_keeper.h
View file

@ -35,6 +35,7 @@ typedef struct _join_bucket_keeper_t {
// ----------------------------------------------------------------
join_bucket_keeper_t* join_bucket_keeper_alloc(
char* prepipe,
char* left_file_name,
char* input_file_format,
int use_mmap_for_read,
@ -48,6 +49,7 @@ join_bucket_keeper_t* join_bucket_keeper_alloc(
join_bucket_keeper_t* join_bucket_keeper_alloc_from_reader(
lrec_reader_t* plrec_reader,
char* prepipe,
char* left_file_name,
slls_t* pleft_field_names);

7
c/input/file_reader_mmap.c
View file

@ -12,7 +12,8 @@
static char empty_buf[1] = { 0 };
// ----------------------------------------------------------------
file_reader_mmap_state_t* file_reader_mmap_open(char* file_name) {
file_reader_mmap_state_t* file_reader_mmap_open(char* prepipe, char* file_name) {
// xxx abend if prepipe is non-null
file_reader_mmap_state_t* pstate = mlr_malloc_or_die(sizeof(file_reader_mmap_state_t));
pstate->fd = open(file_name, O_RDONLY);
if (pstate->fd < 0) {
@ -51,8 +52,8 @@ void file_reader_mmap_close(file_reader_mmap_state_t* pstate) {
}
// ----------------------------------------------------------------
void* file_reader_mmap_vopen(void* pvstate, char* file_name) {
return file_reader_mmap_open(file_name);
void* file_reader_mmap_vopen(void* pvstate, char* prepipe, char* file_name) {
return file_reader_mmap_open(prepipe, file_name);
}
// ----------------------------------------------------------------

4
c/input/file_reader_mmap.h
View file

@ -11,10 +11,10 @@ typedef struct _file_reader_mmap_state_t {
int fd;
} file_reader_mmap_state_t;
file_reader_mmap_state_t* file_reader_mmap_open(char* file_name);
file_reader_mmap_state_t* file_reader_mmap_open(char* prepipe, char* file_name);
void file_reader_mmap_close(file_reader_mmap_state_t* pstate);
void* file_reader_mmap_vopen(void* pvstate, char* file_name);
void* file_reader_mmap_vopen(void* pvstate, char* prepipe, char* file_name);
void file_reader_mmap_vclose(void* pvstate, void* pvhandle);
#endif // FILE_READER_MMAP_H

2
c/input/file_reader_stdio.c
View file

@ -10,7 +10,7 @@
#include "file_reader_stdio.h"
// ----------------------------------------------------------------
void* file_reader_stdio_vopen(void* pvstate, char* filename) {
void* file_reader_stdio_vopen(void* pvstate, char* prepipe, char* filename) {
FILE* input_stream = stdin;
if (!streq(filename, "-")) {

2
c/input/file_reader_stdio.h
View file

@ -5,7 +5,7 @@
#ifndef FILE_READER_STDIO_H
#define FILE_READER_STDIO_H
void* file_reader_stdio_vopen(void* pvstate, char* file_name);
void* file_reader_stdio_vopen(void* pvstate, char* prepipe, char* file_name);
void file_reader_stdio_vclose(void* pvstate, void* pvhandle);
#endif // FILE_READER_STDIO_H

2
c/input/lrec_reader.h
View file

@ -7,7 +7,7 @@
#include "input/file_reader_mmap.h"
// The void* pvhandle is either FILE* for stdio readers or file_reader_mmap_state_t* for mmap readers.
typedef void* lrec_reader_open_func_t(void* pvstate, char* filename);
typedef void* lrec_reader_open_func_t(void* pvstate, char* prepipe, char* filename);
typedef void lrec_reader_close_func_t(void* pvstate, void* pvhandle);
typedef lrec_t* lrec_reader_process_func_t(void* pvstate, void* pvhandle, context_t* pctx);
typedef void lrec_reader_sof_func_t(void* pvstate);

5
c/input/lrec_reader_in_memory.c
View file

@ -13,7 +13,7 @@ typedef struct _lrec_reader_in_memory_state_t {
static void lrec_reader_in_memory_free(void* pvstate);
static void lrec_reader_in_memory_sof(void* pvstate);
static lrec_t* lrec_reader_in_memory_process(void* pvstate, void* pvhandle, context_t* pctx);
static void* lrec_reader_in_memory_vopen(void* pvstate, char* filename);
static void* lrec_reader_in_memory_vopen(void* pvstate, char* prepipe, char* filename);
static void lrec_reader_in_memory_vclose(void* pvstate, void* pvhandle);
// ----------------------------------------------------------------
@ -51,7 +51,8 @@ static lrec_t* lrec_reader_in_memory_process(void* pvstate, void* pvhandle, cont
return sllv_pop(pstate->precords);
}
static void* lrec_reader_in_memory_vopen(void* pvstate, char* filename) {
static void* lrec_reader_in_memory_vopen(void* pvstate, char* prepipe, char* filename) {
// xxx abend if prepipe != NULL
return NULL;
}

6
c/input/lrec_reader_mmap_csv.c
View file

@ -68,7 +68,7 @@ typedef struct _lrec_reader_mmap_csv_state_t {
static void lrec_reader_mmap_csv_free(void* pvstate);
static void lrec_reader_mmap_csv_sof(void* pvstate);
static void* lrec_reader_mmap_csv_vopen(void* pvstate, char* file_name);
static void* lrec_reader_mmap_csv_vopen(void* pvstate, char* prepipe, char* file_name);
static lrec_t* lrec_reader_mmap_csv_process(void* pvstate, void* pvhandle, context_t* pctx);
static int lrec_reader_mmap_csv_get_fields(lrec_reader_mmap_csv_state_t* pstate,
rslls_t* pfields, file_reader_mmap_state_t* phandle);
@ -148,8 +148,8 @@ static void lrec_reader_mmap_csv_sof(void* pvstate) {
}
// ----------------------------------------------------------------
static void* lrec_reader_mmap_csv_vopen(void* pvstate, char* file_name) {
void* pvhandle = file_reader_mmap_open(file_name);
static void* lrec_reader_mmap_csv_vopen(void* pvstate, char* prepipe, char* file_name) {
void* pvhandle = file_reader_mmap_open(prepipe, file_name);
file_reader_mmap_state_t* phandle = pvhandle;
*phandle->eof = EOF;
return pvhandle;

4
c/input/lrec_reader_stdio_csv.c
View file

@ -77,7 +77,7 @@ static lrec_t* paste_indices_and_data(lrec_reader_stdio_csv_state_t* pstate, rsl
context_t* pctx);
static lrec_t* paste_header_and_data(lrec_reader_stdio_csv_state_t* pstate, rslls_t* pdata_fields,
context_t* pctx);
static void* lrec_reader_stdio_csv_open(void* pvstate, char* filename);
static void* lrec_reader_stdio_csv_open(void* pvstate, char* prepipe, char* filename);
static void lrec_reader_stdio_csv_close(void* pvstate, void* pvhandle);
// ----------------------------------------------------------------
@ -368,7 +368,7 @@ static lrec_t* paste_header_and_data(lrec_reader_stdio_csv_state_t* pstate, rsll
}
// ----------------------------------------------------------------
static void* lrec_reader_stdio_csv_open(void* pvstate, char* filename) {
static void* lrec_reader_stdio_csv_open(void* pvstate, char* prepipe, char* filename) {
lrec_reader_stdio_csv_state_t* pstate = pvstate;
pstate->pfr->pbr->popen_func(pstate->pfr->pbr, filename);
pfr_reset(pstate->pfr);

8
c/mapping/mapper_join.c
View file

@ -24,6 +24,7 @@ typedef struct _mapper_join_opts_t {
int emit_left_unpairables;
int emit_right_unpairables;
char* prepipe;
char* left_file_name;
// These allow the joiner to have its own different format/delimiter for
@ -117,6 +118,7 @@ static mapper_t* mapper_join_parse_cli(int* pargi, int argc, char** argv) {
mapper_join_opts_t* popts = mlr_malloc_or_die(sizeof(mapper_join_opts_t));
popts->left_prefix = NULL;
popts->right_prefix = NULL;
popts->prepipe = NULL;
popts->left_file_name = NULL;
popts->poutput_join_field_names = NULL;
popts->pleft_join_field_names = NULL;
@ -138,6 +140,7 @@ static mapper_t* mapper_join_parse_cli(int* pargi, int argc, char** argv) {
char* verb = argv[(*pargi)++];
ap_state_t* pstate = ap_alloc();
ap_define_string_flag(pstate, "--prepipe", &popts->prepipe);
ap_define_string_flag(pstate, "-f", &popts->left_file_name);
ap_define_string_list_flag(pstate, "-j", &popts->poutput_join_field_names);
ap_define_string_list_flag(pstate, "-l", &popts->pleft_join_field_names);
@ -246,6 +249,7 @@ static sllv_t* mapper_join_process_sorted(lrec_t* pright_rec, context_t* pctx, v
mapper_join_opts_t* popts = pstate->popts;
merge_options(pstate->popts);
pstate->pjoin_bucket_keeper = join_bucket_keeper_alloc(
popts->prepipe,
popts->left_file_name,
popts->input_file_format,
popts->use_mmap_for_read,
@ -466,7 +470,9 @@ static void ingest_left_file(mapper_join_state_t* pstate) {
popts->irs, popts->ifs, popts->allow_repeat_ifs, popts->ips, popts->allow_repeat_ips,
popts->use_implicit_csv_header);
void* pvhandle = plrec_reader->popen_func(plrec_reader->pvstate, pstate->popts->left_file_name);
// xxx override ...
void* pvhandle = plrec_reader->popen_func(plrec_reader->pvstate, pstate->popts->prepipe,
pstate->popts->left_file_name);
plrec_reader->psof_func(plrec_reader->pvstate);
context_t ctx = { .nr = 0, .fnr = 0, .filenum = 1, .filename = pstate->popts->left_file_name };

3
c/mlrmain.c
View file

@ -27,12 +27,13 @@ int main(int argc, char** argv) {
cli_opts_t* popts = parse_command_line(argc, argv);
mlr_global_init(argv[0], popts->ofmt, popts);
char* prepipe = popts->prepipe;
lrec_reader_t* plrec_reader = popts->plrec_reader;
sllv_t* pmapper_list = popts->pmapper_list;
lrec_writer_t* plrec_writer = popts->plrec_writer;
char** filenames = popts->filenames;
int ok = do_stream_chained(filenames, plrec_reader, pmapper_list, plrec_writer, popts->ofmt);
int ok = do_stream_chained(prepipe, filenames, plrec_reader, pmapper_list, plrec_writer, popts->ofmt);
cli_opts_free(popts);

13
c/stream/stream.c
View file

@ -10,7 +10,7 @@
#include "mapping/mappers.h"
#include "output/lrec_writers.h"
static int do_file_chained(char* filename, context_t* pctx,
static int do_file_chained(char* prepipe, char* filename, context_t* pctx,
lrec_reader_t* plrec_reader, sllv_t* pmapper_list, lrec_writer_t* plrec_writer, FILE* output_stream);
static sllv_t* chain_map(lrec_t* pinrec, context_t* pctx, sllve_t* pmapper_list_head);
@ -19,7 +19,7 @@ static void drive_lrec(lrec_t* pinrec, context_t* pctx, sllve_t* pmapper_list_he
FILE* output_stream);
// ----------------------------------------------------------------
int do_stream_chained(char** filenames, lrec_reader_t* plrec_reader, sllv_t* pmapper_list,
int do_stream_chained(char* prepipe, char** filenames, lrec_reader_t* plrec_reader, sllv_t* pmapper_list,
lrec_writer_t* plrec_writer, char* ofmt)
{
FILE* output_stream = stdout;
@ -36,7 +36,7 @@ int do_stream_chained(char** filenames, lrec_reader_t* plrec_reader, sllv_t* pma
ctx.filenum++;
ctx.filename = "(stdin)";
ctx.fnr = 0;
ok = do_file_chained("-", &ctx, plrec_reader, pmapper_list, plrec_writer, output_stream) && ok;
ok = do_file_chained(prepipe, "-", &ctx, plrec_reader, pmapper_list, plrec_writer, output_stream) && ok;
} else {
for (char** pfilename = filenames; *pfilename != NULL; pfilename++) {
ctx.filenum++;
@ -44,7 +44,8 @@ int do_stream_chained(char** filenames, lrec_reader_t* plrec_reader, sllv_t* pma
ctx.fnr = 0;
// Start-of-file hook, e.g. expecting CSV headers on input.
plrec_reader->psof_func(plrec_reader->pvstate);
ok = do_file_chained(*pfilename, &ctx, plrec_reader, pmapper_list, plrec_writer, output_stream) && ok;
ok = do_file_chained(prepipe, *pfilename, &ctx, plrec_reader, pmapper_list,
plrec_writer, output_stream) && ok;
}
}
@ -59,10 +60,10 @@ int do_stream_chained(char** filenames, lrec_reader_t* plrec_reader, sllv_t* pma
}
// ----------------------------------------------------------------
static int do_file_chained(char* filename, context_t* pctx,
static int do_file_chained(char* prepipe, char* filename, context_t* pctx,
lrec_reader_t* plrec_reader, sllv_t* pmapper_list, lrec_writer_t* plrec_writer, FILE* output_stream)
{
void* pvhandle = plrec_reader->popen_func(plrec_reader->pvstate, filename);
void* pvhandle = plrec_reader->popen_func(plrec_reader->pvstate, prepipe, filename);
while (1) {
lrec_t* pinrec = plrec_reader->pprocess_func(plrec_reader->pvstate, pvhandle, pctx);

2
c/stream/stream.h
View file

@ -7,7 +7,7 @@
#include "output/lrec_writers.h"
// The filenames array must be null-terminated
int do_stream_chained(char** filenames, lrec_reader_t* plrec_reader, sllv_t* pmapper_list,
int do_stream_chained(char* prepipe, char** filenames, lrec_reader_t* plrec_reader, sllv_t* pmapper_list,
lrec_writer_t* plrec_writer, char* ofmt);
#endif // STREAM_H

1
c/todo.txt
View file

@ -17,6 +17,7 @@ TOP OF LIST
- popen w/ specified args, e.g. zcat (or sh -c zcat ... or env zcat ...)
- doc why not libs
- consider popen output, but probably just suggest pipe.
- UT join w/ mixed left/right compressed/not
* more perf tuning:
- strict power-of-two malloc/strdups thruout?!? profile.

24
c/unit_test/test_join_bucket_keeper.c
View file

@ -122,7 +122,7 @@ static char* test_left_empty_right_empty() {
slls_t* pleft_field_names;
lrec_reader_t* preader;
set_up(make_records_empty(), &pleft_field_names, &preader);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, pleft_field_names);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, NULL, pleft_field_names);
sllv_t* precords_paired;
sllv_t* precords_left_unpaired;
@ -144,7 +144,7 @@ static char* test_left_empty() {
slls_t* pleft_field_names;
lrec_reader_t* preader;
set_up(make_records_empty(), &pleft_field_names, &preader);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, pleft_field_names);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, NULL, pleft_field_names);
sllv_t* precords_paired;
sllv_t* precords_left_unpaired;
@ -172,7 +172,7 @@ static char* test_right_empty() {
slls_t* pleft_field_names;
lrec_reader_t* preader;
set_up(make_records_113335(), &pleft_field_names, &preader);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, pleft_field_names);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, NULL, pleft_field_names);
sllv_t* precords_paired;
sllv_t* precords_left_unpaired;
@ -194,7 +194,7 @@ static char* test_unpaired_before_left_start() {
slls_t* pleft_field_names;
lrec_reader_t* preader;
set_up(make_records_113335(), &pleft_field_names, &preader);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, pleft_field_names);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, NULL, pleft_field_names);
sllv_t* precords_paired;
sllv_t* precords_left_unpaired;
@ -222,7 +222,7 @@ static char* test_unpaired_after_left_end() {
slls_t* pleft_field_names;
lrec_reader_t* preader;
set_up(make_records_113335(), &pleft_field_names, &preader);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, pleft_field_names);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, NULL, pleft_field_names);
sllv_t* precords_paired;
sllv_t* precords_left_unpaired;
@ -250,7 +250,7 @@ static char* test_middle_pairings() {
slls_t* pleft_field_names;
lrec_reader_t* preader;
set_up(make_records_113335(), &pleft_field_names, &preader);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, pleft_field_names);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, NULL, pleft_field_names);
sllv_t* precords_paired;
sllv_t* precords_left_unpaired;
@ -290,7 +290,7 @@ static char* test_middle() {
slls_t* pleft_field_names;
lrec_reader_t* preader;
set_up(make_records_113335(), &pleft_field_names, &preader);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, pleft_field_names);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, NULL, pleft_field_names);
sllv_t* precords_paired;
sllv_t* precords_left_unpaired;
@ -330,7 +330,7 @@ static char* test_walk_through_all() {
slls_t* pleft_field_names;
lrec_reader_t* preader;
set_up(make_records_113335(), &pleft_field_names, &preader);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, pleft_field_names);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, NULL, pleft_field_names);
sllv_t* precords_paired;
sllv_t* precords_left_unpaired;
@ -435,7 +435,7 @@ static char* test_het_unpaired_before_left_start() {
slls_t* pleft_field_names;
lrec_reader_t* preader;
set_up(make_records_het(), &pleft_field_names, &preader);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, pleft_field_names);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, NULL, pleft_field_names);
sllv_t* precords_paired;
sllv_t* precords_left_unpaired;
@ -461,7 +461,7 @@ static char* test_het_unpaired_after_left_end() {
slls_t* pleft_field_names;
lrec_reader_t* preader;
set_up(make_records_het(), &pleft_field_names, &preader);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, pleft_field_names);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, NULL, pleft_field_names);
sllv_t* precords_paired;
sllv_t* precords_left_unpaired;
@ -487,7 +487,7 @@ static char* test_het_initial_pairing() {
slls_t* pleft_field_names;
lrec_reader_t* preader;
set_up(make_records_het(), &pleft_field_names, &preader);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, pleft_field_names);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, NULL, pleft_field_names);
sllv_t* precords_paired;
sllv_t* precords_left_unpaired;
@ -514,7 +514,7 @@ static char* test_het_middle_pairing() {
slls_t* pleft_field_names;
lrec_reader_t* preader;
set_up(make_records_het(), &pleft_field_names, &preader);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, pleft_field_names);
join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, NULL, pleft_field_names);
sllv_t* precords_paired;
sllv_t* precords_left_unpaired;