From 55cabc3ea42b285fd7550d7941b6facf40c2d3fd Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sun, 13 Dec 2015 18:42:39 -0700 Subject: [PATCH] zcat iterate --- c/cli/mlrcli.c | 26 ++++++++++++++++++++++++++ c/todo.txt | 3 +++ doc/mkman.rb | 3 +++ doc/mlr.1.premade | 27 +++++++++++++++++++++++++-- doc/reference.html | 17 +++++++++++++++++ 5 files changed, 74 insertions(+), 2 deletions(-) diff --git a/c/cli/mlrcli.c b/c/cli/mlrcli.c index ee912aca4..f392f2545 100644 --- a/c/cli/mlrcli.c +++ b/c/cli/mlrcli.c @@ -299,6 +299,25 @@ static void main_usage_data_format_options(FILE* o, char* argv0) { fprintf(o, " DKVP-formatted input and pretty-printed output.\n"); } +static void main_usage_compressed_data_options(FILE* o, char* argv0) { + fprintf(o, " --prepipe {command} This allows Miller to handle compressed inputs. You can do\n"); + fprintf(o, " without this for single input files, e.g. \"gunzip < myfile.csv.gz | %s ...\".\n", + argv0); + fprintf(o, " However, when multiple input files are present, between-file separations are\n"); + fprintf(o, " lost; also, the FILENAME variable doesn't iterate. Using --prepipe you can\n"); + fprintf(o, " specify an action to be taken on each input file.\n"); + fprintf(o, " Examples:\n"); + fprintf(o, " %s --prepipe 'gunzip <'\n", argv0); + fprintf(o, " %s --prepipe 'zcat -cf <'\n", argv0); + fprintf(o, " %s --prepipe 'xz -cd <'\n", argv0); + fprintf(o, " %s --prepipe cat\n", argv0); + fprintf(o, " %s --prepipe 'cat <'\n", argv0); + fprintf(o, " Note that this feature is quite general and is not limited to decompression\n"); + fprintf(o, " utilities. You can use it to apply per-file filters of your choice.\n"); + fprintf(o, " For output compression (or other) utilities, simply pipe the output:\n"); + fprintf(o, " %s ... | {your compression command}\n", argv0); +} + static void main_usage_separator_options(FILE* o, char* argv0) { fprintf(o, " --rs --irs --ors Record separators, e.g. 'lf' or '\\r\\n'\n"); fprintf(o, " --fs --ifs --ofs --repifs Field separators, e.g. comma\n"); @@ -420,6 +439,10 @@ static void main_usage(FILE* o, char* argv0) { main_usage_data_format_options(o, argv0); fprintf(o, "\n"); + fprintf(o, "Compressed-data options:\n"); + main_usage_compressed_data_options(o, argv0); + fprintf(o, "\n"); + fprintf(o, "Separator options, for input, output, or both:\n"); main_usage_separator_options(o, argv0); fprintf(o, "\n"); @@ -580,6 +603,9 @@ cli_opts_t* parse_command_line(int argc, char** argv) { } else if (streq(argv[argi], "--usage-data-format-options")) { main_usage_data_format_options(stdout, argv[0]); exit(0); + } else if (streq(argv[argi], "--usage-compressed-data-options")) { + main_usage_compressed_data_options(stdout, argv[0]); + exit(0); } else if (streq(argv[argi], "--usage-separator-options")) { main_usage_separator_options(stdout, argv[0]); exit(0); diff --git a/c/todo.txt b/c/todo.txt index f85e7e1d8..d75764ecf 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -19,6 +19,9 @@ TOP OF LIST - consider popen output, but probably just suggest pipe. - UT join w/ mixed left/right compressed/not - mlr -h, mld section, etc. + o zcat -cf < + o gunzip < + o xz -cd < * more perf tuning: - strict power-of-two malloc/strdups thruout?!? profile. diff --git a/doc/mkman.rb b/doc/mkman.rb index 99697f437..7bec5de0d 100755 --- a/doc/mkman.rb +++ b/doc/mkman.rb @@ -63,6 +63,9 @@ output separator to the given value.""" print make_subsection('I/O FORMATTING', []) print make_code_block(`mlr --usage-data-format-options`) + print make_subsection('COMPRESSED I/O', []) + print make_code_block(`mlr --usage-compressed-data-options`) + print make_subsection('SEPARATORS', []) print make_code_block(`mlr --usage-separator-options`) diff --git a/doc/mlr.1.premade b/doc/mlr.1.premade index 2a6c1862a..bfb9fd6c2 100644 --- a/doc/mlr.1.premade +++ b/doc/mlr.1.premade @@ -2,12 +2,12 @@ .\" Title: mlr .\" Author: [see the "AUTHOR" section] .\" Generator: ./mkman.rb -.\" Date: 2015-12-12 +.\" Date: 2015-12-14 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "MILLER" "1" "2015-12-12" "\ \&" "\ \&" +.TH "MILLER" "1" "2015-12-14" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Portability definitions .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -172,6 +172,29 @@ Please use "mlr --help-all-functions" or "mlr -f" for help on all functions. .fi .if n \{\ .RE +.SS "COMPRESSED I/O" +.if n \{\ +.RS 0 +.\} +.nf + --prepipe {command} This allows Miller to handle compressed inputs. You can do + without this for single input files, e.g. "gunzip < myfile.csv.gz | mlr ...". + However, when multiple input files are present, between-file separations are + lost; also, the FILENAME variable doesn't iterate. Using --prepipe you can + specify an action to be taken on each input file. + Examples: + mlr --prepipe 'gunzip <' + mlr --prepipe 'zcat -cf <' + mlr --prepipe 'xz -cd <' + mlr --prepipe cat + mlr --prepipe 'cat <' + Note that this feature is quite general and is not limited to decompression + utilities. You can use it to apply per-file filters of your choice. + For output compression (or other) utilities, simply pipe the output: + mlr ... | {your compression command} +.fi +.if n \{\ +.RE .SS "SEPARATORS" .if n \{\ .RS 0 diff --git a/doc/reference.html b/doc/reference.html index a76093486..12095767e 100644 --- a/doc/reference.html +++ b/doc/reference.html @@ -366,6 +366,23 @@ Data-format options, for input, output, or both: Examples: --csv for CSV-formatted input and output; --idkvp --opprint for DKVP-formatted input and pretty-printed output. +Compressed-data options: + --prepipe {command} This allows Miller to handle compressed inputs. You can do + without this for single input files, e.g. "gunzip < myfile.csv.gz | mlr ...". + However, when multiple input files are present, between-file separations are + lost; also, the FILENAME variable doesn't iterate. Using --prepipe you can + specify an action to be taken on each input file. + Examples: + mlr --prepipe 'gunzip <' + mlr --prepipe 'zcat -cf <' + mlr --prepipe 'xz -cd <' + mlr --prepipe cat + mlr --prepipe 'cat <' + Note that this feature is quite general and is not limited to decompression + utilities. You can use it to apply per-file filters of your choice. + For output compression (or other) utilities, simply pipe the output: + mlr ... | {your compression command} + Separator options, for input, output, or both: --rs --irs --ors Record separators, e.g. 'lf' or '\r\n' --fs --ifs --ofs --repifs Field separators, e.g. comma