zcat iterate

c/cli/mlrcli.c

@@ -299,6 +299,25 @@ static void main_usage_data_format_options(FILE* o, char* argv0) {
 	fprintf(o, "  DKVP-formatted input and pretty-printed output.\n");
 }
 
+static void main_usage_compressed_data_options(FILE* o, char* argv0) {
+	fprintf(o, "  --prepipe {command} This allows Miller to handle compressed inputs. You can do\n");
+	fprintf(o, "  without this for single input files, e.g. \"gunzip < myfile.csv.gz | %s ...\".\n",
+		argv0);
+	fprintf(o, "  However, when multiple input files are present, between-file separations are\n");
+	fprintf(o, "  lost; also, the FILENAME variable doesn't iterate. Using --prepipe you can\n");
+	fprintf(o, "  specify an action to be taken on each input file.\n");
+	fprintf(o, "  Examples:\n");
+	fprintf(o, "    %s --prepipe 'gunzip <'\n", argv0);
+	fprintf(o, "    %s --prepipe 'zcat -cf <'\n", argv0);
+	fprintf(o, "    %s --prepipe 'xz -cd <'\n", argv0);
+	fprintf(o, "    %s --prepipe cat\n", argv0);
+	fprintf(o, "    %s --prepipe 'cat <'\n", argv0);
+	fprintf(o, "  Note that this feature is quite general and is not limited to decompression\n");
+	fprintf(o, "  utilities. You can use it to apply per-file filters of your choice.\n");
+	fprintf(o, "  For output compression (or other) utilities, simply pipe the output:\n");
+	fprintf(o, "    %s ... | {your compression command}\n", argv0);
+}
+
 static void main_usage_separator_options(FILE* o, char* argv0) {
 	fprintf(o, "  --rs     --irs     --ors              Record separators, e.g. 'lf' or '\\r\\n'\n");
 	fprintf(o, "  --fs     --ifs     --ofs  --repifs    Field separators, e.g. comma\n");
@@ -420,6 +439,10 @@ static void main_usage(FILE* o, char* argv0) {
 	main_usage_data_format_options(o, argv0);
 	fprintf(o, "\n");
 
+	fprintf(o, "Compressed-data options:\n");
+	main_usage_compressed_data_options(o, argv0);
+	fprintf(o, "\n");
+
 	fprintf(o, "Separator options, for input, output, or both:\n");
 	main_usage_separator_options(o, argv0);
 	fprintf(o, "\n");
@@ -580,6 +603,9 @@ cli_opts_t* parse_command_line(int argc, char** argv) {
 		} else if (streq(argv[argi], "--usage-data-format-options")) {
 			main_usage_data_format_options(stdout, argv[0]);
 			exit(0);
+		} else if (streq(argv[argi], "--usage-compressed-data-options")) {
+			main_usage_compressed_data_options(stdout, argv[0]);
+			exit(0);
 		} else if (streq(argv[argi], "--usage-separator-options")) {
 			main_usage_separator_options(stdout, argv[0]);
 			exit(0);

c/todo.txt

@@ -19,6 +19,9 @@ TOP OF LIST
   - consider popen output, but probably just suggest pipe.
   - UT join w/ mixed left/right compressed/not
   - mlr -h, mld section, etc.
+    o zcat -cf <
+    o gunzip <
+    o xz -cd <
 
 * more perf tuning:
   - strict power-of-two malloc/strdups thruout?!? profile.

doc/mkman.rb

@@ -63,6 +63,9 @@ output separator to the given value."""
 	print make_subsection('I/O FORMATTING', [])
 	print make_code_block(`mlr --usage-data-format-options`)
 
+	print make_subsection('COMPRESSED I/O', [])
+	print make_code_block(`mlr --usage-compressed-data-options`)
+
 	print make_subsection('SEPARATORS', [])
 	print make_code_block(`mlr --usage-separator-options`)
 

doc/mlr.1.premade

@@ -2,12 +2,12 @@
 .\"     Title: mlr
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: ./mkman.rb
-.\"      Date: 2015-12-12
+.\"      Date: 2015-12-14
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "MILLER" "1" "2015-12-12" "\ \&" "\ \&"
+.TH "MILLER" "1" "2015-12-14" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Portability definitions
 .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -172,6 +172,29 @@ Please use "mlr --help-all-functions" or "mlr -f" for help on all functions.
 .fi
 .if n \{\
 .RE
+.SS "COMPRESSED I/O"
+.if n \{\
+.RS 0
+.\}
+.nf
+  --prepipe {command} This allows Miller to handle compressed inputs. You can do
+  without this for single input files, e.g. "gunzip < myfile.csv.gz | mlr ...".
+  However, when multiple input files are present, between-file separations are
+  lost; also, the FILENAME variable doesn't iterate. Using --prepipe you can
+  specify an action to be taken on each input file.
+  Examples:
+    mlr --prepipe 'gunzip <'
+    mlr --prepipe 'zcat -cf <'
+    mlr --prepipe 'xz -cd <'
+    mlr --prepipe cat
+    mlr --prepipe 'cat <'
+  Note that this feature is quite general and is not limited to decompression
+  utilities. You can use it to apply per-file filters of your choice.
+  For output compression (or other) utilities, simply pipe the output:
+    mlr ... | {your compression command}
+.fi
+.if n \{\
+.RE
 .SS "SEPARATORS"
 .if n \{\
 .RS 0

doc/reference.html

@@ -366,6 +366,23 @@ Data-format options, for input, output, or both:
   Examples: --csv for CSV-formatted input and output; --idkvp --opprint for
   DKVP-formatted input and pretty-printed output.
 
+Compressed-data options:
+  --prepipe {command} This allows Miller to handle compressed inputs. You can do
+  without this for single input files, e.g. "gunzip < myfile.csv.gz | mlr ...".
+  However, when multiple input files are present, between-file separations are
+  lost; also, the FILENAME variable doesn't iterate. Using --prepipe you can
+  specify an action to be taken on each input file.
+  Examples:
+    mlr --prepipe 'gunzip <'
+    mlr --prepipe 'zcat -cf <'
+    mlr --prepipe 'xz -cd <'
+    mlr --prepipe cat
+    mlr --prepipe 'cat <'
+  Note that this feature is quite general and is not limited to decompression
+  utilities. You can use it to apply per-file filters of your choice.
+  For output compression (or other) utilities, simply pipe the output:
+    mlr ... | {your compression command}
+
 Separator options, for input, output, or both:
   --rs     --irs     --ors              Record separators, e.g. 'lf' or '\r\n'
   --fs     --ifs     --ofs  --repifs    Field separators, e.g. comma