diff --git a/c/cli/mlrcli.c b/c/cli/mlrcli.c index f392f2545..979d34137 100644 --- a/c/cli/mlrcli.c +++ b/c/cli/mlrcli.c @@ -305,13 +305,14 @@ static void main_usage_compressed_data_options(FILE* o, char* argv0) { argv0); fprintf(o, " However, when multiple input files are present, between-file separations are\n"); fprintf(o, " lost; also, the FILENAME variable doesn't iterate. Using --prepipe you can\n"); - fprintf(o, " specify an action to be taken on each input file.\n"); + fprintf(o, " specify an action to be taken on each input file. This pre-pipe command must\n"); + fprintf(o, " be able to read from standard input; it will be invoked with\n"); + fprintf(o, " {command} < {filename}.\n"); fprintf(o, " Examples:\n"); - fprintf(o, " %s --prepipe 'gunzip <'\n", argv0); - fprintf(o, " %s --prepipe 'zcat -cf <'\n", argv0); - fprintf(o, " %s --prepipe 'xz -cd <'\n", argv0); + fprintf(o, " %s --prepipe 'gunzip'\n", argv0); + fprintf(o, " %s --prepipe 'zcat -cf'\n", argv0); + fprintf(o, " %s --prepipe 'xz -cd'\n", argv0); fprintf(o, " %s --prepipe cat\n", argv0); - fprintf(o, " %s --prepipe 'cat <'\n", argv0); fprintf(o, " Note that this feature is quite general and is not limited to decompression\n"); fprintf(o, " utilities. You can use it to apply per-file filters of your choice.\n"); fprintf(o, " For output compression (or other) utilities, simply pipe the output:\n"); diff --git a/c/input/file_reader_stdio.c b/c/input/file_reader_stdio.c index cfdd563c0..eed73bef5 100644 --- a/c/input/file_reader_stdio.c +++ b/c/input/file_reader_stdio.c @@ -23,17 +23,18 @@ void* file_reader_stdio_vopen(void* pvstate, char* prepipe, char* filename) { } } } else { - char* command = mlr_malloc_or_die(strlen(prepipe) + 1 + strlen(filename) + 1); + char* command = mlr_malloc_or_die(strlen(prepipe) + 3 + strlen(filename) + 1); if (streq(filename, "-")) sprintf(command, "%s", prepipe); else - sprintf(command, "%s %s", prepipe, filename); + sprintf(command, "%s < %s", prepipe, filename); input_stream = popen(command, "r"); if (input_stream == NULL) { fprintf(stderr, "%s: Couldn't popen \"%s\" for read.\n", MLR_GLOBALS.argv0, command); perror(command); exit(1); } + free(command); } return input_stream; } diff --git a/c/input/stdio_byte_reader.c b/c/input/stdio_byte_reader.c index 0434f32ea..fcedcb064 100644 --- a/c/input/stdio_byte_reader.c +++ b/c/input/stdio_byte_reader.c @@ -51,17 +51,18 @@ static int stdio_byte_reader_open_func(struct _byte_reader_t* pbr, char* prepipe } } } else { - char* command = mlr_malloc_or_die(strlen(prepipe) + 1 + strlen(filename) + 1); + char* command = mlr_malloc_or_die(strlen(prepipe) + 3 + strlen(filename) + 1); if (streq(filename, "-")) sprintf(command, "%s", prepipe); else - sprintf(command, "%s %s", prepipe, filename); + sprintf(command, "%s < %s", prepipe, filename); pstate->fp = popen(command, "r"); if (pstate->fp == NULL) { fprintf(stderr, "%s: Couldn't popen \"%s\" for read.\n", MLR_GLOBALS.argv0, command); perror(command); exit(1); } + free(command); } pbr->pvstate = pstate; diff --git a/c/reg_test/expected/out b/c/reg_test/expected/out index e623c9f1d..c8075e7ab 100644 --- a/c/reg_test/expected/out +++ b/c/reg_test/expected/out @@ -10223,6 +10223,46 @@ n2 -1.8446744e+19 n3 -2.7670116e+19 +================================================================ +COMPRESSED INPUT + +mlr --csv --prepipe cat cat ./reg_test/input/rfc-csv/simple.csv +a,b,c +1,x,3 +4,5,6 +x,y"yy,z + +mlr --dkvp --prepipe cat cat ./reg_test/input/abixy +a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533 +a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797 +a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776 +a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463 +a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729 +a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697 +a=eks,b=zee,i=7,x=0.6117840605678454,y=0.1878849191181694 +a=zee,b=wye,i=8,x=0.5985540091064224,y=0.976181385699006 +a=hat,b=wye,i=9,x=0.03144187646093577,y=0.7495507603507059 +a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864 + +mlr --csv --prepipe cat cat +a,b,c +1,x,3 +4,5,6 +x,y"yy,z + +mlr --dkvp --prepipe cat cat +a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533 +a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797 +a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776 +a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463 +a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729 +a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697 +a=eks,b=zee,i=7,x=0.6117840605678454,y=0.1878849191181694 +a=zee,b=wye,i=8,x=0.5985540091064224,y=0.976181385699006 +a=hat,b=wye,i=9,x=0.03144187646093577,y=0.7495507603507059 +a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864 + + ================================================================ STDIN diff --git a/c/reg_test/run b/c/reg_test/run index 5f32b88ce..b22a5dec8 100755 --- a/c/reg_test/run +++ b/c/reg_test/run @@ -1096,6 +1096,14 @@ run_mlr --oxtab --ofmt '%.8g' put '$n0=$n+0;$n1=$n+1;$n2=$n+2;$n3=$n+3' $indir/i run_mlr --oxtab --ofmt '%.8g' put '$n0=$n-0;$n1=$n-1;$n2=$n-2;$n3=$n-3' $indir/int64arith.dkvp run_mlr --oxtab --ofmt '%.8g' put '$n0=$n*0;$n1=$n*1;$n2=$n*2;$n3=$n*3' $indir/int64arith.dkvp +# ================================================================ +announce COMPRESSED INPUT + +run_mlr --csv --prepipe 'cat' cat $indir/rfc-csv/simple.csv +run_mlr --dkvp --prepipe 'cat' cat $indir/abixy +run_mlr --csv --prepipe 'cat' cat < $indir/rfc-csv/simple.csv +run_mlr --dkvp --prepipe 'cat' cat < $indir/abixy + # ================================================================ announce STDIN diff --git a/doc/mlr.1.premade b/doc/mlr.1.premade index bfb9fd6c2..b2b0eb3ab 100644 --- a/doc/mlr.1.premade +++ b/doc/mlr.1.premade @@ -181,13 +181,14 @@ Please use "mlr --help-all-functions" or "mlr -f" for help on all functions. without this for single input files, e.g. "gunzip < myfile.csv.gz | mlr ...". However, when multiple input files are present, between-file separations are lost; also, the FILENAME variable doesn't iterate. Using --prepipe you can - specify an action to be taken on each input file. + specify an action to be taken on each input file. This pre-pipe command must + be able to read from standard input; it will be invoked with + {command} < {filename}. Examples: - mlr --prepipe 'gunzip <' - mlr --prepipe 'zcat -cf <' - mlr --prepipe 'xz -cd <' + mlr --prepipe 'gunzip' + mlr --prepipe 'zcat -cf' + mlr --prepipe 'xz -cd' mlr --prepipe cat - mlr --prepipe 'cat <' Note that this feature is quite general and is not limited to decompression utilities. You can use it to apply per-file filters of your choice. For output compression (or other) utilities, simply pipe the output: diff --git a/doc/reference.html b/doc/reference.html index 12095767e..67c55fb8b 100644 --- a/doc/reference.html +++ b/doc/reference.html @@ -371,13 +371,14 @@ Compressed-data options: without this for single input files, e.g. "gunzip < myfile.csv.gz | mlr ...". However, when multiple input files are present, between-file separations are lost; also, the FILENAME variable doesn't iterate. Using --prepipe you can - specify an action to be taken on each input file. + specify an action to be taken on each input file. This pre-pipe command must + be able to read from standard input; it will be invoked with + {command} < {filename}. Examples: - mlr --prepipe 'gunzip <' - mlr --prepipe 'zcat -cf <' - mlr --prepipe 'xz -cd <' + mlr --prepipe 'gunzip' + mlr --prepipe 'zcat -cf' + mlr --prepipe 'xz -cd' mlr --prepipe cat - mlr --prepipe 'cat <' Note that this feature is quite general and is not limited to decompression utilities. You can use it to apply per-file filters of your choice. For output compression (or other) utilities, simply pipe the output: