Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-18 17:04:50 +00:00
Code Issues Projects Releases Wiki Activity

zcat iterate

Browse source
This commit is contained in:
John Kerl 2015-12-13 18:54:15 -07:00
parent 55cabc3ea4
commit 8fbafefff9
7 changed files with 72 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

11
c/cli/mlrcli.c
View file

@ -305,13 +305,14 @@ static void main_usage_compressed_data_options(FILE* o, char* argv0) {
argv0);
fprintf(o, " However, when multiple input files are present, between-file separations are\n");
fprintf(o, " lost; also, the FILENAME variable doesn't iterate. Using --prepipe you can\n");
fprintf(o, " specify an action to be taken on each input file.\n");
fprintf(o, " specify an action to be taken on each input file. This pre-pipe command must\n");
fprintf(o, " be able to read from standard input; it will be invoked with\n");
fprintf(o, " {command} < {filename}.\n");
fprintf(o, " Examples:\n");
fprintf(o, " %s --prepipe 'gunzip <'\n", argv0);
fprintf(o, " %s --prepipe 'zcat -cf <'\n", argv0);
fprintf(o, " %s --prepipe 'xz -cd <'\n", argv0);
fprintf(o, " %s --prepipe 'gunzip'\n", argv0);
fprintf(o, " %s --prepipe 'zcat -cf'\n", argv0);
fprintf(o, " %s --prepipe 'xz -cd'\n", argv0);
fprintf(o, " %s --prepipe cat\n", argv0);
fprintf(o, " %s --prepipe 'cat <'\n", argv0);
fprintf(o, " Note that this feature is quite general and is not limited to decompression\n");
fprintf(o, " utilities. You can use it to apply per-file filters of your choice.\n");
fprintf(o, " For output compression (or other) utilities, simply pipe the output:\n");

5
c/input/file_reader_stdio.c
View file

@ -23,17 +23,18 @@ void* file_reader_stdio_vopen(void* pvstate, char* prepipe, char* filename) {
}
}
} else {
char* command = mlr_malloc_or_die(strlen(prepipe) + 1 + strlen(filename) + 1);
char* command = mlr_malloc_or_die(strlen(prepipe) + 3 + strlen(filename) + 1);
if (streq(filename, "-"))
sprintf(command, "%s", prepipe);
else
sprintf(command, "%s %s", prepipe, filename);
sprintf(command, "%s < %s", prepipe, filename);
input_stream = popen(command, "r");
if (input_stream == NULL) {
fprintf(stderr, "%s: Couldn't popen \"%s\" for read.\n", MLR_GLOBALS.argv0, command);
perror(command);
exit(1);
}
free(command);
}
return input_stream;
}

5
c/input/stdio_byte_reader.c
View file

@ -51,17 +51,18 @@ static int stdio_byte_reader_open_func(struct _byte_reader_t* pbr, char* prepipe
}
}
} else {
char* command = mlr_malloc_or_die(strlen(prepipe) + 1 + strlen(filename) + 1);
char* command = mlr_malloc_or_die(strlen(prepipe) + 3 + strlen(filename) + 1);
if (streq(filename, "-"))
sprintf(command, "%s", prepipe);
else
sprintf(command, "%s %s", prepipe, filename);
sprintf(command, "%s < %s", prepipe, filename);
pstate->fp = popen(command, "r");
if (pstate->fp == NULL) {
fprintf(stderr, "%s: Couldn't popen \"%s\" for read.\n", MLR_GLOBALS.argv0, command);
perror(command);
exit(1);
}
free(command);
}
pbr->pvstate = pstate;

40
c/reg_test/expected/out
View file

@ -10223,6 +10223,46 @@ n2 -1.8446744e+19
n3 -2.7670116e+19
================================================================
COMPRESSED INPUT
mlr --csv --prepipe cat cat ./reg_test/input/rfc-csv/simple.csv
a,b,c
1,x,3
4,5,6
x,y"yy,z
mlr --dkvp --prepipe cat cat ./reg_test/input/abixy
a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697
a=eks,b=zee,i=7,x=0.6117840605678454,y=0.1878849191181694
a=zee,b=wye,i=8,x=0.5985540091064224,y=0.976181385699006
a=hat,b=wye,i=9,x=0.03144187646093577,y=0.7495507603507059
a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864
mlr --csv --prepipe cat cat
a,b,c
1,x,3
4,5,6
x,y"yy,z
mlr --dkvp --prepipe cat cat
a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697
a=eks,b=zee,i=7,x=0.6117840605678454,y=0.1878849191181694
a=zee,b=wye,i=8,x=0.5985540091064224,y=0.976181385699006
a=hat,b=wye,i=9,x=0.03144187646093577,y=0.7495507603507059
a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864
================================================================
STDIN

8
c/reg_test/run
View file

@ -1096,6 +1096,14 @@ run_mlr --oxtab --ofmt '%.8g' put '$n0=$n+0;$n1=$n+1;$n2=$n+2;$n3=$n+3' $indir/i
run_mlr --oxtab --ofmt '%.8g' put '$n0=$n-0;$n1=$n-1;$n2=$n-2;$n3=$n-3' $indir/int64arith.dkvp
run_mlr --oxtab --ofmt '%.8g' put '$n0=$n*0;$n1=$n*1;$n2=$n*2;$n3=$n*3' $indir/int64arith.dkvp
# ================================================================
announce COMPRESSED INPUT
run_mlr --csv --prepipe 'cat' cat $indir/rfc-csv/simple.csv
run_mlr --dkvp --prepipe 'cat' cat $indir/abixy
run_mlr --csv --prepipe 'cat' cat < $indir/rfc-csv/simple.csv
run_mlr --dkvp --prepipe 'cat' cat < $indir/abixy
# ================================================================
announce STDIN

11
doc/mlr.1.premade
View file

@ -181,13 +181,14 @@ Please use "mlr --help-all-functions" or "mlr -f" for help on all functions.
without this for single input files, e.g. "gunzip < myfile.csv.gz | mlr ...".
However, when multiple input files are present, between-file separations are
lost; also, the FILENAME variable doesn't iterate. Using --prepipe you can
specify an action to be taken on each input file.
specify an action to be taken on each input file. This pre-pipe command must
be able to read from standard input; it will be invoked with
{command} < {filename}.
Examples:
mlr --prepipe 'gunzip <'
mlr --prepipe 'zcat -cf <'
mlr --prepipe 'xz -cd <'
mlr --prepipe 'gunzip'
mlr --prepipe 'zcat -cf'
mlr --prepipe 'xz -cd'
mlr --prepipe cat
mlr --prepipe 'cat <'
Note that this feature is quite general and is not limited to decompression
utilities. You can use it to apply per-file filters of your choice.
For output compression (or other) utilities, simply pipe the output:

11
doc/reference.html
View file

@ -371,13 +371,14 @@ Compressed-data options:
without this for single input files, e.g. "gunzip &lt; myfile.csv.gz | mlr ...".
However, when multiple input files are present, between-file separations are
lost; also, the FILENAME variable doesn't iterate. Using --prepipe you can
specify an action to be taken on each input file.
specify an action to be taken on each input file. This pre-pipe command must
be able to read from standard input; it will be invoked with
{command} &lt; {filename}.
Examples:
mlr --prepipe 'gunzip &lt;'
mlr --prepipe 'zcat -cf &lt;'
mlr --prepipe 'xz -cd &lt;'
mlr --prepipe 'gunzip'
mlr --prepipe 'zcat -cf'
mlr --prepipe 'xz -cd'
mlr --prepipe cat
mlr --prepipe 'cat &lt;'
Note that this feature is quite general and is not limited to decompression
utilities. You can use it to apply per-file filters of your choice.
For output compression (or other) utilities, simply pipe the output: