Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-18 00:45:47 +00:00
Code Issues Projects Releases Wiki Activity

autodetect doc updates

Browse source
This commit is contained in:
John Kerl 2017-02-01 23:30:15 -05:00
parent 5bfbadd631
commit fa1feaeb9f
24 changed files with 424 additions and 574 deletions
Download patch file Download diff file Expand all files Collapse all files

18
c/cli/mlrcli.c
View file

@ -374,11 +374,7 @@ static lhmss_t* get_default_rses() {
lhmss_put(singleton_default_rses, "json", "auto", NO_FREE);
lhmss_put(singleton_default_rses, "nidx", "auto", NO_FREE);
char* csv_rs = "auto";
char* env_default = getenv("MLR_CSV_DEFAULT_RS");
if (env_default != NULL && !streq(env_default, ""))
csv_rs = cli_sep_from_arg(env_default);
lhmss_put(singleton_default_rses, "csv", csv_rs, NO_FREE);
lhmss_put(singleton_default_rses, "csv", "auto", NO_FREE);
lhmss_put(singleton_default_rses, "csvlite", "auto", NO_FREE);
lhmss_put(singleton_default_rses, "markdown", "auto", NO_FREE);
lhmss_put(singleton_default_rses, "pprint", "auto", NO_FREE);
@ -397,7 +393,7 @@ static lhmss_t* get_default_fses() {
lhmss_put(singleton_default_fses, "csvlite", ",", NO_FREE);
lhmss_put(singleton_default_fses, "markdown", "(N/A)", NO_FREE);
lhmss_put(singleton_default_fses, "pprint", " ", NO_FREE);
lhmss_put(singleton_default_fses, "xtab", "\n", NO_FREE);
lhmss_put(singleton_default_fses, "xtab", "auto", NO_FREE);
}
return singleton_default_fses;
}
@ -725,11 +721,6 @@ static void main_usage_data_format_options(FILE* o, char* argv0) {
fprintf(o, "\n");
fprintf(o, " Examples: --csv for CSV-formatted input and output; --idkvp --opprint for\n");
fprintf(o, " DKVP-formatted input and pretty-printed output.\n");
fprintf(o, "\n");
fprintf(o, " PLEASE USE \"%s --csv --rs lf\" FOR NATIVE UN*X (LINEFEED-TERMINATED) CSV FILES.\n", argv0);
fprintf(o, " You can also have MLR_CSV_DEFAULT_RS=lf in your shell environment, e.g.\n");
fprintf(o, " \"export MLR_CSV_DEFAULT_RS=lf\" or \"setenv MLR_CSV_DEFAULT_RS lf\" depending on\n");
fprintf(o, " which shell you use.\n");
}
static void main_usage_format_conversion_keystroke_saver_options(FILE* o, char* argv0) {
@ -785,11 +776,6 @@ static void main_usage_separator_options(FILE* o, char* argv0) {
fprintf(o, " platform-native text data. In particular, this means LF line-terminators\n");
fprintf(o, " by default on Linux/OSX. You can use \"--dkvp --rs crlf\" for\n");
fprintf(o, " CRLF-terminated DKVP files, and so on.\n");
fprintf(o, " * CSV is intended to handle RFC-4180-compliant data. In particular, this means\n");
fprintf(o, " it uses CRLF line-terminators by default. You can use \"--csv --rs lf\" for\n");
fprintf(o, " Linux-native CSV files. You can also have \"MLR_CSV_DEFAULT_RS=lf\" in your\n");
fprintf(o, " shell environment, e.g. \"export MLR_CSV_DEFAULT_RS=lf\" or \"setenv\n");
fprintf(o, " MLR_CSV_DEFAULT_RS lf\" depending on which shell you use.\n");
fprintf(o, " * TSV is simply CSV using tab as field separator (\"--fs tab\").\n");
fprintf(o, " * FS/PS are ignored for markdown format; RS is used.\n");
fprintf(o, " * All RS/FS/PS options are ignored for JSON format: JSON doesn't allow\n");

511
c/reg_test/expected/out
View file

@ -43439,42 +43439,6 @@ a
1
4
mlr --csv cut -f a ./reg_test/input/rfc-csv/simple.csv
mlr: unmatched double quote at line 3.
a
1
4
mlr --csv --rs crlf cut -f a ./reg_test/input/rfc-csv/simple.csv
a
1
4
x
mlr --csv --rs lf cut -f a ./reg_test/input/rfc-csv/simple.csv
mlr: unmatched double quote at line 3.
a
1
4
mlr --csv cut -f a ./reg_test/input/rfc-csv/simple.csv
a
1
4
x
mlr --csv --rs crlf cut -f a ./reg_test/input/rfc-csv/simple.csv
a
1
4
x
mlr --csv --rs lf cut -f a ./reg_test/input/rfc-csv/simple.csv
mlr: unmatched double quote at line 3.
a
1
4
================================================================
MULTI-CHARACTER IRS/IFS/IPS FOR DKVP
@ -44016,8 +43980,7 @@ hi@@@78
mlr --no-mmap --xtab --ips . --ops := cut -x -f b ./reg_test/input/dots.xtab
a:=1
c:=345
defg:=6
autodefg:=6
hi:=78
mlr --xtab --ips : --ops @@@@ put $sum=int($a+$b) ./reg_test/input/multi-ips.dkvp
@ -45956,65 +45919,65 @@ x 0.5026260055412137
y 0.9526183602969864
mlr --ifs auto --xtab cat ./reg_test/input/line-term-crlf.xtab
a pan
b pan
i 1
x 0.3467901443380824
y 0.7268028627434533
a eks
b pan
i 2
x 0.7586799647899636
y 0.5221511083334797
a wye
b wye
i 3
x 0.20460330576630303
y 0.33831852551664776
a eks
b wye
i 4
x 0.38139939387114097
y 0.13418874328430463
a wye
b pan
i 5
x 0.5732889198020006
y 0.8636244699032729
a zee
b pan
i 6
x 0.5271261600918548
y 0.49322128674835697
a eks
b zee
i 7
x 0.6117840605678454
y 0.1878849191181694
a zee
b wye
i 8
x 0.5985540091064224
y 0.976181385699006
a hat
b wye
i 9
x 0.03144187646093577
y 0.7495507603507059
a pan
b wye
i 10
x 0.5026260055412137
y 0.9526183602969864
a pan
b pan
i 1
x 0.3467901443380824
y 0.7268028627434533
a eks
b pan
i 2
x 0.7586799647899636
y 0.5221511083334797
a wye
b wye
i 3
x 0.20460330576630303
y 0.33831852551664776
a eks
b wye
i 4
x 0.38139939387114097
y 0.13418874328430463
a wye
b pan
i 5
x 0.5732889198020006
y 0.8636244699032729
a zee
b pan
i 6
x 0.5271261600918548
y 0.49322128674835697
a eks
b zee
i 7
x 0.6117840605678454
y 0.1878849191181694
a zee
b wye
i 8
x 0.5985540091064224
y 0.976181385699006
a hat
b wye
i 9
x 0.03144187646093577
y 0.7495507603507059
a pan
b wye
i 10
x 0.5026260055412137
y 0.9526183602969864
mlr --fs auto --xtab cat ./reg_test/input/line-term-lf.xtab
a pan
@ -46202,65 +46165,65 @@ x 0.5026260055412137
y 0.9526183602969864
mlr --ifs auto --xtab --no-mmap cat ./reg_test/input/line-term-crlf.xtab
a pan
b pan
i 1
x 0.3467901443380824
y 0.7268028627434533
a eks
b pan
i 2
x 0.7586799647899636
y 0.5221511083334797
a wye
b wye
i 3
x 0.20460330576630303
y 0.33831852551664776
a eks
b wye
i 4
x 0.38139939387114097
y 0.13418874328430463
a wye
b pan
i 5
x 0.5732889198020006
y 0.8636244699032729
a zee
b pan
i 6
x 0.5271261600918548
y 0.49322128674835697
a eks
b zee
i 7
x 0.6117840605678454
y 0.1878849191181694
a zee
b wye
i 8
x 0.5985540091064224
y 0.976181385699006
a hat
b wye
i 9
x 0.03144187646093577
y 0.7495507603507059
a pan
b wye
i 10
x 0.5026260055412137
y 0.9526183602969864
a pan
b pan
i 1
x 0.3467901443380824
y 0.7268028627434533
a eks
b pan
i 2
x 0.7586799647899636
y 0.5221511083334797
a wye
b wye
i 3
x 0.20460330576630303
y 0.33831852551664776
a eks
b wye
i 4
x 0.38139939387114097
y 0.13418874328430463
a wye
b pan
i 5
x 0.5732889198020006
y 0.8636244699032729
a zee
b pan
i 6
x 0.5271261600918548
y 0.49322128674835697
a eks
b zee
i 7
x 0.6117840605678454
y 0.1878849191181694
a zee
b wye
i 8
x 0.5985540091064224
y 0.976181385699006
a hat
b wye
i 9
x 0.03144187646093577
y 0.7495507603507059
a pan
b wye
i 10
x 0.5026260055412137
y 0.9526183602969864
mlr --fs auto --xtab --no-mmap cat ./reg_test/input/line-term-lf.xtab
a pan
@ -46448,65 +46411,65 @@ x 0.5026260055412137
y 0.9526183602969864
mlr --ifs auto --xtab cat ./reg_test/input/line-term-crlf.xtab
a pan
b pan
i 1
x 0.3467901443380824
y 0.7268028627434533
a eks
b pan
i 2
x 0.7586799647899636
y 0.5221511083334797
a wye
b wye
i 3
x 0.20460330576630303
y 0.33831852551664776
a eks
b wye
i 4
x 0.38139939387114097
y 0.13418874328430463
a wye
b pan
i 5
x 0.5732889198020006
y 0.8636244699032729
a zee
b pan
i 6
x 0.5271261600918548
y 0.49322128674835697
a eks
b zee
i 7
x 0.6117840605678454
y 0.1878849191181694
a zee
b wye
i 8
x 0.5985540091064224
y 0.976181385699006
a hat
b wye
i 9
x 0.03144187646093577
y 0.7495507603507059
a pan
b wye
i 10
x 0.5026260055412137
y 0.9526183602969864
a pan
b pan
i 1
x 0.3467901443380824
y 0.7268028627434533
a eks
b pan
i 2
x 0.7586799647899636
y 0.5221511083334797
a wye
b wye
i 3
x 0.20460330576630303
y 0.33831852551664776
a eks
b wye
i 4
x 0.38139939387114097
y 0.13418874328430463
a wye
b pan
i 5
x 0.5732889198020006
y 0.8636244699032729
a zee
b pan
i 6
x 0.5271261600918548
y 0.49322128674835697
a eks
b zee
i 7
x 0.6117840605678454
y 0.1878849191181694
a zee
b wye
i 8
x 0.5985540091064224
y 0.976181385699006
a hat
b wye
i 9
x 0.03144187646093577
y 0.7495507603507059
a pan
b wye
i 10
x 0.5026260055412137
y 0.9526183602969864
mlr --fs auto --xtab cat ./reg_test/input/line-term-lf.xtab
a pan
@ -46694,65 +46657,65 @@ x 0.5026260055412137
y 0.9526183602969864
mlr --ifs auto --xtab --no-mmap cat ./reg_test/input/line-term-crlf.xtab
a pan
b pan
i 1
x 0.3467901443380824
y 0.7268028627434533
a eks
b pan
i 2
x 0.7586799647899636
y 0.5221511083334797
a wye
b wye
i 3
x 0.20460330576630303
y 0.33831852551664776
a eks
b wye
i 4
x 0.38139939387114097
y 0.13418874328430463
a wye
b pan
i 5
x 0.5732889198020006
y 0.8636244699032729
a zee
b pan
i 6
x 0.5271261600918548
y 0.49322128674835697
a eks
b zee
i 7
x 0.6117840605678454
y 0.1878849191181694
a zee
b wye
i 8
x 0.5985540091064224
y 0.976181385699006
a hat
b wye
i 9
x 0.03144187646093577
y 0.7495507603507059
a pan
b wye
i 10
x 0.5026260055412137
y 0.9526183602969864
a pan
b pan
i 1
x 0.3467901443380824
y 0.7268028627434533
a eks
b pan
i 2
x 0.7586799647899636
y 0.5221511083334797
a wye
b wye
i 3
x 0.20460330576630303
y 0.33831852551664776
a eks
b wye
i 4
x 0.38139939387114097
y 0.13418874328430463
a wye
b pan
i 5
x 0.5732889198020006
y 0.8636244699032729
a zee
b pan
i 6
x 0.5271261600918548
y 0.49322128674835697
a eks
b zee
i 7
x 0.6117840605678454
y 0.1878849191181694
a zee
b wye
i 8
x 0.5985540091064224
y 0.976181385699006
a hat
b wye
i 9
x 0.03144187646093577
y 0.7495507603507059
a pan
b wye
i 10
x 0.5026260055412137
y 0.9526183602969864
mlr --fs auto --xtab --no-mmap cat ./reg_test/input/line-term-lf.xtab
a pan

11
c/reg_test/run
View file

@ -2,7 +2,6 @@
# ================================================================
set -e
export MLR_CSV_DEFAULT_RS= # In case set in user's shell
# For building with autoconf:
# * in-directory build:
@ -4886,16 +4885,6 @@ run_mlr --csv cut -f a $indir/rfc-csv/simple.csv
run_mlr --csv --rs crlf cut -f a $indir/rfc-csv/simple.csv
mlr_expect_fail --csv --rs lf cut -f a $indir/rfc-csv/simple.csv
export MLR_CSV_DEFAULT_RS=lf; mlr_expect_fail --csv cut -f a $indir/rfc-csv/simple.csv
export MLR_CSV_DEFAULT_RS=lf; run_mlr --csv --rs crlf cut -f a $indir/rfc-csv/simple.csv
export MLR_CSV_DEFAULT_RS=lf; mlr_expect_fail --csv --rs lf cut -f a $indir/rfc-csv/simple.csv
export MLR_CSV_DEFAULT_RS=crlf; run_mlr --csv cut -f a $indir/rfc-csv/simple.csv
export MLR_CSV_DEFAULT_RS=crlf; run_mlr --csv --rs crlf cut -f a $indir/rfc-csv/simple.csv
export MLR_CSV_DEFAULT_RS=crlf; mlr_expect_fail --csv --rs lf cut -f a $indir/rfc-csv/simple.csv
export MLR_CSV_DEFAULT_RS=
# ----------------------------------------------------------------
announce MULTI-CHARACTER IRS/IFS/IPS FOR DKVP

8
c/todo.txt
View file

@ -38,7 +38,15 @@ OTHER:
================================================================
AUTOTERM
! BUGFIX:
$ mlr --ifs auto --xtab --ips . --ops := cut -x -f b < ./reg_test/input/dots.xtab
a:=1
c:=345
autodefg:=6
hi:=78
! doc updates x many ! incl. --irs auto.
- automan etc. re auto; what about --ors auto
- IRS/ORS variables in the DSL are from autodetect if that is in use
- JSON: manpage re ORS selectable now
- XTAB: --ifs/--ofs instead of --irs/--ors

38
doc/10-min.html
View file

@ -179,7 +179,7 @@ purple,square,0,91,72.3735,8.2430
<p/>
<div class="pokipanel">
<pre>
$ mlr --csv --rs lf cat example.csv
$ mlr --csv cat example.csv
color,shape,flag,index,quantity,rate
yellow,triangle,1,11,43.6498,9.8870
red,square,1,15,79.2778,0.0130
@ -200,7 +200,7 @@ purple,square,0,91,72.3735,8.2430
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --opprint cat example.csv
$ mlr --icsv --opprint cat example.csv
color shape flag index quantity rate
yellow triangle 1 11 43.6498 9.8870
red square 1 15 79.2778 0.0130
@ -222,7 +222,7 @@ header is included either way:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --csv --rs lf head -n 4 example.csv
$ mlr --csv head -n 4 example.csv
color,shape,flag,index,quantity,rate
yellow,triangle,1,11,43.6498,9.8870
red,square,1,15,79.2778,0.0130
@ -234,7 +234,7 @@ red,square,0,48,77.5542,7.4670
<p/>
<div class="pokipanel">
<pre>
$ mlr --csv --rs lf tail -n 4 example.csv
$ mlr --csv tail -n 4 example.csv
color,shape,flag,index,quantity,rate
purple,triangle,0,65,80.1405,5.8240
yellow,circle,1,73,63.9785,4.2370
@ -250,7 +250,7 @@ numerically descending on another field: </boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --opprint sort -f shape -nr index example.csv
$ mlr --icsv --opprint sort -f shape -nr index example.csv
color shape flag index quantity rate
yellow circle 1 87 63.5058 8.3350
yellow circle 1 73 63.9785 4.2370
@ -271,7 +271,7 @@ yellow triangle 1 11 43.6498 9.8870
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --opprint cut -f flag,shape example.csv
$ mlr --icsv --opprint cut -f flag,shape example.csv
shape flag
triangle 1
square 1
@ -292,7 +292,7 @@ square 0
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --opprint cut -o -f flag,shape example.csv
$ mlr --icsv --opprint cut -o -f flag,shape example.csv
flag shape
1 triangle
1 square
@ -313,7 +313,7 @@ flag shape
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --opprint cut -x -f flag,shape example.csv
$ mlr --icsv --opprint cut -x -f flag,shape example.csv
color index quantity rate
yellow 11 43.6498 9.8870
red 15 79.2778 0.0130
@ -334,7 +334,7 @@ purple 91 72.3735 8.2430
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --opprint filter '$color == "red"' example.csv
$ mlr --icsv --opprint filter '$color == "red"' example.csv
color shape flag index quantity rate
red square 1 15 79.2778 0.0130
red circle 1 16 13.8103 2.9010
@ -346,7 +346,7 @@ red square 0 64 77.1991 9.5310
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --opprint filter '$color == "red" &amp;&amp; $flag == 1' example.csv
$ mlr --icsv --opprint filter '$color == "red" &amp;&amp; $flag == 1' example.csv
color shape flag index quantity rate
red square 1 15 79.2778 0.0130
red circle 1 16 13.8103 2.9010
@ -359,7 +359,7 @@ red circle 1 16 13.8103 2.9010
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --opprint put '$ratio = $quantity / $rate; $color_shape = $color . "_" . $shape' example.csv
$ mlr --icsv --opprint put '$ratio = $quantity / $rate; $color_shape = $color . "_" . $shape' example.csv
color shape flag index quantity rate ratio color_shape
yellow triangle 1 11 43.6498 9.8870 4.414868 yellow_triangle
red square 1 15 79.2778 0.0130 6098.292308 red_square
@ -380,7 +380,7 @@ purple square 0 91 72.3735 8.2430 8.779995 purple_square
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --ojson put '$ratio = $quantity/$rate; $shape = toupper($shape)' example.csv
$ mlr --icsv --ojson put '$ratio = $quantity/$rate; $shape = toupper($shape)' example.csv
{ "color": "yellow", "shape": "TRIANGLE", "flag": 1, "index": 11, "quantity": 43.6498, "rate": 9.8870, "ratio": 4.414868 }
{ "color": "red", "shape": "SQUARE", "flag": 1, "index": 15, "quantity": 79.2778, "rate": 0.0130, "ratio": 6098.292308 }
{ "color": "red", "shape": "CIRCLE", "flag": 1, "index": 16, "quantity": 13.8103, "rate": 2.9010, "ratio": 4.760531 }
@ -397,7 +397,7 @@ $ mlr --icsv --irs lf --ojson put '$ratio = $quantity/$rate; $shape = toupper($s
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --ojson --jvstack --jlistwrap tail -n 2 example.csv
$ mlr --icsv --ojson --jvstack --jlistwrap tail -n 2 example.csv
[
{
"color": "yellow",
@ -428,7 +428,7 @@ value of the <tt>shape</tt> field.</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --opprint sort -f shape -nr index then head -n 1 -g shape example.csv
$ mlr --icsv --opprint sort -f shape -nr index then head -n 1 -g shape example.csv
color shape flag index quantity rate
yellow circle 1 87 63.5058 8.3350
purple square 0 91 72.3735 8.2430
@ -444,7 +444,7 @@ have lots of columns:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --oxtab --from example.csv stats1 -a p0,p10,p25,p50,p75,p90,p99,p100 -f rate
$ mlr --icsv --oxtab --from example.csv stats1 -a p0,p10,p25,p50,p75,p90,p99,p100 -f rate
rate_p0 0.013000
rate_p10 2.901000
rate_p25 4.237000
@ -459,7 +459,7 @@ rate_p100 9.887000
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape
$ mlr --icsv --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape
shape quantity_count quantity_min quantity_mean quantity_max
triangle 3 43.649800 68.339767 81.229000
square 4 72.373500 76.601150 79.277800
@ -470,7 +470,7 @@ circle 3 13.810300 47.098200 63.978500
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape,color
$ mlr --icsv --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape,color
shape color quantity_count quantity_min quantity_mean quantity_max
triangle yellow 1 43.649800 43.649800 43.649800
square red 3 77.199100 78.010367 79.277800
@ -488,7 +488,7 @@ per one or more field names:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --csv --rs lf --from example.csv put -q 'tee &gt; $shape.".csv", $*'
$ mlr --csv --from example.csv put -q 'tee &gt; $shape.".csv", $*'
</pre>
</div>
<p/>
@ -590,7 +590,7 @@ different output format.
<p/>I like to produce SQL-query output with header-column and tab delimiter:
this is CSV but with a tab instead of a comma, also known as TSV. Then I
post-process with <tt>mlr --tsv --rs lf</tt> or <tt>mlr --tsvlite</tt>. This
post-process with <tt>mlr --tsv</tt> or <tt>mlr --tsvlite</tt>. This
means I can do some (or all, or none) of my data processing within SQL queries,
and some (or none, or all) of my data processing using Miller &mdash; whichever
is most convenient for my needs at the moment.

38
doc/content-for-10-min.html
View file

@ -8,68 +8,68 @@ POKI_RUN_COMMAND{{cat example.csv}}HERE
<p/><boldmaroon> <tt>mlr cat</tt> is like cat ...</boldmaroon>
POKI_RUN_COMMAND{{mlr --csv --rs lf cat example.csv}}HERE
POKI_RUN_COMMAND{{mlr --csv cat example.csv}}HERE
<p/><boldmaroon>... but it can also do format conversion (here, to pretty-printed tabular format): </boldmaroon>
POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint cat example.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --opprint cat example.csv}}HERE
<p/><boldmaroon> <tt>mlr head</tt> and <tt>mlr tail</tt> count records. The CSV
header is included either way:</boldmaroon>
POKI_RUN_COMMAND{{mlr --csv --rs lf head -n 4 example.csv}}HERE
POKI_RUN_COMMAND{{mlr --csv --rs lf tail -n 4 example.csv}}HERE
POKI_RUN_COMMAND{{mlr --csv head -n 4 example.csv}}HERE
POKI_RUN_COMMAND{{mlr --csv tail -n 4 example.csv}}HERE
<p/><boldmaroon> Sort primarily alphabetically on one field, then secondarily
numerically descending on another field: </boldmaroon>
POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint sort -f shape -nr index example.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --opprint sort -f shape -nr index example.csv}}HERE
<p/><boldmaroon> Use <tt>cut</tt> to retain only specified fields, in input-data order:</boldmaroon>
POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint cut -f flag,shape example.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --opprint cut -f flag,shape example.csv}}HERE
<p/><boldmaroon> Use <tt>cut -o</tt> to retain only specified fields, in your specified order:</boldmaroon>
POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint cut -o -f flag,shape example.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --opprint cut -o -f flag,shape example.csv}}HERE
<p/><boldmaroon> Use <tt>cut -x</tt> to omit specified fields:</boldmaroon>
POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint cut -x -f flag,shape example.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --opprint cut -x -f flag,shape example.csv}}HERE
<p/><boldmaroon> Use <tt>filter</tt> to retain specified records:</boldmaroon>
POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint filter '$color == "red"' example.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint filter '$color == "red" && $flag == 1' example.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --opprint filter '$color == "red"' example.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --opprint filter '$color == "red" && $flag == 1' example.csv}}HERE
<p/><boldmaroon> Use <tt>put</tt> to add/replace fields which are computed from other fields:</boldmaroon>
POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint put '$ratio = $quantity / $rate; $color_shape = $color . "_" . $shape' example.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --opprint put '$ratio = $quantity / $rate; $color_shape = $color . "_" . $shape' example.csv}}HERE
<p/><boldmaroon> JSON output:</boldmaroon>
POKI_RUN_COMMAND{{mlr --icsv --irs lf --ojson put '$ratio = $quantity/$rate; $shape = toupper($shape)' example.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --irs lf --ojson --jvstack --jlistwrap tail -n 2 example.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --ojson put '$ratio = $quantity/$rate; $shape = toupper($shape)' example.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --ojson --jvstack --jlistwrap tail -n 2 example.csv}}HERE
<p/><boldmaroon> Use <tt>then</tt> to pipe commands together. Also, the
<tt>-g</tt> option for many Miller commands is for group-by: here,
<tt>head -n 1 -g shape</tt> outputs the first record for each distinct
value of the <tt>shape</tt> field.</boldmaroon>
POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint sort -f shape -nr index then head -n 1 -g shape example.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --opprint sort -f shape -nr index then head -n 1 -g shape example.csv}}HERE
<p/><boldmaroon> Statistics can be computed with or without group-by field(s). Also, the first of these two
examples uses <tt>--oxtab</tt> output format which is a nice alternative to <tt>--opprint</tt> when you
have lots of columns:</boldmaroon>
POKI_RUN_COMMAND{{mlr --icsv --irs lf --oxtab --from example.csv stats1 -a p0,p10,p25,p50,p75,p90,p99,p100 -f rate}}HERE
POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape}}HERE
POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape,color}}HERE
POKI_RUN_COMMAND{{mlr --icsv --oxtab --from example.csv stats1 -a p0,p10,p25,p50,p75,p90,p99,p100 -f rate}}HERE
POKI_RUN_COMMAND{{mlr --icsv --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape}}HERE
POKI_RUN_COMMAND{{mlr --icsv --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape,color}}HERE
<p/><boldmaroon> Using <tt>tee</tt> within <tt>put</tt>, you can split your input data into separate files
per one or more field names:</boldmaroon>
POKI_RUN_COMMAND{{mlr --csv --rs lf --from example.csv put -q 'tee > $shape.".csv", $*'}}HERE
POKI_RUN_COMMAND{{mlr --csv --from example.csv put -q 'tee > $shape.".csv", $*'}}HERE
<table><tr><td>
POKI_RUN_COMMAND{{cat circle.csv}}HERE
</td><td>
@ -137,7 +137,7 @@ different output format.
<p/>I like to produce SQL-query output with header-column and tab delimiter:
this is CSV but with a tab instead of a comma, also known as TSV. Then I
post-process with <tt>mlr --tsv --rs lf</tt> or <tt>mlr --tsvlite</tt>. This
post-process with <tt>mlr --tsv</tt> or <tt>mlr --tsvlite</tt>. This
means I can do some (or all, or none) of my data processing within SQL queries,
and some (or none, or all) of my data processing using Miller &mdash; whichever
is most convenient for my needs at the moment.

18
doc/content-for-cookbook.html
View file

@ -51,14 +51,14 @@ POKI_INCLUDE_AND_RUN_ESCAPED(data/rect.sh)HERE
<h1>Bulk rename of field names</h1>
POKI_RUN_COMMAND{{cat data/spaces.csv}}HERE
POKI_RUN_COMMAND{{mlr --csv --rs lf rename -r -g ' ,_' data/spaces.csv}}HERE
POKI_RUN_COMMAND{{mlr --csv --irs lf --opprint rename -r -g ' ,_' data/spaces.csv}}HERE
POKI_RUN_COMMAND{{mlr --csv rename -r -g ' ,_' data/spaces.csv}}HERE
POKI_RUN_COMMAND{{mlr --csv --opprint rename -r -g ' ,_' data/spaces.csv}}HERE
<p/>You can also do this with a for-loop but it puts the modified fields after the unmodified fields:
POKI_RUN_COMMAND{{cat data/bulk-rename-for-loop.mlr}}HERE
POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint put -f data/bulk-rename-for-loop.mlr data/spaces.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --opprint put -f data/bulk-rename-for-loop.mlr data/spaces.csv}}HERE
<!-- ================================================================ -->
<h1>Headerless CSV on input or output</h1>
@ -69,14 +69,14 @@ POKI_RUN_COMMAND{{cat data/headerless.csv}}HERE
<p/> You can use Miller to add a header: the <tt>--implicit-csv-header</tt> applies positionally indexed labels:
POKI_RUN_COMMAND{{mlr --csv --rs lf --implicit-csv-header cat data/headerless.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --irs lf --implicit-csv-header --opprint cat data/headerless.csv}}HERE
POKI_RUN_COMMAND{{mlr --csv --implicit-csv-header cat data/headerless.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --implicit-csv-header --opprint cat data/headerless.csv}}HERE
<p/> Following that, you can rename the positionally indexed labels to names with meaning for your context.
For example:
POKI_RUN_COMMAND{{mlr --csv --rs lf --implicit-csv-header label name,age,status data/headerless.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --rs lf --implicit-csv-header --opprint label name,age,status data/headerless.csv}}HERE
POKI_RUN_COMMAND{{mlr --csv --implicit-csv-header label name,age,status data/headerless.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --implicit-csv-header --opprint label name,age,status data/headerless.csv}}HERE
<p/> Likewise, if you need to produce CSV which is lacking its header, you can pipe Miller&rsquo;s output
to the system command <tt>sed 1d</tt>, or you can use Miller&rsquo;s <tt>--headerless-csv-output</tt> option:
@ -209,7 +209,7 @@ POKI_INCLUDE_AND_RUN_ESCAPED(data/two-pass-record-numbers.sh)HERE
<p/>The idea is to retain records having the largest value of <tt>n</tt> in the
following data:
POKI_RUN_COMMAND{{mlr --itsv --irs lf --opprint cat data/maxrows.tsv}}HERE
POKI_RUN_COMMAND{{mlr --itsv --opprint cat data/maxrows.tsv}}HERE
<p/>Of course, the largest value of <tt>n</tt> isn&rsquo;t known until after
all data have been read. Using an out-of-stream variable we can retain all
@ -217,7 +217,7 @@ records as they are read, then filter them at the end:
POKI_RUN_COMMAND{{cat data/maxrows.mlr}}HERE
POKI_RUN_COMMAND{{mlr --itsv --irs lf --opprint put -q -f data/maxrows.mlr data/maxrows.tsv}}HERE
POKI_RUN_COMMAND{{mlr --itsv --opprint put -q -f data/maxrows.mlr data/maxrows.tsv}}HERE
<!-- ================================================================ -->
<h1>Filtering paragraphs of text</h1>

5
doc/content-for-data-examples.html
View file

@ -4,11 +4,6 @@ POKI_PUT_TOC_HERE
<p/> The <a href="data/flins.csv">flins.csv</a> file is some sample data
obtained from <a href="https://support.spatialkey.com/spatialkey-sample-csv-data">https://support.spatialkey.com/spatialkey-sample-csv-data</a>.
<p/><b>Note: please use "mlr --csv --rs lf" for for native Un*x (linefeed-terminated) CSV files.</b>
(You can also have <tt>MLR_CSV_DEFAULT_RS=lf</tt> in your shell environment, e.g.
<tt>export MLR_CSV_DEFAULT_RS=lf</tt> or <tt>setenv MLR_CSV_DEFAULT_RS lf</tt> depending on
which shell you use.)
<p/>Vertical-tabular format is good for a quick look at CSV data layout &mdash; seeing what columns you have to work with:
POKI_RUN_COMMAND{{head -n 2 data/flins.csv | mlr --icsv --oxtab cat}}HERE
<p/> A few simple queries:

22
doc/content-for-faq.html
View file

@ -1,17 +1,5 @@
POKI_PUT_TOC_HERE
<h1>Number one FAQ</h1>
<b>
Please use <tt>mlr --csv --rs lf</tt> for native Un*x (linefeed-terminated) CSV files.
</b>
<p/>Instead of specifying <tt>--rs lf</tt> on each invocation, you can instead
have <tt>MLR_CSV_DEFAULT_RS=lf</tt> in your shell environment: e.g. put
<tt>export MLR_CSV_DEFAULT_RS=lf</tt> in your <tt>~/.bashrc</tt> or
<tt>~/.zshrc</tt>, or <tt>setenv MLR_CSV_DEFAULT_RS lf</tt> in your
<tt>~/.cshrc</tt>, as a one-time setup step.
<h1>No output at all</h1>
<p/>Check the line-terminators of the data, e.g. with the command-line
@ -64,18 +52,18 @@ POKI_RUN_COMMAND{{cat data/then-example.csv}}HERE
Next, run the first step of your command, omitting anything from the first <tt>then</tt> onward:
POKI_RUN_COMMAND{{mlr --icsv --rs lf --opprint count-distinct -f Status,Payment_Type data/then-example.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --opprint count-distinct -f Status,Payment_Type data/then-example.csv}}HERE
After that, run it with the next <tt>then</tt> step included:
POKI_RUN_COMMAND{{mlr --icsv --rs lf --opprint count-distinct -f Status,Payment_Type then sort -nr count data/then-example.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --opprint count-distinct -f Status,Payment_Type then sort -nr count data/then-example.csv}}HERE
Now if you use <tt>then</tt> to include another verb after that, the columns
<tt>Status</tt>, <tt>Payment_Type</tt>, and <tt>count</tt> will be the input to
that verb.
<p/>Note, by the way, that you&rsquo;ll get the same results using pipes:
POKI_RUN_COMMAND{{mlr --csv --rs lf count-distinct -f Status,Payment_Type data/then-example.csv | mlr --icsv --rs lf --opprint sort -nr count}}HERE
POKI_RUN_COMMAND{{mlr --csv count-distinct -f Status,Payment_Type data/then-example.csv | mlr --icsv --opprint sort -nr count}}HERE
<h1>I assigned $9 and it&rsquo;s not 9th</h1>
@ -89,10 +77,10 @@ input data.
POKI_RUN_COMMAND{{echo x,y,z | mlr --dkvp cat}}HERE
POKI_RUN_COMMAND{{echo x,y,z | mlr --dkvp put '$6="a";$4="b";$55="cde"'}}HERE
POKI_RUN_COMMAND{{echo x,y,z | mlr --nidx cat}}HERE
POKI_RUN_COMMAND{{echo x,y,z | mlr --csv --rs lf --implicit-csv-header cat}}HERE
POKI_RUN_COMMAND{{echo x,y,z | mlr --csv --implicit-csv-header cat}}HERE
POKI_RUN_COMMAND{{echo x,y,z | mlr --dkvp rename 2,999}}HERE
POKI_RUN_COMMAND{{echo x,y,z | mlr --dkvp rename 2,newname}}HERE
POKI_RUN_COMMAND{{echo x,y,z | mlr --csv --rs lf --implicit-csv-header reorder -f 3,1,2}}HERE
POKI_RUN_COMMAND{{echo x,y,z | mlr --csv --implicit-csv-header reorder -f 3,1,2}}HERE
<h1>Why doesn&rsquo;t mlr cut put fields in the order I want?</h1>

18
doc/content-for-file-formats.html
View file

@ -36,24 +36,6 @@ POKI_PUT_LINK_FOR_PAGE(reference.html)HERE.)
https://tools.ietf.org/html/rfc4180</a>). This includes CRLF line-terminators by default, regardless
of platform.
<p/>
<b>
Please use <tt>mlr --csv --rs lf</tt> for native Un*x (linefeed-terminated) CSV files.
</b>
<p/>Instead of specifying <tt>--rs lf</tt> on each invocation, you can instead
have <tt>MLR_CSV_DEFAULT_RS=lf</tt> in your shell environment: e.g. put
<tt>export MLR_CSV_DEFAULT_RS=lf</tt> in your <tt>~/.bashrc</tt> or
<tt>~/.zshrc</tt>, or <tt>setenv MLR_CSV_DEFAULT_RS lf</tt> in your
<tt>~/.cshrc</tt>, as a one-time setup step.
<p/>The RFC says, somewhat briefly, that &ldquo;there may be a header
line&rdquo;. Miller&rsquo;s <tt>--implicit-csv-header</tt> option allows you to
read CSV data which lacks a header line, applying column labels <tt>1</tt>,
<tt>2</tt>, <tt>3</tt>, etc. for you. You may also use Miller&rsquo;s
<tt>label</tt> to replace those numerical column names with labels of your
choosing.
<p/>Here are the differences between CSV and CSV-lite:
<ul>

2
doc/content-for-index.html
View file

@ -88,7 +88,7 @@ machine, and expect it to work.
<div class="pokipanel">
<pre>
# Row filter
% mlr --csv --rs lf filter '$status != "down" && $upsec >= 10000' *.csv
% mlr --csv filter '$status != "down" && $upsec >= 10000' *.csv
</pre>
</div>

6
doc/content-for-reference-verbs.html
View file

@ -299,9 +299,9 @@ POKI_INCLUDE_ESCAPED(data/label-example.txt)HERE
<p/>Likewise, if you have CSV/CSV-lite input data which has somehow been bereft of its header line, you can re-add a header line using <tt>--implicit-csv-header</tt> and <tt>label</tt>:
POKI_RUN_COMMAND{{cat data/headerless.csv}}HERE
POKI_RUN_COMMAND{{mlr --csv --rs lf --implicit-csv-header cat data/headerless.csv}}HERE
POKI_RUN_COMMAND{{mlr --csv --rs lf --implicit-csv-header label name,age,status data/headerless.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --rs lf --implicit-csv-header --opprint label name,age,status data/headerless.csv}}HERE
POKI_RUN_COMMAND{{mlr --csv --implicit-csv-header cat data/headerless.csv}}HERE
POKI_RUN_COMMAND{{mlr --csv --implicit-csv-header label name,age,status data/headerless.csv}}HERE
POKI_RUN_COMMAND{{mlr --icsv --implicit-csv-header --opprint label name,age,status data/headerless.csv}}HERE
<!-- ================================================================ -->
<h1>least-frequent</h1>

2
doc/content-for-reference.html
View file

@ -288,7 +288,7 @@ streaming-data programs operate through pipes, each one can use a CPU. Rest
assured you get the same results either way.
<p/>The other reason to use then-chaining is for simplicity: you don&rsquo;t
have re-type formatting flags (e.g. <tt>--csv --rs lf --fs tab</tt>) at every
have re-type formatting flags (e.g. <tt>--csv --fs tab</tt>) at every
pipeline stage.
<!-- ================================================================ -->

120
doc/cookbook.html
View file

@ -322,7 +322,7 @@ a b c,def,g h i
<p/>
<div class="pokipanel">
<pre>
$ mlr --csv --rs lf rename -r -g ' ,_' data/spaces.csv
$ mlr --csv rename -r -g ' ,_' data/spaces.csv
a_b_c,def,g_h_i
123,4567,890
2468,1357,3579
@ -333,7 +333,7 @@ a_b_c,def,g_h_i
<p/>
<div class="pokipanel">
<pre>
$ mlr --csv --irs lf --opprint rename -r -g ' ,_' data/spaces.csv
$ mlr --csv --opprint rename -r -g ' ,_' data/spaces.csv
a_b_c def g_h_i
123 4567 890
2468 1357 3579
@ -362,7 +362,7 @@ for (oldk,v in $*) {
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --opprint put -f data/bulk-rename-for-loop.mlr data/spaces.csv
$ mlr --icsv --opprint put -f data/bulk-rename-for-loop.mlr data/spaces.csv
def a_b_c g_h_i
4567 123 890
1357 2468 3579
@ -393,7 +393,7 @@ Carol,45,present
<p/>
<div class="pokipanel">
<pre>
$ mlr --csv --rs lf --implicit-csv-header cat data/headerless.csv
$ mlr --csv --implicit-csv-header cat data/headerless.csv
1,2,3
John,23,present
Fred,34,present
@ -405,7 +405,7 @@ Carol,45,present
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --irs lf --implicit-csv-header --opprint cat data/headerless.csv
$ mlr --icsv --implicit-csv-header --opprint cat data/headerless.csv
1 2 3
John 23 present
Fred 34 present
@ -421,7 +421,7 @@ For example:
<p/>
<div class="pokipanel">
<pre>
$ mlr --csv --rs lf --implicit-csv-header label name,age,status data/headerless.csv
$ mlr --csv --implicit-csv-header label name,age,status data/headerless.csv
name,age,status
John,23,present
Fred,34,present
@ -433,7 +433,7 @@ Carol,45,present
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --rs lf --implicit-csv-header --opprint label name,age,status data/headerless.csv
$ mlr --icsv --implicit-csv-header --opprint label name,age,status data/headerless.csv
name age status
John 23 present
Fred 34 present
@ -607,33 +607,33 @@ $ mlr --ofmt '%.9lf' --opprint seqgen --start 1 --stop 28 then put '
' then put '$seconds=systime()' then step -a delta -f seconds then cut -x -f seconds
i o fcount seconds_delta
1 1 1 0
2 2 3 0.000036001
3 3 5 0.000014067
4 5 9 0.000016928
5 8 15 0.000023127
6 13 25 0.000033855
7 21 41 0.000049114
8 34 67 0.000073910
9 55 109 0.000118017
10 89 177 0.000186920
11 144 287 0.000299215
12 233 465 0.000472784
13 377 753 0.000731230
14 610 1219 0.001174927
15 987 1973 0.001888990
16 1597 3193 0.003062963
17 2584 5167 0.004909039
18 4181 8361 0.009054899
19 6765 13529 0.013993025
20 10946 21891 0.022313118
21 17711 35421 0.035612822
22 28657 57313 0.054386139
23 46368 92735 0.094619989
24 75025 150049 0.137188911
25 121393 242785 0.219310045
26 196418 392835 0.372972965
27 317811 635621 0.564725161
28 514229 1028457 0.906754971
2 2 3 0.000031948
3 3 5 0.000013113
4 5 9 0.000015974
5 8 15 0.000021935
6 13 25 0.000030994
7 21 41 0.000045061
8 34 67 0.000071049
9 55 109 0.000113010
10 89 177 0.000178814
11 144 287 0.000289202
12 233 465 0.000463009
13 377 753 0.000734806
14 610 1219 0.001182079
15 987 1973 0.001869917
16 1597 3193 0.003251076
17 2584 5167 0.005389929
18 4181 8361 0.008499146
19 6765 13529 0.014052868
20 10946 21891 0.024805069
21 17711 35421 0.038400888
22 28657 57313 0.061202049
23 46368 92735 0.093353987
24 75025 150049 0.147068024
25 121393 242785 0.288206100
26 196418 392835 0.395586014
27 317811 635621 0.595448971
28 514229 1028457 0.958051920
</pre>
</div>
<p/>
@ -665,33 +665,33 @@ $ mlr --ofmt '%.9lf' --opprint seqgen --start 1 --stop 28 then put '
' then put '$seconds=systime()' then step -a delta -f seconds then cut -x -f seconds
i o fcount seconds_delta
1 1 1 0
2 2 3 0.000036001
3 3 3 0.000013113
4 5 3 0.000010967
5 8 3 0.000010967
6 13 3 0.000010014
7 21 3 0.000010014
2 2 3 0.000037193
3 3 3 0.000012875
4 5 3 0.000011921
5 8 3 0.000011206
6 13 3 0.000010967
7 21 3 0.000010967
8 34 3 0.000010967
9 55 3 0.000010967
10 89 3 0.000010014
11 144 3 0.000010014
12 233 3 0.000014067
9 55 3 0.000010014
10 89 3 0.000013113
11 144 3 0.000010967
12 233 3 0.000015020
13 377 3 0.000010967
14 610 3 0.000010014
15 987 3 0.000010967
16 1597 3 0.000010014
17 2584 3 0.000010014
17 2584 3 0.000010967
18 4181 3 0.000010014
19 6765 3 0.000010014
20 10946 3 0.000010014
21 17711 3 0.000011921
20 10946 3 0.000010967
21 17711 3 0.000010014
22 28657 3 0.000010014
23 46368 3 0.000012159
24 75025 3 0.000010967
23 46368 3 0.000012875
24 75025 3 0.000012159
25 121393 3 0.000010014
26 196418 3 0.000010014
27 317811 3 0.000010967
28 514229 3 0.000010014
26 196418 3 0.000010967
27 317811 3 0.000010014
28 514229 3 0.000010967
</pre>
</div>
<p/>
@ -738,6 +738,16 @@ $ mlr --from data/miss-date.csv --icsv \
then put '$datestamp = strptime($date, "%Y-%m-%d")' \
then step -a delta -f datestamp \
| head
n=1,date=2012-03-05,qoh=10055,datestamp=1330905600,datestamp_delta=0
n=2,date=2012-03-06,qoh=10486,datestamp=1330992000,datestamp_delta=86400
n=3,date=2012-03-07,qoh=10430,datestamp=1331078400,datestamp_delta=86400
n=4,date=2012-03-08,qoh=10674,datestamp=1331164800,datestamp_delta=86400
n=5,date=2012-03-09,qoh=10880,datestamp=1331251200,datestamp_delta=86400
n=6,date=2012-03-10,qoh=10718,datestamp=1331337600,datestamp_delta=86400
n=7,date=2012-03-11,qoh=10795,datestamp=1331424000,datestamp_delta=86400
n=8,date=2012-03-12,qoh=11043,datestamp=1331510400,datestamp_delta=86400
n=9,date=2012-03-13,qoh=11177,datestamp=1331596800,datestamp_delta=86400
n=10,date=2012-03-14,qoh=11498,datestamp=1331683200,datestamp_delta=86400
</pre>
</div>
<p/>
@ -752,6 +762,8 @@ $ mlr --from data/miss-date.csv --icsv \
then put '$datestamp = strptime($date, "%Y-%m-%d")' \
then step -a delta -f datestamp \
then filter '$datestamp_delta != 86400 &amp;&amp; $n != 1'
n=774,date=2014-04-19,qoh=130140,datestamp=1397865600,datestamp_delta=259200
n=1119,date=2015-03-31,qoh=181625,datestamp=1427760000,datestamp_delta=172800
</pre>
</div>
<p/>
@ -994,7 +1006,7 @@ following data:
<p/>
<div class="pokipanel">
<pre>
$ mlr --itsv --irs lf --opprint cat data/maxrows.tsv
$ mlr --itsv --opprint cat data/maxrows.tsv
a b n score
purple red 5 0.743231
blue purple 2 0.093710
@ -1062,7 +1074,7 @@ end {
<p/>
<div class="pokipanel">
<pre>
$ mlr --itsv --irs lf --opprint put -q -f data/maxrows.mlr data/maxrows.tsv
$ mlr --itsv --opprint put -q -f data/maxrows.mlr data/maxrows.tsv
a b n score
purple red 5 0.743231
purple red 5 0.389055

5
doc/data-examples.html
View file

@ -152,11 +152,6 @@ Miller commands were run with pretty-print-tabular output format.
<p/> The <a href="data/flins.csv">flins.csv</a> file is some sample data
obtained from <a href="https://support.spatialkey.com/spatialkey-sample-csv-data">https://support.spatialkey.com/spatialkey-sample-csv-data</a>.
<p/><b>Note: please use "mlr --csv --rs lf" for for native Un*x (linefeed-terminated) CSV files.</b>
(You can also have <tt>MLR_CSV_DEFAULT_RS=lf</tt> in your shell environment, e.g.
<tt>export MLR_CSV_DEFAULT_RS=lf</tt> or <tt>setenv MLR_CSV_DEFAULT_RS lf</tt> depending on
which shell you use.)
<p/>Vertical-tabular format is good for a quick look at CSV data layout &mdash; seeing what columns you have to work with:
<p/>
<div class="pokipanel">

23
doc/faq.html
View file

@ -143,7 +143,6 @@ Miller commands were run with pretty-print-tabular output format.
<!-- BODY COPIED FROM content-for-faq.html BY poki -->
<div class="pokitoc">
<center><b>Contents:</b></center>
&bull;&nbsp;<a href="#Number_one_FAQ">Number one FAQ</a><br/>
&bull;&nbsp;<a href="#No_output_at_all">No output at all</a><br/>
&bull;&nbsp;<a href="#Fields_not_selected">Fields not selected</a><br/>
&bull;&nbsp;<a href="#Diagnosing_delimiter_specifications">Diagnosing delimiter specifications</a><br/>
@ -157,18 +156,6 @@ Miller commands were run with pretty-print-tabular output format.
</div>
<p/>
<a id="Number_one_FAQ"/><h1>Number one FAQ</h1>
<b>
Please use <tt>mlr --csv --rs lf</tt> for native Un*x (linefeed-terminated) CSV files.
</b>
<p/>Instead of specifying <tt>--rs lf</tt> on each invocation, you can instead
have <tt>MLR_CSV_DEFAULT_RS=lf</tt> in your shell environment: e.g. put
<tt>export MLR_CSV_DEFAULT_RS=lf</tt> in your <tt>~/.bashrc</tt> or
<tt>~/.zshrc</tt>, or <tt>setenv MLR_CSV_DEFAULT_RS lf</tt> in your
<tt>~/.cshrc</tt>, as a one-time setup step.
<a id="No_output_at_all"/><h1>No output at all</h1>
<p/>Check the line-terminators of the data, e.g. with the command-line
@ -303,7 +290,7 @@ Next, run the first step of your command, omitting anything from the first <tt>t
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --rs lf --opprint count-distinct -f Status,Payment_Type data/then-example.csv
$ mlr --icsv --opprint count-distinct -f Status,Payment_Type data/then-example.csv
Status Payment_Type count
paid cash 2
pending debit 1
@ -318,7 +305,7 @@ After that, run it with the next <tt>then</tt> step included:
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --rs lf --opprint count-distinct -f Status,Payment_Type then sort -nr count data/then-example.csv
$ mlr --icsv --opprint count-distinct -f Status,Payment_Type then sort -nr count data/then-example.csv
Status Payment_Type count
paid cash 2
pending debit 1
@ -336,7 +323,7 @@ that verb.
<p/>
<div class="pokipanel">
<pre>
$ mlr --csv --rs lf count-distinct -f Status,Payment_Type data/then-example.csv | mlr --icsv --rs lf --opprint sort -nr count
$ mlr --csv count-distinct -f Status,Payment_Type data/then-example.csv | mlr --icsv --opprint sort -nr count
Status Payment_Type count
paid cash 2
pending debit 1
@ -382,7 +369,7 @@ x,y,z
<p/>
<div class="pokipanel">
<pre>
$ echo x,y,z | mlr --csv --rs lf --implicit-csv-header cat
$ echo x,y,z | mlr --csv --implicit-csv-header cat
1,2,3
x,y,z
</pre>
@ -407,7 +394,7 @@ $ echo x,y,z | mlr --dkvp rename 2,newname
<p/>
<div class="pokipanel">
<pre>
$ echo x,y,z | mlr --csv --rs lf --implicit-csv-header reorder -f 3,1,2
$ echo x,y,z | mlr --csv --implicit-csv-header reorder -f 3,1,2
3,1,2
z,x,y
</pre>

18
doc/file-formats.html
View file

@ -261,24 +261,6 @@ CSV, use <tt>--ifs tab --ofs comma</tt>, etc. (See also
https://tools.ietf.org/html/rfc4180</a>). This includes CRLF line-terminators by default, regardless
of platform.
<p/>
<b>
Please use <tt>mlr --csv --rs lf</tt> for native Un*x (linefeed-terminated) CSV files.
</b>
<p/>Instead of specifying <tt>--rs lf</tt> on each invocation, you can instead
have <tt>MLR_CSV_DEFAULT_RS=lf</tt> in your shell environment: e.g. put
<tt>export MLR_CSV_DEFAULT_RS=lf</tt> in your <tt>~/.bashrc</tt> or
<tt>~/.zshrc</tt>, or <tt>setenv MLR_CSV_DEFAULT_RS lf</tt> in your
<tt>~/.cshrc</tt>, as a one-time setup step.
<p/>The RFC says, somewhat briefly, that &ldquo;there may be a header
line&rdquo;. Miller&rsquo;s <tt>--implicit-csv-header</tt> option allows you to
read CSV data which lacks a header line, applying column labels <tt>1</tt>,
<tt>2</tt>, <tt>3</tt>, etc. for you. You may also use Miller&rsquo;s
<tt>label</tt> to replace those numerical column names with labels of your
choosing.
<p/>Here are the differences between CSV and CSV-lite:
<ul>

2
doc/index.html
View file

@ -231,7 +231,7 @@ machine, and expect it to work.
<div class="pokipanel">
<pre>
# Row filter
% mlr --csv --rs lf filter '$status != "down" && $upsec >= 10000' *.csv
% mlr --csv filter '$status != "down" && $upsec >= 10000' *.csv
</pre>
</div>

31
doc/manpage.html
View file

@ -324,7 +324,8 @@ OPTIONS
--jvstack Put one key-value pair per line for JSON
output.
--jlistwrap Wrap JSON output in outermost [ ].
--jquoteall Quote map values in JSON output, even if they're
--jknquoteint Do not quote non-string map keys in JSON output.
--jvquoteall Quote map values in JSON output, even if they're
numeric.
--jflatsep {string} Separator for flattening multi-level JSON keys,
e.g. '{"a":{"b":3}}' becomes a:b =&gt; 3 for
@ -335,11 +336,6 @@ OPTIONS
Examples: --csv for CSV-formatted input and output; --idkvp --opprint for
DKVP-formatted input and pretty-printed output.
PLEASE USE "mlr --csv --rs lf" FOR NATIVE UN*X (LINEFEED-TERMINATED) CSV FILES.
You can also have MLR_CSV_DEFAULT_RS=lf in your shell environment, e.g.
"export MLR_CSV_DEFAULT_RS=lf" or "setenv MLR_CSV_DEFAULT_RS lf" depending on
which shell you use.
FORMAT-CONVERSION KEYSTROKE-SAVERS
As keystroke-savers for format-conversion you may use the following:
--c2t --c2d --c2n --c2j --c2x --c2p --c2m
@ -390,11 +386,6 @@ OPTIONS
platform-native text data. In particular, this means LF line-terminators
by default on Linux/OSX. You can use "--dkvp --rs crlf" for
CRLF-terminated DKVP files, and so on.
* CSV is intended to handle RFC-4180-compliant data. In particular, this means
it uses CRLF line-terminators by default. You can use "--csv --rs lf" for
Linux-native CSV files. You can also have "MLR_CSV_DEFAULT_RS=lf" in your
shell environment, e.g. "export MLR_CSV_DEFAULT_RS=lf" or "setenv
MLR_CSV_DEFAULT_RS lf" depending on which shell you use.
* TSV is simply CSV using tab as field separator ("--fs tab").
* FS/PS are ignored for markdown format; RS is used.
* All RS/FS/PS options are ignored for JSON format: JSON doesn't allow
@ -407,14 +398,14 @@ OPTIONS
cr crcr newline lf lflf crlf crlfcrlf tab space comma pipe slash colon semicolon equals
* Default separators by format:
File format RS FS PS
dkvp \n , =
json (N/A) (N/A) (N/A)
nidx \n space (N/A)
csv \r\n , (N/A)
csvlite \n , (N/A)
markdown \n (N/A) (N/A)
pprint \n space (N/A)
xtab (N/A) \n space
dkvp auto , =
json auto (N/A) (N/A)
nidx auto space (N/A)
csv auto , (N/A)
csvlite auto , (N/A)
markdown auto (N/A) (N/A)
pprint auto space (N/A)
xtab (N/A) auto space
CSV-SPECIFIC OPTIONS
--implicit-csv-header Use 1,2,3,... as field labels, rather than from line 1
@ -2001,7 +1992,7 @@ SEE ALSO
2017-01-11 MILLER(1)
2017-02-02 MILLER(1)
</pre>
</div>
<p/>

31
doc/manpage.txt
View file

@ -175,7 +175,8 @@ OPTIONS
--jvstack Put one key-value pair per line for JSON
output.
--jlistwrap Wrap JSON output in outermost [ ].
--jquoteall Quote map values in JSON output, even if they're
--jknquoteint Do not quote non-string map keys in JSON output.
--jvquoteall Quote map values in JSON output, even if they're
numeric.
--jflatsep {string} Separator for flattening multi-level JSON keys,
e.g. '{"a":{"b":3}}' becomes a:b => 3 for
@ -186,11 +187,6 @@ OPTIONS
Examples: --csv for CSV-formatted input and output; --idkvp --opprint for
DKVP-formatted input and pretty-printed output.
PLEASE USE "mlr --csv --rs lf" FOR NATIVE UN*X (LINEFEED-TERMINATED) CSV FILES.
You can also have MLR_CSV_DEFAULT_RS=lf in your shell environment, e.g.
"export MLR_CSV_DEFAULT_RS=lf" or "setenv MLR_CSV_DEFAULT_RS lf" depending on
which shell you use.
FORMAT-CONVERSION KEYSTROKE-SAVERS
As keystroke-savers for format-conversion you may use the following:
--c2t --c2d --c2n --c2j --c2x --c2p --c2m
@ -241,11 +237,6 @@ OPTIONS
platform-native text data. In particular, this means LF line-terminators
by default on Linux/OSX. You can use "--dkvp --rs crlf" for
CRLF-terminated DKVP files, and so on.
* CSV is intended to handle RFC-4180-compliant data. In particular, this means
it uses CRLF line-terminators by default. You can use "--csv --rs lf" for
Linux-native CSV files. You can also have "MLR_CSV_DEFAULT_RS=lf" in your
shell environment, e.g. "export MLR_CSV_DEFAULT_RS=lf" or "setenv
MLR_CSV_DEFAULT_RS lf" depending on which shell you use.
* TSV is simply CSV using tab as field separator ("--fs tab").
* FS/PS are ignored for markdown format; RS is used.
* All RS/FS/PS options are ignored for JSON format: JSON doesn't allow
@ -258,14 +249,14 @@ OPTIONS
cr crcr newline lf lflf crlf crlfcrlf tab space comma pipe slash colon semicolon equals
* Default separators by format:
File format RS FS PS
dkvp \n , =
json (N/A) (N/A) (N/A)
nidx \n space (N/A)
csv \r\n , (N/A)
csvlite \n , (N/A)
markdown \n (N/A) (N/A)
pprint \n space (N/A)
xtab (N/A) \n space
dkvp auto , =
json auto (N/A) (N/A)
nidx auto space (N/A)
csv auto , (N/A)
csvlite auto , (N/A)
markdown auto (N/A) (N/A)
pprint auto space (N/A)
xtab (N/A) auto space
CSV-SPECIFIC OPTIONS
--implicit-csv-header Use 1,2,3,... as field labels, rather than from line 1
@ -1852,4 +1843,4 @@ SEE ALSO
2017-01-11 MILLER(1)
2017-02-02 MILLER(1)

1
doc/mkman.rb
View file

@ -14,7 +14,6 @@
# ----------------------------------------------------------------
def main
ENV['MLR_CSV_DEFAULT_RS'] = "" # In case set in user's shell
print make_top
print make_section('NAME', [

33
doc/mlr.1
View file

@ -2,12 +2,12 @@
.\" Title: mlr
.\" Author: [see the "AUTHOR" section]
.\" Generator: ./mkman.rb
.\" Date: 2017-01-11
.\" Date: 2017-02-02
.\" Manual: \ \&
.\" Source: \ \&
.\" Language: English
.\"
.TH "MILLER" "1" "2017-01-11" "\ \&" "\ \&"
.TH "MILLER" "1" "2017-02-02" "\ \&" "\ \&"
.\" -----------------------------------------------------------------
.\" * Portability definitions
.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -230,7 +230,8 @@ Please use "mlr --help-all-keywords" or "mlr -k" for help on all keywords.
--jvstack Put one key-value pair per line for JSON
output.
--jlistwrap Wrap JSON output in outermost [ ].
--jquoteall Quote map values in JSON output, even if they're
--jknquoteint Do not quote non-string map keys in JSON output.
--jvquoteall Quote map values in JSON output, even if they're
numeric.
--jflatsep {string} Separator for flattening multi-level JSON keys,
e.g. '{"a":{"b":3}}' becomes a:b => 3 for
@ -240,11 +241,6 @@ Please use "mlr --help-all-keywords" or "mlr -k" for help on all keywords.
Examples: --csv for CSV-formatted input and output; --idkvp --opprint for
DKVP-formatted input and pretty-printed output.
PLEASE USE "mlr --csv --rs lf" FOR NATIVE UN*X (LINEFEED-TERMINATED) CSV FILES.
You can also have MLR_CSV_DEFAULT_RS=lf in your shell environment, e.g.
"export MLR_CSV_DEFAULT_RS=lf" or "setenv MLR_CSV_DEFAULT_RS lf" depending on
which shell you use.
.fi
.if n \{\
.RE
@ -314,11 +310,6 @@ is available for output only.
platform-native text data. In particular, this means LF line-terminators
by default on Linux/OSX. You can use "--dkvp --rs crlf" for
CRLF-terminated DKVP files, and so on.
* CSV is intended to handle RFC-4180-compliant data. In particular, this means
it uses CRLF line-terminators by default. You can use "--csv --rs lf" for
Linux-native CSV files. You can also have "MLR_CSV_DEFAULT_RS=lf" in your
shell environment, e.g. "export MLR_CSV_DEFAULT_RS=lf" or "setenv
MLR_CSV_DEFAULT_RS lf" depending on which shell you use.
* TSV is simply CSV using tab as field separator ("--fs tab").
* FS/PS are ignored for markdown format; RS is used.
* All RS/FS/PS options are ignored for JSON format: JSON doesn't allow
@ -331,14 +322,14 @@ is available for output only.
cr crcr newline lf lflf crlf crlfcrlf tab space comma pipe slash colon semicolon equals
* Default separators by format:
File format RS FS PS
dkvp \en , =
json (N/A) (N/A) (N/A)
nidx \en space (N/A)
csv \er\en , (N/A)
csvlite \en , (N/A)
markdown \en (N/A) (N/A)
pprint \en space (N/A)
xtab (N/A) \en space
dkvp auto , =
json auto (N/A) (N/A)
nidx auto space (N/A)
csv auto , (N/A)
csvlite auto , (N/A)
markdown auto (N/A) (N/A)
pprint auto space (N/A)
xtab (N/A) auto space
.fi
.if n \{\
.RE

6
doc/reference-verbs.html
View file

@ -1337,7 +1337,7 @@ Carol,45,present
<p/>
<div class="pokipanel">
<pre>
$ mlr --csv --rs lf --implicit-csv-header cat data/headerless.csv
$ mlr --csv --implicit-csv-header cat data/headerless.csv
1,2,3
John,23,present
Fred,34,present
@ -1349,7 +1349,7 @@ Carol,45,present
<p/>
<div class="pokipanel">
<pre>
$ mlr --csv --rs lf --implicit-csv-header label name,age,status data/headerless.csv
$ mlr --csv --implicit-csv-header label name,age,status data/headerless.csv
name,age,status
John,23,present
Fred,34,present
@ -1361,7 +1361,7 @@ Carol,45,present
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --rs lf --implicit-csv-header --opprint label name,age,status data/headerless.csv
$ mlr --icsv --implicit-csv-header --opprint label name,age,status data/headerless.csv
name age status
John 23 present
Fred 34 present

31
doc/reference.html
View file

@ -424,7 +424,8 @@ Data-format options, for input, output, or both:
--jvstack Put one key-value pair per line for JSON
output.
--jlistwrap Wrap JSON output in outermost [ ].
--jquoteall Quote map values in JSON output, even if they're
--jknquoteint Do not quote non-string map keys in JSON output.
--jvquoteall Quote map values in JSON output, even if they're
numeric.
--jflatsep {string} Separator for flattening multi-level JSON keys,
e.g. '{"a":{"b":3}}' becomes a:b =&gt; 3 for
@ -435,11 +436,6 @@ Data-format options, for input, output, or both:
Examples: --csv for CSV-formatted input and output; --idkvp --opprint for
DKVP-formatted input and pretty-printed output.
PLEASE USE "mlr --csv --rs lf" FOR NATIVE UN*X (LINEFEED-TERMINATED) CSV FILES.
You can also have MLR_CSV_DEFAULT_RS=lf in your shell environment, e.g.
"export MLR_CSV_DEFAULT_RS=lf" or "setenv MLR_CSV_DEFAULT_RS lf" depending on
which shell you use.
Format-conversion keystroke-saver options, for input, output, or both:
As keystroke-savers for format-conversion you may use the following:
--c2t --c2d --c2n --c2j --c2x --c2p --c2m
@ -490,11 +486,6 @@ Separator options, for input, output, or both:
platform-native text data. In particular, this means LF line-terminators
by default on Linux/OSX. You can use "--dkvp --rs crlf" for
CRLF-terminated DKVP files, and so on.
* CSV is intended to handle RFC-4180-compliant data. In particular, this means
it uses CRLF line-terminators by default. You can use "--csv --rs lf" for
Linux-native CSV files. You can also have "MLR_CSV_DEFAULT_RS=lf" in your
shell environment, e.g. "export MLR_CSV_DEFAULT_RS=lf" or "setenv
MLR_CSV_DEFAULT_RS lf" depending on which shell you use.
* TSV is simply CSV using tab as field separator ("--fs tab").
* FS/PS are ignored for markdown format; RS is used.
* All RS/FS/PS options are ignored for JSON format: JSON doesn't allow
@ -507,14 +498,14 @@ Separator options, for input, output, or both:
cr crcr newline lf lflf crlf crlfcrlf tab space comma pipe slash colon semicolon equals
* Default separators by format:
File format RS FS PS
dkvp \n , =
json (N/A) (N/A) (N/A)
nidx \n space (N/A)
csv \r\n , (N/A)
csvlite \n , (N/A)
markdown \n (N/A) (N/A)
pprint \n space (N/A)
xtab (N/A) \n space
dkvp auto , =
json auto (N/A) (N/A)
nidx auto space (N/A)
csv auto , (N/A)
csvlite auto , (N/A)
markdown auto (N/A) (N/A)
pprint auto space (N/A)
xtab (N/A) auto space
Relevant to CSV/CSV-lite input only:
--implicit-csv-header Use 1,2,3,... as field labels, rather than from line 1
@ -863,7 +854,7 @@ streaming-data programs operate through pipes, each one can use a CPU. Rest
assured you get the same results either way.
<p/>The other reason to use then-chaining is for simplicity: you don&rsquo;t
have re-type formatting flags (e.g. <tt>--csv --rs lf --fs tab</tt>) at every
have re-type formatting flags (e.g. <tt>--csv --fs tab</tt>) at every
pipeline stage.
<!-- ================================================================ -->