diff --git a/c/cli/mlrcli.c b/c/cli/mlrcli.c index 52026d1e3..489141ae0 100644 --- a/c/cli/mlrcli.c +++ b/c/cli/mlrcli.c @@ -374,11 +374,7 @@ static lhmss_t* get_default_rses() { lhmss_put(singleton_default_rses, "json", "auto", NO_FREE); lhmss_put(singleton_default_rses, "nidx", "auto", NO_FREE); - char* csv_rs = "auto"; - char* env_default = getenv("MLR_CSV_DEFAULT_RS"); - if (env_default != NULL && !streq(env_default, "")) - csv_rs = cli_sep_from_arg(env_default); - lhmss_put(singleton_default_rses, "csv", csv_rs, NO_FREE); + lhmss_put(singleton_default_rses, "csv", "auto", NO_FREE); lhmss_put(singleton_default_rses, "csvlite", "auto", NO_FREE); lhmss_put(singleton_default_rses, "markdown", "auto", NO_FREE); lhmss_put(singleton_default_rses, "pprint", "auto", NO_FREE); @@ -397,7 +393,7 @@ static lhmss_t* get_default_fses() { lhmss_put(singleton_default_fses, "csvlite", ",", NO_FREE); lhmss_put(singleton_default_fses, "markdown", "(N/A)", NO_FREE); lhmss_put(singleton_default_fses, "pprint", " ", NO_FREE); - lhmss_put(singleton_default_fses, "xtab", "\n", NO_FREE); + lhmss_put(singleton_default_fses, "xtab", "auto", NO_FREE); } return singleton_default_fses; } @@ -725,11 +721,6 @@ static void main_usage_data_format_options(FILE* o, char* argv0) { fprintf(o, "\n"); fprintf(o, " Examples: --csv for CSV-formatted input and output; --idkvp --opprint for\n"); fprintf(o, " DKVP-formatted input and pretty-printed output.\n"); - fprintf(o, "\n"); - fprintf(o, " PLEASE USE \"%s --csv --rs lf\" FOR NATIVE UN*X (LINEFEED-TERMINATED) CSV FILES.\n", argv0); - fprintf(o, " You can also have MLR_CSV_DEFAULT_RS=lf in your shell environment, e.g.\n"); - fprintf(o, " \"export MLR_CSV_DEFAULT_RS=lf\" or \"setenv MLR_CSV_DEFAULT_RS lf\" depending on\n"); - fprintf(o, " which shell you use.\n"); } static void main_usage_format_conversion_keystroke_saver_options(FILE* o, char* argv0) { @@ -785,11 +776,6 @@ static void main_usage_separator_options(FILE* o, char* argv0) { fprintf(o, " platform-native text data. In particular, this means LF line-terminators\n"); fprintf(o, " by default on Linux/OSX. You can use \"--dkvp --rs crlf\" for\n"); fprintf(o, " CRLF-terminated DKVP files, and so on.\n"); - fprintf(o, " * CSV is intended to handle RFC-4180-compliant data. In particular, this means\n"); - fprintf(o, " it uses CRLF line-terminators by default. You can use \"--csv --rs lf\" for\n"); - fprintf(o, " Linux-native CSV files. You can also have \"MLR_CSV_DEFAULT_RS=lf\" in your\n"); - fprintf(o, " shell environment, e.g. \"export MLR_CSV_DEFAULT_RS=lf\" or \"setenv\n"); - fprintf(o, " MLR_CSV_DEFAULT_RS lf\" depending on which shell you use.\n"); fprintf(o, " * TSV is simply CSV using tab as field separator (\"--fs tab\").\n"); fprintf(o, " * FS/PS are ignored for markdown format; RS is used.\n"); fprintf(o, " * All RS/FS/PS options are ignored for JSON format: JSON doesn't allow\n"); diff --git a/c/reg_test/expected/out b/c/reg_test/expected/out index 6a16a2d00..a8a22a015 100644 --- a/c/reg_test/expected/out +++ b/c/reg_test/expected/out @@ -43439,42 +43439,6 @@ a 1 4 -mlr --csv cut -f a ./reg_test/input/rfc-csv/simple.csv -mlr: unmatched double quote at line 3. -a -1 -4 - -mlr --csv --rs crlf cut -f a ./reg_test/input/rfc-csv/simple.csv -a -1 -4 -x - -mlr --csv --rs lf cut -f a ./reg_test/input/rfc-csv/simple.csv -mlr: unmatched double quote at line 3. -a -1 -4 - -mlr --csv cut -f a ./reg_test/input/rfc-csv/simple.csv -a -1 -4 -x - -mlr --csv --rs crlf cut -f a ./reg_test/input/rfc-csv/simple.csv -a -1 -4 -x - -mlr --csv --rs lf cut -f a ./reg_test/input/rfc-csv/simple.csv -mlr: unmatched double quote at line 3. -a -1 -4 - ================================================================ MULTI-CHARACTER IRS/IFS/IPS FOR DKVP @@ -44016,8 +43980,7 @@ hi@@@78 mlr --no-mmap --xtab --ips . --ops := cut -x -f b ./reg_test/input/dots.xtab a:=1 c:=345 - -defg:=6 +autodefg:=6 hi:=78 mlr --xtab --ips : --ops @@@@ put $sum=int($a+$b) ./reg_test/input/multi-ips.dkvp @@ -45956,65 +45919,65 @@ x 0.5026260055412137 y 0.9526183602969864 mlr --ifs auto --xtab cat ./reg_test/input/line-term-crlf.xtab -a pan -b pan -i 1 -x 0.3467901443380824 -y 0.7268028627434533 - -a eks -b pan -i 2 -x 0.7586799647899636 -y 0.5221511083334797 - -a wye -b wye -i 3 -x 0.20460330576630303 -y 0.33831852551664776 - -a eks -b wye -i 4 -x 0.38139939387114097 -y 0.13418874328430463 - -a wye -b pan -i 5 -x 0.5732889198020006 -y 0.8636244699032729 - -a zee -b pan -i 6 -x 0.5271261600918548 -y 0.49322128674835697 - -a eks -b zee -i 7 -x 0.6117840605678454 -y 0.1878849191181694 - -a zee -b wye -i 8 -x 0.5985540091064224 -y 0.976181385699006 - -a hat -b wye -i 9 -x 0.03144187646093577 -y 0.7495507603507059 - -a pan -b wye -i 10 -x 0.5026260055412137 -y 0.9526183602969864 +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 + +a wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +b wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 + +a wye +b pan +i 5 +x 0.5732889198020006 +y 0.8636244699032729 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 + +a eks +b zee +i 7 +x 0.6117840605678454 +y 0.1878849191181694 + +a zee +b wye +i 8 +x 0.5985540091064224 +y 0.976181385699006 + +a hat +b wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 mlr --fs auto --xtab cat ./reg_test/input/line-term-lf.xtab a pan @@ -46202,65 +46165,65 @@ x 0.5026260055412137 y 0.9526183602969864 mlr --ifs auto --xtab --no-mmap cat ./reg_test/input/line-term-crlf.xtab -a pan -b pan -i 1 -x 0.3467901443380824 -y 0.7268028627434533 - -a eks -b pan -i 2 -x 0.7586799647899636 -y 0.5221511083334797 - -a wye -b wye -i 3 -x 0.20460330576630303 -y 0.33831852551664776 - -a eks -b wye -i 4 -x 0.38139939387114097 -y 0.13418874328430463 - -a wye -b pan -i 5 -x 0.5732889198020006 -y 0.8636244699032729 - -a zee -b pan -i 6 -x 0.5271261600918548 -y 0.49322128674835697 - -a eks -b zee -i 7 -x 0.6117840605678454 -y 0.1878849191181694 - -a zee -b wye -i 8 -x 0.5985540091064224 -y 0.976181385699006 - -a hat -b wye -i 9 -x 0.03144187646093577 -y 0.7495507603507059 - -a pan -b wye -i 10 -x 0.5026260055412137 -y 0.9526183602969864 +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 + +a wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +b wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 + +a wye +b pan +i 5 +x 0.5732889198020006 +y 0.8636244699032729 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 + +a eks +b zee +i 7 +x 0.6117840605678454 +y 0.1878849191181694 + +a zee +b wye +i 8 +x 0.5985540091064224 +y 0.976181385699006 + +a hat +b wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 mlr --fs auto --xtab --no-mmap cat ./reg_test/input/line-term-lf.xtab a pan @@ -46448,65 +46411,65 @@ x 0.5026260055412137 y 0.9526183602969864 mlr --ifs auto --xtab cat ./reg_test/input/line-term-crlf.xtab -a pan -b pan -i 1 -x 0.3467901443380824 -y 0.7268028627434533 - -a eks -b pan -i 2 -x 0.7586799647899636 -y 0.5221511083334797 - -a wye -b wye -i 3 -x 0.20460330576630303 -y 0.33831852551664776 - -a eks -b wye -i 4 -x 0.38139939387114097 -y 0.13418874328430463 - -a wye -b pan -i 5 -x 0.5732889198020006 -y 0.8636244699032729 - -a zee -b pan -i 6 -x 0.5271261600918548 -y 0.49322128674835697 - -a eks -b zee -i 7 -x 0.6117840605678454 -y 0.1878849191181694 - -a zee -b wye -i 8 -x 0.5985540091064224 -y 0.976181385699006 - -a hat -b wye -i 9 -x 0.03144187646093577 -y 0.7495507603507059 - -a pan -b wye -i 10 -x 0.5026260055412137 -y 0.9526183602969864 +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 + +a wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +b wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 + +a wye +b pan +i 5 +x 0.5732889198020006 +y 0.8636244699032729 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 + +a eks +b zee +i 7 +x 0.6117840605678454 +y 0.1878849191181694 + +a zee +b wye +i 8 +x 0.5985540091064224 +y 0.976181385699006 + +a hat +b wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 mlr --fs auto --xtab cat ./reg_test/input/line-term-lf.xtab a pan @@ -46694,65 +46657,65 @@ x 0.5026260055412137 y 0.9526183602969864 mlr --ifs auto --xtab --no-mmap cat ./reg_test/input/line-term-crlf.xtab -a pan -b pan -i 1 -x 0.3467901443380824 -y 0.7268028627434533 - -a eks -b pan -i 2 -x 0.7586799647899636 -y 0.5221511083334797 - -a wye -b wye -i 3 -x 0.20460330576630303 -y 0.33831852551664776 - -a eks -b wye -i 4 -x 0.38139939387114097 -y 0.13418874328430463 - -a wye -b pan -i 5 -x 0.5732889198020006 -y 0.8636244699032729 - -a zee -b pan -i 6 -x 0.5271261600918548 -y 0.49322128674835697 - -a eks -b zee -i 7 -x 0.6117840605678454 -y 0.1878849191181694 - -a zee -b wye -i 8 -x 0.5985540091064224 -y 0.976181385699006 - -a hat -b wye -i 9 -x 0.03144187646093577 -y 0.7495507603507059 - -a pan -b wye -i 10 -x 0.5026260055412137 -y 0.9526183602969864 +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 + +a wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +b wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 + +a wye +b pan +i 5 +x 0.5732889198020006 +y 0.8636244699032729 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 + +a eks +b zee +i 7 +x 0.6117840605678454 +y 0.1878849191181694 + +a zee +b wye +i 8 +x 0.5985540091064224 +y 0.976181385699006 + +a hat +b wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 mlr --fs auto --xtab --no-mmap cat ./reg_test/input/line-term-lf.xtab a pan diff --git a/c/reg_test/run b/c/reg_test/run index 68a104392..dc2aa8bd9 100755 --- a/c/reg_test/run +++ b/c/reg_test/run @@ -2,7 +2,6 @@ # ================================================================ set -e -export MLR_CSV_DEFAULT_RS= # In case set in user's shell # For building with autoconf: # * in-directory build: @@ -4886,16 +4885,6 @@ run_mlr --csv cut -f a $indir/rfc-csv/simple.csv run_mlr --csv --rs crlf cut -f a $indir/rfc-csv/simple.csv mlr_expect_fail --csv --rs lf cut -f a $indir/rfc-csv/simple.csv -export MLR_CSV_DEFAULT_RS=lf; mlr_expect_fail --csv cut -f a $indir/rfc-csv/simple.csv -export MLR_CSV_DEFAULT_RS=lf; run_mlr --csv --rs crlf cut -f a $indir/rfc-csv/simple.csv -export MLR_CSV_DEFAULT_RS=lf; mlr_expect_fail --csv --rs lf cut -f a $indir/rfc-csv/simple.csv - -export MLR_CSV_DEFAULT_RS=crlf; run_mlr --csv cut -f a $indir/rfc-csv/simple.csv -export MLR_CSV_DEFAULT_RS=crlf; run_mlr --csv --rs crlf cut -f a $indir/rfc-csv/simple.csv -export MLR_CSV_DEFAULT_RS=crlf; mlr_expect_fail --csv --rs lf cut -f a $indir/rfc-csv/simple.csv - -export MLR_CSV_DEFAULT_RS= - # ---------------------------------------------------------------- announce MULTI-CHARACTER IRS/IFS/IPS FOR DKVP diff --git a/c/todo.txt b/c/todo.txt index 9243440a5..da0a13ced 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -38,7 +38,15 @@ OTHER: ================================================================ AUTOTERM +! BUGFIX: + $ mlr --ifs auto --xtab --ips . --ops := cut -x -f b < ./reg_test/input/dots.xtab + a:=1 + c:=345 + autodefg:=6 + hi:=78 + ! doc updates x many ! incl. --irs auto. + - automan etc. re auto; what about --ors auto - IRS/ORS variables in the DSL are from autodetect if that is in use - JSON: manpage re ORS selectable now - XTAB: --ifs/--ofs instead of --irs/--ors diff --git a/doc/10-min.html b/doc/10-min.html index af3d24698..afb1a0cb7 100644 --- a/doc/10-min.html +++ b/doc/10-min.html @@ -179,7 +179,7 @@ purple,square,0,91,72.3735,8.2430

-$ mlr --csv --rs lf cat example.csv
+$ mlr --csv cat example.csv
 color,shape,flag,index,quantity,rate
 yellow,triangle,1,11,43.6498,9.8870
 red,square,1,15,79.2778,0.0130
@@ -200,7 +200,7 @@ purple,square,0,91,72.3735,8.2430
 

-$ mlr --icsv --irs lf --opprint cat example.csv
+$ mlr --icsv --opprint cat example.csv
 color  shape    flag index quantity rate
 yellow triangle 1    11    43.6498  9.8870
 red    square   1    15    79.2778  0.0130
@@ -222,7 +222,7 @@ header is included either way:
 

-$ mlr --csv --rs lf head -n 4 example.csv
+$ mlr --csv head -n 4 example.csv
 color,shape,flag,index,quantity,rate
 yellow,triangle,1,11,43.6498,9.8870
 red,square,1,15,79.2778,0.0130
@@ -234,7 +234,7 @@ red,square,0,48,77.5542,7.4670
 

-$ mlr --csv --rs lf tail -n 4 example.csv
+$ mlr --csv tail -n 4 example.csv
 color,shape,flag,index,quantity,rate
 purple,triangle,0,65,80.1405,5.8240
 yellow,circle,1,73,63.9785,4.2370
@@ -250,7 +250,7 @@ numerically descending on another field: 
 

-$ mlr --icsv --irs lf --opprint sort -f shape -nr index example.csv
+$ mlr --icsv --opprint sort -f shape -nr index example.csv
 color  shape    flag index quantity rate
 yellow circle   1    87    63.5058  8.3350
 yellow circle   1    73    63.9785  4.2370
@@ -271,7 +271,7 @@ yellow triangle 1    11    43.6498  9.8870
 

-$ mlr --icsv --irs lf --opprint cut -f flag,shape example.csv
+$ mlr --icsv --opprint cut -f flag,shape example.csv
 shape    flag
 triangle 1
 square   1
@@ -292,7 +292,7 @@ square   0
 

-$ mlr --icsv --irs lf --opprint cut -o -f flag,shape example.csv
+$ mlr --icsv --opprint cut -o -f flag,shape example.csv
 flag shape
 1    triangle
 1    square
@@ -313,7 +313,7 @@ flag shape
 

-$ mlr --icsv --irs lf --opprint cut -x -f flag,shape example.csv
+$ mlr --icsv --opprint cut -x -f flag,shape example.csv
 color  index quantity rate
 yellow 11    43.6498  9.8870
 red    15    79.2778  0.0130
@@ -334,7 +334,7 @@ purple 91    72.3735  8.2430
 

-$ mlr --icsv --irs lf --opprint filter '$color == "red"' example.csv
+$ mlr --icsv --opprint filter '$color == "red"' example.csv
 color shape  flag index quantity rate
 red   square 1    15    79.2778  0.0130
 red   circle 1    16    13.8103  2.9010
@@ -346,7 +346,7 @@ red   square 0    64    77.1991  9.5310
 

-$ mlr --icsv --irs lf --opprint filter '$color == "red" && $flag == 1' example.csv
+$ mlr --icsv --opprint filter '$color == "red" && $flag == 1' example.csv
 color shape  flag index quantity rate
 red   square 1    15    79.2778  0.0130
 red   circle 1    16    13.8103  2.9010
@@ -359,7 +359,7 @@ red   circle 1    16    13.8103  2.9010
 

-$ mlr --icsv --irs lf --opprint put '$ratio = $quantity / $rate; $color_shape = $color . "_" . $shape' example.csv
+$ mlr --icsv --opprint put '$ratio = $quantity / $rate; $color_shape = $color . "_" . $shape' example.csv
 color  shape    flag index quantity rate   ratio       color_shape
 yellow triangle 1    11    43.6498  9.8870 4.414868    yellow_triangle
 red    square   1    15    79.2778  0.0130 6098.292308 red_square
@@ -380,7 +380,7 @@ purple square   0    91    72.3735  8.2430 8.779995    purple_square
 

-$ mlr --icsv --irs lf --ojson put '$ratio = $quantity/$rate; $shape = toupper($shape)' example.csv
+$ mlr --icsv --ojson put '$ratio = $quantity/$rate; $shape = toupper($shape)' example.csv
 { "color": "yellow", "shape": "TRIANGLE", "flag": 1, "index": 11, "quantity": 43.6498, "rate": 9.8870, "ratio": 4.414868 }
 { "color": "red", "shape": "SQUARE", "flag": 1, "index": 15, "quantity": 79.2778, "rate": 0.0130, "ratio": 6098.292308 }
 { "color": "red", "shape": "CIRCLE", "flag": 1, "index": 16, "quantity": 13.8103, "rate": 2.9010, "ratio": 4.760531 }
@@ -397,7 +397,7 @@ $ mlr --icsv --irs lf --ojson put '$ratio = $quantity/$rate; $shape = toupper($s
 

-$ mlr --icsv --irs lf --ojson --jvstack --jlistwrap tail -n 2 example.csv
+$ mlr --icsv --ojson --jvstack --jlistwrap tail -n 2 example.csv
 [
 {
   "color": "yellow",
@@ -428,7 +428,7 @@ value of the shape field.
 

-$ mlr --icsv --irs lf --opprint sort -f shape -nr index then head -n 1 -g shape example.csv
+$ mlr --icsv --opprint sort -f shape -nr index then head -n 1 -g shape example.csv
 color  shape    flag index quantity rate
 yellow circle   1    87    63.5058  8.3350
 purple square   0    91    72.3735  8.2430
@@ -444,7 +444,7 @@ have lots of columns:
 

-$ mlr --icsv --irs lf --oxtab --from example.csv stats1 -a p0,p10,p25,p50,p75,p90,p99,p100 -f rate
+$ mlr --icsv --oxtab --from example.csv stats1 -a p0,p10,p25,p50,p75,p90,p99,p100 -f rate
 rate_p0   0.013000
 rate_p10  2.901000
 rate_p25  4.237000
@@ -459,7 +459,7 @@ rate_p100 9.887000
 

-$ mlr --icsv --irs lf --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape
+$ mlr --icsv --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape
 shape    quantity_count quantity_min quantity_mean quantity_max
 triangle 3              43.649800    68.339767     81.229000
 square   4              72.373500    76.601150     79.277800
@@ -470,7 +470,7 @@ circle   3              13.810300    47.098200     63.978500
 

-$ mlr --icsv --irs lf --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape,color
+$ mlr --icsv --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape,color
 shape    color  quantity_count quantity_min quantity_mean quantity_max
 triangle yellow 1              43.649800    43.649800     43.649800
 square   red    3              77.199100    78.010367     79.277800
@@ -488,7 +488,7 @@ per one or more field names:
 

-$ mlr --csv --rs lf --from example.csv put -q 'tee > $shape.".csv", $*'
+$ mlr --csv --from example.csv put -q 'tee > $shape.".csv", $*'
 

@@ -590,7 +590,7 @@ different output format.

I like to produce SQL-query output with header-column and tab delimiter: this is CSV but with a tab instead of a comma, also known as TSV. Then I -post-process with mlr --tsv --rs lf or mlr --tsvlite. This +post-process with mlr --tsv or mlr --tsvlite. This means I can do some (or all, or none) of my data processing within SQL queries, and some (or none, or all) of my data processing using Miller — whichever is most convenient for my needs at the moment. diff --git a/doc/content-for-10-min.html b/doc/content-for-10-min.html index fc4dab77a..ad1bd7caf 100644 --- a/doc/content-for-10-min.html +++ b/doc/content-for-10-min.html @@ -8,68 +8,68 @@ POKI_RUN_COMMAND{{cat example.csv}}HERE

mlr cat is like cat ... -POKI_RUN_COMMAND{{mlr --csv --rs lf cat example.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv cat example.csv}}HERE

... but it can also do format conversion (here, to pretty-printed tabular format): -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint cat example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint cat example.csv}}HERE

mlr head and mlr tail count records. The CSV header is included either way: -POKI_RUN_COMMAND{{mlr --csv --rs lf head -n 4 example.csv}}HERE -POKI_RUN_COMMAND{{mlr --csv --rs lf tail -n 4 example.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv head -n 4 example.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv tail -n 4 example.csv}}HERE

Sort primarily alphabetically on one field, then secondarily numerically descending on another field: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint sort -f shape -nr index example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint sort -f shape -nr index example.csv}}HERE

Use cut to retain only specified fields, in input-data order: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint cut -f flag,shape example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint cut -f flag,shape example.csv}}HERE

Use cut -o to retain only specified fields, in your specified order: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint cut -o -f flag,shape example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint cut -o -f flag,shape example.csv}}HERE

Use cut -x to omit specified fields: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint cut -x -f flag,shape example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint cut -x -f flag,shape example.csv}}HERE

Use filter to retain specified records: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint filter '$color == "red"' example.csv}}HERE -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint filter '$color == "red" && $flag == 1' example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint filter '$color == "red"' example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint filter '$color == "red" && $flag == 1' example.csv}}HERE

Use put to add/replace fields which are computed from other fields: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint put '$ratio = $quantity / $rate; $color_shape = $color . "_" . $shape' example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint put '$ratio = $quantity / $rate; $color_shape = $color . "_" . $shape' example.csv}}HERE

JSON output: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --ojson put '$ratio = $quantity/$rate; $shape = toupper($shape)' example.csv}}HERE -POKI_RUN_COMMAND{{mlr --icsv --irs lf --ojson --jvstack --jlistwrap tail -n 2 example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --ojson put '$ratio = $quantity/$rate; $shape = toupper($shape)' example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --ojson --jvstack --jlistwrap tail -n 2 example.csv}}HERE

Use then to pipe commands together. Also, the -g option for many Miller commands is for group-by: here, head -n 1 -g shape outputs the first record for each distinct value of the shape field. -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint sort -f shape -nr index then head -n 1 -g shape example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint sort -f shape -nr index then head -n 1 -g shape example.csv}}HERE

Statistics can be computed with or without group-by field(s). Also, the first of these two examples uses --oxtab output format which is a nice alternative to --opprint when you have lots of columns: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --oxtab --from example.csv stats1 -a p0,p10,p25,p50,p75,p90,p99,p100 -f rate}}HERE -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape}}HERE -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape,color}}HERE +POKI_RUN_COMMAND{{mlr --icsv --oxtab --from example.csv stats1 -a p0,p10,p25,p50,p75,p90,p99,p100 -f rate}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape,color}}HERE

Using tee within put, you can split your input data into separate files per one or more field names: -POKI_RUN_COMMAND{{mlr --csv --rs lf --from example.csv put -q 'tee > $shape.".csv", $*'}}HERE +POKI_RUN_COMMAND{{mlr --csv --from example.csv put -q 'tee > $shape.".csv", $*'}}HERE
POKI_RUN_COMMAND{{cat circle.csv}}HERE @@ -137,7 +137,7 @@ different output format.

I like to produce SQL-query output with header-column and tab delimiter: this is CSV but with a tab instead of a comma, also known as TSV. Then I -post-process with mlr --tsv --rs lf or mlr --tsvlite. This +post-process with mlr --tsv or mlr --tsvlite. This means I can do some (or all, or none) of my data processing within SQL queries, and some (or none, or all) of my data processing using Miller — whichever is most convenient for my needs at the moment. diff --git a/doc/content-for-cookbook.html b/doc/content-for-cookbook.html index 4359bda61..a4f0f3972 100644 --- a/doc/content-for-cookbook.html +++ b/doc/content-for-cookbook.html @@ -51,14 +51,14 @@ POKI_INCLUDE_AND_RUN_ESCAPED(data/rect.sh)HERE

Bulk rename of field names

POKI_RUN_COMMAND{{cat data/spaces.csv}}HERE -POKI_RUN_COMMAND{{mlr --csv --rs lf rename -r -g ' ,_' data/spaces.csv}}HERE -POKI_RUN_COMMAND{{mlr --csv --irs lf --opprint rename -r -g ' ,_' data/spaces.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv rename -r -g ' ,_' data/spaces.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv --opprint rename -r -g ' ,_' data/spaces.csv}}HERE

You can also do this with a for-loop but it puts the modified fields after the unmodified fields: POKI_RUN_COMMAND{{cat data/bulk-rename-for-loop.mlr}}HERE -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint put -f data/bulk-rename-for-loop.mlr data/spaces.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint put -f data/bulk-rename-for-loop.mlr data/spaces.csv}}HERE

Headerless CSV on input or output

@@ -69,14 +69,14 @@ POKI_RUN_COMMAND{{cat data/headerless.csv}}HERE

You can use Miller to add a header: the --implicit-csv-header applies positionally indexed labels: -POKI_RUN_COMMAND{{mlr --csv --rs lf --implicit-csv-header cat data/headerless.csv}}HERE -POKI_RUN_COMMAND{{mlr --icsv --irs lf --implicit-csv-header --opprint cat data/headerless.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv --implicit-csv-header cat data/headerless.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --implicit-csv-header --opprint cat data/headerless.csv}}HERE

Following that, you can rename the positionally indexed labels to names with meaning for your context. For example: -POKI_RUN_COMMAND{{mlr --csv --rs lf --implicit-csv-header label name,age,status data/headerless.csv}}HERE -POKI_RUN_COMMAND{{mlr --icsv --rs lf --implicit-csv-header --opprint label name,age,status data/headerless.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv --implicit-csv-header label name,age,status data/headerless.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --implicit-csv-header --opprint label name,age,status data/headerless.csv}}HERE

Likewise, if you need to produce CSV which is lacking its header, you can pipe Miller’s output to the system command sed 1d, or you can use Miller’s --headerless-csv-output option: @@ -209,7 +209,7 @@ POKI_INCLUDE_AND_RUN_ESCAPED(data/two-pass-record-numbers.sh)HERE

The idea is to retain records having the largest value of n in the following data: -POKI_RUN_COMMAND{{mlr --itsv --irs lf --opprint cat data/maxrows.tsv}}HERE +POKI_RUN_COMMAND{{mlr --itsv --opprint cat data/maxrows.tsv}}HERE

Of course, the largest value of n isn’t known until after all data have been read. Using an out-of-stream variable we can retain all @@ -217,7 +217,7 @@ records as they are read, then filter them at the end: POKI_RUN_COMMAND{{cat data/maxrows.mlr}}HERE -POKI_RUN_COMMAND{{mlr --itsv --irs lf --opprint put -q -f data/maxrows.mlr data/maxrows.tsv}}HERE +POKI_RUN_COMMAND{{mlr --itsv --opprint put -q -f data/maxrows.mlr data/maxrows.tsv}}HERE

Filtering paragraphs of text

diff --git a/doc/content-for-data-examples.html b/doc/content-for-data-examples.html index 499581dcf..a45d56ad0 100644 --- a/doc/content-for-data-examples.html +++ b/doc/content-for-data-examples.html @@ -4,11 +4,6 @@ POKI_PUT_TOC_HERE

The flins.csv file is some sample data obtained from https://support.spatialkey.com/spatialkey-sample-csv-data. -

Note: please use "mlr --csv --rs lf" for for native Un*x (linefeed-terminated) CSV files. -(You can also have MLR_CSV_DEFAULT_RS=lf in your shell environment, e.g. -export MLR_CSV_DEFAULT_RS=lf or setenv MLR_CSV_DEFAULT_RS lf depending on -which shell you use.) -

Vertical-tabular format is good for a quick look at CSV data layout — seeing what columns you have to work with: POKI_RUN_COMMAND{{head -n 2 data/flins.csv | mlr --icsv --oxtab cat}}HERE

A few simple queries: diff --git a/doc/content-for-faq.html b/doc/content-for-faq.html index 91286116e..ce71ca55b 100644 --- a/doc/content-for-faq.html +++ b/doc/content-for-faq.html @@ -1,17 +1,5 @@ POKI_PUT_TOC_HERE -

Number one FAQ

- - -Please use mlr --csv --rs lf for native Un*x (linefeed-terminated) CSV files. - - -

Instead of specifying --rs lf on each invocation, you can instead -have MLR_CSV_DEFAULT_RS=lf in your shell environment: e.g. put -export MLR_CSV_DEFAULT_RS=lf in your ~/.bashrc or -~/.zshrc, or setenv MLR_CSV_DEFAULT_RS lf in your -~/.cshrc, as a one-time setup step. -

No output at all

Check the line-terminators of the data, e.g. with the command-line @@ -64,18 +52,18 @@ POKI_RUN_COMMAND{{cat data/then-example.csv}}HERE Next, run the first step of your command, omitting anything from the first then onward: -POKI_RUN_COMMAND{{mlr --icsv --rs lf --opprint count-distinct -f Status,Payment_Type data/then-example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint count-distinct -f Status,Payment_Type data/then-example.csv}}HERE After that, run it with the next then step included: -POKI_RUN_COMMAND{{mlr --icsv --rs lf --opprint count-distinct -f Status,Payment_Type then sort -nr count data/then-example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint count-distinct -f Status,Payment_Type then sort -nr count data/then-example.csv}}HERE Now if you use then to include another verb after that, the columns Status, Payment_Type, and count will be the input to that verb.

Note, by the way, that you’ll get the same results using pipes: -POKI_RUN_COMMAND{{mlr --csv --rs lf count-distinct -f Status,Payment_Type data/then-example.csv | mlr --icsv --rs lf --opprint sort -nr count}}HERE +POKI_RUN_COMMAND{{mlr --csv count-distinct -f Status,Payment_Type data/then-example.csv | mlr --icsv --opprint sort -nr count}}HERE

I assigned $9 and it’s not 9th

@@ -89,10 +77,10 @@ input data. POKI_RUN_COMMAND{{echo x,y,z | mlr --dkvp cat}}HERE POKI_RUN_COMMAND{{echo x,y,z | mlr --dkvp put '$6="a";$4="b";$55="cde"'}}HERE POKI_RUN_COMMAND{{echo x,y,z | mlr --nidx cat}}HERE -POKI_RUN_COMMAND{{echo x,y,z | mlr --csv --rs lf --implicit-csv-header cat}}HERE +POKI_RUN_COMMAND{{echo x,y,z | mlr --csv --implicit-csv-header cat}}HERE POKI_RUN_COMMAND{{echo x,y,z | mlr --dkvp rename 2,999}}HERE POKI_RUN_COMMAND{{echo x,y,z | mlr --dkvp rename 2,newname}}HERE -POKI_RUN_COMMAND{{echo x,y,z | mlr --csv --rs lf --implicit-csv-header reorder -f 3,1,2}}HERE +POKI_RUN_COMMAND{{echo x,y,z | mlr --csv --implicit-csv-header reorder -f 3,1,2}}HERE

Why doesn’t mlr cut put fields in the order I want?

diff --git a/doc/content-for-file-formats.html b/doc/content-for-file-formats.html index 763df75f5..a24f5791f 100644 --- a/doc/content-for-file-formats.html +++ b/doc/content-for-file-formats.html @@ -36,24 +36,6 @@ POKI_PUT_LINK_FOR_PAGE(reference.html)HERE.) https://tools.ietf.org/html/rfc4180). This includes CRLF line-terminators by default, regardless of platform. -

- -Please use mlr --csv --rs lf for native Un*x (linefeed-terminated) CSV files. - - -

Instead of specifying --rs lf on each invocation, you can instead -have MLR_CSV_DEFAULT_RS=lf in your shell environment: e.g. put -export MLR_CSV_DEFAULT_RS=lf in your ~/.bashrc or -~/.zshrc, or setenv MLR_CSV_DEFAULT_RS lf in your -~/.cshrc, as a one-time setup step. - -

The RFC says, somewhat briefly, that “there may be a header -line”. Miller’s --implicit-csv-header option allows you to -read CSV data which lacks a header line, applying column labels 1, -2, 3, etc. for you. You may also use Miller’s -label to replace those numerical column names with labels of your -choosing. -

Here are the differences between CSV and CSV-lite: