diff --git a/c/cli/mlrcli.c b/c/cli/mlrcli.c index 52026d1e3..489141ae0 100644 --- a/c/cli/mlrcli.c +++ b/c/cli/mlrcli.c @@ -374,11 +374,7 @@ static lhmss_t* get_default_rses() { lhmss_put(singleton_default_rses, "json", "auto", NO_FREE); lhmss_put(singleton_default_rses, "nidx", "auto", NO_FREE); - char* csv_rs = "auto"; - char* env_default = getenv("MLR_CSV_DEFAULT_RS"); - if (env_default != NULL && !streq(env_default, "")) - csv_rs = cli_sep_from_arg(env_default); - lhmss_put(singleton_default_rses, "csv", csv_rs, NO_FREE); + lhmss_put(singleton_default_rses, "csv", "auto", NO_FREE); lhmss_put(singleton_default_rses, "csvlite", "auto", NO_FREE); lhmss_put(singleton_default_rses, "markdown", "auto", NO_FREE); lhmss_put(singleton_default_rses, "pprint", "auto", NO_FREE); @@ -397,7 +393,7 @@ static lhmss_t* get_default_fses() { lhmss_put(singleton_default_fses, "csvlite", ",", NO_FREE); lhmss_put(singleton_default_fses, "markdown", "(N/A)", NO_FREE); lhmss_put(singleton_default_fses, "pprint", " ", NO_FREE); - lhmss_put(singleton_default_fses, "xtab", "\n", NO_FREE); + lhmss_put(singleton_default_fses, "xtab", "auto", NO_FREE); } return singleton_default_fses; } @@ -725,11 +721,6 @@ static void main_usage_data_format_options(FILE* o, char* argv0) { fprintf(o, "\n"); fprintf(o, " Examples: --csv for CSV-formatted input and output; --idkvp --opprint for\n"); fprintf(o, " DKVP-formatted input and pretty-printed output.\n"); - fprintf(o, "\n"); - fprintf(o, " PLEASE USE \"%s --csv --rs lf\" FOR NATIVE UN*X (LINEFEED-TERMINATED) CSV FILES.\n", argv0); - fprintf(o, " You can also have MLR_CSV_DEFAULT_RS=lf in your shell environment, e.g.\n"); - fprintf(o, " \"export MLR_CSV_DEFAULT_RS=lf\" or \"setenv MLR_CSV_DEFAULT_RS lf\" depending on\n"); - fprintf(o, " which shell you use.\n"); } static void main_usage_format_conversion_keystroke_saver_options(FILE* o, char* argv0) { @@ -785,11 +776,6 @@ static void main_usage_separator_options(FILE* o, char* argv0) { fprintf(o, " platform-native text data. In particular, this means LF line-terminators\n"); fprintf(o, " by default on Linux/OSX. You can use \"--dkvp --rs crlf\" for\n"); fprintf(o, " CRLF-terminated DKVP files, and so on.\n"); - fprintf(o, " * CSV is intended to handle RFC-4180-compliant data. In particular, this means\n"); - fprintf(o, " it uses CRLF line-terminators by default. You can use \"--csv --rs lf\" for\n"); - fprintf(o, " Linux-native CSV files. You can also have \"MLR_CSV_DEFAULT_RS=lf\" in your\n"); - fprintf(o, " shell environment, e.g. \"export MLR_CSV_DEFAULT_RS=lf\" or \"setenv\n"); - fprintf(o, " MLR_CSV_DEFAULT_RS lf\" depending on which shell you use.\n"); fprintf(o, " * TSV is simply CSV using tab as field separator (\"--fs tab\").\n"); fprintf(o, " * FS/PS are ignored for markdown format; RS is used.\n"); fprintf(o, " * All RS/FS/PS options are ignored for JSON format: JSON doesn't allow\n"); diff --git a/c/reg_test/expected/out b/c/reg_test/expected/out index 6a16a2d00..a8a22a015 100644 --- a/c/reg_test/expected/out +++ b/c/reg_test/expected/out @@ -43439,42 +43439,6 @@ a 1 4 -mlr --csv cut -f a ./reg_test/input/rfc-csv/simple.csv -mlr: unmatched double quote at line 3. -a -1 -4 - -mlr --csv --rs crlf cut -f a ./reg_test/input/rfc-csv/simple.csv -a -1 -4 -x - -mlr --csv --rs lf cut -f a ./reg_test/input/rfc-csv/simple.csv -mlr: unmatched double quote at line 3. -a -1 -4 - -mlr --csv cut -f a ./reg_test/input/rfc-csv/simple.csv -a -1 -4 -x - -mlr --csv --rs crlf cut -f a ./reg_test/input/rfc-csv/simple.csv -a -1 -4 -x - -mlr --csv --rs lf cut -f a ./reg_test/input/rfc-csv/simple.csv -mlr: unmatched double quote at line 3. -a -1 -4 - ================================================================ MULTI-CHARACTER IRS/IFS/IPS FOR DKVP @@ -44016,8 +43980,7 @@ hi@@@78 mlr --no-mmap --xtab --ips . --ops := cut -x -f b ./reg_test/input/dots.xtab a:=1 c:=345 - -defg:=6 +autodefg:=6 hi:=78 mlr --xtab --ips : --ops @@@@ put $sum=int($a+$b) ./reg_test/input/multi-ips.dkvp @@ -45956,65 +45919,65 @@ x 0.5026260055412137 y 0.9526183602969864 mlr --ifs auto --xtab cat ./reg_test/input/line-term-crlf.xtab -a pan -b pan -i 1 -x 0.3467901443380824 -y 0.7268028627434533 - -a eks -b pan -i 2 -x 0.7586799647899636 -y 0.5221511083334797 - -a wye -b wye -i 3 -x 0.20460330576630303 -y 0.33831852551664776 - -a eks -b wye -i 4 -x 0.38139939387114097 -y 0.13418874328430463 - -a wye -b pan -i 5 -x 0.5732889198020006 -y 0.8636244699032729 - -a zee -b pan -i 6 -x 0.5271261600918548 -y 0.49322128674835697 - -a eks -b zee -i 7 -x 0.6117840605678454 -y 0.1878849191181694 - -a zee -b wye -i 8 -x 0.5985540091064224 -y 0.976181385699006 - -a hat -b wye -i 9 -x 0.03144187646093577 -y 0.7495507603507059 - -a pan -b wye -i 10 -x 0.5026260055412137 -y 0.9526183602969864 +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 + +a wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +b wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 + +a wye +b pan +i 5 +x 0.5732889198020006 +y 0.8636244699032729 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 + +a eks +b zee +i 7 +x 0.6117840605678454 +y 0.1878849191181694 + +a zee +b wye +i 8 +x 0.5985540091064224 +y 0.976181385699006 + +a hat +b wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 mlr --fs auto --xtab cat ./reg_test/input/line-term-lf.xtab a pan @@ -46202,65 +46165,65 @@ x 0.5026260055412137 y 0.9526183602969864 mlr --ifs auto --xtab --no-mmap cat ./reg_test/input/line-term-crlf.xtab -a pan -b pan -i 1 -x 0.3467901443380824 -y 0.7268028627434533 - -a eks -b pan -i 2 -x 0.7586799647899636 -y 0.5221511083334797 - -a wye -b wye -i 3 -x 0.20460330576630303 -y 0.33831852551664776 - -a eks -b wye -i 4 -x 0.38139939387114097 -y 0.13418874328430463 - -a wye -b pan -i 5 -x 0.5732889198020006 -y 0.8636244699032729 - -a zee -b pan -i 6 -x 0.5271261600918548 -y 0.49322128674835697 - -a eks -b zee -i 7 -x 0.6117840605678454 -y 0.1878849191181694 - -a zee -b wye -i 8 -x 0.5985540091064224 -y 0.976181385699006 - -a hat -b wye -i 9 -x 0.03144187646093577 -y 0.7495507603507059 - -a pan -b wye -i 10 -x 0.5026260055412137 -y 0.9526183602969864 +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 + +a wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +b wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 + +a wye +b pan +i 5 +x 0.5732889198020006 +y 0.8636244699032729 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 + +a eks +b zee +i 7 +x 0.6117840605678454 +y 0.1878849191181694 + +a zee +b wye +i 8 +x 0.5985540091064224 +y 0.976181385699006 + +a hat +b wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 mlr --fs auto --xtab --no-mmap cat ./reg_test/input/line-term-lf.xtab a pan @@ -46448,65 +46411,65 @@ x 0.5026260055412137 y 0.9526183602969864 mlr --ifs auto --xtab cat ./reg_test/input/line-term-crlf.xtab -a pan -b pan -i 1 -x 0.3467901443380824 -y 0.7268028627434533 - -a eks -b pan -i 2 -x 0.7586799647899636 -y 0.5221511083334797 - -a wye -b wye -i 3 -x 0.20460330576630303 -y 0.33831852551664776 - -a eks -b wye -i 4 -x 0.38139939387114097 -y 0.13418874328430463 - -a wye -b pan -i 5 -x 0.5732889198020006 -y 0.8636244699032729 - -a zee -b pan -i 6 -x 0.5271261600918548 -y 0.49322128674835697 - -a eks -b zee -i 7 -x 0.6117840605678454 -y 0.1878849191181694 - -a zee -b wye -i 8 -x 0.5985540091064224 -y 0.976181385699006 - -a hat -b wye -i 9 -x 0.03144187646093577 -y 0.7495507603507059 - -a pan -b wye -i 10 -x 0.5026260055412137 -y 0.9526183602969864 +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 + +a wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +b wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 + +a wye +b pan +i 5 +x 0.5732889198020006 +y 0.8636244699032729 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 + +a eks +b zee +i 7 +x 0.6117840605678454 +y 0.1878849191181694 + +a zee +b wye +i 8 +x 0.5985540091064224 +y 0.976181385699006 + +a hat +b wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 mlr --fs auto --xtab cat ./reg_test/input/line-term-lf.xtab a pan @@ -46694,65 +46657,65 @@ x 0.5026260055412137 y 0.9526183602969864 mlr --ifs auto --xtab --no-mmap cat ./reg_test/input/line-term-crlf.xtab -a pan -b pan -i 1 -x 0.3467901443380824 -y 0.7268028627434533 - -a eks -b pan -i 2 -x 0.7586799647899636 -y 0.5221511083334797 - -a wye -b wye -i 3 -x 0.20460330576630303 -y 0.33831852551664776 - -a eks -b wye -i 4 -x 0.38139939387114097 -y 0.13418874328430463 - -a wye -b pan -i 5 -x 0.5732889198020006 -y 0.8636244699032729 - -a zee -b pan -i 6 -x 0.5271261600918548 -y 0.49322128674835697 - -a eks -b zee -i 7 -x 0.6117840605678454 -y 0.1878849191181694 - -a zee -b wye -i 8 -x 0.5985540091064224 -y 0.976181385699006 - -a hat -b wye -i 9 -x 0.03144187646093577 -y 0.7495507603507059 - -a pan -b wye -i 10 -x 0.5026260055412137 -y 0.9526183602969864 +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 + +a wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +b wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 + +a wye +b pan +i 5 +x 0.5732889198020006 +y 0.8636244699032729 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 + +a eks +b zee +i 7 +x 0.6117840605678454 +y 0.1878849191181694 + +a zee +b wye +i 8 +x 0.5985540091064224 +y 0.976181385699006 + +a hat +b wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 mlr --fs auto --xtab --no-mmap cat ./reg_test/input/line-term-lf.xtab a pan diff --git a/c/reg_test/run b/c/reg_test/run index 68a104392..dc2aa8bd9 100755 --- a/c/reg_test/run +++ b/c/reg_test/run @@ -2,7 +2,6 @@ # ================================================================ set -e -export MLR_CSV_DEFAULT_RS= # In case set in user's shell # For building with autoconf: # * in-directory build: @@ -4886,16 +4885,6 @@ run_mlr --csv cut -f a $indir/rfc-csv/simple.csv run_mlr --csv --rs crlf cut -f a $indir/rfc-csv/simple.csv mlr_expect_fail --csv --rs lf cut -f a $indir/rfc-csv/simple.csv -export MLR_CSV_DEFAULT_RS=lf; mlr_expect_fail --csv cut -f a $indir/rfc-csv/simple.csv -export MLR_CSV_DEFAULT_RS=lf; run_mlr --csv --rs crlf cut -f a $indir/rfc-csv/simple.csv -export MLR_CSV_DEFAULT_RS=lf; mlr_expect_fail --csv --rs lf cut -f a $indir/rfc-csv/simple.csv - -export MLR_CSV_DEFAULT_RS=crlf; run_mlr --csv cut -f a $indir/rfc-csv/simple.csv -export MLR_CSV_DEFAULT_RS=crlf; run_mlr --csv --rs crlf cut -f a $indir/rfc-csv/simple.csv -export MLR_CSV_DEFAULT_RS=crlf; mlr_expect_fail --csv --rs lf cut -f a $indir/rfc-csv/simple.csv - -export MLR_CSV_DEFAULT_RS= - # ---------------------------------------------------------------- announce MULTI-CHARACTER IRS/IFS/IPS FOR DKVP diff --git a/c/todo.txt b/c/todo.txt index 9243440a5..da0a13ced 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -38,7 +38,15 @@ OTHER: ================================================================ AUTOTERM +! BUGFIX: + $ mlr --ifs auto --xtab --ips . --ops := cut -x -f b < ./reg_test/input/dots.xtab + a:=1 + c:=345 + autodefg:=6 + hi:=78 + ! doc updates x many ! incl. --irs auto. + - automan etc. re auto; what about --ors auto - IRS/ORS variables in the DSL are from autodetect if that is in use - JSON: manpage re ORS selectable now - XTAB: --ifs/--ofs instead of --irs/--ors diff --git a/doc/10-min.html b/doc/10-min.html index af3d24698..afb1a0cb7 100644 --- a/doc/10-min.html +++ b/doc/10-min.html @@ -179,7 +179,7 @@ purple,square,0,91,72.3735,8.2430
-$ mlr --csv --rs lf cat example.csv +$ mlr --csv cat example.csv color,shape,flag,index,quantity,rate yellow,triangle,1,11,43.6498,9.8870 red,square,1,15,79.2778,0.0130 @@ -200,7 +200,7 @@ purple,square,0,91,72.3735,8.2430-$ mlr --icsv --irs lf --opprint cat example.csv +$ mlr --icsv --opprint cat example.csv color shape flag index quantity rate yellow triangle 1 11 43.6498 9.8870 red square 1 15 79.2778 0.0130 @@ -222,7 +222,7 @@ header is included either way:-$ mlr --csv --rs lf head -n 4 example.csv +$ mlr --csv head -n 4 example.csv color,shape,flag,index,quantity,rate yellow,triangle,1,11,43.6498,9.8870 red,square,1,15,79.2778,0.0130 @@ -234,7 +234,7 @@ red,square,0,48,77.5542,7.4670-$ mlr --csv --rs lf tail -n 4 example.csv +$ mlr --csv tail -n 4 example.csv color,shape,flag,index,quantity,rate purple,triangle,0,65,80.1405,5.8240 yellow,circle,1,73,63.9785,4.2370 @@ -250,7 +250,7 @@ numerically descending on another field:-$ mlr --icsv --irs lf --opprint sort -f shape -nr index example.csv +$ mlr --icsv --opprint sort -f shape -nr index example.csv color shape flag index quantity rate yellow circle 1 87 63.5058 8.3350 yellow circle 1 73 63.9785 4.2370 @@ -271,7 +271,7 @@ yellow triangle 1 11 43.6498 9.8870-$ mlr --icsv --irs lf --opprint cut -f flag,shape example.csv +$ mlr --icsv --opprint cut -f flag,shape example.csv shape flag triangle 1 square 1 @@ -292,7 +292,7 @@ square 0-$ mlr --icsv --irs lf --opprint cut -o -f flag,shape example.csv +$ mlr --icsv --opprint cut -o -f flag,shape example.csv flag shape 1 triangle 1 square @@ -313,7 +313,7 @@ flag shape-$ mlr --icsv --irs lf --opprint cut -x -f flag,shape example.csv +$ mlr --icsv --opprint cut -x -f flag,shape example.csv color index quantity rate yellow 11 43.6498 9.8870 red 15 79.2778 0.0130 @@ -334,7 +334,7 @@ purple 91 72.3735 8.2430-$ mlr --icsv --irs lf --opprint filter '$color == "red"' example.csv +$ mlr --icsv --opprint filter '$color == "red"' example.csv color shape flag index quantity rate red square 1 15 79.2778 0.0130 red circle 1 16 13.8103 2.9010 @@ -346,7 +346,7 @@ red square 0 64 77.1991 9.5310-$ mlr --icsv --irs lf --opprint filter '$color == "red" && $flag == 1' example.csv +$ mlr --icsv --opprint filter '$color == "red" && $flag == 1' example.csv color shape flag index quantity rate red square 1 15 79.2778 0.0130 red circle 1 16 13.8103 2.9010 @@ -359,7 +359,7 @@ red circle 1 16 13.8103 2.9010-$ mlr --icsv --irs lf --opprint put '$ratio = $quantity / $rate; $color_shape = $color . "_" . $shape' example.csv +$ mlr --icsv --opprint put '$ratio = $quantity / $rate; $color_shape = $color . "_" . $shape' example.csv color shape flag index quantity rate ratio color_shape yellow triangle 1 11 43.6498 9.8870 4.414868 yellow_triangle red square 1 15 79.2778 0.0130 6098.292308 red_square @@ -380,7 +380,7 @@ purple square 0 91 72.3735 8.2430 8.779995 purple_square-$ mlr --icsv --irs lf --ojson put '$ratio = $quantity/$rate; $shape = toupper($shape)' example.csv +$ mlr --icsv --ojson put '$ratio = $quantity/$rate; $shape = toupper($shape)' example.csv { "color": "yellow", "shape": "TRIANGLE", "flag": 1, "index": 11, "quantity": 43.6498, "rate": 9.8870, "ratio": 4.414868 } { "color": "red", "shape": "SQUARE", "flag": 1, "index": 15, "quantity": 79.2778, "rate": 0.0130, "ratio": 6098.292308 } { "color": "red", "shape": "CIRCLE", "flag": 1, "index": 16, "quantity": 13.8103, "rate": 2.9010, "ratio": 4.760531 } @@ -397,7 +397,7 @@ $ mlr --icsv --irs lf --ojson put '$ratio = $quantity/$rate; $shape = toupper($s-$ mlr --icsv --irs lf --ojson --jvstack --jlistwrap tail -n 2 example.csv +$ mlr --icsv --ojson --jvstack --jlistwrap tail -n 2 example.csv [ { "color": "yellow", @@ -428,7 +428,7 @@ value of the shape field.-$ mlr --icsv --irs lf --opprint sort -f shape -nr index then head -n 1 -g shape example.csv +$ mlr --icsv --opprint sort -f shape -nr index then head -n 1 -g shape example.csv color shape flag index quantity rate yellow circle 1 87 63.5058 8.3350 purple square 0 91 72.3735 8.2430 @@ -444,7 +444,7 @@ have lots of columns:-$ mlr --icsv --irs lf --oxtab --from example.csv stats1 -a p0,p10,p25,p50,p75,p90,p99,p100 -f rate +$ mlr --icsv --oxtab --from example.csv stats1 -a p0,p10,p25,p50,p75,p90,p99,p100 -f rate rate_p0 0.013000 rate_p10 2.901000 rate_p25 4.237000 @@ -459,7 +459,7 @@ rate_p100 9.887000-$ mlr --icsv --irs lf --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape +$ mlr --icsv --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape shape quantity_count quantity_min quantity_mean quantity_max triangle 3 43.649800 68.339767 81.229000 square 4 72.373500 76.601150 79.277800 @@ -470,7 +470,7 @@ circle 3 13.810300 47.098200 63.978500-$ mlr --icsv --irs lf --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape,color +$ mlr --icsv --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape,color shape color quantity_count quantity_min quantity_mean quantity_max triangle yellow 1 43.649800 43.649800 43.649800 square red 3 77.199100 78.010367 79.277800 @@ -488,7 +488,7 @@ per one or more field names:@@ -590,7 +590,7 @@ different output format. I like to produce SQL-query output with header-column and tab delimiter: this is CSV but with a tab instead of a comma, also known as TSV. Then I -post-process with mlr --tsv --rs lf or mlr --tsvlite. This +post-process with mlr --tsv or mlr --tsvlite. This means I can do some (or all, or none) of my data processing within SQL queries, and some (or none, or all) of my data processing using Miller — whichever is most convenient for my needs at the moment. diff --git a/doc/content-for-10-min.html b/doc/content-for-10-min.html index fc4dab77a..ad1bd7caf 100644 --- a/doc/content-for-10-min.html +++ b/doc/content-for-10-min.html @@ -8,68 +8,68 @@ POKI_RUN_COMMAND{{cat example.csv}}HERE-$ mlr --csv --rs lf --from example.csv put -q 'tee > $shape.".csv", $*' +$ mlr --csv --from example.csv put -q 'tee > $shape.".csv", $*'mlr cat is like cat ... -POKI_RUN_COMMAND{{mlr --csv --rs lf cat example.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv cat example.csv}}HERE... but it can also do format conversion (here, to pretty-printed tabular format): -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint cat example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint cat example.csv}}HEREmlr head and mlr tail count records. The CSV header is included either way: -POKI_RUN_COMMAND{{mlr --csv --rs lf head -n 4 example.csv}}HERE -POKI_RUN_COMMAND{{mlr --csv --rs lf tail -n 4 example.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv head -n 4 example.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv tail -n 4 example.csv}}HERESort primarily alphabetically on one field, then secondarily numerically descending on another field: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint sort -f shape -nr index example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint sort -f shape -nr index example.csv}}HEREUse cut to retain only specified fields, in input-data order: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint cut -f flag,shape example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint cut -f flag,shape example.csv}}HEREUse cut -o to retain only specified fields, in your specified order: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint cut -o -f flag,shape example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint cut -o -f flag,shape example.csv}}HEREUse cut -x to omit specified fields: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint cut -x -f flag,shape example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint cut -x -f flag,shape example.csv}}HEREUse filter to retain specified records: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint filter '$color == "red"' example.csv}}HERE -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint filter '$color == "red" && $flag == 1' example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint filter '$color == "red"' example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint filter '$color == "red" && $flag == 1' example.csv}}HEREUse put to add/replace fields which are computed from other fields: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint put '$ratio = $quantity / $rate; $color_shape = $color . "_" . $shape' example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint put '$ratio = $quantity / $rate; $color_shape = $color . "_" . $shape' example.csv}}HEREJSON output: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --ojson put '$ratio = $quantity/$rate; $shape = toupper($shape)' example.csv}}HERE -POKI_RUN_COMMAND{{mlr --icsv --irs lf --ojson --jvstack --jlistwrap tail -n 2 example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --ojson put '$ratio = $quantity/$rate; $shape = toupper($shape)' example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --ojson --jvstack --jlistwrap tail -n 2 example.csv}}HEREUse then to pipe commands together. Also, the -g option for many Miller commands is for group-by: here, head -n 1 -g shape outputs the first record for each distinct value of the shape field. -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint sort -f shape -nr index then head -n 1 -g shape example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint sort -f shape -nr index then head -n 1 -g shape example.csv}}HEREStatistics can be computed with or without group-by field(s). Also, the first of these two examples uses --oxtab output format which is a nice alternative to --opprint when you have lots of columns: -POKI_RUN_COMMAND{{mlr --icsv --irs lf --oxtab --from example.csv stats1 -a p0,p10,p25,p50,p75,p90,p99,p100 -f rate}}HERE -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape}}HERE -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape,color}}HERE +POKI_RUN_COMMAND{{mlr --icsv --oxtab --from example.csv stats1 -a p0,p10,p25,p50,p75,p90,p99,p100 -f rate}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape,color}}HEREUsing tee within put, you can split your input data into separate files per one or more field names: -POKI_RUN_COMMAND{{mlr --csv --rs lf --from example.csv put -q 'tee > $shape.".csv", $*'}}HERE +POKI_RUN_COMMAND{{mlr --csv --from example.csv put -q 'tee > $shape.".csv", $*'}}HERE
POKI_RUN_COMMAND{{cat circle.csv}}HERE @@ -137,7 +137,7 @@ different output format. I like to produce SQL-query output with header-column and tab delimiter: this is CSV but with a tab instead of a comma, also known as TSV. Then I -post-process with mlr --tsv --rs lf or mlr --tsvlite. This +post-process with mlr --tsv or mlr --tsvlite. This means I can do some (or all, or none) of my data processing within SQL queries, and some (or none, or all) of my data processing using Miller — whichever is most convenient for my needs at the moment. diff --git a/doc/content-for-cookbook.html b/doc/content-for-cookbook.html index 4359bda61..a4f0f3972 100644 --- a/doc/content-for-cookbook.html +++ b/doc/content-for-cookbook.html @@ -51,14 +51,14 @@ POKI_INCLUDE_AND_RUN_ESCAPED(data/rect.sh)HERE Bulk rename of field names
POKI_RUN_COMMAND{{cat data/spaces.csv}}HERE -POKI_RUN_COMMAND{{mlr --csv --rs lf rename -r -g ' ,_' data/spaces.csv}}HERE -POKI_RUN_COMMAND{{mlr --csv --irs lf --opprint rename -r -g ' ,_' data/spaces.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv rename -r -g ' ,_' data/spaces.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv --opprint rename -r -g ' ,_' data/spaces.csv}}HERE You can also do this with a for-loop but it puts the modified fields after the unmodified fields: POKI_RUN_COMMAND{{cat data/bulk-rename-for-loop.mlr}}HERE -POKI_RUN_COMMAND{{mlr --icsv --irs lf --opprint put -f data/bulk-rename-for-loop.mlr data/spaces.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint put -f data/bulk-rename-for-loop.mlr data/spaces.csv}}HEREHeaderless CSV on input or output
@@ -69,14 +69,14 @@ POKI_RUN_COMMAND{{cat data/headerless.csv}}HERE You can use Miller to add a header: the --implicit-csv-header applies positionally indexed labels: -POKI_RUN_COMMAND{{mlr --csv --rs lf --implicit-csv-header cat data/headerless.csv}}HERE -POKI_RUN_COMMAND{{mlr --icsv --irs lf --implicit-csv-header --opprint cat data/headerless.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv --implicit-csv-header cat data/headerless.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --implicit-csv-header --opprint cat data/headerless.csv}}HERE Following that, you can rename the positionally indexed labels to names with meaning for your context. For example: -POKI_RUN_COMMAND{{mlr --csv --rs lf --implicit-csv-header label name,age,status data/headerless.csv}}HERE -POKI_RUN_COMMAND{{mlr --icsv --rs lf --implicit-csv-header --opprint label name,age,status data/headerless.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv --implicit-csv-header label name,age,status data/headerless.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --implicit-csv-header --opprint label name,age,status data/headerless.csv}}HERE Likewise, if you need to produce CSV which is lacking its header, you can pipe Miller’s output to the system command sed 1d, or you can use Miller’s --headerless-csv-output option: @@ -209,7 +209,7 @@ POKI_INCLUDE_AND_RUN_ESCAPED(data/two-pass-record-numbers.sh)HERE The idea is to retain records having the largest value of n in the following data: -POKI_RUN_COMMAND{{mlr --itsv --irs lf --opprint cat data/maxrows.tsv}}HERE +POKI_RUN_COMMAND{{mlr --itsv --opprint cat data/maxrows.tsv}}HERE Of course, the largest value of n isn’t known until after all data have been read. Using an out-of-stream variable we can retain all @@ -217,7 +217,7 @@ records as they are read, then filter them at the end: POKI_RUN_COMMAND{{cat data/maxrows.mlr}}HERE -POKI_RUN_COMMAND{{mlr --itsv --irs lf --opprint put -q -f data/maxrows.mlr data/maxrows.tsv}}HERE +POKI_RUN_COMMAND{{mlr --itsv --opprint put -q -f data/maxrows.mlr data/maxrows.tsv}}HEREFiltering paragraphs of text
diff --git a/doc/content-for-data-examples.html b/doc/content-for-data-examples.html index 499581dcf..a45d56ad0 100644 --- a/doc/content-for-data-examples.html +++ b/doc/content-for-data-examples.html @@ -4,11 +4,6 @@ POKI_PUT_TOC_HERE The flins.csv file is some sample data obtained from https://support.spatialkey.com/spatialkey-sample-csv-data. -Note: please use "mlr --csv --rs lf" for for native Un*x (linefeed-terminated) CSV files. -(You can also have MLR_CSV_DEFAULT_RS=lf in your shell environment, e.g. -export MLR_CSV_DEFAULT_RS=lf or setenv MLR_CSV_DEFAULT_RS lf depending on -which shell you use.) - Vertical-tabular format is good for a quick look at CSV data layout — seeing what columns you have to work with: POKI_RUN_COMMAND{{head -n 2 data/flins.csv | mlr --icsv --oxtab cat}}HERE A few simple queries: diff --git a/doc/content-for-faq.html b/doc/content-for-faq.html index 91286116e..ce71ca55b 100644 --- a/doc/content-for-faq.html +++ b/doc/content-for-faq.html @@ -1,17 +1,5 @@ POKI_PUT_TOC_HERE -Number one FAQ
- - -Please use mlr --csv --rs lf for native Un*x (linefeed-terminated) CSV files. - - -Instead of specifying --rs lf on each invocation, you can instead -have MLR_CSV_DEFAULT_RS=lf in your shell environment: e.g. put -export MLR_CSV_DEFAULT_RS=lf in your ~/.bashrc or -~/.zshrc, or setenv MLR_CSV_DEFAULT_RS lf in your -~/.cshrc, as a one-time setup step. -No output at all
Check the line-terminators of the data, e.g. with the command-line @@ -64,18 +52,18 @@ POKI_RUN_COMMAND{{cat data/then-example.csv}}HERE Next, run the first step of your command, omitting anything from the first then onward: -POKI_RUN_COMMAND{{mlr --icsv --rs lf --opprint count-distinct -f Status,Payment_Type data/then-example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint count-distinct -f Status,Payment_Type data/then-example.csv}}HERE After that, run it with the next then step included: -POKI_RUN_COMMAND{{mlr --icsv --rs lf --opprint count-distinct -f Status,Payment_Type then sort -nr count data/then-example.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --opprint count-distinct -f Status,Payment_Type then sort -nr count data/then-example.csv}}HERE Now if you use then to include another verb after that, the columns Status, Payment_Type, and count will be the input to that verb. Note, by the way, that you’ll get the same results using pipes: -POKI_RUN_COMMAND{{mlr --csv --rs lf count-distinct -f Status,Payment_Type data/then-example.csv | mlr --icsv --rs lf --opprint sort -nr count}}HERE +POKI_RUN_COMMAND{{mlr --csv count-distinct -f Status,Payment_Type data/then-example.csv | mlr --icsv --opprint sort -nr count}}HEREI assigned $9 and it’s not 9th
@@ -89,10 +77,10 @@ input data. POKI_RUN_COMMAND{{echo x,y,z | mlr --dkvp cat}}HERE POKI_RUN_COMMAND{{echo x,y,z | mlr --dkvp put '$6="a";$4="b";$55="cde"'}}HERE POKI_RUN_COMMAND{{echo x,y,z | mlr --nidx cat}}HERE -POKI_RUN_COMMAND{{echo x,y,z | mlr --csv --rs lf --implicit-csv-header cat}}HERE +POKI_RUN_COMMAND{{echo x,y,z | mlr --csv --implicit-csv-header cat}}HERE POKI_RUN_COMMAND{{echo x,y,z | mlr --dkvp rename 2,999}}HERE POKI_RUN_COMMAND{{echo x,y,z | mlr --dkvp rename 2,newname}}HERE -POKI_RUN_COMMAND{{echo x,y,z | mlr --csv --rs lf --implicit-csv-header reorder -f 3,1,2}}HERE +POKI_RUN_COMMAND{{echo x,y,z | mlr --csv --implicit-csv-header reorder -f 3,1,2}}HEREWhy doesn’t mlr cut put fields in the order I want?
diff --git a/doc/content-for-file-formats.html b/doc/content-for-file-formats.html index 763df75f5..a24f5791f 100644 --- a/doc/content-for-file-formats.html +++ b/doc/content-for-file-formats.html @@ -36,24 +36,6 @@ POKI_PUT_LINK_FOR_PAGE(reference.html)HERE.) https://tools.ietf.org/html/rfc4180). This includes CRLF line-terminators by default, regardless of platform. - - -Please use mlr --csv --rs lf for native Un*x (linefeed-terminated) CSV files. - - -Instead of specifying --rs lf on each invocation, you can instead -have MLR_CSV_DEFAULT_RS=lf in your shell environment: e.g. put -export MLR_CSV_DEFAULT_RS=lf in your ~/.bashrc or -~/.zshrc, or setenv MLR_CSV_DEFAULT_RS lf in your -~/.cshrc, as a one-time setup step. - -The RFC says, somewhat briefly, that “there may be a header -line”. Miller’s --implicit-csv-header option allows you to -read CSV data which lacks a header line, applying column labels 1, -2, 3, etc. for you. You may also use Miller’s -label to replace those numerical column names with labels of your -choosing. - Here are the differences between CSV and CSV-lite:diff --git a/doc/content-for-index.html b/doc/content-for-index.html index e5be0ba73..90e295f49 100644 --- a/doc/content-for-index.html +++ b/doc/content-for-index.html @@ -88,7 +88,7 @@ machine, and expect it to work.
diff --git a/doc/content-for-reference-verbs.html b/doc/content-for-reference-verbs.html index 740104eab..322e63ee3 100644 --- a/doc/content-for-reference-verbs.html +++ b/doc/content-for-reference-verbs.html @@ -299,9 +299,9 @@ POKI_INCLUDE_ESCAPED(data/label-example.txt)HERE Likewise, if you have CSV/CSV-lite input data which has somehow been bereft of its header line, you can re-add a header line using --implicit-csv-header and label: POKI_RUN_COMMAND{{cat data/headerless.csv}}HERE -POKI_RUN_COMMAND{{mlr --csv --rs lf --implicit-csv-header cat data/headerless.csv}}HERE -POKI_RUN_COMMAND{{mlr --csv --rs lf --implicit-csv-header label name,age,status data/headerless.csv}}HERE -POKI_RUN_COMMAND{{mlr --icsv --rs lf --implicit-csv-header --opprint label name,age,status data/headerless.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv --implicit-csv-header cat data/headerless.csv}}HERE +POKI_RUN_COMMAND{{mlr --csv --implicit-csv-header label name,age,status data/headerless.csv}}HERE +POKI_RUN_COMMAND{{mlr --icsv --implicit-csv-header --opprint label name,age,status data/headerless.csv}}HERE# Row filter -% mlr --csv --rs lf filter '$status != "down" && $upsec >= 10000' *.csv +% mlr --csv filter '$status != "down" && $upsec >= 10000' *.csvleast-frequent
diff --git a/doc/content-for-reference.html b/doc/content-for-reference.html index 41eb32847..ed9ba76f7 100644 --- a/doc/content-for-reference.html +++ b/doc/content-for-reference.html @@ -288,7 +288,7 @@ streaming-data programs operate through pipes, each one can use a CPU. Rest assured you get the same results either way. The other reason to use then-chaining is for simplicity: you don’t -have re-type formatting flags (e.g. --csv --rs lf --fs tab) at every +have re-type formatting flags (e.g. --csv --fs tab) at every pipeline stage. diff --git a/doc/cookbook.html b/doc/cookbook.html index 7e9e1fa67..c4acc10a3 100644 --- a/doc/cookbook.html +++ b/doc/cookbook.html @@ -322,7 +322,7 @@ a b c,def,g h i-$ mlr --csv --rs lf rename -r -g ' ,_' data/spaces.csv +$ mlr --csv rename -r -g ' ,_' data/spaces.csv a_b_c,def,g_h_i 123,4567,890 2468,1357,3579 @@ -333,7 +333,7 @@ a_b_c,def,g_h_i-$ mlr --csv --irs lf --opprint rename -r -g ' ,_' data/spaces.csv +$ mlr --csv --opprint rename -r -g ' ,_' data/spaces.csv a_b_c def g_h_i 123 4567 890 2468 1357 3579 @@ -362,7 +362,7 @@ for (oldk,v in $*) {-$ mlr --icsv --irs lf --opprint put -f data/bulk-rename-for-loop.mlr data/spaces.csv +$ mlr --icsv --opprint put -f data/bulk-rename-for-loop.mlr data/spaces.csv def a_b_c g_h_i 4567 123 890 1357 2468 3579 @@ -393,7 +393,7 @@ Carol,45,present@@ -994,7 +1006,7 @@ following data:-$ mlr --csv --rs lf --implicit-csv-header cat data/headerless.csv +$ mlr --csv --implicit-csv-header cat data/headerless.csv 1,2,3 John,23,present Fred,34,present @@ -405,7 +405,7 @@ Carol,45,present@@ -752,6 +762,8 @@ $ mlr --from data/miss-date.csv --icsv \ then put '$datestamp = strptime($date, "%Y-%m-%d")' \ then step -a delta -f datestamp \ then filter '$datestamp_delta != 86400 && $n != 1' +n=774,date=2014-04-19,qoh=130140,datestamp=1397865600,datestamp_delta=259200 +n=1119,date=2015-03-31,qoh=181625,datestamp=1427760000,datestamp_delta=172800-$ mlr --icsv --irs lf --implicit-csv-header --opprint cat data/headerless.csv +$ mlr --icsv --implicit-csv-header --opprint cat data/headerless.csv 1 2 3 John 23 present Fred 34 present @@ -421,7 +421,7 @@ For example:@@ -738,6 +738,16 @@ $ mlr --from data/miss-date.csv --icsv \ then put '$datestamp = strptime($date, "%Y-%m-%d")' \ then step -a delta -f datestamp \ | head +n=1,date=2012-03-05,qoh=10055,datestamp=1330905600,datestamp_delta=0 +n=2,date=2012-03-06,qoh=10486,datestamp=1330992000,datestamp_delta=86400 +n=3,date=2012-03-07,qoh=10430,datestamp=1331078400,datestamp_delta=86400 +n=4,date=2012-03-08,qoh=10674,datestamp=1331164800,datestamp_delta=86400 +n=5,date=2012-03-09,qoh=10880,datestamp=1331251200,datestamp_delta=86400 +n=6,date=2012-03-10,qoh=10718,datestamp=1331337600,datestamp_delta=86400 +n=7,date=2012-03-11,qoh=10795,datestamp=1331424000,datestamp_delta=86400 +n=8,date=2012-03-12,qoh=11043,datestamp=1331510400,datestamp_delta=86400 +n=9,date=2012-03-13,qoh=11177,datestamp=1331596800,datestamp_delta=86400 +n=10,date=2012-03-14,qoh=11498,datestamp=1331683200,datestamp_delta=86400-$ mlr --csv --rs lf --implicit-csv-header label name,age,status data/headerless.csv +$ mlr --csv --implicit-csv-header label name,age,status data/headerless.csv name,age,status John,23,present Fred,34,present @@ -433,7 +433,7 @@ Carol,45,present@@ -665,33 +665,33 @@ $ mlr --ofmt '%.9lf' --opprint seqgen --start 1 --stop 28 then put ' ' then put '$seconds=systime()' then step -a delta -f seconds then cut -x -f seconds i o fcount seconds_delta 1 1 1 0 -2 2 3 0.000036001 -3 3 3 0.000013113 -4 5 3 0.000010967 -5 8 3 0.000010967 -6 13 3 0.000010014 -7 21 3 0.000010014 +2 2 3 0.000037193 +3 3 3 0.000012875 +4 5 3 0.000011921 +5 8 3 0.000011206 +6 13 3 0.000010967 +7 21 3 0.000010967 8 34 3 0.000010967 -9 55 3 0.000010967 -10 89 3 0.000010014 -11 144 3 0.000010014 -12 233 3 0.000014067 +9 55 3 0.000010014 +10 89 3 0.000013113 +11 144 3 0.000010967 +12 233 3 0.000015020 13 377 3 0.000010967 14 610 3 0.000010014 15 987 3 0.000010967 16 1597 3 0.000010014 -17 2584 3 0.000010014 +17 2584 3 0.000010967 18 4181 3 0.000010014 19 6765 3 0.000010014 -20 10946 3 0.000010014 -21 17711 3 0.000011921 +20 10946 3 0.000010967 +21 17711 3 0.000010014 22 28657 3 0.000010014 -23 46368 3 0.000012159 -24 75025 3 0.000010967 +23 46368 3 0.000012875 +24 75025 3 0.000012159 25 121393 3 0.000010014 -26 196418 3 0.000010014 -27 317811 3 0.000010967 -28 514229 3 0.000010014 +26 196418 3 0.000010967 +27 317811 3 0.000010014 +28 514229 3 0.000010967-$ mlr --icsv --rs lf --implicit-csv-header --opprint label name,age,status data/headerless.csv +$ mlr --icsv --implicit-csv-header --opprint label name,age,status data/headerless.csv name age status John 23 present Fred 34 present @@ -607,33 +607,33 @@ $ mlr --ofmt '%.9lf' --opprint seqgen --start 1 --stop 28 then put ' ' then put '$seconds=systime()' then step -a delta -f seconds then cut -x -f seconds i o fcount seconds_delta 1 1 1 0 -2 2 3 0.000036001 -3 3 5 0.000014067 -4 5 9 0.000016928 -5 8 15 0.000023127 -6 13 25 0.000033855 -7 21 41 0.000049114 -8 34 67 0.000073910 -9 55 109 0.000118017 -10 89 177 0.000186920 -11 144 287 0.000299215 -12 233 465 0.000472784 -13 377 753 0.000731230 -14 610 1219 0.001174927 -15 987 1973 0.001888990 -16 1597 3193 0.003062963 -17 2584 5167 0.004909039 -18 4181 8361 0.009054899 -19 6765 13529 0.013993025 -20 10946 21891 0.022313118 -21 17711 35421 0.035612822 -22 28657 57313 0.054386139 -23 46368 92735 0.094619989 -24 75025 150049 0.137188911 -25 121393 242785 0.219310045 -26 196418 392835 0.372972965 -27 317811 635621 0.564725161 -28 514229 1028457 0.906754971 +2 2 3 0.000031948 +3 3 5 0.000013113 +4 5 9 0.000015974 +5 8 15 0.000021935 +6 13 25 0.000030994 +7 21 41 0.000045061 +8 34 67 0.000071049 +9 55 109 0.000113010 +10 89 177 0.000178814 +11 144 287 0.000289202 +12 233 465 0.000463009 +13 377 753 0.000734806 +14 610 1219 0.001182079 +15 987 1973 0.001869917 +16 1597 3193 0.003251076 +17 2584 5167 0.005389929 +18 4181 8361 0.008499146 +19 6765 13529 0.014052868 +20 10946 21891 0.024805069 +21 17711 35421 0.038400888 +22 28657 57313 0.061202049 +23 46368 92735 0.093353987 +24 75025 150049 0.147068024 +25 121393 242785 0.288206100 +26 196418 392835 0.395586014 +27 317811 635621 0.595448971 +28 514229 1028457 0.958051920-$ mlr --itsv --irs lf --opprint cat data/maxrows.tsv +$ mlr --itsv --opprint cat data/maxrows.tsv a b n score purple red 5 0.743231 blue purple 2 0.093710 @@ -1062,7 +1074,7 @@ end {-$ mlr --itsv --irs lf --opprint put -q -f data/maxrows.mlr data/maxrows.tsv +$ mlr --itsv --opprint put -q -f data/maxrows.mlr data/maxrows.tsv a b n score purple red 5 0.743231 purple red 5 0.389055 diff --git a/doc/data-examples.html b/doc/data-examples.html index 44b16a209..c6c435080 100644 --- a/doc/data-examples.html +++ b/doc/data-examples.html @@ -152,11 +152,6 @@ Miller commands were run with pretty-print-tabular output format. The flins.csv file is some sample data obtained from https://support.spatialkey.com/spatialkey-sample-csv-data. -Note: please use "mlr --csv --rs lf" for for native Un*x (linefeed-terminated) CSV files. -(You can also have MLR_CSV_DEFAULT_RS=lf in your shell environment, e.g. -export MLR_CSV_DEFAULT_RS=lf or setenv MLR_CSV_DEFAULT_RS lf depending on -which shell you use.) - Vertical-tabular format is good for a quick look at CSV data layout — seeing what columns you have to work with:diff --git a/doc/faq.html b/doc/faq.html index 50680d1ba..4bde4c944 100644 --- a/doc/faq.html +++ b/doc/faq.html @@ -143,7 +143,6 @@ Miller commands were run with pretty-print-tabular output format.-Contents: -• Number one FAQ
• No output at all
• Fields not selected
• Diagnosing delimiter specifications
@@ -157,18 +156,6 @@ Miller commands were run with pretty-print-tabular output format.Number one FAQ
- - -Please use mlr --csv --rs lf for native Un*x (linefeed-terminated) CSV files. - - -Instead of specifying --rs lf on each invocation, you can instead -have MLR_CSV_DEFAULT_RS=lf in your shell environment: e.g. put -export MLR_CSV_DEFAULT_RS=lf in your ~/.bashrc or -~/.zshrc, or setenv MLR_CSV_DEFAULT_RS lf in your -~/.cshrc, as a one-time setup step. -No output at all
Check the line-terminators of the data, e.g. with the command-line @@ -303,7 +290,7 @@ Next, run the first step of your command, omitting anything from the first t-$ mlr --icsv --rs lf --opprint count-distinct -f Status,Payment_Type data/then-example.csv +$ mlr --icsv --opprint count-distinct -f Status,Payment_Type data/then-example.csv Status Payment_Type count paid cash 2 pending debit 1 @@ -318,7 +305,7 @@ After that, run it with the next then step included:-$ mlr --icsv --rs lf --opprint count-distinct -f Status,Payment_Type then sort -nr count data/then-example.csv +$ mlr --icsv --opprint count-distinct -f Status,Payment_Type then sort -nr count data/then-example.csv Status Payment_Type count paid cash 2 pending debit 1 @@ -336,7 +323,7 @@ that verb.-$ mlr --csv --rs lf count-distinct -f Status,Payment_Type data/then-example.csv | mlr --icsv --rs lf --opprint sort -nr count +$ mlr --csv count-distinct -f Status,Payment_Type data/then-example.csv | mlr --icsv --opprint sort -nr count Status Payment_Type count paid cash 2 pending debit 1 @@ -382,7 +369,7 @@ x,y,z-$ echo x,y,z | mlr --csv --rs lf --implicit-csv-header cat +$ echo x,y,z | mlr --csv --implicit-csv-header cat 1,2,3 x,y,z@@ -407,7 +394,7 @@ $ echo x,y,z | mlr --dkvp rename 2,newnamediff --git a/doc/manpage.txt b/doc/manpage.txt index e43b5820e..32540c7ff 100644 --- a/doc/manpage.txt +++ b/doc/manpage.txt @@ -175,7 +175,8 @@ OPTIONS --jvstack Put one key-value pair per line for JSON output. --jlistwrap Wrap JSON output in outermost [ ]. - --jquoteall Quote map values in JSON output, even if they're + --jknquoteint Do not quote non-string map keys in JSON output. + --jvquoteall Quote map values in JSON output, even if they're numeric. --jflatsep {string} Separator for flattening multi-level JSON keys, e.g. '{"a":{"b":3}}' becomes a:b => 3 for @@ -186,11 +187,6 @@ OPTIONS Examples: --csv for CSV-formatted input and output; --idkvp --opprint for DKVP-formatted input and pretty-printed output. - PLEASE USE "mlr --csv --rs lf" FOR NATIVE UN*X (LINEFEED-TERMINATED) CSV FILES. - You can also have MLR_CSV_DEFAULT_RS=lf in your shell environment, e.g. - "export MLR_CSV_DEFAULT_RS=lf" or "setenv MLR_CSV_DEFAULT_RS lf" depending on - which shell you use. - FORMAT-CONVERSION KEYSTROKE-SAVERS As keystroke-savers for format-conversion you may use the following: --c2t --c2d --c2n --c2j --c2x --c2p --c2m @@ -241,11 +237,6 @@ OPTIONS platform-native text data. In particular, this means LF line-terminators by default on Linux/OSX. You can use "--dkvp --rs crlf" for CRLF-terminated DKVP files, and so on. - * CSV is intended to handle RFC-4180-compliant data. In particular, this means - it uses CRLF line-terminators by default. You can use "--csv --rs lf" for - Linux-native CSV files. You can also have "MLR_CSV_DEFAULT_RS=lf" in your - shell environment, e.g. "export MLR_CSV_DEFAULT_RS=lf" or "setenv - MLR_CSV_DEFAULT_RS lf" depending on which shell you use. * TSV is simply CSV using tab as field separator ("--fs tab"). * FS/PS are ignored for markdown format; RS is used. * All RS/FS/PS options are ignored for JSON format: JSON doesn't allow @@ -258,14 +249,14 @@ OPTIONS cr crcr newline lf lflf crlf crlfcrlf tab space comma pipe slash colon semicolon equals * Default separators by format: File format RS FS PS - dkvp \n , = - json (N/A) (N/A) (N/A) - nidx \n space (N/A) - csv \r\n , (N/A) - csvlite \n , (N/A) - markdown \n (N/A) (N/A) - pprint \n space (N/A) - xtab (N/A) \n space + dkvp auto , = + json auto (N/A) (N/A) + nidx auto space (N/A) + csv auto , (N/A) + csvlite auto , (N/A) + markdown auto (N/A) (N/A) + pprint auto space (N/A) + xtab (N/A) auto space CSV-SPECIFIC OPTIONS --implicit-csv-header Use 1,2,3,... as field labels, rather than from line 1 @@ -1852,4 +1843,4 @@ SEE ALSO - 2017-01-11 MILLER(1) + 2017-02-02 MILLER(1) diff --git a/doc/mkman.rb b/doc/mkman.rb index 7001208e6..49be6e510 100755 --- a/doc/mkman.rb +++ b/doc/mkman.rb @@ -14,7 +14,6 @@ # ---------------------------------------------------------------- def main - ENV['MLR_CSV_DEFAULT_RS'] = "" # In case set in user's shell print make_top print make_section('NAME', [ diff --git a/doc/mlr.1 b/doc/mlr.1 index 82ac1060f..0947aa9e8 100644 --- a/doc/mlr.1 +++ b/doc/mlr.1 @@ -2,12 +2,12 @@ .\" Title: mlr .\" Author: [see the "AUTHOR" section] .\" Generator: ./mkman.rb -.\" Date: 2017-01-11 +.\" Date: 2017-02-02 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "MILLER" "1" "2017-01-11" "\ \&" "\ \&" +.TH "MILLER" "1" "2017-02-02" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Portability definitions .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -230,7 +230,8 @@ Please use "mlr --help-all-keywords" or "mlr -k" for help on all keywords. --jvstack Put one key-value pair per line for JSON output. --jlistwrap Wrap JSON output in outermost [ ]. - --jquoteall Quote map values in JSON output, even if they're + --jknquoteint Do not quote non-string map keys in JSON output. + --jvquoteall Quote map values in JSON output, even if they're numeric. --jflatsep {string} Separator for flattening multi-level JSON keys, e.g. '{"a":{"b":3}}' becomes a:b => 3 for @@ -240,11 +241,6 @@ Please use "mlr --help-all-keywords" or "mlr -k" for help on all keywords. Examples: --csv for CSV-formatted input and output; --idkvp --opprint for DKVP-formatted input and pretty-printed output. - - PLEASE USE "mlr --csv --rs lf" FOR NATIVE UN*X (LINEFEED-TERMINATED) CSV FILES. - You can also have MLR_CSV_DEFAULT_RS=lf in your shell environment, e.g. - "export MLR_CSV_DEFAULT_RS=lf" or "setenv MLR_CSV_DEFAULT_RS lf" depending on - which shell you use. .fi .if n \{\ .RE @@ -314,11 +310,6 @@ is available for output only. platform-native text data. In particular, this means LF line-terminators by default on Linux/OSX. You can use "--dkvp --rs crlf" for CRLF-terminated DKVP files, and so on. - * CSV is intended to handle RFC-4180-compliant data. In particular, this means - it uses CRLF line-terminators by default. You can use "--csv --rs lf" for - Linux-native CSV files. You can also have "MLR_CSV_DEFAULT_RS=lf" in your - shell environment, e.g. "export MLR_CSV_DEFAULT_RS=lf" or "setenv - MLR_CSV_DEFAULT_RS lf" depending on which shell you use. * TSV is simply CSV using tab as field separator ("--fs tab"). * FS/PS are ignored for markdown format; RS is used. * All RS/FS/PS options are ignored for JSON format: JSON doesn't allow @@ -331,14 +322,14 @@ is available for output only. cr crcr newline lf lflf crlf crlfcrlf tab space comma pipe slash colon semicolon equals * Default separators by format: File format RS FS PS - dkvp \en , = - json (N/A) (N/A) (N/A) - nidx \en space (N/A) - csv \er\en , (N/A) - csvlite \en , (N/A) - markdown \en (N/A) (N/A) - pprint \en space (N/A) - xtab (N/A) \en space + dkvp auto , = + json auto (N/A) (N/A) + nidx auto space (N/A) + csv auto , (N/A) + csvlite auto , (N/A) + markdown auto (N/A) (N/A) + pprint auto space (N/A) + xtab (N/A) auto space .fi .if n \{\ .RE diff --git a/doc/reference-verbs.html b/doc/reference-verbs.html index 5f82f1350..ea171e052 100644 --- a/doc/reference-verbs.html +++ b/doc/reference-verbs.html @@ -1337,7 +1337,7 @@ Carol,45,present-$ echo x,y,z | mlr --csv --rs lf --implicit-csv-header reorder -f 3,1,2 +$ echo x,y,z | mlr --csv --implicit-csv-header reorder -f 3,1,2 3,1,2 z,x,ydiff --git a/doc/file-formats.html b/doc/file-formats.html index 1f6ff0b1f..89a24b35d 100644 --- a/doc/file-formats.html +++ b/doc/file-formats.html @@ -261,24 +261,6 @@ CSV, use --ifs tab --ofs comma, etc. (See also https://tools.ietf.org/html/rfc4180). This includes CRLF line-terminators by default, regardless of platform. - - -Please use mlr --csv --rs lf for native Un*x (linefeed-terminated) CSV files. - - -Instead of specifying --rs lf on each invocation, you can instead -have MLR_CSV_DEFAULT_RS=lf in your shell environment: e.g. put -export MLR_CSV_DEFAULT_RS=lf in your ~/.bashrc or -~/.zshrc, or setenv MLR_CSV_DEFAULT_RS lf in your -~/.cshrc, as a one-time setup step. - -The RFC says, somewhat briefly, that “there may be a header -line”. Miller’s --implicit-csv-header option allows you to -read CSV data which lacks a header line, applying column labels 1, -2, 3, etc. for you. You may also use Miller’s -label to replace those numerical column names with labels of your -choosing. - Here are the differences between CSV and CSV-lite:diff --git a/doc/index.html b/doc/index.html index 3c1142cbb..4958510f6 100644 --- a/doc/index.html +++ b/doc/index.html @@ -231,7 +231,7 @@ machine, and expect it to work.
diff --git a/doc/manpage.html b/doc/manpage.html index 5e75ffc85..d9e877d5b 100644 --- a/doc/manpage.html +++ b/doc/manpage.html @@ -324,7 +324,8 @@ OPTIONS --jvstack Put one key-value pair per line for JSON output. --jlistwrap Wrap JSON output in outermost [ ]. - --jquoteall Quote map values in JSON output, even if they're + --jknquoteint Do not quote non-string map keys in JSON output. + --jvquoteall Quote map values in JSON output, even if they're numeric. --jflatsep {string} Separator for flattening multi-level JSON keys, e.g. '{"a":{"b":3}}' becomes a:b => 3 for @@ -335,11 +336,6 @@ OPTIONS Examples: --csv for CSV-formatted input and output; --idkvp --opprint for DKVP-formatted input and pretty-printed output. - PLEASE USE "mlr --csv --rs lf" FOR NATIVE UN*X (LINEFEED-TERMINATED) CSV FILES. - You can also have MLR_CSV_DEFAULT_RS=lf in your shell environment, e.g. - "export MLR_CSV_DEFAULT_RS=lf" or "setenv MLR_CSV_DEFAULT_RS lf" depending on - which shell you use. - FORMAT-CONVERSION KEYSTROKE-SAVERS As keystroke-savers for format-conversion you may use the following: --c2t --c2d --c2n --c2j --c2x --c2p --c2m @@ -390,11 +386,6 @@ OPTIONS platform-native text data. In particular, this means LF line-terminators by default on Linux/OSX. You can use "--dkvp --rs crlf" for CRLF-terminated DKVP files, and so on. - * CSV is intended to handle RFC-4180-compliant data. In particular, this means - it uses CRLF line-terminators by default. You can use "--csv --rs lf" for - Linux-native CSV files. You can also have "MLR_CSV_DEFAULT_RS=lf" in your - shell environment, e.g. "export MLR_CSV_DEFAULT_RS=lf" or "setenv - MLR_CSV_DEFAULT_RS lf" depending on which shell you use. * TSV is simply CSV using tab as field separator ("--fs tab"). * FS/PS are ignored for markdown format; RS is used. * All RS/FS/PS options are ignored for JSON format: JSON doesn't allow @@ -407,14 +398,14 @@ OPTIONS cr crcr newline lf lflf crlf crlfcrlf tab space comma pipe slash colon semicolon equals * Default separators by format: File format RS FS PS - dkvp \n , = - json (N/A) (N/A) (N/A) - nidx \n space (N/A) - csv \r\n , (N/A) - csvlite \n , (N/A) - markdown \n (N/A) (N/A) - pprint \n space (N/A) - xtab (N/A) \n space + dkvp auto , = + json auto (N/A) (N/A) + nidx auto space (N/A) + csv auto , (N/A) + csvlite auto , (N/A) + markdown auto (N/A) (N/A) + pprint auto space (N/A) + xtab (N/A) auto space CSV-SPECIFIC OPTIONS --implicit-csv-header Use 1,2,3,... as field labels, rather than from line 1 @@ -2001,7 +1992,7 @@ SEE ALSO - 2017-01-11 MILLER(1) + 2017-02-02 MILLER(1)# Row filter -% mlr --csv --rs lf filter '$status != "down" && $upsec >= 10000' *.csv +% mlr --csv filter '$status != "down" && $upsec >= 10000' *.csv-$ mlr --csv --rs lf --implicit-csv-header cat data/headerless.csv +$ mlr --csv --implicit-csv-header cat data/headerless.csv 1,2,3 John,23,present Fred,34,present @@ -1349,7 +1349,7 @@ Carol,45,present-$ mlr --csv --rs lf --implicit-csv-header label name,age,status data/headerless.csv +$ mlr --csv --implicit-csv-header label name,age,status data/headerless.csv name,age,status John,23,present Fred,34,present @@ -1361,7 +1361,7 @@ Carol,45,present-$ mlr --icsv --rs lf --implicit-csv-header --opprint label name,age,status data/headerless.csv +$ mlr --icsv --implicit-csv-header --opprint label name,age,status data/headerless.csv name age status John 23 present Fred 34 present diff --git a/doc/reference.html b/doc/reference.html index fcb26fd0c..fce7923c7 100644 --- a/doc/reference.html +++ b/doc/reference.html @@ -424,7 +424,8 @@ Data-format options, for input, output, or both: --jvstack Put one key-value pair per line for JSON output. --jlistwrap Wrap JSON output in outermost [ ]. - --jquoteall Quote map values in JSON output, even if they're + --jknquoteint Do not quote non-string map keys in JSON output. + --jvquoteall Quote map values in JSON output, even if they're numeric. --jflatsep {string} Separator for flattening multi-level JSON keys, e.g. '{"a":{"b":3}}' becomes a:b => 3 for @@ -435,11 +436,6 @@ Data-format options, for input, output, or both: Examples: --csv for CSV-formatted input and output; --idkvp --opprint for DKVP-formatted input and pretty-printed output. - PLEASE USE "mlr --csv --rs lf" FOR NATIVE UN*X (LINEFEED-TERMINATED) CSV FILES. - You can also have MLR_CSV_DEFAULT_RS=lf in your shell environment, e.g. - "export MLR_CSV_DEFAULT_RS=lf" or "setenv MLR_CSV_DEFAULT_RS lf" depending on - which shell you use. - Format-conversion keystroke-saver options, for input, output, or both: As keystroke-savers for format-conversion you may use the following: --c2t --c2d --c2n --c2j --c2x --c2p --c2m @@ -490,11 +486,6 @@ Separator options, for input, output, or both: platform-native text data. In particular, this means LF line-terminators by default on Linux/OSX. You can use "--dkvp --rs crlf" for CRLF-terminated DKVP files, and so on. - * CSV is intended to handle RFC-4180-compliant data. In particular, this means - it uses CRLF line-terminators by default. You can use "--csv --rs lf" for - Linux-native CSV files. You can also have "MLR_CSV_DEFAULT_RS=lf" in your - shell environment, e.g. "export MLR_CSV_DEFAULT_RS=lf" or "setenv - MLR_CSV_DEFAULT_RS lf" depending on which shell you use. * TSV is simply CSV using tab as field separator ("--fs tab"). * FS/PS are ignored for markdown format; RS is used. * All RS/FS/PS options are ignored for JSON format: JSON doesn't allow @@ -507,14 +498,14 @@ Separator options, for input, output, or both: cr crcr newline lf lflf crlf crlfcrlf tab space comma pipe slash colon semicolon equals * Default separators by format: File format RS FS PS - dkvp \n , = - json (N/A) (N/A) (N/A) - nidx \n space (N/A) - csv \r\n , (N/A) - csvlite \n , (N/A) - markdown \n (N/A) (N/A) - pprint \n space (N/A) - xtab (N/A) \n space + dkvp auto , = + json auto (N/A) (N/A) + nidx auto space (N/A) + csv auto , (N/A) + csvlite auto , (N/A) + markdown auto (N/A) (N/A) + pprint auto space (N/A) + xtab (N/A) auto space Relevant to CSV/CSV-lite input only: --implicit-csv-header Use 1,2,3,... as field labels, rather than from line 1 @@ -863,7 +854,7 @@ streaming-data programs operate through pipes, each one can use a CPU. Rest assured you get the same results either way. The other reason to use then-chaining is for simplicity: you don’t -have re-type formatting flags (e.g. --csv --rs lf --fs tab) at every +have re-type formatting flags (e.g. --csv --fs tab) at every pipeline stage.