diff --git a/c/reg_test/expected/out b/c/reg_test/expected/out index 7687117e5..c77079127 100644 --- a/c/reg_test/expected/out +++ b/c/reg_test/expected/out @@ -1091,6 +1091,172 @@ a=zee,b=wye,i=8,x=0.5985540091064224,y=0.976181385699006 a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864 +================================================================ +WHITESPACE-REDUCTION + +mlr --icsv --ojson cat ./reg_test/input/clean-whitespace.csv +{ "n": 1, "a": "xy", "b ": 2, " c": 3 } +{ "n": 2, "a": "xy ", "b ": 2, " c": 3 } +{ "n": 3, "a": "xy ", "b ": 2, " c": 3 } +{ "n": 4, "a": "xy ", "b ": 2, " c": 3 } +{ "n": 5, "a": "xy", "b ": 2, " c": 3 } +{ "n": 6, "a": " xy", "b ": 2, " c": 3 } +{ "n": 7, "a": " xy", "b ": 2, " c": 3 } +{ "n": 8, "a": " xy", "b ": 2, " c": 3 } +{ "n": 9, "a": "xy", "b ": 2, " c": 3 } +{ "n": 10, "a": " xy ", "b ": 2, " c": 3 } +{ "n": 11, "a": " xy ", "b ": 2, " c": 3 } +{ "n": 12, "a": " xy ", "b ": 2, " c": 3 } +{ "n": 13, "a": "", "b ": 2, " c": 3 } +{ "n": 14, "a": " ", "b ": 2, " c": 3 } +{ "n": 15, "a": " ", "b ": 2, " c": 3 } +{ "n": 16, "a": " ", "b ": 2, " c": 3 } + +mlr --icsv --ojson put $a = lstrip($a) ./reg_test/input/clean-whitespace.csv +{ "n": 1, "a": "xy", "b ": 2, " c": 3 } +{ "n": 2, "a": "xy ", "b ": 2, " c": 3 } +{ "n": 3, "a": "xy ", "b ": 2, " c": 3 } +{ "n": 4, "a": "xy ", "b ": 2, " c": 3 } +{ "n": 5, "a": "xy", "b ": 2, " c": 3 } +{ "n": 6, "a": "xy", "b ": 2, " c": 3 } +{ "n": 7, "a": "xy", "b ": 2, " c": 3 } +{ "n": 8, "a": "xy", "b ": 2, " c": 3 } +{ "n": 9, "a": "xy", "b ": 2, " c": 3 } +{ "n": 10, "a": "xy ", "b ": 2, " c": 3 } +{ "n": 11, "a": "xy ", "b ": 2, " c": 3 } +{ "n": 12, "a": "xy ", "b ": 2, " c": 3 } +{ "n": 13, "a": "", "b ": 2, " c": 3 } +{ "n": 14, "a": "", "b ": 2, " c": 3 } +{ "n": 15, "a": "", "b ": 2, " c": 3 } +{ "n": 16, "a": "", "b ": 2, " c": 3 } + +mlr --icsv --ojson put $a = rstrip($a) ./reg_test/input/clean-whitespace.csv +{ "n": 1, "a": "xy", "b ": 2, " c": 3 } +{ "n": 2, "a": "xy", "b ": 2, " c": 3 } +{ "n": 3, "a": "xy", "b ": 2, " c": 3 } +{ "n": 4, "a": "xy", "b ": 2, " c": 3 } +{ "n": 5, "a": "xy", "b ": 2, " c": 3 } +{ "n": 6, "a": " xy", "b ": 2, " c": 3 } +{ "n": 7, "a": " xy", "b ": 2, " c": 3 } +{ "n": 8, "a": " xy", "b ": 2, " c": 3 } +{ "n": 9, "a": "xy", "b ": 2, " c": 3 } +{ "n": 10, "a": " xy", "b ": 2, " c": 3 } +{ "n": 11, "a": " xy", "b ": 2, " c": 3 } +{ "n": 12, "a": " xy", "b ": 2, " c": 3 } +{ "n": 13, "a": "", "b ": 2, " c": 3 } +{ "n": 14, "a": "", "b ": 2, " c": 3 } +{ "n": 15, "a": "", "b ": 2, " c": 3 } +{ "n": 16, "a": "", "b ": 2, " c": 3 } + +mlr --icsv --ojson put $a = strip($a) ./reg_test/input/clean-whitespace.csv +{ "n": 1, "a": "xy", "b ": 2, " c": 3 } +{ "n": 2, "a": "xy", "b ": 2, " c": 3 } +{ "n": 3, "a": "xy", "b ": 2, " c": 3 } +{ "n": 4, "a": "xy", "b ": 2, " c": 3 } +{ "n": 5, "a": "xy", "b ": 2, " c": 3 } +{ "n": 6, "a": "xy", "b ": 2, " c": 3 } +{ "n": 7, "a": "xy", "b ": 2, " c": 3 } +{ "n": 8, "a": "xy", "b ": 2, " c": 3 } +{ "n": 9, "a": "xy", "b ": 2, " c": 3 } +{ "n": 10, "a": "xy", "b ": 2, " c": 3 } +{ "n": 11, "a": "xy", "b ": 2, " c": 3 } +{ "n": 12, "a": "xy", "b ": 2, " c": 3 } +{ "n": 13, "a": "", "b ": 2, " c": 3 } +{ "n": 14, "a": "", "b ": 2, " c": 3 } +{ "n": 15, "a": "", "b ": 2, " c": 3 } +{ "n": 16, "a": "", "b ": 2, " c": 3 } + +mlr --icsv --ojson put $a = collapse_whitespace($a) ./reg_test/input/clean-whitespace.csv +{ "n": 1, "a": "xy", "b ": 2, " c": 3 } +{ "n": 2, "a": "xy ", "b ": 2, " c": 3 } +{ "n": 3, "a": "xy ", "b ": 2, " c": 3 } +{ "n": 4, "a": "xy ", "b ": 2, " c": 3 } +{ "n": 5, "a": "xy", "b ": 2, " c": 3 } +{ "n": 6, "a": " xy", "b ": 2, " c": 3 } +{ "n": 7, "a": " xy", "b ": 2, " c": 3 } +{ "n": 8, "a": " xy", "b ": 2, " c": 3 } +{ "n": 9, "a": "xy", "b ": 2, " c": 3 } +{ "n": 10, "a": " xy ", "b ": 2, " c": 3 } +{ "n": 11, "a": " xy ", "b ": 2, " c": 3 } +{ "n": 12, "a": " xy ", "b ": 2, " c": 3 } +{ "n": 13, "a": "", "b ": 2, " c": 3 } +{ "n": 14, "a": " ", "b ": 2, " c": 3 } +{ "n": 15, "a": " ", "b ": 2, " c": 3 } +{ "n": 16, "a": " ", "b ": 2, " c": 3 } + +mlr --icsv --ojson put $a = clean_whitespace($a) ./reg_test/input/clean-whitespace.csv +{ "n": 1, "a": "xy", "b ": 2, " c": 3 } +{ "n": 2, "a": "xy", "b ": 2, " c": 3 } +{ "n": 3, "a": "xy", "b ": 2, " c": 3 } +{ "n": 4, "a": "xy", "b ": 2, " c": 3 } +{ "n": 5, "a": "xy", "b ": 2, " c": 3 } +{ "n": 6, "a": "xy", "b ": 2, " c": 3 } +{ "n": 7, "a": "xy", "b ": 2, " c": 3 } +{ "n": 8, "a": "xy", "b ": 2, " c": 3 } +{ "n": 9, "a": "xy", "b ": 2, " c": 3 } +{ "n": 10, "a": "xy", "b ": 2, " c": 3 } +{ "n": 11, "a": "xy", "b ": 2, " c": 3 } +{ "n": 12, "a": "xy", "b ": 2, " c": 3 } +{ "n": 13, "a": "", "b ": 2, " c": 3 } +{ "n": 14, "a": "", "b ": 2, " c": 3 } +{ "n": 15, "a": "", "b ": 2, " c": 3 } +{ "n": 16, "a": "", "b ": 2, " c": 3 } + +mlr --icsv --ojson clean-whitespace -k ./reg_test/input/clean-whitespace.csv +{ "n": 1, "a": "xy", "b": 2, "c": 3 } +{ "n": 2, "a": "xy ", "b": 2, "c": 3 } +{ "n": 3, "a": "xy ", "b": 2, "c": 3 } +{ "n": 4, "a": "xy ", "b": 2, "c": 3 } +{ "n": 5, "a": "xy", "b": 2, "c": 3 } +{ "n": 6, "a": " xy", "b": 2, "c": 3 } +{ "n": 7, "a": " xy", "b": 2, "c": 3 } +{ "n": 8, "a": " xy", "b": 2, "c": 3 } +{ "n": 9, "a": "xy", "b": 2, "c": 3 } +{ "n": 10, "a": " xy ", "b": 2, "c": 3 } +{ "n": 11, "a": " xy ", "b": 2, "c": 3 } +{ "n": 12, "a": " xy ", "b": 2, "c": 3 } +{ "n": 13, "a": "", "b": 2, "c": 3 } +{ "n": 14, "a": " ", "b": 2, "c": 3 } +{ "n": 15, "a": " ", "b": 2, "c": 3 } +{ "n": 16, "a": " ", "b": 2, "c": 3 } + +mlr --icsv --ojson clean-whitespace -v ./reg_test/input/clean-whitespace.csv +{ "n": 1, "a": "xy", "b ": 2, " c": 3 } +{ "n": 2, "a": "xy", "b ": 2, " c": 3 } +{ "n": 3, "a": "xy", "b ": 2, " c": 3 } +{ "n": 4, "a": "xy", "b ": 2, " c": 3 } +{ "n": 5, "a": "xy", "b ": 2, " c": 3 } +{ "n": 6, "a": "xy", "b ": 2, " c": 3 } +{ "n": 7, "a": "xy", "b ": 2, " c": 3 } +{ "n": 8, "a": "xy", "b ": 2, " c": 3 } +{ "n": 9, "a": "xy", "b ": 2, " c": 3 } +{ "n": 10, "a": "xy", "b ": 2, " c": 3 } +{ "n": 11, "a": "xy", "b ": 2, " c": 3 } +{ "n": 12, "a": "xy", "b ": 2, " c": 3 } +{ "n": 13, "a": "", "b ": 2, " c": 3 } +{ "n": 14, "a": "", "b ": 2, " c": 3 } +{ "n": 15, "a": "", "b ": 2, " c": 3 } +{ "n": 16, "a": "", "b ": 2, " c": 3 } + +mlr --icsv --ojson clean-whitespace ./reg_test/input/clean-whitespace.csv +{ "n": 1, "a": "xy", "b": 2, "c": 3 } +{ "n": 2, "a": "xy", "b": 2, "c": 3 } +{ "n": 3, "a": "xy", "b": 2, "c": 3 } +{ "n": 4, "a": "xy", "b": 2, "c": 3 } +{ "n": 5, "a": "xy", "b": 2, "c": 3 } +{ "n": 6, "a": "xy", "b": 2, "c": 3 } +{ "n": 7, "a": "xy", "b": 2, "c": 3 } +{ "n": 8, "a": "xy", "b": 2, "c": 3 } +{ "n": 9, "a": "xy", "b": 2, "c": 3 } +{ "n": 10, "a": "xy", "b": 2, "c": 3 } +{ "n": 11, "a": "xy", "b": 2, "c": 3 } +{ "n": 12, "a": "xy", "b": 2, "c": 3 } +{ "n": 13, "a": "", "b": 2, "c": 3 } +{ "n": 14, "a": "", "b": 2, "c": 3 } +{ "n": 15, "a": "", "b": 2, "c": 3 } +{ "n": 16, "a": "", "b": 2, "c": 3 } + + ================================================================ SORT diff --git a/c/reg_test/input/Makefile.am b/c/reg_test/input/Makefile.am index 2b7ff0d5e..dce594e81 100644 --- a/c/reg_test/input/Makefile.am +++ b/c/reg_test/input/Makefile.am @@ -27,6 +27,7 @@ EXTRA_DIST= \ c.pprint \ capture-lengths.dkvp \ capture.dkvp \ + clean-whitespace.csv \ comments \ d.csv \ d.pprint \ diff --git a/c/reg_test/input/clean-whitespace.csv b/c/reg_test/input/clean-whitespace.csv new file mode 100644 index 000000000..420c41b11 --- /dev/null +++ b/c/reg_test/input/clean-whitespace.csv @@ -0,0 +1,17 @@ +n,a,b , c +1,xy,2,3 +2,xy ,2,3 +3,xy ,2,3 +4,xy ,2,3 +5,xy,2,3 +6, xy,2,3 +7, xy,2,3 +8, xy,2,3 +9,xy,2,3 +10, xy ,2,3 +11, xy ,2,3 +12, xy ,2,3 +13,,2,3 +14, ,2,3 +15, ,2,3 +16, ,2,3 diff --git a/c/reg_test/run b/c/reg_test/run index 987ab7103..ff747f40b 100755 --- a/c/reg_test/run +++ b/c/reg_test/run @@ -337,6 +337,20 @@ run_mlr decimate -g a -n 2 $indir/abixy run_mlr decimate -g a -b -n 2 $indir/abixy run_mlr decimate -g a -e -n 2 $indir/abixy +# ---------------------------------------------------------------- +announce WHITESPACE-REDUCTION + +run_mlr --icsv --ojson cat $indir/clean-whitespace.csv +run_mlr --icsv --ojson put '$a = lstrip($a)' $indir/clean-whitespace.csv +run_mlr --icsv --ojson put '$a = rstrip($a)' $indir/clean-whitespace.csv +run_mlr --icsv --ojson put '$a = strip($a)' $indir/clean-whitespace.csv +run_mlr --icsv --ojson put '$a = collapse_whitespace($a)' $indir/clean-whitespace.csv +run_mlr --icsv --ojson put '$a = clean_whitespace($a)' $indir/clean-whitespace.csv + +run_mlr --icsv --ojson clean-whitespace -k $indir/clean-whitespace.csv +run_mlr --icsv --ojson clean-whitespace -v $indir/clean-whitespace.csv +run_mlr --icsv --ojson clean-whitespace $indir/clean-whitespace.csv + # ---------------------------------------------------------------- announce SORT