From f5553c095e2d2dad8e12cbfbcb29add8747243ef Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 25 Jan 2021 00:06:14 -0500 Subject: [PATCH] mlr clean-whitespace --- c/mapping/mapper_clean_whitespace.c | 5 +- go/cases-to-do.txt | 1 - ...espace.sh => case-dsl-clean-whitespace.sh} | 0 .../case-c-dsl-clean-whitespace.sh.out | 163 ---- .../expected/case-dsl-clean-whitespace.sh.out | 883 ++++++++++++++++++ go/src/miller/cli/mlrcli_transformers.go | 1 + .../miller/transformers/clean-whitespace.go | 185 ++++ go/todo.txt | 2 +- 8 files changed, 1073 insertions(+), 167 deletions(-) rename go/reg-test/cases/{case-c-dsl-clean-whitespace.sh => case-dsl-clean-whitespace.sh} (100%) delete mode 100644 go/reg-test/expected/case-c-dsl-clean-whitespace.sh.out create mode 100644 go/reg-test/expected/case-dsl-clean-whitespace.sh.out create mode 100644 go/src/miller/transformers/clean-whitespace.go diff --git a/c/mapping/mapper_clean_whitespace.c b/c/mapping/mapper_clean_whitespace.c index 9f887ede9..c832afa15 100644 --- a/c/mapping/mapper_clean_whitespace.c +++ b/c/mapping/mapper_clean_whitespace.c @@ -31,7 +31,7 @@ mapper_setup_t mapper_clean_whitespace_setup = { // ---------------------------------------------------------------- static void mapper_clean_whitespace_usage(FILE* o, char* argv0, char* verb) { - fprintf(o, "Usage: %s %s [options] {old1,new1,old2,new2,...}\n", argv0, verb); + fprintf(o, "Usage: %s %s [options]\n", argv0, verb); fprintf(o, "For each record, for each field in the record, whitespace-cleans the keys and\n"); fprintf(o, "values. Whitespace-cleaning entails stripping leading and trailing whitespace,\n"); fprintf(o, "and replacing multiple whitespace with singles. For finer-grained control,\n"); @@ -41,7 +41,8 @@ static void mapper_clean_whitespace_usage(FILE* o, char* argv0, char* verb) { fprintf(o, "Options:\n"); fprintf(o, "-k|--keys-only Do not touch values.\n"); fprintf(o, "-v|--values-only Do not touch keys.\n"); - fprintf(o, "It is an error to specify -k as well as -v.\n"); + fprintf(o, "It is an error to specify -k as well as -v -- to clean keys and values,\n"); + fprintf(o, "leave off -k as well as -v.\n"); } static mapper_t* mapper_clean_whitespace_parse_cli(int* pargi, int argc, char** argv, diff --git a/go/cases-to-do.txt b/go/cases-to-do.txt index 136f7272b..8935c2a2e 100644 --- a/go/cases-to-do.txt +++ b/go/cases-to-do.txt @@ -30,7 +30,6 @@ rrv -C20 ./reg-test/cases/case-c-dsl-local-map-variable-write-read.sh ================================================================ PARTIALLY IMPLEMENTED: -rrv ./reg-test/cases/case-c-clean-whitespace.sh rrv ./reg-test/cases/case-c-cut.sh rrv ./reg-test/cases/case-c-dsl-scientific-notation.sh rrv ./reg-test/cases/case-c-implicit-header-csv-input.sh diff --git a/go/reg-test/cases/case-c-dsl-clean-whitespace.sh b/go/reg-test/cases/case-dsl-clean-whitespace.sh similarity index 100% rename from go/reg-test/cases/case-c-dsl-clean-whitespace.sh rename to go/reg-test/cases/case-dsl-clean-whitespace.sh diff --git a/go/reg-test/expected/case-c-dsl-clean-whitespace.sh.out b/go/reg-test/expected/case-c-dsl-clean-whitespace.sh.out deleted file mode 100644 index becbe3ea7..000000000 --- a/go/reg-test/expected/case-c-dsl-clean-whitespace.sh.out +++ /dev/null @@ -1,163 +0,0 @@ - -mlr --icsv --ojson cat ./reg-test/input/clean-whitespace.csv -{ "n": 1, "a": "xy", "b ": 2, " c": 3 } -{ "n": 2, "a": "xy ", "b ": 2, " c": 3 } -{ "n": 3, "a": "xy ", "b ": 2, " c": 3 } -{ "n": 4, "a": "xy ", "b ": 2, " c": 3 } -{ "n": 5, "a": "xy", "b ": 2, " c": 3 } -{ "n": 6, "a": " xy", "b ": 2, " c": 3 } -{ "n": 7, "a": " xy", "b ": 2, " c": 3 } -{ "n": 8, "a": " xy", "b ": 2, " c": 3 } -{ "n": 9, "a": "xy", "b ": 2, " c": 3 } -{ "n": 10, "a": " xy ", "b ": 2, " c": 3 } -{ "n": 11, "a": " xy ", "b ": 2, " c": 3 } -{ "n": 12, "a": " xy ", "b ": 2, " c": 3 } -{ "n": 13, "a": "", "b ": 2, " c": 3 } -{ "n": 14, "a": " ", "b ": 2, " c": 3 } -{ "n": 15, "a": " ", "b ": 2, " c": 3 } -{ "n": 16, "a": " ", "b ": 2, " c": 3 } - -mlr --icsv --ojson put $a = lstrip($a) ./reg-test/input/clean-whitespace.csv -{ "n": 1, "a": "xy", "b ": 2, " c": 3 } -{ "n": 2, "a": "xy ", "b ": 2, " c": 3 } -{ "n": 3, "a": "xy ", "b ": 2, " c": 3 } -{ "n": 4, "a": "xy ", "b ": 2, " c": 3 } -{ "n": 5, "a": "xy", "b ": 2, " c": 3 } -{ "n": 6, "a": "xy", "b ": 2, " c": 3 } -{ "n": 7, "a": "xy", "b ": 2, " c": 3 } -{ "n": 8, "a": "xy", "b ": 2, " c": 3 } -{ "n": 9, "a": "xy", "b ": 2, " c": 3 } -{ "n": 10, "a": "xy ", "b ": 2, " c": 3 } -{ "n": 11, "a": "xy ", "b ": 2, " c": 3 } -{ "n": 12, "a": "xy ", "b ": 2, " c": 3 } -{ "n": 13, "a": "", "b ": 2, " c": 3 } -{ "n": 14, "a": "", "b ": 2, " c": 3 } -{ "n": 15, "a": "", "b ": 2, " c": 3 } -{ "n": 16, "a": "", "b ": 2, " c": 3 } - -mlr --icsv --ojson put $a = rstrip($a) ./reg-test/input/clean-whitespace.csv -{ "n": 1, "a": "xy", "b ": 2, " c": 3 } -{ "n": 2, "a": "xy", "b ": 2, " c": 3 } -{ "n": 3, "a": "xy", "b ": 2, " c": 3 } -{ "n": 4, "a": "xy", "b ": 2, " c": 3 } -{ "n": 5, "a": "xy", "b ": 2, " c": 3 } -{ "n": 6, "a": " xy", "b ": 2, " c": 3 } -{ "n": 7, "a": " xy", "b ": 2, " c": 3 } -{ "n": 8, "a": " xy", "b ": 2, " c": 3 } -{ "n": 9, "a": "xy", "b ": 2, " c": 3 } -{ "n": 10, "a": " xy", "b ": 2, " c": 3 } -{ "n": 11, "a": " xy", "b ": 2, " c": 3 } -{ "n": 12, "a": " xy", "b ": 2, " c": 3 } -{ "n": 13, "a": "", "b ": 2, " c": 3 } -{ "n": 14, "a": "", "b ": 2, " c": 3 } -{ "n": 15, "a": "", "b ": 2, " c": 3 } -{ "n": 16, "a": "", "b ": 2, " c": 3 } - -mlr --icsv --ojson put $a = strip($a) ./reg-test/input/clean-whitespace.csv -{ "n": 1, "a": "xy", "b ": 2, " c": 3 } -{ "n": 2, "a": "xy", "b ": 2, " c": 3 } -{ "n": 3, "a": "xy", "b ": 2, " c": 3 } -{ "n": 4, "a": "xy", "b ": 2, " c": 3 } -{ "n": 5, "a": "xy", "b ": 2, " c": 3 } -{ "n": 6, "a": "xy", "b ": 2, " c": 3 } -{ "n": 7, "a": "xy", "b ": 2, " c": 3 } -{ "n": 8, "a": "xy", "b ": 2, " c": 3 } -{ "n": 9, "a": "xy", "b ": 2, " c": 3 } -{ "n": 10, "a": "xy", "b ": 2, " c": 3 } -{ "n": 11, "a": "xy", "b ": 2, " c": 3 } -{ "n": 12, "a": "xy", "b ": 2, " c": 3 } -{ "n": 13, "a": "", "b ": 2, " c": 3 } -{ "n": 14, "a": "", "b ": 2, " c": 3 } -{ "n": 15, "a": "", "b ": 2, " c": 3 } -{ "n": 16, "a": "", "b ": 2, " c": 3 } - -mlr --icsv --ojson put $a = collapse_whitespace($a) ./reg-test/input/clean-whitespace.csv -{ "n": 1, "a": "xy", "b ": 2, " c": 3 } -{ "n": 2, "a": "xy ", "b ": 2, " c": 3 } -{ "n": 3, "a": "xy ", "b ": 2, " c": 3 } -{ "n": 4, "a": "xy ", "b ": 2, " c": 3 } -{ "n": 5, "a": "xy", "b ": 2, " c": 3 } -{ "n": 6, "a": " xy", "b ": 2, " c": 3 } -{ "n": 7, "a": " xy", "b ": 2, " c": 3 } -{ "n": 8, "a": " xy", "b ": 2, " c": 3 } -{ "n": 9, "a": "xy", "b ": 2, " c": 3 } -{ "n": 10, "a": " xy ", "b ": 2, " c": 3 } -{ "n": 11, "a": " xy ", "b ": 2, " c": 3 } -{ "n": 12, "a": " xy ", "b ": 2, " c": 3 } -{ "n": 13, "a": "", "b ": 2, " c": 3 } -{ "n": 14, "a": " ", "b ": 2, " c": 3 } -{ "n": 15, "a": " ", "b ": 2, " c": 3 } -{ "n": 16, "a": " ", "b ": 2, " c": 3 } - -mlr --icsv --ojson put $a = clean_whitespace($a) ./reg-test/input/clean-whitespace.csv -{ "n": 1, "a": "xy", "b ": 2, " c": 3 } -{ "n": 2, "a": "xy", "b ": 2, " c": 3 } -{ "n": 3, "a": "xy", "b ": 2, " c": 3 } -{ "n": 4, "a": "xy", "b ": 2, " c": 3 } -{ "n": 5, "a": "xy", "b ": 2, " c": 3 } -{ "n": 6, "a": "xy", "b ": 2, " c": 3 } -{ "n": 7, "a": "xy", "b ": 2, " c": 3 } -{ "n": 8, "a": "xy", "b ": 2, " c": 3 } -{ "n": 9, "a": "xy", "b ": 2, " c": 3 } -{ "n": 10, "a": "xy", "b ": 2, " c": 3 } -{ "n": 11, "a": "xy", "b ": 2, " c": 3 } -{ "n": 12, "a": "xy", "b ": 2, " c": 3 } -{ "n": 13, "a": "", "b ": 2, " c": 3 } -{ "n": 14, "a": "", "b ": 2, " c": 3 } -{ "n": 15, "a": "", "b ": 2, " c": 3 } -{ "n": 16, "a": "", "b ": 2, " c": 3 } - -mlr --icsv --ojson clean-whitespace -k ./reg-test/input/clean-whitespace.csv -{ "n": 1, "a": "xy", "b": 2, "c": 3 } -{ "n": 2, "a": "xy ", "b": 2, "c": 3 } -{ "n": 3, "a": "xy ", "b": 2, "c": 3 } -{ "n": 4, "a": "xy ", "b": 2, "c": 3 } -{ "n": 5, "a": "xy", "b": 2, "c": 3 } -{ "n": 6, "a": " xy", "b": 2, "c": 3 } -{ "n": 7, "a": " xy", "b": 2, "c": 3 } -{ "n": 8, "a": " xy", "b": 2, "c": 3 } -{ "n": 9, "a": "xy", "b": 2, "c": 3 } -{ "n": 10, "a": " xy ", "b": 2, "c": 3 } -{ "n": 11, "a": " xy ", "b": 2, "c": 3 } -{ "n": 12, "a": " xy ", "b": 2, "c": 3 } -{ "n": 13, "a": "", "b": 2, "c": 3 } -{ "n": 14, "a": " ", "b": 2, "c": 3 } -{ "n": 15, "a": " ", "b": 2, "c": 3 } -{ "n": 16, "a": " ", "b": 2, "c": 3 } - -mlr --icsv --ojson clean-whitespace -v ./reg-test/input/clean-whitespace.csv -{ "n": 1, "a": "xy", "b ": 2, " c": 3 } -{ "n": 2, "a": "xy", "b ": 2, " c": 3 } -{ "n": 3, "a": "xy", "b ": 2, " c": 3 } -{ "n": 4, "a": "xy", "b ": 2, " c": 3 } -{ "n": 5, "a": "xy", "b ": 2, " c": 3 } -{ "n": 6, "a": "xy", "b ": 2, " c": 3 } -{ "n": 7, "a": "xy", "b ": 2, " c": 3 } -{ "n": 8, "a": "xy", "b ": 2, " c": 3 } -{ "n": 9, "a": "xy", "b ": 2, " c": 3 } -{ "n": 10, "a": "xy", "b ": 2, " c": 3 } -{ "n": 11, "a": "xy", "b ": 2, " c": 3 } -{ "n": 12, "a": "xy", "b ": 2, " c": 3 } -{ "n": 13, "a": "", "b ": 2, " c": 3 } -{ "n": 14, "a": "", "b ": 2, " c": 3 } -{ "n": 15, "a": "", "b ": 2, " c": 3 } -{ "n": 16, "a": "", "b ": 2, " c": 3 } - -mlr --icsv --ojson clean-whitespace ./reg-test/input/clean-whitespace.csv -{ "n": 1, "a": "xy", "b": 2, "c": 3 } -{ "n": 2, "a": "xy", "b": 2, "c": 3 } -{ "n": 3, "a": "xy", "b": 2, "c": 3 } -{ "n": 4, "a": "xy", "b": 2, "c": 3 } -{ "n": 5, "a": "xy", "b": 2, "c": 3 } -{ "n": 6, "a": "xy", "b": 2, "c": 3 } -{ "n": 7, "a": "xy", "b": 2, "c": 3 } -{ "n": 8, "a": "xy", "b": 2, "c": 3 } -{ "n": 9, "a": "xy", "b": 2, "c": 3 } -{ "n": 10, "a": "xy", "b": 2, "c": 3 } -{ "n": 11, "a": "xy", "b": 2, "c": 3 } -{ "n": 12, "a": "xy", "b": 2, "c": 3 } -{ "n": 13, "a": "", "b": 2, "c": 3 } -{ "n": 14, "a": "", "b": 2, "c": 3 } -{ "n": 15, "a": "", "b": 2, "c": 3 } -{ "n": 16, "a": "", "b": 2, "c": 3 } - diff --git a/go/reg-test/expected/case-dsl-clean-whitespace.sh.out b/go/reg-test/expected/case-dsl-clean-whitespace.sh.out new file mode 100644 index 000000000..f4c64b643 --- /dev/null +++ b/go/reg-test/expected/case-dsl-clean-whitespace.sh.out @@ -0,0 +1,883 @@ + +mlr --icsv --ojson cat ./reg-test/input/clean-whitespace.csv +{ + "n": 1, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 2, + "a": "xy ", + "b ": 2, + " c": 3 +} +{ + "n": 3, + "a": "xy ", + "b ": 2, + " c": 3 +} +{ + "n": 4, + "a": "xy ", + "b ": 2, + " c": 3 +} +{ + "n": 5, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 6, + "a": " xy", + "b ": 2, + " c": 3 +} +{ + "n": 7, + "a": " xy", + "b ": 2, + " c": 3 +} +{ + "n": 8, + "a": " xy", + "b ": 2, + " c": 3 +} +{ + "n": 9, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 10, + "a": " xy ", + "b ": 2, + " c": 3 +} +{ + "n": 11, + "a": " xy ", + "b ": 2, + " c": 3 +} +{ + "n": 12, + "a": " xy ", + "b ": 2, + " c": 3 +} +{ + "n": 13, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 14, + "a": " ", + "b ": 2, + " c": 3 +} +{ + "n": 15, + "a": " ", + "b ": 2, + " c": 3 +} +{ + "n": 16, + "a": " ", + "b ": 2, + " c": 3 +} + +mlr --icsv --ojson put $a = lstrip($a) ./reg-test/input/clean-whitespace.csv +{ + "n": 1, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 2, + "a": "xy ", + "b ": 2, + " c": 3 +} +{ + "n": 3, + "a": "xy ", + "b ": 2, + " c": 3 +} +{ + "n": 4, + "a": "xy ", + "b ": 2, + " c": 3 +} +{ + "n": 5, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 6, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 7, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 8, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 9, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 10, + "a": "xy ", + "b ": 2, + " c": 3 +} +{ + "n": 11, + "a": "xy ", + "b ": 2, + " c": 3 +} +{ + "n": 12, + "a": "xy ", + "b ": 2, + " c": 3 +} +{ + "n": 13, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 14, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 15, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 16, + "a": "", + "b ": 2, + " c": 3 +} + +mlr --icsv --ojson put $a = rstrip($a) ./reg-test/input/clean-whitespace.csv +{ + "n": 1, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 2, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 3, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 4, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 5, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 6, + "a": " xy", + "b ": 2, + " c": 3 +} +{ + "n": 7, + "a": " xy", + "b ": 2, + " c": 3 +} +{ + "n": 8, + "a": " xy", + "b ": 2, + " c": 3 +} +{ + "n": 9, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 10, + "a": " xy", + "b ": 2, + " c": 3 +} +{ + "n": 11, + "a": " xy", + "b ": 2, + " c": 3 +} +{ + "n": 12, + "a": " xy", + "b ": 2, + " c": 3 +} +{ + "n": 13, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 14, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 15, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 16, + "a": "", + "b ": 2, + " c": 3 +} + +mlr --icsv --ojson put $a = strip($a) ./reg-test/input/clean-whitespace.csv +{ + "n": 1, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 2, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 3, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 4, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 5, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 6, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 7, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 8, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 9, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 10, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 11, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 12, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 13, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 14, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 15, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 16, + "a": "", + "b ": 2, + " c": 3 +} + +mlr --icsv --ojson put $a = collapse_whitespace($a) ./reg-test/input/clean-whitespace.csv +{ + "n": 1, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 2, + "a": "xy ", + "b ": 2, + " c": 3 +} +{ + "n": 3, + "a": "xy ", + "b ": 2, + " c": 3 +} +{ + "n": 4, + "a": "xy ", + "b ": 2, + " c": 3 +} +{ + "n": 5, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 6, + "a": " xy", + "b ": 2, + " c": 3 +} +{ + "n": 7, + "a": " xy", + "b ": 2, + " c": 3 +} +{ + "n": 8, + "a": " xy", + "b ": 2, + " c": 3 +} +{ + "n": 9, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 10, + "a": " xy ", + "b ": 2, + " c": 3 +} +{ + "n": 11, + "a": " xy ", + "b ": 2, + " c": 3 +} +{ + "n": 12, + "a": " xy ", + "b ": 2, + " c": 3 +} +{ + "n": 13, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 14, + "a": " ", + "b ": 2, + " c": 3 +} +{ + "n": 15, + "a": " ", + "b ": 2, + " c": 3 +} +{ + "n": 16, + "a": " ", + "b ": 2, + " c": 3 +} + +mlr --icsv --ojson put $a = clean_whitespace($a) ./reg-test/input/clean-whitespace.csv +{ + "n": 1, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 2, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 3, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 4, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 5, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 6, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 7, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 8, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 9, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 10, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 11, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 12, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 13, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 14, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 15, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 16, + "a": "", + "b ": 2, + " c": 3 +} + +mlr --icsv --ojson clean-whitespace -k ./reg-test/input/clean-whitespace.csv +{ + "n": 1, + "a": "xy", + "b": 2, + "c": 3 +} +{ + "n": 2, + "a": "xy ", + "b": 2, + "c": 3 +} +{ + "n": 3, + "a": "xy ", + "b": 2, + "c": 3 +} +{ + "n": 4, + "a": "xy ", + "b": 2, + "c": 3 +} +{ + "n": 5, + "a": "xy", + "b": 2, + "c": 3 +} +{ + "n": 6, + "a": " xy", + "b": 2, + "c": 3 +} +{ + "n": 7, + "a": " xy", + "b": 2, + "c": 3 +} +{ + "n": 8, + "a": " xy", + "b": 2, + "c": 3 +} +{ + "n": 9, + "a": "xy", + "b": 2, + "c": 3 +} +{ + "n": 10, + "a": " xy ", + "b": 2, + "c": 3 +} +{ + "n": 11, + "a": " xy ", + "b": 2, + "c": 3 +} +{ + "n": 12, + "a": " xy ", + "b": 2, + "c": 3 +} +{ + "n": 13, + "a": "", + "b": 2, + "c": 3 +} +{ + "n": 14, + "a": " ", + "b": 2, + "c": 3 +} +{ + "n": 15, + "a": " ", + "b": 2, + "c": 3 +} +{ + "n": 16, + "a": " ", + "b": 2, + "c": 3 +} + +mlr --icsv --ojson clean-whitespace -v ./reg-test/input/clean-whitespace.csv +{ + "n": 1, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 2, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 3, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 4, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 5, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 6, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 7, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 8, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 9, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 10, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 11, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 12, + "a": "xy", + "b ": 2, + " c": 3 +} +{ + "n": 13, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 14, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 15, + "a": "", + "b ": 2, + " c": 3 +} +{ + "n": 16, + "a": "", + "b ": 2, + " c": 3 +} + +mlr --icsv --ojson clean-whitespace ./reg-test/input/clean-whitespace.csv +{ + "n": 1, + "a": "xy", + "b": 2, + "c": 3 +} +{ + "n": 2, + "a": "xy", + "b": 2, + "c": 3 +} +{ + "n": 3, + "a": "xy", + "b": 2, + "c": 3 +} +{ + "n": 4, + "a": "xy", + "b": 2, + "c": 3 +} +{ + "n": 5, + "a": "xy", + "b": 2, + "c": 3 +} +{ + "n": 6, + "a": "xy", + "b": 2, + "c": 3 +} +{ + "n": 7, + "a": "xy", + "b": 2, + "c": 3 +} +{ + "n": 8, + "a": "xy", + "b": 2, + "c": 3 +} +{ + "n": 9, + "a": "xy", + "b": 2, + "c": 3 +} +{ + "n": 10, + "a": "xy", + "b": 2, + "c": 3 +} +{ + "n": 11, + "a": "xy", + "b": 2, + "c": 3 +} +{ + "n": 12, + "a": "xy", + "b": 2, + "c": 3 +} +{ + "n": 13, + "a": "", + "b": 2, + "c": 3 +} +{ + "n": 14, + "a": "", + "b": 2, + "c": 3 +} +{ + "n": 15, + "a": "", + "b": 2, + "c": 3 +} +{ + "n": 16, + "a": "", + "b": 2, + "c": 3 +} + diff --git a/go/src/miller/cli/mlrcli_transformers.go b/go/src/miller/cli/mlrcli_transformers.go index ba7d74c9a..2d06d1e10 100644 --- a/go/src/miller/cli/mlrcli_transformers.go +++ b/go/src/miller/cli/mlrcli_transformers.go @@ -15,6 +15,7 @@ var MAPPER_LOOKUP_TABLE = []transforming.TransformerSetup{ transformers.BootstrapSetup, transformers.CatSetup, transformers.CheckSetup, + transformers.CleanWhitespaceSetup, transformers.CountSetup, transformers.CountSimilarSetup, transformers.CutSetup, diff --git a/go/src/miller/transformers/clean-whitespace.go b/go/src/miller/transformers/clean-whitespace.go new file mode 100644 index 000000000..5d23c13a7 --- /dev/null +++ b/go/src/miller/transformers/clean-whitespace.go @@ -0,0 +1,185 @@ +package transformers + +import ( + "flag" + "fmt" + "os" + "strings" + + "miller/clitypes" + "miller/transforming" + "miller/types" +) + +// ---------------------------------------------------------------- +var CleanWhitespaceSetup = transforming.TransformerSetup{ + Verb: "clean-whitespace", + ParseCLIFunc: transformerCleanWhitespaceParseCLI, + IgnoresInput: false, +} + +func transformerCleanWhitespaceParseCLI( + pargi *int, + argc int, + args []string, + errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError + _ *clitypes.TReaderOptions, + __ *clitypes.TWriterOptions, +) transforming.IRecordTransformer { + + doKeys := true + doValues := true + + // Get the verb name from the current spot in the mlr command line + argi := *pargi + verb := args[argi] + argi++ + + for argi < argc /* variable increment: 1 or 2 depending on flag */ { + if !strings.HasPrefix(args[argi], "-") { + break // No more flag options to process + + } else if args[argi] == "-h" || args[argi] == "--help" { + transformerPutUsage(os.Stdout, 0, errorHandling, args[0], verb) + return nil // help intentionally requested + + } else if args[argi] == "-k" || args[argi] == "--keys-only" { + doKeys = true + doValues = false + argi++ + } else if args[argi] == "-v" || args[argi] == "--values-only" { + doKeys = false + doValues = true + argi++ + + } else { + transformerPutUsage(os.Stderr, 1, flag.ExitOnError, args[0], verb) + os.Exit(1) + } + } + + if !doKeys && !doValues { + transformerCleanWhitespaceUsage(os.Stderr, 1, flag.ExitOnError, args[0], verb) + os.Exit(1) + } + + transformer, _ := NewTransformerCleanWhitespace( + doKeys, + doValues, + ) + + *pargi = argi + return transformer +} + +func transformerCleanWhitespaceUsage( + o *os.File, + exitCode int, + errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError + argv0 string, + verb string, +) { + fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb) + fmt.Fprintf(o, "For each record, for each field in the record, whitespace-cleans the keys and/or\n") + fmt.Fprintf(o, "values. Whitespace-cleaning entails stripping leading and trailing whitespace,\n") + fmt.Fprintf(o, "and replacing multiple whitespace with singles. For finer-grained control,\n") + fmt.Fprintf(o, "please see the DSL functions lstrip, rstrip, strip, collapse_whitespace,\n") + fmt.Fprintf(o, "and clean_whitespace.\n") + fmt.Fprintf(o, "\n") + fmt.Fprintf(o, "Options:\n") + fmt.Fprintf(o, "-k|--keys-only Do not touch values.\n") + fmt.Fprintf(o, "-v|--values-only Do not touch keys.\n") + fmt.Fprintf(o, "It is an error to specify -k as well as -v -- to clean keys and values,\n") + fmt.Fprintf(o, "leave off -k as well as -v.\n") +} + +// ---------------------------------------------------------------- +type TransformerCleanWhitespace struct { + recordTransformerFunc transforming.RecordTransformerFunc +} + +// ---------------------------------------------------------------- +func NewTransformerCleanWhitespace( + doKeys bool, + doValues bool, +) (*TransformerCleanWhitespace, error) { + + this := &TransformerCleanWhitespace{} + + if doKeys && doValues { + this.recordTransformerFunc = this.cleanWhitespaceInKeysAndValues + } else if doKeys { + this.recordTransformerFunc = this.cleanWhitespaceInKeys + } else { + this.recordTransformerFunc = this.cleanWhitespaceInValues + } + + return this, nil +} + +// ---------------------------------------------------------------- +func (this *TransformerCleanWhitespace) Transform( + inrecAndContext *types.RecordAndContext, + outputChannel chan<- *types.RecordAndContext, +) { + this.recordTransformerFunc(inrecAndContext, outputChannel) +} + +// ---------------------------------------------------------------- +func (this *TransformerCleanWhitespace) cleanWhitespaceInKeysAndValues( + inrecAndContext *types.RecordAndContext, + outputChannel chan<- *types.RecordAndContext, +) { + if !inrecAndContext.EndOfStream { + newrec := types.NewMlrmapAsRecord() + + for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { + oldKey := types.MlrvalFromString(pe.Key) + newKey := types.MlrvalCleanWhitespace(&oldKey) + newValue := types.MlrvalCleanWhitespace(pe.Value) + // Transferring ownership from old record to new record; no copy needed + newrec.PutReference(newKey.String(), &newValue) + } + + outputChannel <- types.NewRecordAndContext(newrec, &inrecAndContext.Context) + } else { + outputChannel <- inrecAndContext + } +} + +// ---------------------------------------------------------------- +func (this *TransformerCleanWhitespace) cleanWhitespaceInKeys( + inrecAndContext *types.RecordAndContext, + outputChannel chan<- *types.RecordAndContext, +) { + if !inrecAndContext.EndOfStream { + newrec := types.NewMlrmapAsRecord() + + for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { + oldKey := types.MlrvalFromString(pe.Key) + newKey := types.MlrvalCleanWhitespace(&oldKey) + // Transferring ownership from old record to new record; no copy needed + newrec.PutReference(newKey.String(), pe.Value) + } + + outputChannel <- types.NewRecordAndContext(newrec, &inrecAndContext.Context) + } else { + outputChannel <- inrecAndContext + } +} + +// ---------------------------------------------------------------- +func (this *TransformerCleanWhitespace) cleanWhitespaceInValues( + inrecAndContext *types.RecordAndContext, + outputChannel chan<- *types.RecordAndContext, +) { + if !inrecAndContext.EndOfStream { + for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { + newValue := types.MlrvalCleanWhitespace(pe.Value) + pe.Value = &newValue + } + outputChannel <- inrecAndContext + } else { + outputChannel <- inrecAndContext + } +} diff --git a/go/todo.txt b/go/todo.txt index f57ec23c6..616acaeb7 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -187,9 +187,9 @@ no need to bootstrap a parser for the parser-generator language REMAINING VERBS: having-fields +-> regex dependency in order to complete it bar -clean-whitespace count-distinct format-values fraction