Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-25 17:04:01 +00:00
Code Issues Projects Releases Wiki Activity

mlr clean-whitespace

Browse source
This commit is contained in:
John Kerl 2021-01-25 00:06:14 -05:00
parent afce46f08e
commit f5553c095e
8 changed files with 1073 additions and 167 deletions
Download patch file Download diff file Expand all files Collapse all files

5
c/mapping/mapper_clean_whitespace.c
View file

@ -31,7 +31,7 @@ mapper_setup_t mapper_clean_whitespace_setup = {
// ----------------------------------------------------------------
static void mapper_clean_whitespace_usage(FILE* o, char* argv0, char* verb) {
fprintf(o, "Usage: %s %s [options] {old1,new1,old2,new2,...}\n", argv0, verb);
fprintf(o, "Usage: %s %s [options]\n", argv0, verb);
fprintf(o, "For each record, for each field in the record, whitespace-cleans the keys and\n");
fprintf(o, "values. Whitespace-cleaning entails stripping leading and trailing whitespace,\n");
fprintf(o, "and replacing multiple whitespace with singles. For finer-grained control,\n");
@ -41,7 +41,8 @@ static void mapper_clean_whitespace_usage(FILE* o, char* argv0, char* verb) {
fprintf(o, "Options:\n");
fprintf(o, "-k|--keys-only Do not touch values.\n");
fprintf(o, "-v|--values-only Do not touch keys.\n");
fprintf(o, "It is an error to specify -k as well as -v.\n");
fprintf(o, "It is an error to specify -k as well as -v -- to clean keys and values,\n");
fprintf(o, "leave off -k as well as -v.\n");
}
static mapper_t* mapper_clean_whitespace_parse_cli(int* pargi, int argc, char** argv,

1
go/cases-to-do.txt
View file

@ -30,7 +30,6 @@ rrv -C20 ./reg-test/cases/case-c-dsl-local-map-variable-write-read.sh
================================================================
PARTIALLY IMPLEMENTED:
rrv ./reg-test/cases/case-c-clean-whitespace.sh
rrv ./reg-test/cases/case-c-cut.sh
rrv ./reg-test/cases/case-c-dsl-scientific-notation.sh
rrv ./reg-test/cases/case-c-implicit-header-csv-input.sh

0
go/reg-test/cases/case-c-dsl-clean-whitespace.sh → go/reg-test/cases/case-dsl-clean-whitespace.sh
View file

163
go/reg-test/expected/case-c-dsl-clean-whitespace.sh.out
View file

@ -1,163 +0,0 @@
mlr --icsv --ojson cat ./reg-test/input/clean-whitespace.csv
{ "n": 1, "a": "xy", "b ": 2, " c": 3 }
{ "n": 2, "a": "xy ", "b ": 2, " c": 3 }
{ "n": 3, "a": "xy ", "b ": 2, " c": 3 }
{ "n": 4, "a": "xy ", "b ": 2, " c": 3 }
{ "n": 5, "a": "xy", "b ": 2, " c": 3 }
{ "n": 6, "a": " xy", "b ": 2, " c": 3 }
{ "n": 7, "a": " xy", "b ": 2, " c": 3 }
{ "n": 8, "a": " xy", "b ": 2, " c": 3 }
{ "n": 9, "a": "xy", "b ": 2, " c": 3 }
{ "n": 10, "a": " xy ", "b ": 2, " c": 3 }
{ "n": 11, "a": " xy ", "b ": 2, " c": 3 }
{ "n": 12, "a": " xy ", "b ": 2, " c": 3 }
{ "n": 13, "a": "", "b ": 2, " c": 3 }
{ "n": 14, "a": " ", "b ": 2, " c": 3 }
{ "n": 15, "a": " ", "b ": 2, " c": 3 }
{ "n": 16, "a": " ", "b ": 2, " c": 3 }
mlr --icsv --ojson put $a = lstrip($a) ./reg-test/input/clean-whitespace.csv
{ "n": 1, "a": "xy", "b ": 2, " c": 3 }
{ "n": 2, "a": "xy ", "b ": 2, " c": 3 }
{ "n": 3, "a": "xy ", "b ": 2, " c": 3 }
{ "n": 4, "a": "xy ", "b ": 2, " c": 3 }
{ "n": 5, "a": "xy", "b ": 2, " c": 3 }
{ "n": 6, "a": "xy", "b ": 2, " c": 3 }
{ "n": 7, "a": "xy", "b ": 2, " c": 3 }
{ "n": 8, "a": "xy", "b ": 2, " c": 3 }
{ "n": 9, "a": "xy", "b ": 2, " c": 3 }
{ "n": 10, "a": "xy ", "b ": 2, " c": 3 }
{ "n": 11, "a": "xy ", "b ": 2, " c": 3 }
{ "n": 12, "a": "xy ", "b ": 2, " c": 3 }
{ "n": 13, "a": "", "b ": 2, " c": 3 }
{ "n": 14, "a": "", "b ": 2, " c": 3 }
{ "n": 15, "a": "", "b ": 2, " c": 3 }
{ "n": 16, "a": "", "b ": 2, " c": 3 }
mlr --icsv --ojson put $a = rstrip($a) ./reg-test/input/clean-whitespace.csv
{ "n": 1, "a": "xy", "b ": 2, " c": 3 }
{ "n": 2, "a": "xy", "b ": 2, " c": 3 }
{ "n": 3, "a": "xy", "b ": 2, " c": 3 }
{ "n": 4, "a": "xy", "b ": 2, " c": 3 }
{ "n": 5, "a": "xy", "b ": 2, " c": 3 }
{ "n": 6, "a": " xy", "b ": 2, " c": 3 }
{ "n": 7, "a": " xy", "b ": 2, " c": 3 }
{ "n": 8, "a": " xy", "b ": 2, " c": 3 }
{ "n": 9, "a": "xy", "b ": 2, " c": 3 }
{ "n": 10, "a": " xy", "b ": 2, " c": 3 }
{ "n": 11, "a": " xy", "b ": 2, " c": 3 }
{ "n": 12, "a": " xy", "b ": 2, " c": 3 }
{ "n": 13, "a": "", "b ": 2, " c": 3 }
{ "n": 14, "a": "", "b ": 2, " c": 3 }
{ "n": 15, "a": "", "b ": 2, " c": 3 }
{ "n": 16, "a": "", "b ": 2, " c": 3 }
mlr --icsv --ojson put $a = strip($a) ./reg-test/input/clean-whitespace.csv
{ "n": 1, "a": "xy", "b ": 2, " c": 3 }
{ "n": 2, "a": "xy", "b ": 2, " c": 3 }
{ "n": 3, "a": "xy", "b ": 2, " c": 3 }
{ "n": 4, "a": "xy", "b ": 2, " c": 3 }
{ "n": 5, "a": "xy", "b ": 2, " c": 3 }
{ "n": 6, "a": "xy", "b ": 2, " c": 3 }
{ "n": 7, "a": "xy", "b ": 2, " c": 3 }
{ "n": 8, "a": "xy", "b ": 2, " c": 3 }
{ "n": 9, "a": "xy", "b ": 2, " c": 3 }
{ "n": 10, "a": "xy", "b ": 2, " c": 3 }
{ "n": 11, "a": "xy", "b ": 2, " c": 3 }
{ "n": 12, "a": "xy", "b ": 2, " c": 3 }
{ "n": 13, "a": "", "b ": 2, " c": 3 }
{ "n": 14, "a": "", "b ": 2, " c": 3 }
{ "n": 15, "a": "", "b ": 2, " c": 3 }
{ "n": 16, "a": "", "b ": 2, " c": 3 }
mlr --icsv --ojson put $a = collapse_whitespace($a) ./reg-test/input/clean-whitespace.csv
{ "n": 1, "a": "xy", "b ": 2, " c": 3 }
{ "n": 2, "a": "xy ", "b ": 2, " c": 3 }
{ "n": 3, "a": "xy ", "b ": 2, " c": 3 }
{ "n": 4, "a": "xy ", "b ": 2, " c": 3 }
{ "n": 5, "a": "xy", "b ": 2, " c": 3 }
{ "n": 6, "a": " xy", "b ": 2, " c": 3 }
{ "n": 7, "a": " xy", "b ": 2, " c": 3 }
{ "n": 8, "a": " xy", "b ": 2, " c": 3 }
{ "n": 9, "a": "xy", "b ": 2, " c": 3 }
{ "n": 10, "a": " xy ", "b ": 2, " c": 3 }
{ "n": 11, "a": " xy ", "b ": 2, " c": 3 }
{ "n": 12, "a": " xy ", "b ": 2, " c": 3 }
{ "n": 13, "a": "", "b ": 2, " c": 3 }
{ "n": 14, "a": " ", "b ": 2, " c": 3 }
{ "n": 15, "a": " ", "b ": 2, " c": 3 }
{ "n": 16, "a": " ", "b ": 2, " c": 3 }
mlr --icsv --ojson put $a = clean_whitespace($a) ./reg-test/input/clean-whitespace.csv
{ "n": 1, "a": "xy", "b ": 2, " c": 3 }
{ "n": 2, "a": "xy", "b ": 2, " c": 3 }
{ "n": 3, "a": "xy", "b ": 2, " c": 3 }
{ "n": 4, "a": "xy", "b ": 2, " c": 3 }
{ "n": 5, "a": "xy", "b ": 2, " c": 3 }
{ "n": 6, "a": "xy", "b ": 2, " c": 3 }
{ "n": 7, "a": "xy", "b ": 2, " c": 3 }
{ "n": 8, "a": "xy", "b ": 2, " c": 3 }
{ "n": 9, "a": "xy", "b ": 2, " c": 3 }
{ "n": 10, "a": "xy", "b ": 2, " c": 3 }
{ "n": 11, "a": "xy", "b ": 2, " c": 3 }
{ "n": 12, "a": "xy", "b ": 2, " c": 3 }
{ "n": 13, "a": "", "b ": 2, " c": 3 }
{ "n": 14, "a": "", "b ": 2, " c": 3 }
{ "n": 15, "a": "", "b ": 2, " c": 3 }
{ "n": 16, "a": "", "b ": 2, " c": 3 }
mlr --icsv --ojson clean-whitespace -k ./reg-test/input/clean-whitespace.csv
{ "n": 1, "a": "xy", "b": 2, "c": 3 }
{ "n": 2, "a": "xy ", "b": 2, "c": 3 }
{ "n": 3, "a": "xy ", "b": 2, "c": 3 }
{ "n": 4, "a": "xy ", "b": 2, "c": 3 }
{ "n": 5, "a": "xy", "b": 2, "c": 3 }
{ "n": 6, "a": " xy", "b": 2, "c": 3 }
{ "n": 7, "a": " xy", "b": 2, "c": 3 }
{ "n": 8, "a": " xy", "b": 2, "c": 3 }
{ "n": 9, "a": "xy", "b": 2, "c": 3 }
{ "n": 10, "a": " xy ", "b": 2, "c": 3 }
{ "n": 11, "a": " xy ", "b": 2, "c": 3 }
{ "n": 12, "a": " xy ", "b": 2, "c": 3 }
{ "n": 13, "a": "", "b": 2, "c": 3 }
{ "n": 14, "a": " ", "b": 2, "c": 3 }
{ "n": 15, "a": " ", "b": 2, "c": 3 }
{ "n": 16, "a": " ", "b": 2, "c": 3 }
mlr --icsv --ojson clean-whitespace -v ./reg-test/input/clean-whitespace.csv
{ "n": 1, "a": "xy", "b ": 2, " c": 3 }
{ "n": 2, "a": "xy", "b ": 2, " c": 3 }
{ "n": 3, "a": "xy", "b ": 2, " c": 3 }
{ "n": 4, "a": "xy", "b ": 2, " c": 3 }
{ "n": 5, "a": "xy", "b ": 2, " c": 3 }
{ "n": 6, "a": "xy", "b ": 2, " c": 3 }
{ "n": 7, "a": "xy", "b ": 2, " c": 3 }
{ "n": 8, "a": "xy", "b ": 2, " c": 3 }
{ "n": 9, "a": "xy", "b ": 2, " c": 3 }
{ "n": 10, "a": "xy", "b ": 2, " c": 3 }
{ "n": 11, "a": "xy", "b ": 2, " c": 3 }
{ "n": 12, "a": "xy", "b ": 2, " c": 3 }
{ "n": 13, "a": "", "b ": 2, " c": 3 }
{ "n": 14, "a": "", "b ": 2, " c": 3 }
{ "n": 15, "a": "", "b ": 2, " c": 3 }
{ "n": 16, "a": "", "b ": 2, " c": 3 }
mlr --icsv --ojson clean-whitespace ./reg-test/input/clean-whitespace.csv
{ "n": 1, "a": "xy", "b": 2, "c": 3 }
{ "n": 2, "a": "xy", "b": 2, "c": 3 }
{ "n": 3, "a": "xy", "b": 2, "c": 3 }
{ "n": 4, "a": "xy", "b": 2, "c": 3 }
{ "n": 5, "a": "xy", "b": 2, "c": 3 }
{ "n": 6, "a": "xy", "b": 2, "c": 3 }
{ "n": 7, "a": "xy", "b": 2, "c": 3 }
{ "n": 8, "a": "xy", "b": 2, "c": 3 }
{ "n": 9, "a": "xy", "b": 2, "c": 3 }
{ "n": 10, "a": "xy", "b": 2, "c": 3 }
{ "n": 11, "a": "xy", "b": 2, "c": 3 }
{ "n": 12, "a": "xy", "b": 2, "c": 3 }
{ "n": 13, "a": "", "b": 2, "c": 3 }
{ "n": 14, "a": "", "b": 2, "c": 3 }
{ "n": 15, "a": "", "b": 2, "c": 3 }
{ "n": 16, "a": "", "b": 2, "c": 3 }

883
go/reg-test/expected/case-dsl-clean-whitespace.sh.out Normal file
View file

@ -0,0 +1,883 @@
mlr --icsv --ojson cat ./reg-test/input/clean-whitespace.csv
{
"n": 1,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 2,
"a": "xy ",
"b ": 2,
" c": 3
}
{
"n": 3,
"a": "xy ",
"b ": 2,
" c": 3
}
{
"n": 4,
"a": "xy ",
"b ": 2,
" c": 3
}
{
"n": 5,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 6,
"a": " xy",
"b ": 2,
" c": 3
}
{
"n": 7,
"a": " xy",
"b ": 2,
" c": 3
}
{
"n": 8,
"a": " xy",
"b ": 2,
" c": 3
}
{
"n": 9,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 10,
"a": " xy ",
"b ": 2,
" c": 3
}
{
"n": 11,
"a": " xy ",
"b ": 2,
" c": 3
}
{
"n": 12,
"a": " xy ",
"b ": 2,
" c": 3
}
{
"n": 13,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 14,
"a": " ",
"b ": 2,
" c": 3
}
{
"n": 15,
"a": " ",
"b ": 2,
" c": 3
}
{
"n": 16,
"a": " ",
"b ": 2,
" c": 3
}
mlr --icsv --ojson put $a = lstrip($a) ./reg-test/input/clean-whitespace.csv
{
"n": 1,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 2,
"a": "xy ",
"b ": 2,
" c": 3
}
{
"n": 3,
"a": "xy ",
"b ": 2,
" c": 3
}
{
"n": 4,
"a": "xy ",
"b ": 2,
" c": 3
}
{
"n": 5,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 6,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 7,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 8,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 9,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 10,
"a": "xy ",
"b ": 2,
" c": 3
}
{
"n": 11,
"a": "xy ",
"b ": 2,
" c": 3
}
{
"n": 12,
"a": "xy ",
"b ": 2,
" c": 3
}
{
"n": 13,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 14,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 15,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 16,
"a": "",
"b ": 2,
" c": 3
}
mlr --icsv --ojson put $a = rstrip($a) ./reg-test/input/clean-whitespace.csv
{
"n": 1,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 2,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 3,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 4,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 5,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 6,
"a": " xy",
"b ": 2,
" c": 3
}
{
"n": 7,
"a": " xy",
"b ": 2,
" c": 3
}
{
"n": 8,
"a": " xy",
"b ": 2,
" c": 3
}
{
"n": 9,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 10,
"a": " xy",
"b ": 2,
" c": 3
}
{
"n": 11,
"a": " xy",
"b ": 2,
" c": 3
}
{
"n": 12,
"a": " xy",
"b ": 2,
" c": 3
}
{
"n": 13,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 14,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 15,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 16,
"a": "",
"b ": 2,
" c": 3
}
mlr --icsv --ojson put $a = strip($a) ./reg-test/input/clean-whitespace.csv
{
"n": 1,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 2,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 3,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 4,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 5,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 6,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 7,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 8,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 9,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 10,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 11,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 12,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 13,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 14,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 15,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 16,
"a": "",
"b ": 2,
" c": 3
}
mlr --icsv --ojson put $a = collapse_whitespace($a) ./reg-test/input/clean-whitespace.csv
{
"n": 1,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 2,
"a": "xy ",
"b ": 2,
" c": 3
}
{
"n": 3,
"a": "xy ",
"b ": 2,
" c": 3
}
{
"n": 4,
"a": "xy ",
"b ": 2,
" c": 3
}
{
"n": 5,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 6,
"a": " xy",
"b ": 2,
" c": 3
}
{
"n": 7,
"a": " xy",
"b ": 2,
" c": 3
}
{
"n": 8,
"a": " xy",
"b ": 2,
" c": 3
}
{
"n": 9,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 10,
"a": " xy ",
"b ": 2,
" c": 3
}
{
"n": 11,
"a": " xy ",
"b ": 2,
" c": 3
}
{
"n": 12,
"a": " xy ",
"b ": 2,
" c": 3
}
{
"n": 13,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 14,
"a": " ",
"b ": 2,
" c": 3
}
{
"n": 15,
"a": " ",
"b ": 2,
" c": 3
}
{
"n": 16,
"a": " ",
"b ": 2,
" c": 3
}
mlr --icsv --ojson put $a = clean_whitespace($a) ./reg-test/input/clean-whitespace.csv
{
"n": 1,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 2,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 3,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 4,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 5,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 6,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 7,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 8,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 9,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 10,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 11,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 12,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 13,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 14,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 15,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 16,
"a": "",
"b ": 2,
" c": 3
}
mlr --icsv --ojson clean-whitespace -k ./reg-test/input/clean-whitespace.csv
{
"n": 1,
"a": "xy",
"b": 2,
"c": 3
}
{
"n": 2,
"a": "xy ",
"b": 2,
"c": 3
}
{
"n": 3,
"a": "xy ",
"b": 2,
"c": 3
}
{
"n": 4,
"a": "xy ",
"b": 2,
"c": 3
}
{
"n": 5,
"a": "xy",
"b": 2,
"c": 3
}
{
"n": 6,
"a": " xy",
"b": 2,
"c": 3
}
{
"n": 7,
"a": " xy",
"b": 2,
"c": 3
}
{
"n": 8,
"a": " xy",
"b": 2,
"c": 3
}
{
"n": 9,
"a": "xy",
"b": 2,
"c": 3
}
{
"n": 10,
"a": " xy ",
"b": 2,
"c": 3
}
{
"n": 11,
"a": " xy ",
"b": 2,
"c": 3
}
{
"n": 12,
"a": " xy ",
"b": 2,
"c": 3
}
{
"n": 13,
"a": "",
"b": 2,
"c": 3
}
{
"n": 14,
"a": " ",
"b": 2,
"c": 3
}
{
"n": 15,
"a": " ",
"b": 2,
"c": 3
}
{
"n": 16,
"a": " ",
"b": 2,
"c": 3
}
mlr --icsv --ojson clean-whitespace -v ./reg-test/input/clean-whitespace.csv
{
"n": 1,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 2,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 3,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 4,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 5,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 6,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 7,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 8,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 9,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 10,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 11,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 12,
"a": "xy",
"b ": 2,
" c": 3
}
{
"n": 13,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 14,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 15,
"a": "",
"b ": 2,
" c": 3
}
{
"n": 16,
"a": "",
"b ": 2,
" c": 3
}
mlr --icsv --ojson clean-whitespace ./reg-test/input/clean-whitespace.csv
{
"n": 1,
"a": "xy",
"b": 2,
"c": 3
}
{
"n": 2,
"a": "xy",
"b": 2,
"c": 3
}
{
"n": 3,
"a": "xy",
"b": 2,
"c": 3
}
{
"n": 4,
"a": "xy",
"b": 2,
"c": 3
}
{
"n": 5,
"a": "xy",
"b": 2,
"c": 3
}
{
"n": 6,
"a": "xy",
"b": 2,
"c": 3
}
{
"n": 7,
"a": "xy",
"b": 2,
"c": 3
}
{
"n": 8,
"a": "xy",
"b": 2,
"c": 3
}
{
"n": 9,
"a": "xy",
"b": 2,
"c": 3
}
{
"n": 10,
"a": "xy",
"b": 2,
"c": 3
}
{
"n": 11,
"a": "xy",
"b": 2,
"c": 3
}
{
"n": 12,
"a": "xy",
"b": 2,
"c": 3
}
{
"n": 13,
"a": "",
"b": 2,
"c": 3
}
{
"n": 14,
"a": "",
"b": 2,
"c": 3
}
{
"n": 15,
"a": "",
"b": 2,
"c": 3
}
{
"n": 16,
"a": "",
"b": 2,
"c": 3
}

1
go/src/miller/cli/mlrcli_transformers.go
View file

@ -15,6 +15,7 @@ var MAPPER_LOOKUP_TABLE = []transforming.TransformerSetup{
transformers.BootstrapSetup,
transformers.CatSetup,
transformers.CheckSetup,
transformers.CleanWhitespaceSetup,
transformers.CountSetup,
transformers.CountSimilarSetup,
transformers.CutSetup,

185
go/src/miller/transformers/clean-whitespace.go Normal file
View file

@ -0,0 +1,185 @@
package transformers
import (
"flag"
"fmt"
"os"
"strings"
"miller/clitypes"
"miller/transforming"
"miller/types"
)
// ----------------------------------------------------------------
var CleanWhitespaceSetup = transforming.TransformerSetup{
Verb: "clean-whitespace",
ParseCLIFunc: transformerCleanWhitespaceParseCLI,
IgnoresInput: false,
}
func transformerCleanWhitespaceParseCLI(
pargi *int,
argc int,
args []string,
errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError
_ *clitypes.TReaderOptions,
__ *clitypes.TWriterOptions,
) transforming.IRecordTransformer {
doKeys := true
doValues := true
// Get the verb name from the current spot in the mlr command line
argi := *pargi
verb := args[argi]
argi++
for argi < argc /* variable increment: 1 or 2 depending on flag */ {
if !strings.HasPrefix(args[argi], "-") {
break // No more flag options to process
} else if args[argi] == "-h" || args[argi] == "--help" {
transformerPutUsage(os.Stdout, 0, errorHandling, args[0], verb)
return nil // help intentionally requested
} else if args[argi] == "-k" || args[argi] == "--keys-only" {
doKeys = true
doValues = false
argi++
} else if args[argi] == "-v" || args[argi] == "--values-only" {
doKeys = false
doValues = true
argi++
} else {
transformerPutUsage(os.Stderr, 1, flag.ExitOnError, args[0], verb)
os.Exit(1)
}
}
if !doKeys && !doValues {
transformerCleanWhitespaceUsage(os.Stderr, 1, flag.ExitOnError, args[0], verb)
os.Exit(1)
}
transformer, _ := NewTransformerCleanWhitespace(
doKeys,
doValues,
)
*pargi = argi
return transformer
}
func transformerCleanWhitespaceUsage(
o *os.File,
exitCode int,
errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError
argv0 string,
verb string,
) {
fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb)
fmt.Fprintf(o, "For each record, for each field in the record, whitespace-cleans the keys and/or\n")
fmt.Fprintf(o, "values. Whitespace-cleaning entails stripping leading and trailing whitespace,\n")
fmt.Fprintf(o, "and replacing multiple whitespace with singles. For finer-grained control,\n")
fmt.Fprintf(o, "please see the DSL functions lstrip, rstrip, strip, collapse_whitespace,\n")
fmt.Fprintf(o, "and clean_whitespace.\n")
fmt.Fprintf(o, "\n")
fmt.Fprintf(o, "Options:\n")
fmt.Fprintf(o, "-k|--keys-only Do not touch values.\n")
fmt.Fprintf(o, "-v|--values-only Do not touch keys.\n")
fmt.Fprintf(o, "It is an error to specify -k as well as -v -- to clean keys and values,\n")
fmt.Fprintf(o, "leave off -k as well as -v.\n")
}
// ----------------------------------------------------------------
type TransformerCleanWhitespace struct {
recordTransformerFunc transforming.RecordTransformerFunc
}
// ----------------------------------------------------------------
func NewTransformerCleanWhitespace(
doKeys bool,
doValues bool,
) (*TransformerCleanWhitespace, error) {
this := &TransformerCleanWhitespace{}
if doKeys && doValues {
this.recordTransformerFunc = this.cleanWhitespaceInKeysAndValues
} else if doKeys {
this.recordTransformerFunc = this.cleanWhitespaceInKeys
} else {
this.recordTransformerFunc = this.cleanWhitespaceInValues
}
return this, nil
}
// ----------------------------------------------------------------
func (this *TransformerCleanWhitespace) Transform(
inrecAndContext *types.RecordAndContext,
outputChannel chan<- *types.RecordAndContext,
) {
this.recordTransformerFunc(inrecAndContext, outputChannel)
}
// ----------------------------------------------------------------
func (this *TransformerCleanWhitespace) cleanWhitespaceInKeysAndValues(
inrecAndContext *types.RecordAndContext,
outputChannel chan<- *types.RecordAndContext,
) {
if !inrecAndContext.EndOfStream {
newrec := types.NewMlrmapAsRecord()
for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next {
oldKey := types.MlrvalFromString(pe.Key)
newKey := types.MlrvalCleanWhitespace(&oldKey)
newValue := types.MlrvalCleanWhitespace(pe.Value)
// Transferring ownership from old record to new record; no copy needed
newrec.PutReference(newKey.String(), &newValue)
}
outputChannel <- types.NewRecordAndContext(newrec, &inrecAndContext.Context)
} else {
outputChannel <- inrecAndContext
}
}
// ----------------------------------------------------------------
func (this *TransformerCleanWhitespace) cleanWhitespaceInKeys(
inrecAndContext *types.RecordAndContext,
outputChannel chan<- *types.RecordAndContext,
) {
if !inrecAndContext.EndOfStream {
newrec := types.NewMlrmapAsRecord()
for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next {
oldKey := types.MlrvalFromString(pe.Key)
newKey := types.MlrvalCleanWhitespace(&oldKey)
// Transferring ownership from old record to new record; no copy needed
newrec.PutReference(newKey.String(), pe.Value)
}
outputChannel <- types.NewRecordAndContext(newrec, &inrecAndContext.Context)
} else {
outputChannel <- inrecAndContext
}
}
// ----------------------------------------------------------------
func (this *TransformerCleanWhitespace) cleanWhitespaceInValues(
inrecAndContext *types.RecordAndContext,
outputChannel chan<- *types.RecordAndContext,
) {
if !inrecAndContext.EndOfStream {
for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next {
newValue := types.MlrvalCleanWhitespace(pe.Value)
pe.Value = &newValue
}
outputChannel <- inrecAndContext
} else {
outputChannel <- inrecAndContext
}
}

2
go/todo.txt
View file

@ -187,9 +187,9 @@ no need to bootstrap a parser for the parser-generator language
REMAINING VERBS:
having-fields
-> regex dependency in order to complete it
bar
clean-whitespace
count-distinct
format-values
fraction