From dbef8bb0f8a658eccbd78345f5c0dc30c29f806e Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sun, 14 Oct 2018 11:49:33 -0400 Subject: [PATCH] clean-whitespace iterate --- c/cli/mlrcli.c | 1 + c/lib/mvfuncs.c | 7 +- c/mapping/Makefile.am | 1 + c/mapping/mapper_clean_whitespace.c | 171 ++++++++++++++++++++++++++++ c/mapping/mappers.h | 1 + 5 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 c/mapping/mapper_clean_whitespace.c diff --git a/c/cli/mlrcli.c b/c/cli/mlrcli.c index 9df3f7753..b29577b45 100644 --- a/c/cli/mlrcli.c +++ b/c/cli/mlrcli.c @@ -43,6 +43,7 @@ static mapper_setup_t* mapper_lookup_table[] = { &mapper_bootstrap_setup, &mapper_cat_setup, &mapper_check_setup, + &mapper_clean_whitespace_setup, &mapper_count_distinct_setup, &mapper_count_similar_setup, &mapper_cut_setup, diff --git a/c/lib/mvfuncs.c b/c/lib/mvfuncs.c index f3b46f4f5..52e9737aa 100644 --- a/c/lib/mvfuncs.c +++ b/c/lib/mvfuncs.c @@ -400,6 +400,7 @@ mv_t s_x_typeof_func(mv_t* pval1) { return rv; } +// ---------------------------------------------------------------- mv_t s_s_tolower_func(mv_t* pval1) { char* string = mlr_strdup_or_die(pval1->u.strv); for (char* c = string; *c; c++) @@ -420,6 +421,7 @@ mv_t s_s_toupper_func(mv_t* pval1) { return mv_from_string_with_free(string); } +// ---------------------------------------------------------------- mv_t s_s_lstrip_func(mv_t* pval1) { if (!isspace(pval1->u.strv[0])) { return *pval1; @@ -439,12 +441,14 @@ mv_t s_s_rstrip_func(mv_t* pval1) { while ((start <= last_non_space) && isspace(*last_non_space)) last_non_space--; if (last_non_space < start) { + mv_free(pval1); return mv_empty(); } else { int newlen = (last_non_space - start) + 1; char* retval = mlr_malloc_or_die(newlen + 1); memcpy(retval, start, newlen); - retval[newlen+1] = 0; + retval[newlen] = 0; + mv_free(pval1); return mv_from_string(retval, FREE_ENTRY_VALUE); } } @@ -469,6 +473,7 @@ mv_t s_s_collapse_whitespace_func(mv_t* pval1) { last_was_space = current_is_space; } *pdst = 0; + mv_free(pval1); return mv_from_string(retval, FREE_ENTRY_VALUE); } diff --git a/c/mapping/Makefile.am b/c/mapping/Makefile.am index 50b38e9b8..3d84af505 100644 --- a/c/mapping/Makefile.am +++ b/c/mapping/Makefile.am @@ -6,6 +6,7 @@ libmapping_la_SOURCES= \ mapper_bootstrap.c \ mapper_cat.c \ mapper_check.c \ + mapper_clean_whitespace.c \ mapper_count_similar.c \ mapper_cut.c \ mapper_decimate.c \ diff --git a/c/mapping/mapper_clean_whitespace.c b/c/mapping/mapper_clean_whitespace.c new file mode 100644 index 000000000..9f887ede9 --- /dev/null +++ b/c/mapping/mapper_clean_whitespace.c @@ -0,0 +1,171 @@ +#include "lib/mlrutil.h" +#include "lib/mvfuncs.h" +#include "containers/sllv.h" +#include "mapping/mappers.h" +#include "cli/argparse.h" + +#define RENAME_SB_ALLOC_LENGTH 16 + +typedef struct _mapper_clean_whitespace_state_t { + ap_state_t* pargp; + int do_keys; + int do_values; +} mapper_clean_whitespace_state_t; + +static void mapper_clean_whitespace_usage(FILE* o, char* argv0, char* verb); +static mapper_t* mapper_clean_whitespace_parse_cli(int* pargi, int argc, char** argv, + cli_reader_opts_t* _, cli_writer_opts_t* __); +static mapper_t* mapper_clean_whitespace_alloc(ap_state_t* pargp, int do_keys, int do_values); +static void mapper_clean_whitespace_free(mapper_t* pmapper, context_t* _); +static sllv_t* mapper_clean_whitespace_kvprocess(lrec_t* pinrec, context_t* pctx, void* pvstate); +static sllv_t* mapper_clean_whitespace_kprocess(lrec_t* pinrec, context_t* pctx, void* pvstate); +static sllv_t* mapper_clean_whitespace_vprocess(lrec_t* pinrec, context_t* pctx, void* pvstate); + +// ---------------------------------------------------------------- +mapper_setup_t mapper_clean_whitespace_setup = { + .verb = "clean-whitespace", + .pusage_func = mapper_clean_whitespace_usage, + .pparse_func = mapper_clean_whitespace_parse_cli, + .ignores_input = FALSE, +}; + +// ---------------------------------------------------------------- +static void mapper_clean_whitespace_usage(FILE* o, char* argv0, char* verb) { + fprintf(o, "Usage: %s %s [options] {old1,new1,old2,new2,...}\n", argv0, verb); + fprintf(o, "For each record, for each field in the record, whitespace-cleans the keys and\n"); + fprintf(o, "values. Whitespace-cleaning entails stripping leading and trailing whitespace,\n"); + fprintf(o, "and replacing multiple whitespace with singles. For finer-grained control,\n"); + fprintf(o, "please see the DSL functions lstrip, rstrip, strip, collapse_whitespace,\n"); + fprintf(o, "and clean_whitespace.\n"); + fprintf(o, "\n"); + fprintf(o, "Options:\n"); + fprintf(o, "-k|--keys-only Do not touch values.\n"); + fprintf(o, "-v|--values-only Do not touch keys.\n"); + fprintf(o, "It is an error to specify -k as well as -v.\n"); +} + +static mapper_t* mapper_clean_whitespace_parse_cli(int* pargi, int argc, char** argv, + cli_reader_opts_t* _, cli_writer_opts_t* __) +{ + int kflag = FALSE; + int vflag = FALSE; + + char* verb = argv[(*pargi)++]; + + ap_state_t* pstate = ap_alloc(); + ap_define_true_flag(pstate, "-k", &kflag); + ap_define_true_flag(pstate, "--keys-only", &kflag); + ap_define_true_flag(pstate, "-v", &vflag); + ap_define_true_flag(pstate, "--values-only", &vflag); + + if (!ap_parse(pstate, verb, pargi, argc, argv)) { + mapper_clean_whitespace_usage(stderr, argv[0], verb); + return NULL; + } + + int do_keys = TRUE; + int do_values = TRUE; + if (kflag && vflag) { + mapper_clean_whitespace_usage(stderr, argv[0], verb); + return NULL; + } else if (kflag) { + do_values = FALSE; + } else if (vflag) { + do_keys = FALSE; + } + + return mapper_clean_whitespace_alloc(pstate, do_keys, do_values); +} + +// ---------------------------------------------------------------- +static mapper_t* mapper_clean_whitespace_alloc(ap_state_t* pargp, int do_keys, int do_values) { + mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t)); + + mapper_clean_whitespace_state_t* pstate = mlr_malloc_or_die(sizeof(mapper_clean_whitespace_state_t)); + + pstate->pargp = pargp; + if (do_keys && do_values) { + pmapper->pprocess_func = mapper_clean_whitespace_kvprocess; + } else if (do_keys) { + pmapper->pprocess_func = mapper_clean_whitespace_kprocess; + } else if (do_values) { + pmapper->pprocess_func = mapper_clean_whitespace_vprocess; + } + pmapper->pfree_func = mapper_clean_whitespace_free; + + pmapper->pvstate = (void*)pstate; + return pmapper; +} + +static void mapper_clean_whitespace_free(mapper_t* pmapper, context_t* _) { + mapper_clean_whitespace_state_t* pstate = pmapper->pvstate; + ap_free(pstate->pargp); + free(pstate); + free(pmapper); +} + +// ---------------------------------------------------------------- +static sllv_t* mapper_clean_whitespace_kvprocess(lrec_t* pinrec, context_t* pctx, void* pvstate) { + if (pinrec != NULL) { + lrec_t* poutrec = lrec_unbacked_alloc(); + for (lrece_t* pe = pinrec->phead; pe != NULL; pe = pe->pnext) { + mv_t old_key = mv_from_string_with_free(mlr_strdup_or_die(pe->key)); + mv_t old_value = mv_from_string_with_free(mlr_strdup_or_die(pe->value)); + mv_t new_key = s_s_clean_whitespace_func(&old_key); + mv_t new_value = s_s_clean_whitespace_func(&old_value); + char free_flags = 0; + if (new_key.free_flags & FREE_ENTRY_VALUE) + free_flags |= FREE_ENTRY_KEY; + if (new_value.free_flags & FREE_ENTRY_VALUE) + free_flags |= FREE_ENTRY_VALUE; + lrec_put(poutrec, new_key.u.strv, new_value.u.strv, free_flags); + } + lrec_free(pinrec); + return sllv_single(poutrec); + } + else { + return sllv_single(NULL); + } +} + +// ---------------------------------------------------------------- +static sllv_t* mapper_clean_whitespace_kprocess(lrec_t* pinrec, context_t* pctx, void* pvstate) { + if (pinrec != NULL) { + lrec_t* poutrec = lrec_unbacked_alloc(); + for (lrece_t* pe = pinrec->phead; pe != NULL; pe = pe->pnext) { + mv_t old_key = mv_from_string_with_free(mlr_strdup_or_die(pe->key)); + mv_t value = mv_from_string_with_free(mlr_strdup_or_die(pe->value)); + mv_t new_key = s_s_clean_whitespace_func(&old_key); + char free_flags = FREE_ENTRY_VALUE; + if (new_key.free_flags & FREE_ENTRY_VALUE) + free_flags |= FREE_ENTRY_KEY; + lrec_put(poutrec, new_key.u.strv, value.u.strv, free_flags); + } + lrec_free(pinrec); + return sllv_single(poutrec); + } + else { + return sllv_single(NULL); + } +} + +// ---------------------------------------------------------------- +static sllv_t* mapper_clean_whitespace_vprocess(lrec_t* pinrec, context_t* pctx, void* pvstate) { + if (pinrec != NULL) { + lrec_t* poutrec = lrec_unbacked_alloc(); + for (lrece_t* pe = pinrec->phead; pe != NULL; pe = pe->pnext) { + mv_t key = mv_from_string_with_free(mlr_strdup_or_die(pe->key)); + mv_t old_value = mv_from_string_with_free(mlr_strdup_or_die(pe->value)); + mv_t new_value = s_s_clean_whitespace_func(&old_value); + char free_flags = FREE_ENTRY_KEY; + if (new_value.free_flags & FREE_ENTRY_VALUE) + free_flags |= FREE_ENTRY_VALUE; + lrec_put(poutrec, key.u.strv, new_value.u.strv, free_flags); + } + lrec_free(pinrec); + return sllv_single(poutrec); + } + else { + return sllv_single(NULL); + } +} diff --git a/c/mapping/mappers.h b/c/mapping/mappers.h index 7b26f0c14..0a88ad2e0 100644 --- a/c/mapping/mappers.h +++ b/c/mapping/mappers.h @@ -9,6 +9,7 @@ extern mapper_setup_t mapper_bar_setup; extern mapper_setup_t mapper_bootstrap_setup; extern mapper_setup_t mapper_cat_setup; extern mapper_setup_t mapper_check_setup; +extern mapper_setup_t mapper_clean_whitespace_setup; extern mapper_setup_t mapper_count_distinct_setup; extern mapper_setup_t mapper_count_similar_setup; extern mapper_setup_t mapper_cut_setup;