From 20dc5d3fa96f917d53e7ea6c1f573e1fc3f3442a Mon Sep 17 00:00:00 2001 From: John Kerl Date: Thu, 9 Mar 2017 22:06:26 -0500 Subject: [PATCH] strtok -> strmsep --- c/lib/mlrutil.c | 17 +++++++++++++++++ c/lib/mlrutil.h | 5 +++++ c/mapping/mapper_nest.c | 21 +++++++++++++++++---- c/output/lrec_writer_json.c | 7 +++++-- c/reg_test/expected/out | 14 ++++++++++---- c/todo.txt | 5 +++++ 6 files changed, 59 insertions(+), 10 deletions(-) diff --git a/c/lib/mlrutil.c b/c/lib/mlrutil.c index 155a9fe03..06890ebef 100644 --- a/c/lib/mlrutil.c +++ b/c/lib/mlrutil.c @@ -26,6 +26,23 @@ void mlr_internal_coding_error_unless(int v, char* file, int line) { } } +// ---------------------------------------------------------------- +char* mlr_strmsep(char **pstring, const char *sep, int seplen) { + char* string = *pstring; + if (string == NULL) { + return NULL; + } + char* pnext = strstr(string, sep); + if (pnext == NULL) { + *pstring = NULL; + return string; + } else { + *pnext = 0; + *pstring = pnext + seplen; + return string; + } +} + // ---------------------------------------------------------------- int mlr_bsearch_double_for_insert(double* array, int size, double value) { int lo = 0; diff --git a/c/lib/mlrutil.h b/c/lib/mlrutil.h index 65a792647..a7d3ad6cf 100644 --- a/c/lib/mlrutil.h +++ b/c/lib/mlrutil.h @@ -73,6 +73,11 @@ static inline int streqn(char* a, char* b, int n) { #endif } +// ---------------------------------------------------------------- +// Like strsep but the sep argument is a multi-character delimiter, +// not a set of single-character delimiters. +char* mlr_strmsep(char **pstring, const char *sep, int seplen); + // ---------------------------------------------------------------- int mlr_bsearch_double_for_insert(double* array, int size, double value); diff --git a/c/mapping/mapper_nest.c b/c/mapping/mapper_nest.c index bcd3b77b7..a57844e58 100644 --- a/c/mapping/mapper_nest.c +++ b/c/mapping/mapper_nest.c @@ -250,14 +250,18 @@ static sllv_t* mapper_nest_explode_values_across_fields(lrec_t* pinrec, context_ lrece_t* porig = pentry; char* sep = pstate->nested_fs; + int seplen = strlen(sep); int i = 1; - for (char* piece = strtok(field_value, sep); piece != NULL; piece = strtok(NULL, sep), i++) { + char* walker = field_value; + char* piece = NULL; + while ((piece = mlr_strmsep(&walker, sep, seplen)) != NULL) { char istring_free_flags; char* istring = low_int_to_string(i, &istring_free_flags); char* new_key = mlr_paste_3_strings(pstate->field_name, "_", istring); if (istring_free_flags & FREE_ENTRY_KEY) free(istring); pentry = lrec_put_after(pinrec, pentry, new_key, mlr_strdup_or_die(piece), FREE_ENTRY_KEY|FREE_ENTRY_VALUE); + i++; } lrec_unlink_and_free(pinrec, porig); return sllv_single(pinrec);; @@ -314,7 +318,10 @@ static sllv_t* mapper_nest_explode_values_across_records(lrec_t* pinrec, context sllv_t* poutrecs = sllv_alloc(); char* sep = pstate->nested_fs; - for (char* piece = strtok(field_value, sep); piece != NULL; piece = strtok(NULL, sep)) { + int seplen = strlen(sep); + char* walker = field_value; + char* piece = NULL; + while ((piece = mlr_strmsep(&walker, sep, seplen)) != NULL) { lrec_t* poutrec = lrec_copy(pinrec); lrec_put(poutrec, pstate->field_name, mlr_strdup_or_die(piece), FREE_ENTRY_VALUE); sllv_append(poutrecs, poutrec); @@ -400,7 +407,10 @@ static sllv_t* mapper_nest_explode_pairs_across_fields(lrec_t* pinrec, context_t lrece_t* porig = pentry; char* sep = pstate->nested_fs; - for (char* piece = strtok(field_value, sep); piece != NULL; piece = strtok(NULL, sep)) { + int seplen = strlen(sep); + char* walker = field_value; + char* piece = NULL; + while ((piece = mlr_strmsep(&walker, sep, seplen)) != NULL) { char* found_sep = strstr(piece, pstate->nested_ps); if (found_sep != NULL) { // there is a pair *found_sep = 0; @@ -430,7 +440,10 @@ static sllv_t* mapper_nest_explode_pairs_across_records(lrec_t* pinrec, context_ sllv_t* poutrecs = sllv_alloc(); char* sep = pstate->nested_fs; - for (char* piece = strtok(field_value, sep); piece != NULL; piece = strtok(NULL, sep)) { + int seplen = strlen(sep); + char* walker = field_value; + char* piece = NULL; + while ((piece = mlr_strmsep(&walker, sep, seplen)) != NULL) { char* found_sep = strstr(piece, pstate->nested_ps); lrec_t* poutrec = lrec_copy(pinrec); lrece_t* pe = NULL; diff --git a/c/output/lrec_writer_json.c b/c/output/lrec_writer_json.c index 0ac91654b..fea3d0458 100644 --- a/c/output/lrec_writer_json.c +++ b/c/output/lrec_writer_json.c @@ -117,15 +117,18 @@ static void lrec_writer_json_process(void* pvstate, FILE* output_stream, lrec_t* mlhmmv_root_t* pmap = mlhmmv_root_alloc(); char* sep = pstate->output_json_flatten_separator; + int seplen = strlen(sep); for (lrece_t* pe = prec->phead; pe != NULL; pe = pe->pnext) { - // strdup since strtok is destructive and CSV/PPRINT header fields + // strdup since strmsep is destructive and CSV/PPRINT header fields // are shared across multiple records char* lkey = mlr_strdup_or_die(pe->key); char* lvalue = pe->value; sllmv_t* pmvkeys = sllmv_alloc(); - for (char* piece = strtok(lkey, sep); piece != NULL; piece = strtok(NULL, sep)) { + char* walker = lkey; + char* piece = NULL; + while ((piece = mlr_strmsep(&walker, sep, seplen)) != NULL) { mv_t mvkey = mv_from_string(piece, NO_FREE); sllmv_append_no_free(pmvkeys, &mvkey); } diff --git a/c/reg_test/expected/out b/c/reg_test/expected/out index b2b8fd3eb..dfe77bb4c 100644 --- a/c/reg_test/expected/out +++ b/c/reg_test/expected/out @@ -3262,25 +3262,25 @@ x=a=4|b=5,y=d=70 mlr nest --explode --values --across-fields -f x ./reg_test/input/nest-explode.dkvp x_1=a:1,x_2=b:2,x_3=c:3,y=d:40 -y=d:50 +x_1=,y=d:50 u=100,y=d:60 x_1=a:4,x_2=b:5,y=d:70 mlr nest --explode --values --across-fields -f x --nested-fs pipe --nested-ps = ./reg_test/input/nest-explode-vary-fs-ps.dkvp x_1=a=1,x_2=b=2,x_3=c=3,y=d=40 -y=d=50 +x_1=,y=d=50 u=100,y=d=60 x_1=a=4,x_2=b=5,y=d=70 mlr nest --explode --values --across-fields -f x then nest --implode --values --across-fields -f x ./reg_test/input/nest-explode.dkvp x=a:1;b:2;c:3,y=d:40 -y=d:50 +x=,y=d:50 u=100,y=d:60 x=a:4;b:5,y=d:70 mlr nest --explode --values --across-fields -f x --nested-fs pipe --nested-ps = then nest --implode --values --across-fields -f x --nested-fs pipe --nested-ps = ./reg_test/input/nest-explode-vary-fs-ps.dkvp x=a=1|b=2|c=3,y=d=40 -y=d=50 +x=,y=d=50 u=100,y=d=60 x=a=4|b=5,y=d=70 @@ -3288,6 +3288,7 @@ mlr nest --explode --values --across-records -f x ./reg_test/input/nest-explode. x=a:1,y=d:40 x=b:2,y=d:40 x=c:3,y=d:40 +x=,y=d:50 u=100,y=d:60 x=a:4,y=d:70 x=b:5,y=d:70 @@ -3296,6 +3297,7 @@ mlr nest --explode --values --across-records -f x --nested-fs pipe --nested-ps = x=a=1,y=d=40 x=b=2,y=d=40 x=c=3,y=d=40 +x=,y=d=50 u=100,y=d=60 x=a=4,y=d=70 x=b=5,y=d=70 @@ -3303,11 +3305,13 @@ x=b=5,y=d=70 mlr nest --explode --values --across-records -f x then nest --implode --values --across-records -f x ./reg_test/input/nest-explode.dkvp u=100,y=d:60 x=a:1;b:2;c:3,y=d:40 +x=,y=d:50 x=a:4;b:5,y=d:70 mlr nest --explode --values --across-records -f x --nested-fs pipe --nested-ps = then nest --implode --values --across-records -f x --nested-fs pipe --nested-ps = ./reg_test/input/nest-explode-vary-fs-ps.dkvp u=100,y=d=60 x=a=1|b=2|c=3,y=d=40 +x=,y=d=50 x=a=4|b=5,y=d=70 mlr nest --explode --pairs --across-fields -f x ./reg_test/input/nest-explode.dkvp @@ -3326,6 +3330,7 @@ mlr nest --explode --pairs --across-records -f x ./reg_test/input/nest-explode.d a=1,y=d:40 b=2,y=d:40 c=3,y=d:40 +y=d:50 u=100,y=d:60 a=4,y=d:70 b=5,y=d:70 @@ -3334,6 +3339,7 @@ mlr nest --explode --pairs --across-records -f x --nested-fs pipe --nested-ps = a=1,y=d=40 b=2,y=d=40 c=3,y=d=40 +y=d=50 u=100,y=d=60 a=4,y=d=70 b=5,y=d=70 diff --git a/c/todo.txt b/c/todo.txt index 1a2ed19bb..ff45ac98a 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -15,10 +15,13 @@ BUGFIXES * xtab.wtf ? tsv->json blank line -- ? needs repro. +* nidx with repifs & leading spaces :^/ + ================================================================ 5.0.0 POST-RELEASE TO DO: https://github.com/johnkerl/miller/issues/132 +-> strmsep UTs ! cook: @@ -296,6 +299,8 @@ COMPARES: * mlr paste (sjackman) * dump @records after @records[NR]=$*: json "1" rather than 1 indices are b04k3d. make a cliopt. +* regexes for IPS/IFS/IRS; also in mlr nest etc. + !!?? disallow implicit assignment? force w/ var at least?? - w/ implicit: for (...) {sum += x} FUBAR - w/o implicit: sad panda cannot type 'x=1'