From dbf34e73ea4adfeb0b7ad510f87e53eb07bede7c Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 15 Feb 2016 15:42:00 -0500 Subject: [PATCH] mlr nest iterate --- c/mapping/mapper_nest.c | 86 ++------------------------------ c/todo.txt | 108 +++++++++++++++++++--------------------- 2 files changed, 55 insertions(+), 139 deletions(-) diff --git a/c/mapping/mapper_nest.c b/c/mapping/mapper_nest.c index 3cf266d14..1209e7f39 100644 --- a/c/mapping/mapper_nest.c +++ b/c/mapping/mapper_nest.c @@ -1,5 +1,4 @@ #include "lib/mlrutil.h" -#include "lib/mlrregex.h" #include "lib/string_builder.h" #include "containers/lhmss.h" #include "containers/sllv.h" @@ -39,15 +38,8 @@ typedef struct _mapper_nest_state_t { ap_state_t* pargp; - // -e/-i/--explode/--implode: required: -- via process func - // --values/--pairs: required: - // --fields/--records: streaming & non - // --nested-fs: required - // --nested-fp: required - // for wide-to-long: slls_t* input_field_names; - sllv_t* input_field_regexes; char* output_key_field_name; char* output_value_field_name; @@ -67,14 +59,12 @@ static mapper_t* mapper_nest_parse_cli(int* pargi, int argc, char** argv); static mapper_t* mapper_nest_alloc( ap_state_t* pargp, slls_t* input_field_names, - slls_t* input_field_regex_strings, char* output_key_field_name, char* output_value_field_name, char* split_out_key_field_name, char* split_out_value_field_name); static void mapper_nest_free(mapper_t* pmapper); static sllv_t* mapper_nest_wide_to_long_no_regex_process(lrec_t* pinrec, context_t* pctx, void* pvstate); -static sllv_t* mapper_nest_wide_to_long_regex_process(lrec_t* pinrec, context_t* pctx, void* pvstate); static sllv_t* mapper_nest_long_to_wide_process(lrec_t* pinrec, context_t* pctx, void* pvstate); static nest_bucket_t* nest_bucket_alloc(lrec_t* prepresentative); @@ -93,7 +83,6 @@ static void mapper_nest_usage(FILE* o, char* argv0, char* verb) { fprintf(o, "-- xxx temp in-progress copy from reshape --\n"); fprintf(o, "Wide-to-long options:\n"); fprintf(o, " -i {input field names}\n"); - fprintf(o, " -r {input field regexes}\n"); fprintf(o, " These pivot/nest the input data such that the input fields are removed\n"); fprintf(o, " and separate records are emitted for each key/value pair.\n"); fprintf(o, " Note: this works with tail -f and produces output records for each input\n"); @@ -148,7 +137,6 @@ static void mapper_nest_usage(FILE* o, char* argv0, char* verb) { static mapper_t* mapper_nest_parse_cli(int* pargi, int argc, char** argv) { slls_t* input_field_names = NULL; - slls_t* input_field_regex_strings = NULL; slls_t* output_field_names = NULL; slls_t* split_out_field_names = NULL; @@ -156,7 +144,6 @@ static mapper_t* mapper_nest_parse_cli(int* pargi, int argc, char** argv) { ap_state_t* pstate = ap_alloc(); ap_define_string_list_flag(pstate, "-i", &input_field_names); - ap_define_string_list_flag(pstate, "-r", &input_field_regex_strings); ap_define_string_list_flag(pstate, "-o", &output_field_names); ap_define_string_list_flag(pstate, "-s", &split_out_field_names); @@ -172,7 +159,7 @@ static mapper_t* mapper_nest_parse_cli(int* pargi, int argc, char** argv) { if (split_out_field_names == NULL) { // wide to long - if (input_field_names == NULL && input_field_regex_strings == NULL) { + if (input_field_names == NULL) { mapper_nest_usage(stderr, argv[0], verb); return NULL; } @@ -199,7 +186,7 @@ static mapper_t* mapper_nest_parse_cli(int* pargi, int argc, char** argv) { slls_free(split_out_field_names); } - return mapper_nest_alloc(pstate, input_field_names, input_field_regex_strings, + return mapper_nest_alloc(pstate, input_field_names, output_key_field_name, output_value_field_name, split_out_key_field_name, split_out_value_field_name); } @@ -208,7 +195,6 @@ static mapper_t* mapper_nest_parse_cli(int* pargi, int argc, char** argv) { static mapper_t* mapper_nest_alloc( ap_state_t* pargp, slls_t* input_field_names, - slls_t* input_field_regex_strings, char* output_key_field_name, char* output_value_field_name, char* split_out_key_field_name, @@ -225,23 +211,8 @@ static mapper_t* mapper_nest_alloc( pstate->split_out_key_field_name = split_out_key_field_name; pstate->split_out_value_field_name = split_out_value_field_name; - if (input_field_regex_strings == NULL) { - pstate->input_field_regexes = NULL; - } else { - pstate->input_field_regexes = sllv_alloc(); - for (sllse_t* pe = input_field_regex_strings->phead; pe != NULL; pe = pe->pnext) { - regex_t* pregex = mlr_malloc_or_die(sizeof(regex_t)); - regcomp_or_die(pregex, pe->value, 0); - sllv_append(pstate->input_field_regexes, pregex); - } - slls_free(input_field_regex_strings); - } - if (split_out_key_field_name == NULL) { - if (pstate->input_field_regexes == NULL) - pmapper->pprocess_func = mapper_nest_wide_to_long_no_regex_process; - else - pmapper->pprocess_func = mapper_nest_wide_to_long_regex_process; + pmapper->pprocess_func = mapper_nest_wide_to_long_no_regex_process; pstate->other_keys_to_other_values_to_buckets = NULL; } else { pmapper->pprocess_func = mapper_nest_long_to_wide_process; @@ -265,14 +236,6 @@ static void mapper_nest_free(mapper_t* pmapper) { free(pstate->split_out_key_field_name); free(pstate->split_out_value_field_name); - if (pstate->input_field_regexes != NULL) { - for (sllve_t* pe = pstate->input_field_regexes->phead; pe != NULL; pe = pe->pnext) { - regex_t* pregex = pe->pvvalue; - regfree(pregex); - } - sllv_free(pstate->input_field_regexes); - } - if (pstate->other_keys_to_other_values_to_buckets != NULL) { for (lhmslve_t* pe = pstate->other_keys_to_other_values_to_buckets->phead; pe != NULL; pe = pe->pnext) { lhmslv_t* other_values_to_buckets = pe->pvvalue; @@ -329,49 +292,6 @@ static sllv_t* mapper_nest_wide_to_long_no_regex_process(lrec_t* pinrec, context return poutrecs; } -// ---------------------------------------------------------------- -static sllv_t* mapper_nest_wide_to_long_regex_process(lrec_t* pinrec, context_t* pctx, void* pvstate) { - if (pinrec == NULL) // End of input stream - return sllv_single(NULL); - - mapper_nest_state_t* pstate = (mapper_nest_state_t*)pvstate; - - sllv_t* poutrecs = sllv_alloc(); - lhmss_t* pairs = lhmss_alloc(); - - for (lrece_t* pe = pinrec->phead; pe != NULL; pe = pe->pnext) { - for (sllve_t* pf = pstate->input_field_regexes->phead; pf != NULL; pf = pf->pnext) { - regex_t* pregex = pf->pvvalue; - if (regmatch_or_die(pregex, pe->key, 0, NULL)) { - // Ownership-transfer of the about-to-be-freed key-value pairs from lrec to lhmss - lhmss_put(pairs, pe->key, pe->value, pe->free_flags); - pe->free_flags = NO_FREE; - break; - } - } - } - - // Unset the lrec keys after iterating over them, rather than during - for (lhmsse_t* pg = pairs->phead; pg != NULL; pg = pg->pnext) - lrec_remove(pinrec, pg->key); - - if (pairs->num_occupied == 0) { - sllv_append(poutrecs, pinrec); - } else { - for (lhmsse_t* pf = pairs->phead; pf != NULL; pf = pf->pnext) { - lrec_t* poutrec = lrec_copy(pinrec); - lrec_put(poutrec, pstate->output_key_field_name, mlr_strdup_or_die(pf->key), FREE_ENTRY_VALUE); - lrec_put(poutrec, pstate->output_value_field_name, mlr_strdup_or_die(pf->value), FREE_ENTRY_VALUE); - sllv_append(poutrecs, poutrec); - } - lrec_free(pinrec); - } - - lhmss_free(pairs); - - return poutrecs; -} - // ---------------------------------------------------------------- static sllv_t* mapper_nest_long_to_wide_process(lrec_t* pinrec, context_t* pctx, void* pvstate) { mapper_nest_state_t* pstate = (mapper_nest_state_t*)pvstate; diff --git a/c/todo.txt b/c/todo.txt index 5e233704d..93df4ed08 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -40,13 +40,12 @@ count_min=1,count_max=1 EXPLODE/UNNEST/ETC: ---------------------------------------------------------------- -explode across records by value (forward): - mlr nest - --explode - --values - --across-records - --nested-fs semicolon - -f x +mlr nest + --explode + --values + --across-records + --nested-fs semicolon + -f x x=a;b;c,y=d @@ -54,13 +53,12 @@ explode across records by value (forward): x=b,y=d x=c,y=d -implode across records by value (reverse): - mlr nest - --implode - --values - --across-records - --nested-fs semicolon - -f x +mlr nest + --implode + --values + --across-records + --nested-fs semicolon + -f x x=a,y=d x=b,y=d @@ -69,38 +67,36 @@ implode across records by value (reverse): x=a;b;c,y=d ---------------------------------------------------------------- -explode across fields by value (forward): - mlr nest - --explode - --values - --across-fields - --nested-fs semicolon - -f x +mlr nest + --explode + --values + --across-fields + --nested-fs semicolon + -f x x=a;b;c,y=d x_1=a,x_2=b,x_3=cy=d -implode across fields by value (reverse): - mlr nest - --implode - --values - --across-fields - --nested-fs semicolon - -f x +mlr nest + --implode + --values + --across-fields + --nested-fs semicolon + -f x x_1=a,x_2=b,x_3=cy=d x=a;b;c,y=d ---------------------------------------------------------------- -explode across records by key-value (forward): - mlr nest - --explode - --pairs - --across-records - --nested-ps colon - --nested-fs semicolon +mlr nest + --explode + --pairs + --across-records + --nested-ps colon + --nested-fs semicolon + -f x x=a:1;b:2;c:3,y=d @@ -108,13 +104,13 @@ explode across records by key-value (forward): b=2,y=d c=3,y=d -implode across records by key-value (reverse): - mlr nest - --implode - --pairs - --across-records - --nested-ps colon - --nested-fs semicolon +mlr nest + --implode + --pairs + --across-records + --nested-ps colon + --nested-fs semicolon + -f x a=1,y=d b=2,y=d @@ -123,25 +119,25 @@ implode across records by key-value (reverse): x=a:1;b:2;c:3,y=d ---------------------------------------------------------------- -explode across fields by key-value (forward): - mlr nest - --explode - --pairs - --across-fields - --nested-ps colon - --nested-fs semicolon +mlr nest + --explode + --pairs + --across-fields + --nested-ps colon + --nested-fs semicolon + -f x x=a:1;b:2;c:3,y=d a=1,b=2,c=3,y=d -implode across fields by key-value (reverse): - mlr nest - --implode - --pairs - --across-fields - --nested-ps colon - --nested-fs semicolon +mlr nest + --implode + --pairs + --across-fields + --nested-ps colon + --nested-fs semicolon + -f x a=1,b=2,c=3,y=d