From 759b22dec18d279b91dc473bba7905af80a5754e Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sat, 9 Jan 2016 17:49:02 +0000 Subject: [PATCH] mlr step -a decay --- c/mapping/mapper_step.c | 109 ++++++++++++++++++++++++++++++---------- c/todo.txt | 2 +- 2 files changed, 84 insertions(+), 27 deletions(-) diff --git a/c/mapping/mapper_step.c b/c/mapping/mapper_step.c index bc38c60ec..12ccd9626 100644 --- a/c/mapping/mapper_step.c +++ b/c/mapping/mapper_step.c @@ -32,16 +32,17 @@ typedef struct _step_t { step_free_func_t* pfree_func; } step_t; -typedef step_t* step_alloc_func_t(char* input_field_name, int allow_int_float); +typedef step_t* step_alloc_func_t(char* input_field_name, int allow_int_float, char* string_alpha); typedef struct _mapper_step_state_t { - ap_state_t* pargp; - slls_t* pstepper_names; - string_array_t* pvalue_field_names; // parameter - string_array_t* pvalue_field_values; // scratch space used per-record - slls_t* pgroup_by_field_names; // parameter - lhmslv_t* groups; - int allow_int_float; + ap_state_t* pargp; + slls_t* pstepper_names; + string_array_t* pvalue_field_names; // parameter + string_array_t* pvalue_field_values; // scratch space used per-record + slls_t* pgroup_by_field_names; // parameter + lhmslv_t* groups; + int allow_int_float; + char* string_alpha; } mapper_step_state_t; // Multilevel hashmap structure: @@ -70,17 +71,18 @@ typedef struct _mapper_step_state_t { static void mapper_step_usage(FILE* o, char* argv0, char* verb); static mapper_t* mapper_step_parse_cli(int* pargi, int argc, char** argv); static mapper_t* mapper_step_alloc(ap_state_t* pargp, slls_t* pstepper_names, string_array_t* pvalue_field_names, - slls_t* pgroup_by_field_names, int allow_int_float); + slls_t* pgroup_by_field_names, int allow_int_float, char* string_alpha); static void mapper_step_free(mapper_t* pmapper); static sllv_t* mapper_step_process(lrec_t* pinrec, context_t* pctx, void* pvstate); -static step_t* step_delta_alloc(char* input_field_name, int allow_int_float); -static step_t* step_from_first_alloc(char* input_field_name, int allow_int_float); -static step_t* step_ratio_alloc(char* input_field_name, int allow_int_float); -static step_t* step_rsum_alloc(char* input_field_name, int allow_int_float); -static step_t* step_counter_alloc(char* input_field_name, int allow_int_float); +static step_t* step_delta_alloc (char* input_field_name, int allow_int_float, char* unused); +static step_t* step_from_first_alloc (char* input_field_name, int allow_int_float, char* unused); +static step_t* step_ratio_alloc (char* input_field_name, int allow_int_float, char* unused); +static step_t* step_rsum_alloc (char* input_field_name, int allow_int_float, char* unused); +static step_t* step_counter_alloc (char* input_field_name, int allow_int_float, char* unused); +static step_t* step_decay_alloc (char* input_field_name, int unused, char* string_alpha); -static step_t* make_step(char* step_name, char* input_field_name, int allow_int_float); +static step_t* make_step(char* step_name, char* input_field_name, int allow_int_float, char* string_alpha); typedef struct _step_lookup_t { char* name; @@ -93,6 +95,7 @@ static step_lookup_t step_lookup_table[] = { {"ratio", step_ratio_alloc, "Compute ratios in field(s) between successive records"}, {"rsum", step_rsum_alloc, "Compute running sums of field(s) between successive records"}, {"counter", step_counter_alloc, "Count instances of field(s) between successive records"}, + {"decay", step_decay_alloc, "xxx doc line goes here"}, }; static int step_lookup_table_length = sizeof(step_lookup_table) / sizeof(step_lookup_table[0]); @@ -121,7 +124,8 @@ static mapper_t* mapper_step_parse_cli(int* pargi, int argc, char** argv) { slls_t* pstepper_names = NULL; string_array_t* pvalue_field_names = NULL; slls_t* pgroup_by_field_names = slls_alloc(); - int allow_int_float = TRUE; + char* string_alpha = "0.5"; // xxx null default w/ check ... or #define dflt & into usg + int allow_int_float = TRUE; char* verb = argv[(*pargi)++]; @@ -129,6 +133,7 @@ static mapper_t* mapper_step_parse_cli(int* pargi, int argc, char** argv) { ap_define_string_list_flag(pstate, "-a", &pstepper_names); ap_define_string_array_flag(pstate, "-f", &pvalue_field_names); ap_define_string_list_flag(pstate, "-g", &pgroup_by_field_names); + ap_define_string_flag(pstate, "-d", &string_alpha); ap_define_false_flag(pstate, "-F", &allow_int_float); if (!ap_parse(pstate, verb, pargi, argc, argv)) { @@ -141,12 +146,13 @@ static mapper_t* mapper_step_parse_cli(int* pargi, int argc, char** argv) { return NULL; } - return mapper_step_alloc(pstate, pstepper_names, pvalue_field_names, pgroup_by_field_names, allow_int_float); + return mapper_step_alloc(pstate, pstepper_names, pvalue_field_names, pgroup_by_field_names, + allow_int_float, string_alpha); } // ---------------------------------------------------------------- static mapper_t* mapper_step_alloc(ap_state_t* pargp, slls_t* pstepper_names, string_array_t* pvalue_field_names, - slls_t* pgroup_by_field_names, int allow_int_float) + slls_t* pgroup_by_field_names, int allow_int_float, char* string_alpha) { mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t)); @@ -159,6 +165,7 @@ static mapper_t* mapper_step_alloc(ap_state_t* pargp, slls_t* pstepper_names, st pstate->pgroup_by_field_names = pgroup_by_field_names; pstate->groups = lhmslv_alloc(); pstate->allow_int_float = allow_int_float; + pstate->string_alpha = string_alpha; pmapper->pvstate = pstate; pmapper->pprocess_func = mapper_step_process; @@ -241,7 +248,7 @@ static sllv_t* mapper_step_process(lrec_t* pinrec, context_t* pctx, void* pvstat char* step_name = pc->value; step_t* pstep = lhmsv_get(pacc_field_to_acc_state, step_name); if (pstep == NULL) { - pstep = make_step(step_name, value_field_name, pstate->allow_int_float); + pstep = make_step(step_name, value_field_name, pstate->allow_int_float, pstate->string_alpha); if (pstep == NULL) { fprintf(stderr, "mlr step: stepper \"%s\" not found.\n", step_name); @@ -283,10 +290,10 @@ static sllv_t* mapper_step_process(lrec_t* pinrec, context_t* pctx, void* pvstat return sllv_single(pinrec); } -static step_t* make_step(char* step_name, char* input_field_name, int allow_int_float) { +static step_t* make_step(char* step_name, char* input_field_name, int allow_int_float, char* string_alpha) { for (int i = 0; i < step_lookup_table_length; i++) if (streq(step_name, step_lookup_table[i].name)) - return step_lookup_table[i].palloc_func(input_field_name, allow_int_float); + return step_lookup_table[i].palloc_func(input_field_name, allow_int_float, string_alpha); return NULL; } @@ -317,7 +324,7 @@ static void step_delta_free(step_t* pstep) { free(pstate); free(pstep); } -static step_t* step_delta_alloc(char* input_field_name, int allow_int_float) { +static step_t* step_delta_alloc(char* input_field_name, int allow_int_float, char* unused) { step_t* pstep = mlr_malloc_or_die(sizeof(step_t)); step_delta_state_t* pstate = mlr_malloc_or_die(sizeof(step_delta_state_t)); pstate->prev = mv_from_null(); @@ -359,7 +366,7 @@ static void step_from_first_free(step_t* pstep) { free(pstate); free(pstep); } -static step_t* step_from_first_alloc(char* input_field_name, int allow_int_float) { +static step_t* step_from_first_alloc(char* input_field_name, int allow_int_float, char* unused) { step_t* pstep = mlr_malloc_or_die(sizeof(step_t)); step_from_first_state_t* pstate = mlr_malloc_or_die(sizeof(step_from_first_state_t)); pstate->first = mv_from_null(); @@ -402,7 +409,7 @@ static void step_ratio_free(step_t* pstep) { free(pstate); free(pstep); } -static step_t* step_ratio_alloc(char* input_field_name, int allow_int_float) { +static step_t* step_ratio_alloc(char* input_field_name, int allow_int_float, char* unused) { step_t* pstep = mlr_malloc_or_die(sizeof(step_t)); step_ratio_state_t* pstate = mlr_malloc_or_die(sizeof(step_ratio_state_t)); pstate->prev = -999.0; @@ -440,7 +447,7 @@ static void step_rsum_free(step_t* pstep) { free(pstate); free(pstep); } -static step_t* step_rsum_alloc(char* input_field_name, int allow_int_float) { +static step_t* step_rsum_alloc(char* input_field_name, int allow_int_float, char* unused) { step_t* pstep = mlr_malloc_or_die(sizeof(step_t)); step_rsum_state_t* pstate = mlr_malloc_or_die(sizeof(step_rsum_state_t)); pstate->allow_int_float = allow_int_float; @@ -477,7 +484,7 @@ static void step_counter_free(step_t* pstep) { free(pstate); free(pstep); } -static step_t* step_counter_alloc(char* input_field_name, int allow_int_float) { +static step_t* step_counter_alloc(char* input_field_name, int allow_int_float, char* unused) { step_t* pstep = mlr_malloc_or_die(sizeof(step_t)); step_counter_state_t* pstate = mlr_malloc_or_die(sizeof(step_counter_state_t)); pstate->counter = allow_int_float ? mv_from_int(0LL) : mv_from_float(0.0); @@ -492,3 +499,53 @@ static step_t* step_counter_alloc(char* input_field_name, int allow_int_float) { pstep->pfree_func = step_counter_free; return pstep; } + +// ---------------------------------------------------------------- +typedef struct _step_decay_state_t { + double alpha; + double alphacompl; + int have_prev; + double prev; + char* output_field_name; +} step_decay_state_t; +static void step_decay_dprocess(void* pvstate, double fltv, lrec_t* prec) { + step_decay_state_t* pstate = pvstate; + double curr = fltv; + if (pstate->have_prev) { + curr = pstate->alpha * curr + pstate->alphacompl * pstate->prev; + } else { + pstate->have_prev = TRUE; + } + lrec_put(prec, pstate->output_field_name, mlr_alloc_string_from_double(curr, MLR_GLOBALS.ofmt), + FREE_ENTRY_VALUE); + pstate->prev = curr; +} +static void step_decay_zprocess(void* pvstate, lrec_t* prec) { + step_decay_state_t* pstate = pvstate; + lrec_put(prec, pstate->output_field_name, "", NO_FREE); +} +static void step_decay_free(step_t* pstep) { + step_decay_state_t* pstate = pstep->pvstate; + free(pstate->output_field_name); + free(pstate); + free(pstep); +} +static step_t* step_decay_alloc(char* input_field_name, int unused, char* string_alpha) { + step_t* pstep = mlr_malloc_or_die(sizeof(step_t)); + + step_decay_state_t* pstate = mlr_malloc_or_die(sizeof(step_decay_state_t)); + pstate->alpha = mlr_double_from_string_or_die(string_alpha); + pstate->alphacompl = 1.0 - pstate->alpha; + pstate->have_prev = FALSE; + pstate->prev = 0.0; + // xxx support slls ... output_field_names ... + pstate->output_field_name = mlr_paste_3_strings(input_field_name, "_decay_", string_alpha); + + pstep->pvstate = (void*)pstate; + pstep->pdprocess_func = step_decay_dprocess; + pstep->pnprocess_func = NULL; + pstep->psprocess_func = NULL; + pstep->pzprocess_func = step_decay_zprocess; + pstep->pfree_func = step_decay_free; + return pstep; +} diff --git a/c/todo.txt b/c/todo.txt index d5a46d1b8..bc43f96b1 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -14,7 +14,7 @@ TOP OF LIST - linregs; corr/cov? ? also, option of weighted stats w/ explicit weights field? -* debian screenshot +debian screenshot * ruby @ optextdep @ mld; poki+mkman * profile new streqn vs. strncmp ...