From 6e4df862e1247538ff15010cddca9828a2ab6191 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Wed, 25 Nov 2015 09:08:52 -0500 Subject: [PATCH] mlr step -a from-first --- c/mapping/mapper_step.c | 40 +++++++++++++++++++++++++++++---- c/reg_test/expected/out | 18 +++++++++++++++ c/reg_test/input/Makefile.am | 1 + c/reg_test/input/from-first.csv | 7 ++++++ c/reg_test/run | 3 +++ c/todo.txt | 4 ++-- doc/mlr.1.premade | 7 +++--- doc/reference.html | 3 ++- 8 files changed, 73 insertions(+), 10 deletions(-) create mode 100644 c/reg_test/input/from-first.csv diff --git a/c/mapping/mapper_step.c b/c/mapping/mapper_step.c index 42286ab5d..fef9d71ee 100644 --- a/c/mapping/mapper_step.c +++ b/c/mapping/mapper_step.c @@ -68,6 +68,7 @@ static void mapper_step_free(void* pvstate); static sllv_t* mapper_step_process(lrec_t* pinrec, context_t* pctx, void* pvstate); static step_t* step_delta_alloc(char* input_field_name, int allow_int_float); +static step_t* step_from_first_alloc(char* input_field_name, int allow_int_float); static step_t* step_ratio_alloc(char* input_field_name, int allow_int_float); static step_t* step_rsum_alloc(char* input_field_name, int allow_int_float); static step_t* step_counter_alloc(char* input_field_name, int allow_int_float); @@ -80,10 +81,11 @@ typedef struct _step_lookup_t { char* desc; } step_lookup_t; static step_lookup_t step_lookup_table[] = { - {"delta", step_delta_alloc, "Compute differences in field(s) between successive records"}, - {"ratio", step_ratio_alloc, "Compute ratios in field(s) between successive records"}, - {"rsum", step_rsum_alloc, "Compute running sums of field(s) between successive records"}, - {"counter", step_counter_alloc, "Count instances of field(s) between successive records"}, + {"delta", step_delta_alloc, "Compute differences in field(s) between successive records"}, + {"from-first", step_from_first_alloc, "Compute differences in field(s) from first record"}, + {"ratio", step_ratio_alloc, "Compute ratios in field(s) between successive records"}, + {"rsum", step_rsum_alloc, "Compute running sums of field(s) between successive records"}, + {"counter", step_counter_alloc, "Count instances of field(s) between successive records"}, }; static int step_lookup_table_length = sizeof(step_lookup_table) / sizeof(step_lookup_table[0]); @@ -287,6 +289,36 @@ static step_t* step_delta_alloc(char* input_field_name, int allow_int_float) { } // xxx step_delta_free et al. +// ---------------------------------------------------------------- +typedef struct _step_from_first_state_t { + mv_t first; + char* output_field_name; + int allow_int_float; +} step_from_first_state_t; +static void step_from_first_nprocess(void* pvstate, mv_t* pnumv, lrec_t* prec) { + step_from_first_state_t* pstate = pvstate; + mv_t from_first; + if (mv_is_null(&pstate->first)) { + from_first = pstate->allow_int_float ? mv_from_int(0LL) : mv_from_float(0.0); + pstate->first = *pnumv; + } else { + from_first = n_nn_minus_func(pnumv, &pstate->first); + } + lrec_put(prec, pstate->output_field_name, mv_format_val(&from_first), LREC_FREE_ENTRY_VALUE); +} +static step_t* step_from_first_alloc(char* input_field_name, int allow_int_float) { + step_t* pstep = mlr_malloc_or_die(sizeof(step_t)); + step_from_first_state_t* pstate = mlr_malloc_or_die(sizeof(step_from_first_state_t)); + pstate->first = mv_from_null(); + pstate->allow_int_float = allow_int_float; + pstate->output_field_name = mlr_paste_2_strings(input_field_name, "_from_first"); + pstep->pvstate = (void*)pstate; + pstep->pdprocess_func = NULL; + pstep->pnprocess_func = step_from_first_nprocess; + pstep->psprocess_func = NULL; + return pstep; +} + // ---------------------------------------------------------------- typedef struct _step_ratio_state_t { double prev; diff --git a/c/reg_test/expected/out b/c/reg_test/expected/out index deae9d4b9..52b5f8733 100644 --- a/c/reg_test/expected/out +++ b/c/reg_test/expected/out @@ -4893,6 +4893,24 @@ a=zee,b=wye,i=8,x=0.5985540091064224,yyy=0.976181385699006,x_rsum=1.125680,x_del aaa=hat,bbb=wye,i=9,x=0.03144187646093577,y=0.7495507603507059 a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864,x_rsum=0.849416,x_delta=0.155836,x_counter=2,y_rsum=1.679421,y_delta=0.225815,y_counter=2 +mlr --icsvlite --opprint step -a from-first -f x ./reg_test/input/from-first.csv +x g x_from_first +100 red 0 +203 red 103 +101 green 1 +307 red 207 +209 green 109 +314 green 214 + +mlr --icsvlite --opprint step -a from-first -f x -g g ./reg_test/input/from-first.csv +x g x_from_first +100 red 0 +203 red 103 +101 green 0 +307 red 207 +209 green 108 +314 green 213 + mlr --opprint histogram -f x,y --lo 0 --hi 1 --nbins 20 ./reg_test/input/small bin_lo bin_hi x_count y_count 0.000000 0.050000 1 0 diff --git a/c/reg_test/input/Makefile.am b/c/reg_test/input/Makefile.am index 4d37a0574..c82988db3 100644 --- a/c/reg_test/input/Makefile.am +++ b/c/reg_test/input/Makefile.am @@ -21,6 +21,7 @@ EXTRA_DIST= \ e.pprint \ f.csv \ f.pprint \ + from-first.csv \ fsec2xhms \ g.csv \ g.pprint \ diff --git a/c/reg_test/input/from-first.csv b/c/reg_test/input/from-first.csv new file mode 100644 index 000000000..5e8474bbe --- /dev/null +++ b/c/reg_test/input/from-first.csv @@ -0,0 +1,7 @@ +x,g +100,red +203,red +101,green +307,red +209,green +314,green diff --git a/c/reg_test/run b/c/reg_test/run index 3e1066fef..a8f9b367d 100755 --- a/c/reg_test/run +++ b/c/reg_test/run @@ -331,6 +331,9 @@ run_mlr --opprint step -a rsum,delta,counter -f x,y -g a $indir/abixy run_mlr --odkvp step -a rsum,delta,counter -f x,y $indir/abixy-het run_mlr --odkvp step -a rsum,delta,counter -f x,y -g a $indir/abixy-het +run_mlr --icsvlite --opprint step -a from-first -f x $indir/from-first.csv +run_mlr --icsvlite --opprint step -a from-first -f x -g g $indir/from-first.csv + run_mlr --opprint histogram -f x,y --lo 0 --hi 1 --nbins 20 $indir/small # ---------------------------------------------------------------- diff --git a/c/todo.txt b/c/todo.txt index df8d1137e..544435377 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -8,8 +8,8 @@ TOP OF LIST ---------------------------------------------------------------- -! 1e6 accepted from field value but not literal. DSLs, and UTs. -* mlr step from-first (check against delta + rsum ...) +! scinot@literals: cut a 3.0.1 + * bar --auto (w/ hold & emit & left-right labels) * faqents: diff --git a/doc/mlr.1.premade b/doc/mlr.1.premade index d78905a7c..33867e023 100644 --- a/doc/mlr.1.premade +++ b/doc/mlr.1.premade @@ -2,12 +2,12 @@ .\" Title: mlr .\" Author: [see the "AUTHOR" section] .\" Generator: ./mkman.rb -.\" Date: 2015-11-24 +.\" Date: 2015-11-25 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "MILLER" "1" "2015-11-24" "\ \&" "\ \&" +.TH "MILLER" "1" "2015-11-25" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Portability definitions .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -38,7 +38,7 @@ on integer-indexed fields: if the natural data structure for the latter is the array, then Miller's natural data structure is the insertion-ordered hash map. This encompasses a variety of data formats, including but not limited to the familiar CSV. (Miller can handle positionally-indexed data as a special -case.) This manpage documents Miller v3.0.0. +case.) This manpage documents Miller v3.0.0-dev. .SH "EXAMPLES" .sp @@ -753,6 +753,7 @@ Computes values dependent on the previous record, optionally grouped by category. -a {delta,rsum,...} Names of steppers: comma-separated, one or more of: delta Compute differences in field(s) between successive records + from-first Compute differences in field(s) from first record ratio Compute ratios in field(s) between successive records rsum Compute running sums of field(s) between successive records counter Count instances of field(s) between successive records diff --git a/doc/reference.html b/doc/reference.html index 00b8771bf..a5513bccc 100644 --- a/doc/reference.html +++ b/doc/reference.html @@ -432,7 +432,7 @@ Output of one verb may be chained as input to another using "then", e.g. mlr stats1 -a min,mean,max -f flag,u,v -g color then sort -f color For more information please see http://johnkerl.org/miller/doc and/or -http://github.com/johnkerl/miller. This is Miller version v3.0.0. +http://github.com/johnkerl/miller. This is Miller version v3.0.0-dev.

@@ -2572,6 +2572,7 @@ Computes values dependent on the previous record, optionally grouped by category. -a {delta,rsum,...} Names of steppers: comma-separated, one or more of: delta Compute differences in field(s) between successive records + from-first Compute differences in field(s) from first record ratio Compute ratios in field(s) between successive records rsum Compute running sums of field(s) between successive records counter Count instances of field(s) between successive records