diff --git a/c/containers/lhmsi.c b/c/containers/lhmsi.c index d0075f63d..7c9d0a1ff 100644 --- a/c/containers/lhmsi.c +++ b/c/containers/lhmsi.c @@ -182,7 +182,21 @@ int lhmsi_get(lhmsi_t* pmap, char* key) { if (pmap->states[index] == OCCUPIED) return pe->value; else if (pmap->states[index] == EMPTY) - return -999; + return -999; // xxx fix me ... extend the api to handle nullity. + else { + fprintf(stderr, "lhmsi_find_index_for_key did not find end of chain"); + exit(1); + } +} + +lhmsie_t* lhmsi_get_entry(lhmsi_t* pmap, char* key) { + int index = lhmsi_find_index_for_key(pmap, key); + lhmsie_t* pe = &pmap->entries[index]; + + if (pmap->states[index] == OCCUPIED) + return pe; + else if (pmap->states[index] == EMPTY) + return NULL; else { fprintf(stderr, "lhmsi_find_index_for_key did not find end of chain"); exit(1); diff --git a/c/containers/lhmsi.h b/c/containers/lhmsi.h index c5102b99d..6b39b37af 100644 --- a/c/containers/lhmsi.h +++ b/c/containers/lhmsi.h @@ -22,7 +22,7 @@ typedef struct _lhmsie_t { int ideal_index; char* key; - int value; + int value; // xxx make ull struct _lhmsie_t *pprev; struct _lhmsie_t *pnext; } lhmsie_t; @@ -45,6 +45,7 @@ lhmsi_t* lhmsi_copy(lhmsi_t* pmap); void lhmsi_free(lhmsi_t* pmap); void lhmsi_put(lhmsi_t* pmap, char* key, int value); int lhmsi_get(lhmsi_t* pmap, char* key); +lhmsie_t* lhmsi_get_entry(lhmsi_t* pmap, char* key); int lhmsi_has_key(lhmsi_t* pmap, char* key); void lhmsi_remove(lhmsi_t* pmap, char* key); diff --git a/c/containers/lhmsv.c b/c/containers/lhmsv.c index 2195c44d8..9c31bb290 100644 --- a/c/containers/lhmsv.c +++ b/c/containers/lhmsv.c @@ -1,5 +1,5 @@ // ================================================================ -// Array-only (open addressing) string-to-string linked hash map with linear +// Array-only (open addressing) string-to-void linked hash map with linear // probing for collisions. // // John Kerl 2012-08-13 diff --git a/c/containers/lhmsv.h b/c/containers/lhmsv.h index bb11fdfa9..b9650138b 100644 --- a/c/containers/lhmsv.h +++ b/c/containers/lhmsv.h @@ -4,7 +4,7 @@ #include "containers/sllv.h" // ================================================================ -// Array-only (open addressing) string-to-string linked hash map with linear +// Array-only (open addressing) string-to-void linked hash map with linear // probing for collisions. // // John Kerl 2012-08-13 diff --git a/c/mapping/mapper_stats1.c b/c/mapping/mapper_stats1.c index b3739fe0b..99044e5b6 100644 --- a/c/mapping/mapper_stats1.c +++ b/c/mapping/mapper_stats1.c @@ -8,6 +8,7 @@ #include "containers/slls.h" #include "containers/lhmslv.h" #include "containers/lhmsv.h" +#include "containers/lhmsi.h" #include "containers/mixutil.h" #include "mapping/mappers.h" #include "cli/argparse.h" @@ -24,6 +25,7 @@ // char* get(); // ================================================================ +// put -> ingest/consume/something typedef void acc_dput_func_t(void* pvstate, double val); typedef void acc_sput_func_t(void* pvstate, char* val); typedef char* acc_get_func_t (void* pvstate, char* pfree_flags); @@ -251,6 +253,52 @@ acc_t* acc_max_alloc(static_context_t* pstatx) { return pacc; } +// ---------------------------------------------------------------- +typedef struct _acc_mode_state_t { + lhmsi_t* pcounts_for_value; + static_context_t* pstatx; +} acc_mode_state_t; +// mode on strings? what about "1.0" and "1" and "1.0000" ?? +void acc_mode_sput(void* pvstate, char* val) { + acc_mode_state_t* pstate = pvstate; + lhmsie_t* pe = lhmsi_get_entry(pstate->pcounts_for_value, val); + if (pe == NULL) { + // xxx at the moment, lhmsi does a strdup so we needn't. + lhmsi_put(pstate->pcounts_for_value, val, 1); + } else { + pe->value++; + } +} +char* acc_mode_get(void* pvstate, char* pfree_flags) { + acc_mode_state_t* pstate = pvstate; + int max_count = 0; + char* max_key = NULL; + for (lhmsie_t* pe = pstate->pcounts_for_value->phead; pe != NULL; pe = pe->pnext) { + int count = pe->value; + if (count > max_count) { + max_key = pe->key; + max_count = count; + } + } + return max_key; +} +// xxx somewhere note a crucial assumption: while pctx is passed in +// to the mapper on a per-row basis, we stash it here on first use and +// use it on subsequent rows. assumptions: +// * the address doesn't change +// * the content we use (namely, ofmt) isn't row-dependent +acc_t* acc_mode_alloc(static_context_t* pstatx) { + acc_t* pacc = mlr_malloc_or_die(sizeof(acc_t)); + acc_mode_state_t* pstate = mlr_malloc_or_die(sizeof(acc_mode_state_t)); + pstate->pcounts_for_value = lhmsi_alloc(); + pstate->pstatx = pstatx; + pacc->pvstate = (void*)pstate; + pacc->psput_func = &acc_mode_sput; + pacc->pdput_func = NULL; + pacc->pget_func = &acc_mode_get; + return pacc; +} + // ---------------------------------------------------------------- typedef struct _acc_lookup_t { char* name; @@ -264,6 +312,7 @@ static acc_lookup_t acc_lookup_table[] = { {"avgeb", acc_avgeb_alloc}, {"min", acc_min_alloc}, {"max", acc_max_alloc}, + {"mode", acc_mode_alloc}, }; static int acc_lookup_table_length = sizeof(acc_lookup_table) / sizeof(acc_lookup_table[0]); @@ -281,7 +330,7 @@ typedef struct _mapper_stats1_state_t { slls_t* pvalue_field_names; slls_t* pgroup_by_field_names; - lhmslv_t* pmaps_level_1; + lhmslv_t* groups; } mapper_stats1_state_t; @@ -293,10 +342,10 @@ typedef struct _mapper_stats1_state_t { // s t 5 6 u w 1 7 1 9 // u w 7 9 -// ["s","t"] |--> "x" |--> "sum" |--> acc_sum_t* (as void*) +// ["s","t"] |--> "x" |--> "sum" |--> acc_t* (as void*) // level_1 level_2 level_3 // lhmslv_t lhmsv_t lhmsv_t -// acc_sum_t implements interface: +// acc_t implements interface: // void init(); // void dacc(double dval); // void sacc(char* sval); @@ -323,10 +372,10 @@ sllv_t* mapper_stats1_func(lrec_t* pinrec, context_t* pctx, void* pvstate) { return NULL; } - lhmsv_t* pmaps_level_2 = lhmslv_get(pstate->pmaps_level_1, pgroup_by_field_values); - if (pmaps_level_2 == NULL) { - pmaps_level_2 = lhmsv_alloc(); - lhmslv_put(pstate->pmaps_level_1, slls_copy(pgroup_by_field_values), pmaps_level_2); + lhmsv_t* group_to_acc_field = lhmslv_get(pstate->groups, pgroup_by_field_values); + if (group_to_acc_field == NULL) { + group_to_acc_field = lhmsv_alloc(); + lhmslv_put(pstate->groups, slls_copy(pgroup_by_field_values), group_to_acc_field); } sllse_t* pa = pstate->pvalue_field_names->phead; @@ -338,17 +387,17 @@ sllv_t* mapper_stats1_func(lrec_t* pinrec, context_t* pctx, void* pvstate) { int have_dval = FALSE; double value_field_dval = -999.0; - lhmsv_t* pmaps_level_3 = lhmsv_get(pmaps_level_2, value_field_name); - if (pmaps_level_3 == NULL) { - pmaps_level_3 = lhmsv_alloc(); - lhmsv_put(pmaps_level_2, value_field_name, pmaps_level_3); + lhmsv_t* acc_field_to_acc_state = lhmsv_get(group_to_acc_field, value_field_name); + if (acc_field_to_acc_state == NULL) { + acc_field_to_acc_state = lhmsv_alloc(); + lhmsv_put(group_to_acc_field, value_field_name, acc_field_to_acc_state); } // for "sum", "count" sllse_t* pc = pstate->paccumulator_names->phead; for ( ; pc != NULL; pc = pc->pnext) { char* acc_name = pc->value; - acc_t* pacc = lhmsv_get(pmaps_level_3, acc_name); + acc_t* pacc = lhmsv_get(acc_field_to_acc_state, acc_name); if (pacc == NULL) { pacc = make_acc(acc_name, &pctx->statx); if (pacc == NULL) { @@ -356,7 +405,7 @@ sllv_t* mapper_stats1_func(lrec_t* pinrec, context_t* pctx, void* pvstate) { acc_name); exit(1); } - lhmsv_put(pmaps_level_3, acc_name, pacc); + lhmsv_put(acc_field_to_acc_state, acc_name, pacc); } if (pacc == NULL) { // xxx needs argv[0] somewhere ... @@ -383,7 +432,7 @@ sllv_t* mapper_stats1_func(lrec_t* pinrec, context_t* pctx, void* pvstate) { else { sllv_t* poutrecs = sllv_alloc(); - for (lhmslve_t* pa = pstate->pmaps_level_1->phead; pa != NULL; pa = pa->pnext) { + for (lhmslve_t* pa = pstate->groups->phead; pa != NULL; pa = pa->pnext) { lrec_t* poutrec = lrec_unbacked_alloc(); slls_t* pgroup_by_field_values = pa->key; @@ -396,13 +445,13 @@ sllv_t* mapper_stats1_func(lrec_t* pinrec, context_t* pctx, void* pvstate) { } // Add in fields such as x_sum=#, y_count=#, etc.: - lhmsv_t* pmaps_level_2 = pa->value; + lhmsv_t* group_to_acc_field = pa->value; // for "x", "y" - for (lhmsve_t* pd = pmaps_level_2->phead; pd != NULL; pd = pd->pnext) { + for (lhmsve_t* pd = group_to_acc_field->phead; pd != NULL; pd = pd->pnext) { char* value_field_name = pd->key; - lhmsv_t* pmaps_level_3 = pd->value; + lhmsv_t* acc_field_to_acc_state = pd->value; // for "count", "sum" - for (lhmsve_t* pe = pmaps_level_3->phead; pe != NULL; pe = pe->pnext) { + for (lhmsve_t* pe = acc_field_to_acc_state->phead; pe != NULL; pe = pe->pnext) { char* acc_name = pe->key; acc_t* pacc = pe->value; @@ -427,7 +476,7 @@ static void mapper_stats1_free(void* pvstate) { slls_free(pstate->pvalue_field_names); slls_free(pstate->pgroup_by_field_names); // xxx free the level-2's 1st - lhmslv_free(pstate->pmaps_level_1); + lhmslv_free(pstate->groups); } mapper_t* mapper_stats1_alloc(slls_t* paccumulator_names, slls_t* pvalue_field_names, slls_t* pgroup_by_field_names) { @@ -438,7 +487,7 @@ mapper_t* mapper_stats1_alloc(slls_t* paccumulator_names, slls_t* pvalue_field_n pstate->paccumulator_names = paccumulator_names; pstate->pvalue_field_names = pvalue_field_names; pstate->pgroup_by_field_names = pgroup_by_field_names; - pstate->pmaps_level_1 = lhmslv_alloc(); + pstate->groups = lhmslv_alloc(); pmapper->pvstate = pstate; pmapper->pmapper_process_func = mapper_stats1_func; diff --git a/c/t1.rb b/c/t1.rb index 9ead58959..43eb3e957 100755 --- a/c/t1.rb +++ b/c/t1.rb @@ -77,7 +77,7 @@ run('CORRMLRV', "mlr --csv stats2 -a corr -f x,y -g a,b < data/big.csv > puts puts -run("sortsort1", "sort -t, -k 1,2 < data/big > /dev/null") +run("sortsort1", "sort -t= -k 1,2 < data/big > /dev/null") run("SORTMLR1", "mlr sort -f a,b < data/big > /dev/null") puts @@ -90,5 +90,5 @@ run("sortsort1v", "sort -t, -k 1,4 < data/big.csv > /dev/null") run("SORTMLR1V", "mlr --csv sort -f a,x < data/big.csv > /dev/null") puts -run("sortsort2v", "sort -t, -k 4,5 < data/big.csv > /dev/null") +run("sortsort2v", "sort -t, -n -k 4,5 < data/big.csv > /dev/null") run("SORTMLR2V", "mlr --csv sort -n x,y < data/big.csv > /dev/null") diff --git a/c/test/expected/out b/c/test/expected/out index 4d61a3de5..90161d541 100644 --- a/c/test/expected/out +++ b/c/test/expected/out @@ -402,18 +402,18 @@ a=zee,b=wye,count=2 a=hat,b=wye,count=2 a=pan,b=wye,count=2 -./test/../mlr --opprint stats1 -a avg,sum,count,min,max -f i,x,y -g a,b ./test/input/abixy -a b i_avg i_sum i_count i_min i_max x_avg x_sum x_count x_min x_max y_avg y_sum y_count y_min y_max -pan pan 1.000000 1.000000 1 1.000000 1.000000 0.346790 0.346790 1 0.346790 0.346790 0.726803 0.726803 1 0.726803 0.726803 -eks pan 2.000000 2.000000 1 2.000000 2.000000 0.758680 0.758680 1 0.758680 0.758680 0.522151 0.522151 1 0.522151 0.522151 -wye wye 3.000000 3.000000 1 3.000000 3.000000 0.204603 0.204603 1 0.204603 0.204603 0.338319 0.338319 1 0.338319 0.338319 -eks wye 4.000000 4.000000 1 4.000000 4.000000 0.381399 0.381399 1 0.381399 0.381399 0.134189 0.134189 1 0.134189 0.134189 -wye pan 5.000000 5.000000 1 5.000000 5.000000 0.573289 0.573289 1 0.573289 0.573289 0.863624 0.863624 1 0.863624 0.863624 -zee pan 6.000000 6.000000 1 6.000000 6.000000 0.527126 0.527126 1 0.527126 0.527126 0.493221 0.493221 1 0.493221 0.493221 -eks zee 7.000000 7.000000 1 7.000000 7.000000 0.611784 0.611784 1 0.611784 0.611784 0.187885 0.187885 1 0.187885 0.187885 -zee wye 8.000000 8.000000 1 8.000000 8.000000 0.598554 0.598554 1 0.598554 0.598554 0.976181 0.976181 1 0.976181 0.976181 -hat wye 9.000000 9.000000 1 9.000000 9.000000 0.031442 0.031442 1 0.031442 0.031442 0.749551 0.749551 1 0.749551 0.749551 -pan wye 10.000000 10.000000 1 10.000000 10.000000 0.502626 0.502626 1 0.502626 0.502626 0.952618 0.952618 1 0.952618 0.952618 +./test/../mlr --opprint stats1 -a avg,sum,count,min,max,mode -f i,x,y -g a,b ./test/input/abixy +a b i_avg i_sum i_count i_min i_max i_mode x_avg x_sum x_count x_min x_max x_mode y_avg y_sum y_count y_min y_max y_mode +pan pan 1.000000 1.000000 1 1.000000 1.000000 1 0.346790 0.346790 1 0.346790 0.346790 0.3467901443380824 0.726803 0.726803 1 0.726803 0.726803 0.7268028627434533 +eks pan 2.000000 2.000000 1 2.000000 2.000000 2 0.758680 0.758680 1 0.758680 0.758680 0.7586799647899636 0.522151 0.522151 1 0.522151 0.522151 0.5221511083334797 +wye wye 3.000000 3.000000 1 3.000000 3.000000 3 0.204603 0.204603 1 0.204603 0.204603 0.20460330576630303 0.338319 0.338319 1 0.338319 0.338319 0.33831852551664776 +eks wye 4.000000 4.000000 1 4.000000 4.000000 4 0.381399 0.381399 1 0.381399 0.381399 0.38139939387114097 0.134189 0.134189 1 0.134189 0.134189 0.13418874328430463 +wye pan 5.000000 5.000000 1 5.000000 5.000000 5 0.573289 0.573289 1 0.573289 0.573289 0.5732889198020006 0.863624 0.863624 1 0.863624 0.863624 0.8636244699032729 +zee pan 6.000000 6.000000 1 6.000000 6.000000 6 0.527126 0.527126 1 0.527126 0.527126 0.5271261600918548 0.493221 0.493221 1 0.493221 0.493221 0.49322128674835697 +eks zee 7.000000 7.000000 1 7.000000 7.000000 7 0.611784 0.611784 1 0.611784 0.611784 0.6117840605678454 0.187885 0.187885 1 0.187885 0.187885 0.1878849191181694 +zee wye 8.000000 8.000000 1 8.000000 8.000000 8 0.598554 0.598554 1 0.598554 0.598554 0.5985540091064224 0.976181 0.976181 1 0.976181 0.976181 0.976181385699006 +hat wye 9.000000 9.000000 1 9.000000 9.000000 9 0.031442 0.031442 1 0.031442 0.031442 0.03144187646093577 0.749551 0.749551 1 0.749551 0.749551 0.7495507603507059 +pan wye 10.000000 10.000000 1 10.000000 10.000000 10 0.502626 0.502626 1 0.502626 0.502626 0.5026260055412137 0.952618 0.952618 1 0.952618 0.952618 0.9526183602969864 ./test/../mlr --opprint stats2 -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2,x2,x2 -g a,b ./test/input/abixy-wide a b x_y_ols_m x_y_ols_b x_y_pca_m x_y_pca_b x_y_pca_quality x_y_r2 x_y_corr x_y_cov xy_y2_ols_m xy_y2_ols_b xy_y2_pca_m xy_y2_pca_b xy_y2_pca_quality xy_y2_r2 xy_y2_corr xy_y2_cov x2_x2_ols_m x2_x2_ols_b x2_x2_pca_m x2_x2_pca_b x2_x2_pca_quality x2_x2_r2 x2_x2_corr x2_x2_cov diff --git a/c/test/output/out b/c/test/output/out index 4d61a3de5..90161d541 100644 --- a/c/test/output/out +++ b/c/test/output/out @@ -402,18 +402,18 @@ a=zee,b=wye,count=2 a=hat,b=wye,count=2 a=pan,b=wye,count=2 -./test/../mlr --opprint stats1 -a avg,sum,count,min,max -f i,x,y -g a,b ./test/input/abixy -a b i_avg i_sum i_count i_min i_max x_avg x_sum x_count x_min x_max y_avg y_sum y_count y_min y_max -pan pan 1.000000 1.000000 1 1.000000 1.000000 0.346790 0.346790 1 0.346790 0.346790 0.726803 0.726803 1 0.726803 0.726803 -eks pan 2.000000 2.000000 1 2.000000 2.000000 0.758680 0.758680 1 0.758680 0.758680 0.522151 0.522151 1 0.522151 0.522151 -wye wye 3.000000 3.000000 1 3.000000 3.000000 0.204603 0.204603 1 0.204603 0.204603 0.338319 0.338319 1 0.338319 0.338319 -eks wye 4.000000 4.000000 1 4.000000 4.000000 0.381399 0.381399 1 0.381399 0.381399 0.134189 0.134189 1 0.134189 0.134189 -wye pan 5.000000 5.000000 1 5.000000 5.000000 0.573289 0.573289 1 0.573289 0.573289 0.863624 0.863624 1 0.863624 0.863624 -zee pan 6.000000 6.000000 1 6.000000 6.000000 0.527126 0.527126 1 0.527126 0.527126 0.493221 0.493221 1 0.493221 0.493221 -eks zee 7.000000 7.000000 1 7.000000 7.000000 0.611784 0.611784 1 0.611784 0.611784 0.187885 0.187885 1 0.187885 0.187885 -zee wye 8.000000 8.000000 1 8.000000 8.000000 0.598554 0.598554 1 0.598554 0.598554 0.976181 0.976181 1 0.976181 0.976181 -hat wye 9.000000 9.000000 1 9.000000 9.000000 0.031442 0.031442 1 0.031442 0.031442 0.749551 0.749551 1 0.749551 0.749551 -pan wye 10.000000 10.000000 1 10.000000 10.000000 0.502626 0.502626 1 0.502626 0.502626 0.952618 0.952618 1 0.952618 0.952618 +./test/../mlr --opprint stats1 -a avg,sum,count,min,max,mode -f i,x,y -g a,b ./test/input/abixy +a b i_avg i_sum i_count i_min i_max i_mode x_avg x_sum x_count x_min x_max x_mode y_avg y_sum y_count y_min y_max y_mode +pan pan 1.000000 1.000000 1 1.000000 1.000000 1 0.346790 0.346790 1 0.346790 0.346790 0.3467901443380824 0.726803 0.726803 1 0.726803 0.726803 0.7268028627434533 +eks pan 2.000000 2.000000 1 2.000000 2.000000 2 0.758680 0.758680 1 0.758680 0.758680 0.7586799647899636 0.522151 0.522151 1 0.522151 0.522151 0.5221511083334797 +wye wye 3.000000 3.000000 1 3.000000 3.000000 3 0.204603 0.204603 1 0.204603 0.204603 0.20460330576630303 0.338319 0.338319 1 0.338319 0.338319 0.33831852551664776 +eks wye 4.000000 4.000000 1 4.000000 4.000000 4 0.381399 0.381399 1 0.381399 0.381399 0.38139939387114097 0.134189 0.134189 1 0.134189 0.134189 0.13418874328430463 +wye pan 5.000000 5.000000 1 5.000000 5.000000 5 0.573289 0.573289 1 0.573289 0.573289 0.5732889198020006 0.863624 0.863624 1 0.863624 0.863624 0.8636244699032729 +zee pan 6.000000 6.000000 1 6.000000 6.000000 6 0.527126 0.527126 1 0.527126 0.527126 0.5271261600918548 0.493221 0.493221 1 0.493221 0.493221 0.49322128674835697 +eks zee 7.000000 7.000000 1 7.000000 7.000000 7 0.611784 0.611784 1 0.611784 0.611784 0.6117840605678454 0.187885 0.187885 1 0.187885 0.187885 0.1878849191181694 +zee wye 8.000000 8.000000 1 8.000000 8.000000 8 0.598554 0.598554 1 0.598554 0.598554 0.5985540091064224 0.976181 0.976181 1 0.976181 0.976181 0.976181385699006 +hat wye 9.000000 9.000000 1 9.000000 9.000000 9 0.031442 0.031442 1 0.031442 0.031442 0.03144187646093577 0.749551 0.749551 1 0.749551 0.749551 0.7495507603507059 +pan wye 10.000000 10.000000 1 10.000000 10.000000 10 0.502626 0.502626 1 0.502626 0.502626 0.5026260055412137 0.952618 0.952618 1 0.952618 0.952618 0.9526183602969864 ./test/../mlr --opprint stats2 -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2,x2,x2 -g a,b ./test/input/abixy-wide a b x_y_ols_m x_y_ols_b x_y_pca_m x_y_pca_b x_y_pca_quality x_y_r2 x_y_corr x_y_cov xy_y2_ols_m xy_y2_ols_b xy_y2_pca_m xy_y2_pca_b xy_y2_pca_quality xy_y2_r2 xy_y2_corr xy_y2_cov x2_x2_ols_m x2_x2_ols_b x2_x2_pca_m x2_x2_pca_b x2_x2_pca_quality x2_x2_r2 x2_x2_corr x2_x2_cov diff --git a/c/test/run b/c/test/run index c9e1db8d2..edc8bc54c 100755 --- a/c/test/run +++ b/c/test/run @@ -102,7 +102,7 @@ run_command $mlr sort -f a -r b -nf x -nr y $indir/abixy announce STATS run_command $mlr count-distinct -f a,b $indir/small $indir/abixy -run_command $mlr --opprint stats1 -a avg,sum,count,min,max -f i,x,y -g a,b $indir/abixy +run_command $mlr --opprint stats1 -a avg,sum,count,min,max,mode -f i,x,y -g a,b $indir/abixy run_command $mlr --opprint stats2 -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2,x2,x2 -g a,b $indir/abixy-wide run_command $mlr --opprint step -a rsum,delta,counter -f x,y -g a $indir/abixy run_command $mlr --opprint histogram -f x,y --lo 0 --hi 1 --nbins 20 $indir/small diff --git a/c/todo.txt b/c/todo.txt index d5ce14894..e725116dc 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -6,6 +6,9 @@ ================================================================ FEATURES !! quantiles !! + -> be sure to include p99/p50 example (with then-chaining) in mlrwik +!! mode + !! reminder pgr legend is broken !! http://en.wikipedia.org/wiki/Order_statistic_tree !! dkvp as generalization of nidx. restructure mlrwik to emphasize this. @@ -18,7 +21,6 @@ FEATURES ! ordered cut (a la reorder). either a new command (yeck) or cut option (e.g. cut -o) * stats1 mode: lhmsi & then sort. what about "1"=="1.0"? * mod op (either c-like, or sane) and put into wikidoc if so. -* impl/doc mlr check: no output * linreg-quality 2nd pass -- code it up in stats2 w/ -m {m} -b {b} -- ? ================================================================ @@ -66,6 +68,7 @@ TESTING * reorder reg-test cases * ut cases: multifile & multimapper often. e.g. wrapping everything in cat then X then cat ... * ut cases: make should-fail machinery & use it for null-key dkvp cases. +* ut case for stats1 mode ================================================================ DOC diff --git a/doc/content-for-reference.html b/doc/content-for-reference.html index 45dc9585a..e226fa9a4 100644 --- a/doc/content-for-reference.html +++ b/doc/content-for-reference.html @@ -89,6 +89,8 @@ Miller. For example: POKI_CARDIFY(mlr cut --complement -f os_version *.dat | mlr sort -f hostname,uptime)HERE +POKI_INCLUDE_ESCAPED(data/then-chaining-performance.txt)HERE +
For better performance (avoiding redundant string-parsing and string-formatting when you pipe Miller commands together) you can, if you like, instead simply @@ -449,6 +451,49 @@ POKI_RUN_COMMAND{{mlr --oxtab stats1 -a count,sum,avg -f x,y data/medium}}HERE POKI_RUN_COMMAND{{mlr --opprint stats1 -a avg -f x,y -g b then sort -f b data/medium}}HERE + ++% cat piped.sh +mlr cut -x -f i,y data/big | mlr sort -n y > /dev/null + +% time sh piped.sh +real 0m2.828s +user 0m3.183s +sys 0m0.137s + + +% cat chained.sh +mlr cut -x -f i,y then sort -n y data/big > /dev/null + +% time sh chained.sh +real 0m2.082s +user 0m1.933s +sys 0m0.137s ++