Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-18 00:45:47 +00:00
Code Issues Projects Releases Wiki Activity

stats1 mode; perf note

Browse source
This commit is contained in:
John Kerl 2015-05-09 17:30:45 -07:00
parent 868397bfd3
commit 80093e6efc
13 changed files with 246 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

16
c/containers/lhmsi.c
View file

@ -182,7 +182,21 @@ int lhmsi_get(lhmsi_t* pmap, char* key) {
if (pmap->states[index] == OCCUPIED)
return pe->value;
else if (pmap->states[index] == EMPTY)
return -999;
return -999; // xxx fix me ... extend the api to handle nullity.
else {
fprintf(stderr, "lhmsi_find_index_for_key did not find end of chain");
exit(1);
}
}
lhmsie_t* lhmsi_get_entry(lhmsi_t* pmap, char* key) {
int index = lhmsi_find_index_for_key(pmap, key);
lhmsie_t* pe = &pmap->entries[index];
if (pmap->states[index] == OCCUPIED)
return pe;
else if (pmap->states[index] == EMPTY)
return NULL;
else {
fprintf(stderr, "lhmsi_find_index_for_key did not find end of chain");
exit(1);

3
c/containers/lhmsi.h
View file

@ -22,7 +22,7 @@
typedef struct _lhmsie_t {
int ideal_index;
char* key;
int value;
int value; // xxx make ull
struct _lhmsie_t *pprev;
struct _lhmsie_t *pnext;
} lhmsie_t;
@ -45,6 +45,7 @@ lhmsi_t* lhmsi_copy(lhmsi_t* pmap);
void lhmsi_free(lhmsi_t* pmap);
void lhmsi_put(lhmsi_t* pmap, char* key, int value);
int lhmsi_get(lhmsi_t* pmap, char* key);
lhmsie_t* lhmsi_get_entry(lhmsi_t* pmap, char* key);
int lhmsi_has_key(lhmsi_t* pmap, char* key);
void lhmsi_remove(lhmsi_t* pmap, char* key);

2
c/containers/lhmsv.c
View file

@ -1,5 +1,5 @@
// ================================================================
// Array-only (open addressing) string-to-string linked hash map with linear
// Array-only (open addressing) string-to-void linked hash map with linear
// probing for collisions.
//
// John Kerl 2012-08-13

2
c/containers/lhmsv.h
View file

@ -4,7 +4,7 @@
#include "containers/sllv.h"
// ================================================================
// Array-only (open addressing) string-to-string linked hash map with linear
// Array-only (open addressing) string-to-void linked hash map with linear
// probing for collisions.
//
// John Kerl 2012-08-13

89
c/mapping/mapper_stats1.c
View file

@ -8,6 +8,7 @@
#include "containers/slls.h"
#include "containers/lhmslv.h"
#include "containers/lhmsv.h"
#include "containers/lhmsi.h"
#include "containers/mixutil.h"
#include "mapping/mappers.h"
#include "cli/argparse.h"
@ -24,6 +25,7 @@
// char* get();
// ================================================================
// put -> ingest/consume/something
typedef void acc_dput_func_t(void* pvstate, double val);
typedef void acc_sput_func_t(void* pvstate, char* val);
typedef char* acc_get_func_t (void* pvstate, char* pfree_flags);
@ -251,6 +253,52 @@ acc_t* acc_max_alloc(static_context_t* pstatx) {
return pacc;
}
// ----------------------------------------------------------------
typedef struct _acc_mode_state_t {
lhmsi_t* pcounts_for_value;
static_context_t* pstatx;
} acc_mode_state_t;
// mode on strings? what about "1.0" and "1" and "1.0000" ??
void acc_mode_sput(void* pvstate, char* val) {
acc_mode_state_t* pstate = pvstate;
lhmsie_t* pe = lhmsi_get_entry(pstate->pcounts_for_value, val);
if (pe == NULL) {
// xxx at the moment, lhmsi does a strdup so we needn't.
lhmsi_put(pstate->pcounts_for_value, val, 1);
} else {
pe->value++;
}
}
char* acc_mode_get(void* pvstate, char* pfree_flags) {
acc_mode_state_t* pstate = pvstate;
int max_count = 0;
char* max_key = NULL;
for (lhmsie_t* pe = pstate->pcounts_for_value->phead; pe != NULL; pe = pe->pnext) {
int count = pe->value;
if (count > max_count) {
max_key = pe->key;
max_count = count;
}
}
return max_key;
}
// xxx somewhere note a crucial assumption: while pctx is passed in
// to the mapper on a per-row basis, we stash it here on first use and
// use it on subsequent rows. assumptions:
// * the address doesn't change
// * the content we use (namely, ofmt) isn't row-dependent
acc_t* acc_mode_alloc(static_context_t* pstatx) {
acc_t* pacc = mlr_malloc_or_die(sizeof(acc_t));
acc_mode_state_t* pstate = mlr_malloc_or_die(sizeof(acc_mode_state_t));
pstate->pcounts_for_value = lhmsi_alloc();
pstate->pstatx = pstatx;
pacc->pvstate = (void*)pstate;
pacc->psput_func = &acc_mode_sput;
pacc->pdput_func = NULL;
pacc->pget_func = &acc_mode_get;
return pacc;
}
// ----------------------------------------------------------------
typedef struct _acc_lookup_t {
char* name;
@ -264,6 +312,7 @@ static acc_lookup_t acc_lookup_table[] = {
{"avgeb", acc_avgeb_alloc},
{"min", acc_min_alloc},
{"max", acc_max_alloc},
{"mode", acc_mode_alloc},
};
static int acc_lookup_table_length = sizeof(acc_lookup_table) / sizeof(acc_lookup_table[0]);
@ -281,7 +330,7 @@ typedef struct _mapper_stats1_state_t {
slls_t* pvalue_field_names;
slls_t* pgroup_by_field_names;
lhmslv_t* pmaps_level_1;
lhmslv_t* groups;
} mapper_stats1_state_t;
@ -293,10 +342,10 @@ typedef struct _mapper_stats1_state_t {
// s t 5 6 u w 1 7 1 9
// u w 7 9
// ["s","t"] |--> "x" |--> "sum" |--> acc_sum_t* (as void*)
// ["s","t"] |--> "x" |--> "sum" |--> acc_t* (as void*)
// level_1 level_2 level_3
// lhmslv_t lhmsv_t lhmsv_t
// acc_sum_t implements interface:
// acc_t implements interface:
// void init();
// void dacc(double dval);
// void sacc(char* sval);
@ -323,10 +372,10 @@ sllv_t* mapper_stats1_func(lrec_t* pinrec, context_t* pctx, void* pvstate) {
return NULL;
}
lhmsv_t* pmaps_level_2 = lhmslv_get(pstate->pmaps_level_1, pgroup_by_field_values);
if (pmaps_level_2 == NULL) {
pmaps_level_2 = lhmsv_alloc();
lhmslv_put(pstate->pmaps_level_1, slls_copy(pgroup_by_field_values), pmaps_level_2);
lhmsv_t* group_to_acc_field = lhmslv_get(pstate->groups, pgroup_by_field_values);
if (group_to_acc_field == NULL) {
group_to_acc_field = lhmsv_alloc();
lhmslv_put(pstate->groups, slls_copy(pgroup_by_field_values), group_to_acc_field);
}
sllse_t* pa = pstate->pvalue_field_names->phead;
@ -338,17 +387,17 @@ sllv_t* mapper_stats1_func(lrec_t* pinrec, context_t* pctx, void* pvstate) {
int have_dval = FALSE;
double value_field_dval = -999.0;
lhmsv_t* pmaps_level_3 = lhmsv_get(pmaps_level_2, value_field_name);
if (pmaps_level_3 == NULL) {
pmaps_level_3 = lhmsv_alloc();
lhmsv_put(pmaps_level_2, value_field_name, pmaps_level_3);
lhmsv_t* acc_field_to_acc_state = lhmsv_get(group_to_acc_field, value_field_name);
if (acc_field_to_acc_state == NULL) {
acc_field_to_acc_state = lhmsv_alloc();
lhmsv_put(group_to_acc_field, value_field_name, acc_field_to_acc_state);
}
// for "sum", "count"
sllse_t* pc = pstate->paccumulator_names->phead;
for ( ; pc != NULL; pc = pc->pnext) {
char* acc_name = pc->value;
acc_t* pacc = lhmsv_get(pmaps_level_3, acc_name);
acc_t* pacc = lhmsv_get(acc_field_to_acc_state, acc_name);
if (pacc == NULL) {
pacc = make_acc(acc_name, &pctx->statx);
if (pacc == NULL) {
@ -356,7 +405,7 @@ sllv_t* mapper_stats1_func(lrec_t* pinrec, context_t* pctx, void* pvstate) {
acc_name);
exit(1);
}
lhmsv_put(pmaps_level_3, acc_name, pacc);
lhmsv_put(acc_field_to_acc_state, acc_name, pacc);
}
if (pacc == NULL) {
// xxx needs argv[0] somewhere ...
@ -383,7 +432,7 @@ sllv_t* mapper_stats1_func(lrec_t* pinrec, context_t* pctx, void* pvstate) {
else {
sllv_t* poutrecs = sllv_alloc();
for (lhmslve_t* pa = pstate->pmaps_level_1->phead; pa != NULL; pa = pa->pnext) {
for (lhmslve_t* pa = pstate->groups->phead; pa != NULL; pa = pa->pnext) {
lrec_t* poutrec = lrec_unbacked_alloc();
slls_t* pgroup_by_field_values = pa->key;
@ -396,13 +445,13 @@ sllv_t* mapper_stats1_func(lrec_t* pinrec, context_t* pctx, void* pvstate) {
}
// Add in fields such as x_sum=#, y_count=#, etc.:
lhmsv_t* pmaps_level_2 = pa->value;
lhmsv_t* group_to_acc_field = pa->value;
// for "x", "y"
for (lhmsve_t* pd = pmaps_level_2->phead; pd != NULL; pd = pd->pnext) {
for (lhmsve_t* pd = group_to_acc_field->phead; pd != NULL; pd = pd->pnext) {
char* value_field_name = pd->key;
lhmsv_t* pmaps_level_3 = pd->value;
lhmsv_t* acc_field_to_acc_state = pd->value;
// for "count", "sum"
for (lhmsve_t* pe = pmaps_level_3->phead; pe != NULL; pe = pe->pnext) {
for (lhmsve_t* pe = acc_field_to_acc_state->phead; pe != NULL; pe = pe->pnext) {
char* acc_name = pe->key;
acc_t* pacc = pe->value;
@ -427,7 +476,7 @@ static void mapper_stats1_free(void* pvstate) {
slls_free(pstate->pvalue_field_names);
slls_free(pstate->pgroup_by_field_names);
// xxx free the level-2's 1st
lhmslv_free(pstate->pmaps_level_1);
lhmslv_free(pstate->groups);
}
mapper_t* mapper_stats1_alloc(slls_t* paccumulator_names, slls_t* pvalue_field_names, slls_t* pgroup_by_field_names) {
@ -438,7 +487,7 @@ mapper_t* mapper_stats1_alloc(slls_t* paccumulator_names, slls_t* pvalue_field_n
pstate->paccumulator_names = paccumulator_names;
pstate->pvalue_field_names = pvalue_field_names;
pstate->pgroup_by_field_names = pgroup_by_field_names;
pstate->pmaps_level_1 = lhmslv_alloc();
pstate->groups = lhmslv_alloc();
pmapper->pvstate = pstate;
pmapper->pmapper_process_func = mapper_stats1_func;

4
c/t1.rb
View file

@ -77,7 +77,7 @@ run('CORRMLRV', "mlr --csv stats2 -a corr -f x,y -g a,b < data/big.csv >
puts
puts
run("sortsort1", "sort -t, -k 1,2 < data/big > /dev/null")
run("sortsort1", "sort -t= -k 1,2 < data/big > /dev/null")
run("SORTMLR1", "mlr sort -f a,b < data/big > /dev/null")
puts
@ -90,5 +90,5 @@ run("sortsort1v", "sort -t, -k 1,4 < data/big.csv > /dev/null")
run("SORTMLR1V", "mlr --csv sort -f a,x < data/big.csv > /dev/null")
puts
run("sortsort2v", "sort -t, -k 4,5 < data/big.csv > /dev/null")
run("sortsort2v", "sort -t, -n -k 4,5 < data/big.csv > /dev/null")
run("SORTMLR2V", "mlr --csv sort -n x,y < data/big.csv > /dev/null")

24
c/test/expected/out
View file

@ -402,18 +402,18 @@ a=zee,b=wye,count=2
a=hat,b=wye,count=2
a=pan,b=wye,count=2
./test/../mlr --opprint stats1 -a avg,sum,count,min,max -f i,x,y -g a,b ./test/input/abixy
a b i_avg i_sum i_count i_min i_max x_avg x_sum x_count x_min x_max y_avg y_sum y_count y_min y_max
pan pan 1.000000 1.000000 1 1.000000 1.000000 0.346790 0.346790 1 0.346790 0.346790 0.726803 0.726803 1 0.726803 0.726803
eks pan 2.000000 2.000000 1 2.000000 2.000000 0.758680 0.758680 1 0.758680 0.758680 0.522151 0.522151 1 0.522151 0.522151
wye wye 3.000000 3.000000 1 3.000000 3.000000 0.204603 0.204603 1 0.204603 0.204603 0.338319 0.338319 1 0.338319 0.338319
eks wye 4.000000 4.000000 1 4.000000 4.000000 0.381399 0.381399 1 0.381399 0.381399 0.134189 0.134189 1 0.134189 0.134189
wye pan 5.000000 5.000000 1 5.000000 5.000000 0.573289 0.573289 1 0.573289 0.573289 0.863624 0.863624 1 0.863624 0.863624
zee pan 6.000000 6.000000 1 6.000000 6.000000 0.527126 0.527126 1 0.527126 0.527126 0.493221 0.493221 1 0.493221 0.493221
eks zee 7.000000 7.000000 1 7.000000 7.000000 0.611784 0.611784 1 0.611784 0.611784 0.187885 0.187885 1 0.187885 0.187885
zee wye 8.000000 8.000000 1 8.000000 8.000000 0.598554 0.598554 1 0.598554 0.598554 0.976181 0.976181 1 0.976181 0.976181
hat wye 9.000000 9.000000 1 9.000000 9.000000 0.031442 0.031442 1 0.031442 0.031442 0.749551 0.749551 1 0.749551 0.749551
pan wye 10.000000 10.000000 1 10.000000 10.000000 0.502626 0.502626 1 0.502626 0.502626 0.952618 0.952618 1 0.952618 0.952618
./test/../mlr --opprint stats1 -a avg,sum,count,min,max,mode -f i,x,y -g a,b ./test/input/abixy
a b i_avg i_sum i_count i_min i_max i_mode x_avg x_sum x_count x_min x_max x_mode y_avg y_sum y_count y_min y_max y_mode
pan pan 1.000000 1.000000 1 1.000000 1.000000 1 0.346790 0.346790 1 0.346790 0.346790 0.3467901443380824 0.726803 0.726803 1 0.726803 0.726803 0.7268028627434533
eks pan 2.000000 2.000000 1 2.000000 2.000000 2 0.758680 0.758680 1 0.758680 0.758680 0.7586799647899636 0.522151 0.522151 1 0.522151 0.522151 0.5221511083334797
wye wye 3.000000 3.000000 1 3.000000 3.000000 3 0.204603 0.204603 1 0.204603 0.204603 0.20460330576630303 0.338319 0.338319 1 0.338319 0.338319 0.33831852551664776
eks wye 4.000000 4.000000 1 4.000000 4.000000 4 0.381399 0.381399 1 0.381399 0.381399 0.38139939387114097 0.134189 0.134189 1 0.134189 0.134189 0.13418874328430463
wye pan 5.000000 5.000000 1 5.000000 5.000000 5 0.573289 0.573289 1 0.573289 0.573289 0.5732889198020006 0.863624 0.863624 1 0.863624 0.863624 0.8636244699032729
zee pan 6.000000 6.000000 1 6.000000 6.000000 6 0.527126 0.527126 1 0.527126 0.527126 0.5271261600918548 0.493221 0.493221 1 0.493221 0.493221 0.49322128674835697
eks zee 7.000000 7.000000 1 7.000000 7.000000 7 0.611784 0.611784 1 0.611784 0.611784 0.6117840605678454 0.187885 0.187885 1 0.187885 0.187885 0.1878849191181694
zee wye 8.000000 8.000000 1 8.000000 8.000000 8 0.598554 0.598554 1 0.598554 0.598554 0.5985540091064224 0.976181 0.976181 1 0.976181 0.976181 0.976181385699006
hat wye 9.000000 9.000000 1 9.000000 9.000000 9 0.031442 0.031442 1 0.031442 0.031442 0.03144187646093577 0.749551 0.749551 1 0.749551 0.749551 0.7495507603507059
pan wye 10.000000 10.000000 1 10.000000 10.000000 10 0.502626 0.502626 1 0.502626 0.502626 0.5026260055412137 0.952618 0.952618 1 0.952618 0.952618 0.9526183602969864
./test/../mlr --opprint stats2 -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2,x2,x2 -g a,b ./test/input/abixy-wide
a b x_y_ols_m x_y_ols_b x_y_pca_m x_y_pca_b x_y_pca_quality x_y_r2 x_y_corr x_y_cov xy_y2_ols_m xy_y2_ols_b xy_y2_pca_m xy_y2_pca_b xy_y2_pca_quality xy_y2_r2 xy_y2_corr xy_y2_cov x2_x2_ols_m x2_x2_ols_b x2_x2_pca_m x2_x2_pca_b x2_x2_pca_quality x2_x2_r2 x2_x2_corr x2_x2_cov

24
c/test/output/out
View file

@ -402,18 +402,18 @@ a=zee,b=wye,count=2
a=hat,b=wye,count=2
a=pan,b=wye,count=2
./test/../mlr --opprint stats1 -a avg,sum,count,min,max -f i,x,y -g a,b ./test/input/abixy
a b i_avg i_sum i_count i_min i_max x_avg x_sum x_count x_min x_max y_avg y_sum y_count y_min y_max
pan pan 1.000000 1.000000 1 1.000000 1.000000 0.346790 0.346790 1 0.346790 0.346790 0.726803 0.726803 1 0.726803 0.726803
eks pan 2.000000 2.000000 1 2.000000 2.000000 0.758680 0.758680 1 0.758680 0.758680 0.522151 0.522151 1 0.522151 0.522151
wye wye 3.000000 3.000000 1 3.000000 3.000000 0.204603 0.204603 1 0.204603 0.204603 0.338319 0.338319 1 0.338319 0.338319
eks wye 4.000000 4.000000 1 4.000000 4.000000 0.381399 0.381399 1 0.381399 0.381399 0.134189 0.134189 1 0.134189 0.134189
wye pan 5.000000 5.000000 1 5.000000 5.000000 0.573289 0.573289 1 0.573289 0.573289 0.863624 0.863624 1 0.863624 0.863624
zee pan 6.000000 6.000000 1 6.000000 6.000000 0.527126 0.527126 1 0.527126 0.527126 0.493221 0.493221 1 0.493221 0.493221
eks zee 7.000000 7.000000 1 7.000000 7.000000 0.611784 0.611784 1 0.611784 0.611784 0.187885 0.187885 1 0.187885 0.187885
zee wye 8.000000 8.000000 1 8.000000 8.000000 0.598554 0.598554 1 0.598554 0.598554 0.976181 0.976181 1 0.976181 0.976181
hat wye 9.000000 9.000000 1 9.000000 9.000000 0.031442 0.031442 1 0.031442 0.031442 0.749551 0.749551 1 0.749551 0.749551
pan wye 10.000000 10.000000 1 10.000000 10.000000 0.502626 0.502626 1 0.502626 0.502626 0.952618 0.952618 1 0.952618 0.952618
./test/../mlr --opprint stats1 -a avg,sum,count,min,max,mode -f i,x,y -g a,b ./test/input/abixy
a b i_avg i_sum i_count i_min i_max i_mode x_avg x_sum x_count x_min x_max x_mode y_avg y_sum y_count y_min y_max y_mode
pan pan 1.000000 1.000000 1 1.000000 1.000000 1 0.346790 0.346790 1 0.346790 0.346790 0.3467901443380824 0.726803 0.726803 1 0.726803 0.726803 0.7268028627434533
eks pan 2.000000 2.000000 1 2.000000 2.000000 2 0.758680 0.758680 1 0.758680 0.758680 0.7586799647899636 0.522151 0.522151 1 0.522151 0.522151 0.5221511083334797
wye wye 3.000000 3.000000 1 3.000000 3.000000 3 0.204603 0.204603 1 0.204603 0.204603 0.20460330576630303 0.338319 0.338319 1 0.338319 0.338319 0.33831852551664776
eks wye 4.000000 4.000000 1 4.000000 4.000000 4 0.381399 0.381399 1 0.381399 0.381399 0.38139939387114097 0.134189 0.134189 1 0.134189 0.134189 0.13418874328430463
wye pan 5.000000 5.000000 1 5.000000 5.000000 5 0.573289 0.573289 1 0.573289 0.573289 0.5732889198020006 0.863624 0.863624 1 0.863624 0.863624 0.8636244699032729
zee pan 6.000000 6.000000 1 6.000000 6.000000 6 0.527126 0.527126 1 0.527126 0.527126 0.5271261600918548 0.493221 0.493221 1 0.493221 0.493221 0.49322128674835697
eks zee 7.000000 7.000000 1 7.000000 7.000000 7 0.611784 0.611784 1 0.611784 0.611784 0.6117840605678454 0.187885 0.187885 1 0.187885 0.187885 0.1878849191181694
zee wye 8.000000 8.000000 1 8.000000 8.000000 8 0.598554 0.598554 1 0.598554 0.598554 0.5985540091064224 0.976181 0.976181 1 0.976181 0.976181 0.976181385699006
hat wye 9.000000 9.000000 1 9.000000 9.000000 9 0.031442 0.031442 1 0.031442 0.031442 0.03144187646093577 0.749551 0.749551 1 0.749551 0.749551 0.7495507603507059
pan wye 10.000000 10.000000 1 10.000000 10.000000 10 0.502626 0.502626 1 0.502626 0.502626 0.5026260055412137 0.952618 0.952618 1 0.952618 0.952618 0.9526183602969864
./test/../mlr --opprint stats2 -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2,x2,x2 -g a,b ./test/input/abixy-wide
a b x_y_ols_m x_y_ols_b x_y_pca_m x_y_pca_b x_y_pca_quality x_y_r2 x_y_corr x_y_cov xy_y2_ols_m xy_y2_ols_b xy_y2_pca_m xy_y2_pca_b xy_y2_pca_quality xy_y2_r2 xy_y2_corr xy_y2_cov x2_x2_ols_m x2_x2_ols_b x2_x2_pca_m x2_x2_pca_b x2_x2_pca_quality x2_x2_r2 x2_x2_corr x2_x2_cov

2
c/test/run
View file

@ -102,7 +102,7 @@ run_command $mlr sort -f a -r b -nf x -nr y $indir/abixy
announce STATS
run_command $mlr count-distinct -f a,b $indir/small $indir/abixy
run_command $mlr --opprint stats1 -a avg,sum,count,min,max -f i,x,y -g a,b $indir/abixy
run_command $mlr --opprint stats1 -a avg,sum,count,min,max,mode -f i,x,y -g a,b $indir/abixy
run_command $mlr --opprint stats2 -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2,x2,x2 -g a,b $indir/abixy-wide
run_command $mlr --opprint step -a rsum,delta,counter -f x,y -g a $indir/abixy
run_command $mlr --opprint histogram -f x,y --lo 0 --hi 1 --nbins 20 $indir/small

5
c/todo.txt
View file

@ -6,6 +6,9 @@
================================================================
FEATURES
!! quantiles !!
-> be sure to include p99/p50 example (with then-chaining) in mlrwik
!! mode
!! reminder pgr legend is broken !!
http://en.wikipedia.org/wiki/Order_statistic_tree
!! dkvp as generalization of nidx. restructure mlrwik to emphasize this.
@ -18,7 +21,6 @@ FEATURES
! ordered cut (a la reorder). either a new command (yeck) or cut option (e.g. cut -o)
* stats1 mode: lhmsi & then sort. what about "1"=="1.0"?
* mod op (either c-like, or sane) and put into wikidoc if so.
* impl/doc mlr check: no output
* linreg-quality 2nd pass -- code it up in stats2 w/ -m {m} -b {b} -- ?
================================================================
@ -66,6 +68,7 @@ TESTING
* reorder reg-test cases
* ut cases: multifile & multimapper often. e.g. wrapping everything in cat then X then cat ...
* ut cases: make should-fail machinery & use it for null-key dkvp cases.
* ut case for stats1 mode
================================================================
DOC

45
doc/content-for-reference.html
View file

@ -89,6 +89,8 @@ Miller. For example:
POKI_CARDIFY(mlr cut --complement -f os_version *.dat | mlr sort -f hostname,uptime)HERE
POKI_INCLUDE_ESCAPED(data/then-chaining-performance.txt)HERE
<p/>
For better performance (avoiding redundant string-parsing and string-formatting
when you pipe Miller commands together) you can, if you like, instead simply
@ -449,6 +451,49 @@ POKI_RUN_COMMAND{{mlr --oxtab stats1 -a count,sum,avg -f x,y data/medium}}HERE
POKI_RUN_COMMAND{{mlr --opprint stats1 -a avg -f x,y -g b then sort -f b data/medium}}HERE
</td></tr></table>
<!--
kerl@kerl-mbp[s1j0d1][c]$ mlr --opprint count-distinct -f b then sort -nr count data/small
b count
wye 5
pan 4
zee 1
kerl@kerl-mbp[s0j0d1][c]$ mlr --opprint count-distinct -f a then sort -nr count data/small
a count
eks 3
pan 2
wye 2
zee 2
hat 1
kerl@kerl-mbp[s0j0d1][c]$ mlr --omlr --opprint stats1 -a mode -f a,b data/small
a_mode b_mode
eks wye
kerl@kerl-mbp[s0j0d1][c]$ mlr --opprint sort -f a data/small
a b i x y
eks pan 2 0.7586799647899636 0.5221511083334797
eks wye 4 0.38139939387114097 0.13418874328430463
eks zee 7 0.6117840605678454 0.1878849191181694
hat wye 9 0.03144187646093577 0.7495507603507059
pan pan 1 0.3467901443380824 0.7268028627434533
pan wye 10 0.5026260055412137 0.9526183602969864
wye wye 3 0.20460330576630303 0.33831852551664776
wye pan 5 0.5732889198020006 0.8636244699032729
zee pan 6 0.5271261600918548 0.49322128674835697
zee wye 8 0.5985540091064224 0.976181385699006
kerl@kerl-mbp[s0j0d1][c]$ mlr --opprint sort -f b data/small
a b i x y
pan pan 1 0.3467901443380824 0.7268028627434533
eks pan 2 0.7586799647899636 0.5221511083334797
wye pan 5 0.5732889198020006 0.8636244699032729
zee pan 6 0.5271261600918548 0.49322128674835697
wye wye 3 0.20460330576630303 0.33831852551664776
eks wye 4 0.38139939387114097 0.13418874328430463
zee wye 8 0.5985540091064224 0.976181385699006
hat wye 9 0.03144187646093577 0.7495507603507059
pan wye 10 0.5026260055412137 0.9526183602969864
eks zee 7 0.6117840605678454 0.1878849191181694
-->
<!-- ================================================================ -->
<h2>stats2</h2>

16
doc/data/then-chaining-performance.txt Normal file
View file

@ -0,0 +1,16 @@
% cat piped.sh
mlr cut -x -f i,y data/big | mlr sort -n y > /dev/null
% time sh piped.sh
real 0m2.828s
user 0m3.183s
sys 0m0.137s
% cat chained.sh
mlr cut -x -f i,y then sort -n y data/big > /dev/null
% time sh chained.sh
real 0m2.082s
user 0m1.933s
sys 0m0.137s

66
doc/reference.html
View file

@ -264,6 +264,29 @@ mlr cut --complement -f os_version *.dat | mlr sort -f hostname,uptime
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
% cat piped.sh
mlr cut -x -f i,y data/big | mlr sort -n y &gt; /dev/null
% time sh piped.sh
real 0m2.828s
user 0m3.183s
sys 0m0.137s
% cat chained.sh
mlr cut -x -f i,y then sort -n y data/big &gt; /dev/null
% time sh chained.sh
real 0m2.082s
user 0m1.933s
sys 0m0.137s
</pre>
</div>
<p/>
<p/>
For better performance (avoiding redundant string-parsing and string-formatting
when you pipe Miller commands together) you can, if you like, instead simply
@ -1240,6 +1263,49 @@ zee 0.504242 0.502997
<p/>
</td></tr></table>
<!--
kerl@kerl-mbp[s1j0d1][c]$ mlr --opprint count-distinct -f b then sort -nr count data/small
b count
wye 5
pan 4
zee 1
kerl@kerl-mbp[s0j0d1][c]$ mlr --opprint count-distinct -f a then sort -nr count data/small
a count
eks 3
pan 2
wye 2
zee 2
hat 1
kerl@kerl-mbp[s0j0d1][c]$ mlr --omlr --opprint stats1 -a mode -f a,b data/small
a_mode b_mode
eks wye
kerl@kerl-mbp[s0j0d1][c]$ mlr --opprint sort -f a data/small
a b i x y
eks pan 2 0.7586799647899636 0.5221511083334797
eks wye 4 0.38139939387114097 0.13418874328430463
eks zee 7 0.6117840605678454 0.1878849191181694
hat wye 9 0.03144187646093577 0.7495507603507059
pan pan 1 0.3467901443380824 0.7268028627434533
pan wye 10 0.5026260055412137 0.9526183602969864
wye wye 3 0.20460330576630303 0.33831852551664776
wye pan 5 0.5732889198020006 0.8636244699032729
zee pan 6 0.5271261600918548 0.49322128674835697
zee wye 8 0.5985540091064224 0.976181385699006
kerl@kerl-mbp[s0j0d1][c]$ mlr --opprint sort -f b data/small
a b i x y
pan pan 1 0.3467901443380824 0.7268028627434533
eks pan 2 0.7586799647899636 0.5221511083334797
wye pan 5 0.5732889198020006 0.8636244699032729
zee pan 6 0.5271261600918548 0.49322128674835697
wye wye 3 0.20460330576630303 0.33831852551664776
eks wye 4 0.38139939387114097 0.13418874328430463
zee wye 8 0.5985540091064224 0.976181385699006
hat wye 9 0.03144187646093577 0.7495507603507059
pan wye 10 0.5026260055412137 0.9526183602969864
eks zee 7 0.6117840605678454 0.1878849191181694
-->
<!-- ================================================================ -->
<a id="stats2"/><h2>stats2</h2>