Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-29 02:30:12 +00:00
Code Issues Projects Releases Wiki Activity

-o option for histogram

Browse source
This commit is contained in:
John Kerl 2017-01-08 21:23:30 -05:00
parent 7669d1b8d2
commit 38559853e6
8 changed files with 109 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

47
c/mapping/mapper_histogram.c
View file

@ -23,13 +23,14 @@ typedef struct _mapper_histogram_state_t {
double mul;
lhmsv_t* pcounts_by_field;
lhmsv_t* pvectors_by_field; // For auto-mode
char* output_prefix;
} mapper_histogram_state_t;
static void mapper_histogram_usage(FILE* o, char* argv0, char* verb);
static mapper_t* mapper_histogram_parse_cli(int* pargi, int argc, char** argv,
cli_reader_opts_t* _, cli_writer_opts_t* __);
static mapper_t* mapper_histogram_alloc(ap_state_t* pargp, slls_t* value_field_names, double lo, int nbins, double hi,
int do_auto);
int do_auto, char* output_prefix);
static void mapper_histogram_free(mapper_t* pmapper);
static void mapper_histogram_ingest(lrec_t* pinrec, mapper_histogram_state_t* pstate);
@ -57,6 +58,7 @@ static void mapper_histogram_usage(FILE* o, char* argv0, char* verb) {
fprintf(o, "--nbins {n} Number of histogram bins\n");
fprintf(o, "--auto Automatically computes limits, ignoring --lo and --hi.\n");
fprintf(o, " Holds all values in memory before producing any output.\n");
fprintf(o, "-p {prefix} Prefix for output field name. Default: no prefix.\n");
fprintf(o, "Just a histogram. Input values < lo or > hi are not counted.\n");
}
@ -68,6 +70,7 @@ static mapper_t* mapper_histogram_parse_cli(int* pargi, int argc, char** argv,
double hi = 0.0;
int nbins = 0;
int do_auto = FALSE;
char* output_prefix = NULL;
char* verb = argv[(*pargi)++];
@ -77,6 +80,7 @@ static mapper_t* mapper_histogram_parse_cli(int* pargi, int argc, char** argv,
ap_define_float_flag(pstate, "--hi", &hi);
ap_define_int_flag(pstate, "--nbins", &nbins);
ap_define_true_flag(pstate, "--auto", &do_auto);
ap_define_string_flag(pstate, "-p", &output_prefix);
if (!ap_parse(pstate, verb, pargi, argc, argv)) {
mapper_histogram_usage(stderr, argv[0], verb);
@ -98,12 +102,12 @@ static mapper_t* mapper_histogram_parse_cli(int* pargi, int argc, char** argv,
return NULL;
}
return mapper_histogram_alloc(pstate, value_field_names, lo, nbins, hi, do_auto);
return mapper_histogram_alloc(pstate, value_field_names, lo, nbins, hi, do_auto, output_prefix);
}
// ----------------------------------------------------------------
static mapper_t* mapper_histogram_alloc(ap_state_t* pargp, slls_t* value_field_names,
double lo, int nbins, double hi, int do_auto)
double lo, int nbins, double hi, int do_auto, char* output_prefix)
{
mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t));
@ -133,6 +137,7 @@ static mapper_t* mapper_histogram_alloc(ap_state_t* pargp, slls_t* value_field_n
pstate->hi = hi;
pstate->mul = nbins / (hi - lo);
}
pstate->output_prefix = output_prefix;
pmapper->pvstate = pstate;
pmapper->pprocess_func = do_auto ? mapper_histogram_process_auto : mapper_histogram_process;
@ -200,7 +205,9 @@ static sllv_t* mapper_histogram_emit(mapper_histogram_state_t* pstate) {
lhmss_t* pcount_field_names = lhmss_alloc();
for (sllse_t* pe = pstate->value_field_names->phead; pe != NULL; pe = pe->pnext) {
char* value_field_name = pe->value;
char* count_field_name = mlr_paste_3_strings(value_field_name, "_", "count");
char* count_field_name = (pstate->output_prefix == NULL)
? mlr_paste_3_strings(value_field_name, "_", "count")
: mlr_paste_4_strings(pstate->output_prefix, value_field_name, "_", "count");
lhmss_put(pcount_field_names, mlr_strdup_or_die(value_field_name), count_field_name,
FREE_ENTRY_KEY|FREE_ENTRY_VALUE);
}
@ -209,10 +216,20 @@ static sllv_t* mapper_histogram_emit(mapper_histogram_state_t* pstate) {
lrec_t* poutrec = lrec_unbacked_alloc();
char* value = mlr_alloc_string_from_double(pstate->lo + i / pstate->mul, MLR_GLOBALS.ofmt);
lrec_put(poutrec, "bin_lo", value, FREE_ENTRY_VALUE);
if (pstate->output_prefix == NULL) {
lrec_put(poutrec, "bin_lo", value, FREE_ENTRY_VALUE);
} else {
lrec_put(poutrec, mlr_paste_2_strings(pstate->output_prefix, "bin_lo"), value,
FREE_ENTRY_KEY | FREE_ENTRY_VALUE);
}
value = mlr_alloc_string_from_double(pstate->lo + (i+1) / pstate->mul, MLR_GLOBALS.ofmt);
lrec_put(poutrec, "bin_hi", value, FREE_ENTRY_VALUE);
if (pstate->output_prefix == NULL) {
lrec_put(poutrec, "bin_hi", value, FREE_ENTRY_VALUE);
} else {
lrec_put(poutrec, mlr_paste_2_strings(pstate->output_prefix, "bin_hi"), value,
FREE_ENTRY_KEY | FREE_ENTRY_VALUE);
}
for (sllse_t* pe = pstate->value_field_names->phead; pe != NULL; pe = pe->pnext) {
char* value_field_name = pe->value;
@ -221,7 +238,8 @@ static sllv_t* mapper_histogram_emit(mapper_histogram_state_t* pstate) {
char* count_field_name = lhmss_get(pcount_field_names, value_field_name);
value = mlr_alloc_string_from_ull(pcounts[i]);
lrec_put(poutrec, mlr_strdup_or_die(count_field_name), value, FREE_ENTRY_KEY|FREE_ENTRY_VALUE);
lrec_put(poutrec, mlr_strdup_or_die(count_field_name), value,
FREE_ENTRY_KEY|FREE_ENTRY_VALUE);
}
sllv_append(poutrecs, poutrec);
@ -303,19 +321,26 @@ static sllv_t* mapper_histogram_emit_auto(mapper_histogram_state_t* pstate) {
// Emission pass
sllv_t* poutrecs = sllv_alloc();
lhmss_t* pcount_field_names = lhmss_alloc();
for (sllse_t* pe = pstate->value_field_names->phead; pe != NULL; pe = pe->pnext) {
char* value_field_name = pe->value;
char* count_field_name = mlr_paste_3_strings(value_field_name, "_", "count");
lhmss_put(pcount_field_names, mlr_strdup_or_die(value_field_name), count_field_name, FREE_ENTRY_KEY|FREE_ENTRY_VALUE);
char* count_field_name = (pstate->output_prefix == NULL)
? mlr_paste_3_strings(value_field_name, "_", "count")
: mlr_paste_4_strings(pstate->output_prefix, value_field_name, "_", "count");
lhmss_put(pcount_field_names, mlr_strdup_or_die(value_field_name), count_field_name,
FREE_ENTRY_KEY|FREE_ENTRY_VALUE);
}
for (int i = 0; i < nbins; i++) {
lrec_t* poutrec = lrec_unbacked_alloc();
char* value = mlr_alloc_string_from_double(lo + i / mul, MLR_GLOBALS.ofmt);
lrec_put(poutrec, "bin_lo", value, FREE_ENTRY_VALUE);
if (pstate->output_prefix == NULL) {
lrec_put(poutrec, "bin_lo", value, FREE_ENTRY_VALUE);
} else {
lrec_put(poutrec, mlr_paste_2_strings(pstate->output_prefix, "bin_lo"), value,
FREE_ENTRY_KEY | FREE_ENTRY_VALUE);
}
value = mlr_alloc_string_from_double(lo + (i+1) / mul, MLR_GLOBALS.ofmt);
lrec_put(poutrec, "bin_hi", value, FREE_ENTRY_VALUE);

3
c/todo.txt
View file

@ -16,9 +16,6 @@ FEATURE REQUESTS
mapping/mapper_histogram.c: lrec_put(poutrec, "bin_lo", value, FREE_ENTRY_VALUE);
mapping/mapper_histogram.c: lrec_put(poutrec, "bin_hi", value, FREE_ENTRY_VALUE);
mapping/mapper_top.c: lrec_put(poutrec, "top_idx", mlr_alloc_string_from_ull(i+1), FREE_ENTRY_VALUE);
mapping/mapper_top.c: lrec_put(poutrec, "top_idx", mlr_alloc_string_from_ull(i+1), FREE_ENTRY_VALUE);
================================================================
OTHER:

2
doc/content-for-reference-verbs.html
View file

@ -243,6 +243,8 @@ etc., but bin 9 has <tt>0.9 &le; x &le; 1.0</tt>.
POKI_RUN_COMMAND{{mlr --opprint put '$x2=$x**2;$x3=$x2*$x' then histogram -f x,x2,x3 --lo 0 --hi 1 --nbins 10 data/medium}}HERE
POKI_RUN_COMMAND{{mlr --opprint put '$x2=$x**2;$x3=$x2*$x' then histogram -f x,x2,x3 --lo 0 --hi 1 --nbins 10 -p "my_" data/medium}}HERE
<!-- ================================================================ -->
<h1>join</h1>

96
doc/cookbook.html
View file

@ -607,33 +607,33 @@ $ mlr --ofmt '%.9lf' --opprint seqgen --start 1 --stop 28 then put '
' then put '$seconds=systime()' then step -a delta -f seconds then cut -x -f seconds
i o fcount seconds_delta
1 1 1 0
2 2 3 0.000025988
3 3 5 0.000013113
4 5 9 0.000015974
5 8 15 0.000021935
6 13 25 0.000033140
7 21 41 0.000046015
8 34 67 0.000070810
9 55 109 0.000112057
10 89 177 0.000178099
11 144 287 0.000282049
12 233 465 0.000453949
13 377 753 0.000732899
14 610 1219 0.001173973
15 987 1973 0.002047062
16 1597 3193 0.003067017
17 2584 5167 0.004837036
18 4181 8361 0.007942915
19 6765 13529 0.012510061
20 10946 21891 0.020375967
21 17711 35421 0.036911011
22 28657 57313 0.055205107
23 46368 92735 0.087458849
24 75025 150049 0.139748096
25 121393 242785 0.220874071
26 196418 392835 0.351442814
27 317811 635621 0.577186108
28 514229 1028457 0.909245014
2 2 3 0.000027895
3 3 5 0.000014067
4 5 9 0.000016928
5 8 15 0.000022888
6 13 25 0.000034094
7 21 41 0.000047922
8 34 67 0.000076056
9 55 109 0.000118017
10 89 177 0.000188112
11 144 287 0.000298977
12 233 465 0.000478983
13 377 753 0.000771999
14 610 1219 0.001240969
15 987 1973 0.002471924
16 1597 3193 0.003172159
17 2584 5167 0.005033970
18 4181 8361 0.009949923
19 6765 13529 0.012804985
20 10946 21891 0.019966125
21 17711 35421 0.031900883
22 28657 57313 0.051866055
23 46368 92735 0.085595846
24 75025 150049 0.148756027
25 121393 242785 0.219053030
26 196418 392835 0.352735996
27 317811 635621 0.579770088
28 514229 1028457 0.903336048
</pre>
</div>
<p/>
@ -665,33 +665,33 @@ $ mlr --ofmt '%.9lf' --opprint seqgen --start 1 --stop 28 then put '
' then put '$seconds=systime()' then step -a delta -f seconds then cut -x -f seconds
i o fcount seconds_delta
1 1 1 0
2 2 3 0.000028849
3 3 3 0.000013113
4 5 3 0.000011921
2 2 3 0.000027895
3 3 3 0.000014067
4 5 3 0.000010967
5 8 3 0.000010967
6 13 3 0.000011206
7 21 3 0.000009775
8 34 3 0.000011206
9 55 3 0.000010014
10 89 3 0.000010967
6 13 3 0.000010967
7 21 3 0.000010014
8 34 3 0.000010967
9 55 3 0.000010967
10 89 3 0.000011206
11 144 3 0.000010014
12 233 3 0.000013828
13 377 3 0.000010967
14 610 3 0.000011206
15 987 3 0.000010014
16 1597 3 0.000010014
17 2584 3 0.000010967
18 4181 3 0.000010014
15 987 3 0.000009775
16 1597 3 0.000011206
17 2584 3 0.000010014
18 4181 3 0.000010967
19 6765 3 0.000010014
20 10946 3 0.000009775
21 17711 3 0.000010014
22 28657 3 0.000012159
23 46368 3 0.000012875
24 75025 3 0.000010967
25 121393 3 0.000010014
26 196418 3 0.000010014
27 317811 3 0.000010967
28 514229 3 0.000010014
20 10946 3 0.000010014
21 17711 3 0.000011921
22 28657 3 0.000010967
23 46368 3 0.000011921
24 75025 3 0.000011206
25 121393 3 0.000009775
26 196418 3 0.000011206
27 317811 3 0.000010014
28 514229 3 0.000009775
</pre>
</div>
<p/>

1
doc/manpage.html
View file

@ -656,6 +656,7 @@ VERBS
--nbins {n} Number of histogram bins
--auto Automatically computes limits, ignoring --lo and --hi.
Holds all values in memory before producing any output.
-p {prefix} Prefix for output field name. Default: no prefix.
Just a histogram. Input values &lt; lo or &gt; hi are not counted.
join

1
doc/manpage.txt
View file

@ -507,6 +507,7 @@ VERBS
--nbins {n} Number of histogram bins
--auto Automatically computes limits, ignoring --lo and --hi.
Holds all values in memory before producing any output.
-p {prefix} Prefix for output field name. Default: no prefix.
Just a histogram. Input values < lo or > hi are not counted.
join

1
doc/mlr.1
View file

@ -696,6 +696,7 @@ Usage: mlr histogram [options]
--nbins {n} Number of histogram bins
--auto Automatically computes limits, ignoring --lo and --hi.
Holds all values in memory before producing any output.
-p {prefix} Prefix for output field name. Default: no prefix.
Just a histogram. Input values < lo or > hi are not counted.
.fi
.if n \{\

20
doc/reference-verbs.html
View file

@ -1006,6 +1006,7 @@ Usage: mlr histogram [options]
--nbins {n} Number of histogram bins
--auto Automatically computes limits, ignoring --lo and --hi.
Holds all values in memory before producing any output.
-p {prefix} Prefix for output field name. Default: no prefix.
Just a histogram. Input values &lt; lo or &gt; hi are not counted.
</pre>
</div>
@ -1037,6 +1038,25 @@ bin_lo bin_hi x_count x2_count x3_count
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint put '$x2=$x**2;$x3=$x2*$x' then histogram -f x,x2,x3 --lo 0 --hi 1 --nbins 10 -p "my_" data/medium
my_bin_lo my_bin_hi my_x_count my_x2_count my_x3_count
0.000000 0.100000 1072 3231 4661
0.100000 0.200000 938 1254 1184
0.200000 0.300000 1037 988 845
0.300000 0.400000 988 832 676
0.400000 0.500000 950 774 576
0.500000 0.600000 1002 692 476
0.600000 0.700000 1007 591 438
0.700000 0.800000 1007 560 420
0.800000 0.900000 986 571 383
0.900000 1.000000 1013 507 341
</pre>
</div>
<p/>
<!-- ================================================================ -->
<a id="join"/><h1>join</h1>