mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-24 00:18:39 +00:00
Port merge-fields verb from C to Go (#514)
* Port merge-fields verb from C to Go * stats1 accumulator reset feature for merge-fields * stats1-accumulator value-to-pointer mod * iterate * merge-fields regtest case-neatens * iterating * iterating * merge-fields regtest complete
This commit is contained in:
John Kerl
GitHub
parent
7c9dbf72a3
commit
5d706fcf65
60 changed files with 864 additions and 541 deletions
|
|
@ -1,4 +0,0 @@
|
|||
a_p0 a_min a_p29 a_max a_p100 a_sum b_p0 b_min b_p29 b_max b_p100 b_sum
|
||||
436.000000 436 451.660000 490 490 926 195.000000 195 267.790000 446 446 641
|
||||
320.000000 320 379.740000 526 526 846 780.000000 780 833.070000 963 963 1743
|
||||
220.000000 220 413.720000 888 888 1108 705.000000 705 741.540000 831 831 1536
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
a_in a_out b_in b_out a_p0 a_min a_p29 a_max a_p100 a_sum b_p0 b_min b_p29 b_max b_p100 b_sum
|
||||
436 490 446 195 436.000000 436 451.660000 490 490 926 195.000000 195 267.790000 446 446 641
|
||||
526 320 963 780 320.000000 320 379.740000 526 526 846 780.000000 780 833.070000 963 963 1743
|
||||
220 888 705 831 220.000000 220 413.720000 888 888 1108 705.000000 705 741.540000 831 831 1536
|
||||
|
|
@ -1 +0,0 @@
|
|||
mlr --oxtab merge-fields -k -a p0,min,p29,max,p100,sum,count -r in_,out_ -o bar regtest/input/merge-fields-abxy.dkvp
|
||||
|
|
@ -1,227 +0,0 @@
|
|||
a_in_x 1
|
||||
a_out_x 2
|
||||
b_in_y 4
|
||||
b_out_x 8
|
||||
bar_p0 1
|
||||
bar_min 1
|
||||
bar_p29 2
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 15
|
||||
bar_count 4
|
||||
|
||||
z 1
|
||||
bar_p0
|
||||
bar_min
|
||||
bar_p29
|
||||
bar_max
|
||||
bar_p100
|
||||
bar_sum 0
|
||||
bar_count 0
|
||||
|
||||
a_on_x 1
|
||||
a_out_x 2
|
||||
b_in_y 4
|
||||
b_out_x 8
|
||||
bar_p0 2
|
||||
bar_min 2
|
||||
bar_p29 2
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 14
|
||||
bar_count 3
|
||||
|
||||
a_in_x 1
|
||||
a_oot_x 2
|
||||
b_in_y 4
|
||||
b_out_x 8
|
||||
bar_p0 1
|
||||
bar_min 1
|
||||
bar_p29 1
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 13
|
||||
bar_count 3
|
||||
|
||||
a_in_x 1
|
||||
a_out_x 2
|
||||
b_on_y 4
|
||||
b_out_x 8
|
||||
bar_p0 1
|
||||
bar_min 1
|
||||
bar_p29 1
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 11
|
||||
bar_count 3
|
||||
|
||||
a_in_x 1
|
||||
a_out_x 2
|
||||
b_in_y 4
|
||||
b_oot_x 8
|
||||
bar_p0 1
|
||||
bar_min 1
|
||||
bar_p29 1
|
||||
bar_max 4
|
||||
bar_p100 4
|
||||
bar_sum 7
|
||||
bar_count 3
|
||||
|
||||
z 2
|
||||
bar_p0
|
||||
bar_min
|
||||
bar_p29
|
||||
bar_max
|
||||
bar_p100
|
||||
bar_sum 0
|
||||
bar_count 0
|
||||
|
||||
a_on_x 1
|
||||
a_oot_x 2
|
||||
b_in_y 4
|
||||
b_out_x 8
|
||||
bar_p0 4
|
||||
bar_min 4
|
||||
bar_p29 4
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 12
|
||||
bar_count 2
|
||||
|
||||
a_on_x 1
|
||||
a_out_x 2
|
||||
b_on_y 4
|
||||
b_out_x 8
|
||||
bar_p0 2
|
||||
bar_min 2
|
||||
bar_p29 2
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 10
|
||||
bar_count 2
|
||||
|
||||
a_on_x 1
|
||||
a_out_x 2
|
||||
b_in_y 4
|
||||
b_oot_x 8
|
||||
bar_p0 2
|
||||
bar_min 2
|
||||
bar_p29 2
|
||||
bar_max 4
|
||||
bar_p100 4
|
||||
bar_sum 6
|
||||
bar_count 2
|
||||
|
||||
a_in_x 1
|
||||
a_oot_x 2
|
||||
b_on_y 4
|
||||
b_out_x 8
|
||||
bar_p0 1
|
||||
bar_min 1
|
||||
bar_p29 1
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 9
|
||||
bar_count 2
|
||||
|
||||
a_in_x 1
|
||||
a_oot_x 2
|
||||
b_in_y 4
|
||||
b_oot_x 8
|
||||
bar_p0 1
|
||||
bar_min 1
|
||||
bar_p29 1
|
||||
bar_max 4
|
||||
bar_p100 4
|
||||
bar_sum 5
|
||||
bar_count 2
|
||||
|
||||
a_in_x 1
|
||||
a_out_x 2
|
||||
b_on_y 4
|
||||
b_oot_x 8
|
||||
bar_p0 1
|
||||
bar_min 1
|
||||
bar_p29 1
|
||||
bar_max 2
|
||||
bar_p100 2
|
||||
bar_sum 3
|
||||
bar_count 2
|
||||
|
||||
z 3
|
||||
bar_p0
|
||||
bar_min
|
||||
bar_p29
|
||||
bar_max
|
||||
bar_p100
|
||||
bar_sum 0
|
||||
bar_count 0
|
||||
|
||||
a_in_x 1
|
||||
a_oot_x 2
|
||||
b_on_y 4
|
||||
b_oot_x 8
|
||||
bar_p0 1
|
||||
bar_min 1
|
||||
bar_p29 1
|
||||
bar_max 1
|
||||
bar_p100 1
|
||||
bar_sum 1
|
||||
bar_count 1
|
||||
|
||||
a_on_x 1
|
||||
a_out_x 2
|
||||
b_on_y 4
|
||||
b_oot_x 8
|
||||
bar_p0 2
|
||||
bar_min 2
|
||||
bar_p29 2
|
||||
bar_max 2
|
||||
bar_p100 2
|
||||
bar_sum 2
|
||||
bar_count 1
|
||||
|
||||
a_on_x 1
|
||||
a_oot_x 2
|
||||
b_in_y 4
|
||||
b_oot_x 8
|
||||
bar_p0 4
|
||||
bar_min 4
|
||||
bar_p29 4
|
||||
bar_max 4
|
||||
bar_p100 4
|
||||
bar_sum 4
|
||||
bar_count 1
|
||||
|
||||
a_on_x 1
|
||||
a_oot_x 2
|
||||
b_on_y 4
|
||||
b_out_x 8
|
||||
bar_p0 8
|
||||
bar_min 8
|
||||
bar_p29 8
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 8
|
||||
bar_count 1
|
||||
|
||||
z 4
|
||||
bar_p0
|
||||
bar_min
|
||||
bar_p29
|
||||
bar_max
|
||||
bar_p100
|
||||
bar_sum 0
|
||||
bar_count 0
|
||||
|
||||
a_on_x 1
|
||||
a_oot_x 2
|
||||
b_on_y 4
|
||||
b_oot_x 8
|
||||
bar_p0
|
||||
bar_min
|
||||
bar_p29
|
||||
bar_max
|
||||
bar_p100
|
||||
bar_sum 0
|
||||
bar_count 0
|
||||
|
|
@ -1 +0,0 @@
|
|||
mlr --oxtab merge-fields -i -k -a p0,min,p29,max,p100,sum,count -r in_,out_ -o bar regtest/input/merge-fields-abxy.dkvp
|
||||
|
|
@ -1,227 +0,0 @@
|
|||
a_in_x 1
|
||||
a_out_x 2
|
||||
b_in_y 4
|
||||
b_out_x 8
|
||||
bar_p0 1.000000
|
||||
bar_min 1
|
||||
bar_p29 1.870000
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 15
|
||||
bar_count 4
|
||||
|
||||
z 1
|
||||
bar_p0
|
||||
bar_min
|
||||
bar_p29
|
||||
bar_max
|
||||
bar_p100
|
||||
bar_sum 0
|
||||
bar_count 0
|
||||
|
||||
a_on_x 1
|
||||
a_out_x 2
|
||||
b_in_y 4
|
||||
b_out_x 8
|
||||
bar_p0 2.000000
|
||||
bar_min 2
|
||||
bar_p29 3.160000
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 14
|
||||
bar_count 3
|
||||
|
||||
a_in_x 1
|
||||
a_oot_x 2
|
||||
b_in_y 4
|
||||
b_out_x 8
|
||||
bar_p0 1.000000
|
||||
bar_min 1
|
||||
bar_p29 2.740000
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 13
|
||||
bar_count 3
|
||||
|
||||
a_in_x 1
|
||||
a_out_x 2
|
||||
b_on_y 4
|
||||
b_out_x 8
|
||||
bar_p0 1.000000
|
||||
bar_min 1
|
||||
bar_p29 1.580000
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 11
|
||||
bar_count 3
|
||||
|
||||
a_in_x 1
|
||||
a_out_x 2
|
||||
b_in_y 4
|
||||
b_oot_x 8
|
||||
bar_p0 1.000000
|
||||
bar_min 1
|
||||
bar_p29 1.580000
|
||||
bar_max 4
|
||||
bar_p100 4
|
||||
bar_sum 7
|
||||
bar_count 3
|
||||
|
||||
z 2
|
||||
bar_p0
|
||||
bar_min
|
||||
bar_p29
|
||||
bar_max
|
||||
bar_p100
|
||||
bar_sum 0
|
||||
bar_count 0
|
||||
|
||||
a_on_x 1
|
||||
a_oot_x 2
|
||||
b_in_y 4
|
||||
b_out_x 8
|
||||
bar_p0 4.000000
|
||||
bar_min 4
|
||||
bar_p29 5.160000
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 12
|
||||
bar_count 2
|
||||
|
||||
a_on_x 1
|
||||
a_out_x 2
|
||||
b_on_y 4
|
||||
b_out_x 8
|
||||
bar_p0 2.000000
|
||||
bar_min 2
|
||||
bar_p29 3.740000
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 10
|
||||
bar_count 2
|
||||
|
||||
a_on_x 1
|
||||
a_out_x 2
|
||||
b_in_y 4
|
||||
b_oot_x 8
|
||||
bar_p0 2.000000
|
||||
bar_min 2
|
||||
bar_p29 2.580000
|
||||
bar_max 4
|
||||
bar_p100 4
|
||||
bar_sum 6
|
||||
bar_count 2
|
||||
|
||||
a_in_x 1
|
||||
a_oot_x 2
|
||||
b_on_y 4
|
||||
b_out_x 8
|
||||
bar_p0 1.000000
|
||||
bar_min 1
|
||||
bar_p29 3.030000
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 9
|
||||
bar_count 2
|
||||
|
||||
a_in_x 1
|
||||
a_oot_x 2
|
||||
b_in_y 4
|
||||
b_oot_x 8
|
||||
bar_p0 1.000000
|
||||
bar_min 1
|
||||
bar_p29 1.870000
|
||||
bar_max 4
|
||||
bar_p100 4
|
||||
bar_sum 5
|
||||
bar_count 2
|
||||
|
||||
a_in_x 1
|
||||
a_out_x 2
|
||||
b_on_y 4
|
||||
b_oot_x 8
|
||||
bar_p0 1.000000
|
||||
bar_min 1
|
||||
bar_p29 1.290000
|
||||
bar_max 2
|
||||
bar_p100 2
|
||||
bar_sum 3
|
||||
bar_count 2
|
||||
|
||||
z 3
|
||||
bar_p0
|
||||
bar_min
|
||||
bar_p29
|
||||
bar_max
|
||||
bar_p100
|
||||
bar_sum 0
|
||||
bar_count 0
|
||||
|
||||
a_in_x 1
|
||||
a_oot_x 2
|
||||
b_on_y 4
|
||||
b_oot_x 8
|
||||
bar_p0 1
|
||||
bar_min 1
|
||||
bar_p29 1
|
||||
bar_max 1
|
||||
bar_p100 1
|
||||
bar_sum 1
|
||||
bar_count 1
|
||||
|
||||
a_on_x 1
|
||||
a_out_x 2
|
||||
b_on_y 4
|
||||
b_oot_x 8
|
||||
bar_p0 2
|
||||
bar_min 2
|
||||
bar_p29 2
|
||||
bar_max 2
|
||||
bar_p100 2
|
||||
bar_sum 2
|
||||
bar_count 1
|
||||
|
||||
a_on_x 1
|
||||
a_oot_x 2
|
||||
b_in_y 4
|
||||
b_oot_x 8
|
||||
bar_p0 4
|
||||
bar_min 4
|
||||
bar_p29 4
|
||||
bar_max 4
|
||||
bar_p100 4
|
||||
bar_sum 4
|
||||
bar_count 1
|
||||
|
||||
a_on_x 1
|
||||
a_oot_x 2
|
||||
b_on_y 4
|
||||
b_out_x 8
|
||||
bar_p0 8
|
||||
bar_min 8
|
||||
bar_p29 8
|
||||
bar_max 8
|
||||
bar_p100 8
|
||||
bar_sum 8
|
||||
bar_count 1
|
||||
|
||||
z 4
|
||||
bar_p0
|
||||
bar_min
|
||||
bar_p29
|
||||
bar_max
|
||||
bar_p100
|
||||
bar_sum 0
|
||||
bar_count 0
|
||||
|
||||
a_on_x 1
|
||||
a_oot_x 2
|
||||
b_on_y 4
|
||||
b_oot_x 8
|
||||
bar_p0
|
||||
bar_min
|
||||
bar_p29
|
||||
bar_max
|
||||
bar_p100
|
||||
bar_sum 0
|
||||
bar_count 0
|
||||
|
|
@ -362,6 +362,50 @@ Options:
|
|||
-o {name} Field name for output count. Default "count".
|
||||
See also "mlr most-frequent".
|
||||
================================================================
|
||||
Usage: mlr merge-fields [options]
|
||||
Computes univariate statistics for each input record, accumulated across
|
||||
specified fields.
|
||||
Options:
|
||||
-a {sum,count,...} Names of accumulators. One or more of:
|
||||
count Count instances of fields
|
||||
mode Find most-frequently-occurring values for fields; first-found wins tie
|
||||
antimode Find least-frequently-occurring values for fields; first-found wins tie
|
||||
sum Compute sums of specified fields
|
||||
mean Compute averages (sample means) of specified fields
|
||||
var Compute sample variance of specified fields
|
||||
stddev Compute sample standard deviation of specified fields
|
||||
meaneb Estimate error bars for averages (assuming no sample autocorrelation)
|
||||
skewness Compute sample skewness of specified fields
|
||||
kurtosis Compute sample kurtosis of specified fields
|
||||
min Compute minimum values of specified fields
|
||||
max Compute maximum values of specified fields
|
||||
-f {a,b,c} Value-field names on which to compute statistics. Requires -o.
|
||||
-r {a,b,c} Regular expressions for value-field names on which to compute
|
||||
statistics. Requires -o.
|
||||
-c {a,b,c} Substrings for collapse mode. All fields which have the same names
|
||||
after removing substrings will be accumulated together. Please see
|
||||
examples below.
|
||||
-i Use interpolated percentiles, like R's type=7; default like type=1.
|
||||
Not sensical for string-valued fields.
|
||||
-o {name} Output field basename for -f/-r.
|
||||
-k Keep the input fields which contributed to the output statistics;
|
||||
the default is to omit them.
|
||||
|
||||
String-valued data make sense unless arithmetic on them is required,
|
||||
e.g. for sum, mean, interpolated percentiles, etc. In case of mixed data,
|
||||
numbers are less than strings.
|
||||
|
||||
Example input data: "a_in_x=1,a_out_x=2,b_in_y=4,b_out_x=8".
|
||||
Example: mlr merge-fields -a sum,count -f a_in_x,a_out_x -o foo
|
||||
produces "b_in_y=4,b_out_x=8,foo_sum=3,foo_count=2" since "a_in_x,a_out_x" are
|
||||
summed over.
|
||||
Example: mlr merge-fields -a sum,count -r in_,out_ -o bar
|
||||
produces "bar_sum=15,bar_count=4" since all four fields are summed over.
|
||||
Example: mlr merge-fields -a sum,count -c in_,out_
|
||||
produces "a_x_sum=3,a_x_count=2,b_y_sum=4,b_y_count=1,b_x_sum=8,b_x_count=1"
|
||||
since "a_in_x" and "a_out_x" both collapse to "a_x", "b_in_y" collapses to
|
||||
"b_y", and "b_out_x" collapses to "b_x".
|
||||
================================================================
|
||||
Usage: mlr most-frequent [options]
|
||||
Shows the most frequently occurring distinct values for specified field names.
|
||||
The first entry is the statistical mode; the remaining are runners-up.
|
||||
|
|
|
|||
4
go/regtest/cases/verb-merge-fields/collapse-0003/expout
Normal file
4
go/regtest/cases/verb-merge-fields/collapse-0003/expout
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
a_p0 a_min a_p29 a_max a_p100 a_sum b_p0 b_min b_p29 b_max b_p100 b_sum
|
||||
436 436 451.66 490 490 926 195 195 267.78999999999996 446 446 641
|
||||
320 320 379.74 526 526 846 780 780 833.0699999999999 963 963 1743
|
||||
220 220 413.72 888 888 1108 705 705 741.54 831 831 1536
|
||||
4
go/regtest/cases/verb-merge-fields/collapse-0004/expout
Normal file
4
go/regtest/cases/verb-merge-fields/collapse-0004/expout
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
a_in a_out b_in b_out a_p0 a_min a_p29 a_max a_p100 a_sum b_p0 b_min b_p29 b_max b_p100 b_sum
|
||||
436 490 446 195 436 436 451.66 490 490 926 195 195 267.78999999999996 446 446 641
|
||||
526 320 963 780 320 320 379.74 526 526 846 780 780 833.0699999999999 963 963 1743
|
||||
220 888 705 831 220 220 413.72 888 888 1108 705 705 741.54 831 831 1536
|
||||
|
|
@ -2,9 +2,9 @@ a_in_x 1
|
|||
a_out_x 2
|
||||
b_in_y 4
|
||||
b_out_x 8
|
||||
a_x_p0 1.000000
|
||||
a_x_p0 1
|
||||
a_x_min 1
|
||||
a_x_p29 1.290000
|
||||
a_x_p29 1.29
|
||||
a_x_max 2
|
||||
a_x_p100 2
|
||||
a_x_sum 3
|
||||
|
|
@ -82,9 +82,9 @@ a_in_x 1
|
|||
a_out_x 2
|
||||
b_on_y 4
|
||||
b_out_x 8
|
||||
a_x_p0 1.000000
|
||||
a_x_p0 1
|
||||
a_x_min 1
|
||||
a_x_p29 1.290000
|
||||
a_x_p29 1.29
|
||||
a_x_max 2
|
||||
a_x_p100 2
|
||||
a_x_sum 3
|
||||
|
|
@ -101,9 +101,9 @@ a_in_x 1
|
|||
a_out_x 2
|
||||
b_in_y 4
|
||||
b_oot_x 8
|
||||
a_x_p0 1.000000
|
||||
a_x_p0 1
|
||||
a_x_min 1
|
||||
a_x_p29 1.290000
|
||||
a_x_p29 1.29
|
||||
a_x_max 2
|
||||
a_x_p100 2
|
||||
a_x_sum 3
|
||||
|
|
@ -217,9 +217,9 @@ a_in_x 1
|
|||
a_out_x 2
|
||||
b_on_y 4
|
||||
b_oot_x 8
|
||||
a_x_p0 1.000000
|
||||
a_x_p0 1
|
||||
a_x_min 1
|
||||
a_x_p29 1.290000
|
||||
a_x_p29 1.29
|
||||
a_x_max 2
|
||||
a_x_p100 2
|
||||
a_x_sum 3
|
||||
1
go/regtest/cases/verb-merge-fields/name-0001/cmd
Normal file
1
go/regtest/cases/verb-merge-fields/name-0001/cmd
Normal file
|
|
@ -0,0 +1 @@
|
|||
mlr --oxtab head -n 2 then merge-fields -k -a p0,min,p29,max,p100,sum,count -f x,y -o xy regtest/input/abixy
|
||||
25
go/regtest/cases/verb-merge-fields/name-0001/expout
Normal file
25
go/regtest/cases/verb-merge-fields/name-0001/expout
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
a pan
|
||||
b pan
|
||||
i 1
|
||||
x 0.3467901443380824
|
||||
y 0.7268028627434533
|
||||
xy_p0 0.3467901443380824
|
||||
xy_min 0.3467901443380824
|
||||
xy_p29 0.3467901443380824
|
||||
xy_max 0.7268028627434533
|
||||
xy_p100 0.7268028627434533
|
||||
xy_sum 1.0735930070815356
|
||||
xy_count 2
|
||||
|
||||
a eks
|
||||
b pan
|
||||
i 2
|
||||
x 0.7586799647899636
|
||||
y 0.5221511083334797
|
||||
xy_p0 0.5221511083334797
|
||||
xy_min 0.5221511083334797
|
||||
xy_p29 0.5221511083334797
|
||||
xy_max 0.7586799647899636
|
||||
xy_p100 0.7586799647899636
|
||||
xy_sum 1.2808310731234434
|
||||
xy_count 2
|
||||
1
go/regtest/cases/verb-merge-fields/name-0002/cmd
Normal file
1
go/regtest/cases/verb-merge-fields/name-0002/cmd
Normal file
|
|
@ -0,0 +1 @@
|
|||
mlr --oxtab head -n 2 then merge-fields -k -a p0,min,p29,max,p100,sum,count -f x,v -o xv regtest/input/abixy
|
||||
25
go/regtest/cases/verb-merge-fields/name-0002/expout
Normal file
25
go/regtest/cases/verb-merge-fields/name-0002/expout
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
a pan
|
||||
b pan
|
||||
i 1
|
||||
x 0.3467901443380824
|
||||
y 0.7268028627434533
|
||||
xv_p0 0.3467901443380824
|
||||
xv_min 0.3467901443380824
|
||||
xv_p29 0.3467901443380824
|
||||
xv_max 0.3467901443380824
|
||||
xv_p100 0.3467901443380824
|
||||
xv_sum 0.3467901443380824
|
||||
xv_count 1
|
||||
|
||||
a eks
|
||||
b pan
|
||||
i 2
|
||||
x 0.7586799647899636
|
||||
y 0.5221511083334797
|
||||
xv_p0 0.7586799647899636
|
||||
xv_min 0.7586799647899636
|
||||
xv_p29 0.7586799647899636
|
||||
xv_max 0.7586799647899636
|
||||
xv_p100 0.7586799647899636
|
||||
xv_sum 0.7586799647899636
|
||||
xv_count 1
|
||||
1
go/regtest/cases/verb-merge-fields/name-0003/cmd
Normal file
1
go/regtest/cases/verb-merge-fields/name-0003/cmd
Normal file
|
|
@ -0,0 +1 @@
|
|||
mlr --oxtab head -n 2 then merge-fields -k -a p0,min,p29,max,p100,sum,count -f u,v -o xy regtest/input/abixy
|
||||
25
go/regtest/cases/verb-merge-fields/name-0003/expout
Normal file
25
go/regtest/cases/verb-merge-fields/name-0003/expout
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
a pan
|
||||
b pan
|
||||
i 1
|
||||
x 0.3467901443380824
|
||||
y 0.7268028627434533
|
||||
xy_p0
|
||||
xy_min
|
||||
xy_p29
|
||||
xy_max
|
||||
xy_p100
|
||||
xy_sum 0
|
||||
xy_count 0
|
||||
|
||||
a eks
|
||||
b pan
|
||||
i 2
|
||||
x 0.7586799647899636
|
||||
y 0.5221511083334797
|
||||
xy_p0
|
||||
xy_min
|
||||
xy_p29
|
||||
xy_max
|
||||
xy_p100
|
||||
xy_sum 0
|
||||
xy_count 0
|
||||
0
go/regtest/cases/verb-merge-fields/name-0005/experr
Normal file
0
go/regtest/cases/verb-merge-fields/name-0005/experr
Normal file
|
|
@ -2,9 +2,9 @@ a_in_x 1
|
|||
a_out_x 2
|
||||
b_in_y 4
|
||||
b_out_x 8
|
||||
foo_p0 1.000000
|
||||
foo_p0 1
|
||||
foo_min 1
|
||||
foo_p29 1.290000
|
||||
foo_p29 1.29
|
||||
foo_max 2
|
||||
foo_p100 2
|
||||
foo_sum 3
|
||||
|
|
@ -47,9 +47,9 @@ a_in_x 1
|
|||
a_out_x 2
|
||||
b_on_y 4
|
||||
b_out_x 8
|
||||
foo_p0 1.000000
|
||||
foo_p0 1
|
||||
foo_min 1
|
||||
foo_p29 1.290000
|
||||
foo_p29 1.29
|
||||
foo_max 2
|
||||
foo_p100 2
|
||||
foo_sum 3
|
||||
|
|
@ -59,9 +59,9 @@ a_in_x 1
|
|||
a_out_x 2
|
||||
b_in_y 4
|
||||
b_oot_x 8
|
||||
foo_p0 1.000000
|
||||
foo_p0 1
|
||||
foo_min 1
|
||||
foo_p29 1.290000
|
||||
foo_p29 1.29
|
||||
foo_max 2
|
||||
foo_p100 2
|
||||
foo_sum 3
|
||||
|
|
@ -140,9 +140,9 @@ a_in_x 1
|
|||
a_out_x 2
|
||||
b_on_y 4
|
||||
b_oot_x 8
|
||||
foo_p0 1.000000
|
||||
foo_p0 1
|
||||
foo_min 1
|
||||
foo_p29 1.290000
|
||||
foo_p29 1.29
|
||||
foo_max 2
|
||||
foo_p100 2
|
||||
foo_sum 3
|
||||
1
go/regtest/cases/verb-merge-fields/regex-0001/cmd
Normal file
1
go/regtest/cases/verb-merge-fields/regex-0001/cmd
Normal file
|
|
@ -0,0 +1 @@
|
|||
mlr merge-fields -k -a p0,min,p29,max,p100,sum,count -r in_,out_ -o bar regtest/input/merge-fields-abxy.dkvp
|
||||
0
go/regtest/cases/verb-merge-fields/regex-0001/experr
Normal file
0
go/regtest/cases/verb-merge-fields/regex-0001/experr
Normal file
20
go/regtest/cases/verb-merge-fields/regex-0001/expout
Normal file
20
go/regtest/cases/verb-merge-fields/regex-0001/expout
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
a_in_x=1,a_out_x=2,b_in_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=2,bar_max=8,bar_p100=8,bar_sum=15,bar_count=4
|
||||
z=1,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0
|
||||
a_on_x=1,a_out_x=2,b_in_y=4,b_out_x=8,bar_p0=2,bar_min=2,bar_p29=2,bar_max=8,bar_p100=8,bar_sum=14,bar_count=3
|
||||
a_in_x=1,a_oot_x=2,b_in_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=8,bar_p100=8,bar_sum=13,bar_count=3
|
||||
a_in_x=1,a_out_x=2,b_on_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=8,bar_p100=8,bar_sum=11,bar_count=3
|
||||
a_in_x=1,a_out_x=2,b_in_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=4,bar_p100=4,bar_sum=7,bar_count=3
|
||||
z=2,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0
|
||||
a_on_x=1,a_oot_x=2,b_in_y=4,b_out_x=8,bar_p0=4,bar_min=4,bar_p29=4,bar_max=8,bar_p100=8,bar_sum=12,bar_count=2
|
||||
a_on_x=1,a_out_x=2,b_on_y=4,b_out_x=8,bar_p0=2,bar_min=2,bar_p29=2,bar_max=8,bar_p100=8,bar_sum=10,bar_count=2
|
||||
a_on_x=1,a_out_x=2,b_in_y=4,b_oot_x=8,bar_p0=2,bar_min=2,bar_p29=2,bar_max=4,bar_p100=4,bar_sum=6,bar_count=2
|
||||
a_in_x=1,a_oot_x=2,b_on_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=8,bar_p100=8,bar_sum=9,bar_count=2
|
||||
a_in_x=1,a_oot_x=2,b_in_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=4,bar_p100=4,bar_sum=5,bar_count=2
|
||||
a_in_x=1,a_out_x=2,b_on_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=2,bar_p100=2,bar_sum=3,bar_count=2
|
||||
z=3,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0
|
||||
a_in_x=1,a_oot_x=2,b_on_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=1,bar_p100=1,bar_sum=1,bar_count=1
|
||||
a_on_x=1,a_out_x=2,b_on_y=4,b_oot_x=8,bar_p0=2,bar_min=2,bar_p29=2,bar_max=2,bar_p100=2,bar_sum=2,bar_count=1
|
||||
a_on_x=1,a_oot_x=2,b_in_y=4,b_oot_x=8,bar_p0=4,bar_min=4,bar_p29=4,bar_max=4,bar_p100=4,bar_sum=4,bar_count=1
|
||||
a_on_x=1,a_oot_x=2,b_on_y=4,b_out_x=8,bar_p0=8,bar_min=8,bar_p29=8,bar_max=8,bar_p100=8,bar_sum=8,bar_count=1
|
||||
z=4,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0
|
||||
a_on_x=1,a_oot_x=2,b_on_y=4,b_oot_x=8,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0
|
||||
1
go/regtest/cases/verb-merge-fields/regex-0002/cmd
Normal file
1
go/regtest/cases/verb-merge-fields/regex-0002/cmd
Normal file
|
|
@ -0,0 +1 @@
|
|||
mlr merge-fields -i -k -a p0,min,p29,max,p100,sum,count -r in_,out_ -o bar regtest/input/merge-fields-abxy.dkvp
|
||||
0
go/regtest/cases/verb-merge-fields/regex-0002/experr
Normal file
0
go/regtest/cases/verb-merge-fields/regex-0002/experr
Normal file
20
go/regtest/cases/verb-merge-fields/regex-0002/expout
Normal file
20
go/regtest/cases/verb-merge-fields/regex-0002/expout
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
a_in_x=1,a_out_x=2,b_in_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=1.8699999999999999,bar_max=8,bar_p100=8,bar_sum=15,bar_count=4
|
||||
z=1,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0
|
||||
a_on_x=1,a_out_x=2,b_in_y=4,b_out_x=8,bar_p0=2,bar_min=2,bar_p29=3.16,bar_max=8,bar_p100=8,bar_sum=14,bar_count=3
|
||||
a_in_x=1,a_oot_x=2,b_in_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=2.7399999999999998,bar_max=8,bar_p100=8,bar_sum=13,bar_count=3
|
||||
a_in_x=1,a_out_x=2,b_on_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=1.58,bar_max=8,bar_p100=8,bar_sum=11,bar_count=3
|
||||
a_in_x=1,a_out_x=2,b_in_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1.58,bar_max=4,bar_p100=4,bar_sum=7,bar_count=3
|
||||
z=2,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0
|
||||
a_on_x=1,a_oot_x=2,b_in_y=4,b_out_x=8,bar_p0=4,bar_min=4,bar_p29=5.16,bar_max=8,bar_p100=8,bar_sum=12,bar_count=2
|
||||
a_on_x=1,a_out_x=2,b_on_y=4,b_out_x=8,bar_p0=2,bar_min=2,bar_p29=3.7399999999999998,bar_max=8,bar_p100=8,bar_sum=10,bar_count=2
|
||||
a_on_x=1,a_out_x=2,b_in_y=4,b_oot_x=8,bar_p0=2,bar_min=2,bar_p29=2.58,bar_max=4,bar_p100=4,bar_sum=6,bar_count=2
|
||||
a_in_x=1,a_oot_x=2,b_on_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=3.03,bar_max=8,bar_p100=8,bar_sum=9,bar_count=2
|
||||
a_in_x=1,a_oot_x=2,b_in_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1.8699999999999999,bar_max=4,bar_p100=4,bar_sum=5,bar_count=2
|
||||
a_in_x=1,a_out_x=2,b_on_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1.29,bar_max=2,bar_p100=2,bar_sum=3,bar_count=2
|
||||
z=3,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0
|
||||
a_in_x=1,a_oot_x=2,b_on_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=1,bar_p100=1,bar_sum=1,bar_count=1
|
||||
a_on_x=1,a_out_x=2,b_on_y=4,b_oot_x=8,bar_p0=2,bar_min=2,bar_p29=2,bar_max=2,bar_p100=2,bar_sum=2,bar_count=1
|
||||
a_on_x=1,a_oot_x=2,b_in_y=4,b_oot_x=8,bar_p0=4,bar_min=4,bar_p29=4,bar_max=4,bar_p100=4,bar_sum=4,bar_count=1
|
||||
a_on_x=1,a_oot_x=2,b_on_y=4,b_out_x=8,bar_p0=8,bar_min=8,bar_p29=8,bar_max=8,bar_p100=8,bar_sum=8,bar_count=1
|
||||
z=4,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0
|
||||
a_on_x=1,a_oot_x=2,b_on_y=4,b_oot_x=8,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0
|
||||
|
|
@ -39,6 +39,7 @@ var MAPPER_LOOKUP_TABLE = []transforming.TransformerSetup{
|
|||
transformers.JoinSetup,
|
||||
transformers.LabelSetup,
|
||||
transformers.LeastFrequentSetup,
|
||||
transformers.MergeFieldsSetup,
|
||||
transformers.MostFrequentSetup,
|
||||
transformers.NestSetup,
|
||||
transformers.NothingSetup,
|
||||
|
|
|
|||
534
go/src/transformers/merge-fields.go
Normal file
534
go/src/transformers/merge-fields.go
Normal file
|
|
@ -0,0 +1,534 @@
|
|||
package transformers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"miller/src/cliutil"
|
||||
"miller/src/lib"
|
||||
"miller/src/transformers/utils"
|
||||
"miller/src/transforming"
|
||||
"miller/src/types"
|
||||
)
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
const verbNameMergeFields = "merge-fields"
|
||||
|
||||
var MergeFieldsSetup = transforming.TransformerSetup{
|
||||
Verb: verbNameMergeFields,
|
||||
UsageFunc: transformerMergeFieldsUsage,
|
||||
ParseCLIFunc: transformerMergeFieldsParseCLI,
|
||||
IgnoresInput: false,
|
||||
}
|
||||
|
||||
type mergeByType int
|
||||
|
||||
const (
|
||||
e_MERGE_BY_NAME_LIST mergeByType = iota
|
||||
e_MERGE_BY_NAME_REGEX
|
||||
e_MERGE_BY_COLLAPSING
|
||||
e_MERGE_UNSPECIFIED
|
||||
)
|
||||
|
||||
func transformerMergeFieldsUsage(
|
||||
o *os.File,
|
||||
doExit bool,
|
||||
exitCode int,
|
||||
) {
|
||||
argv0 := lib.MlrExeName()
|
||||
verb := verbNameMergeFields
|
||||
fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb)
|
||||
fmt.Fprintf(o, "Computes univariate statistics for each input record, accumulated across\n")
|
||||
fmt.Fprintf(o, "specified fields.\n")
|
||||
fmt.Fprintf(o, "Options:\n")
|
||||
fmt.Fprintf(o, "-a {sum,count,...} Names of accumulators. One or more of:\n")
|
||||
utils.ListStats1Accumulators(o)
|
||||
fmt.Fprintf(o, "-f {a,b,c} Value-field names on which to compute statistics. Requires -o.\n")
|
||||
fmt.Fprintf(o, "-r {a,b,c} Regular expressions for value-field names on which to compute\n")
|
||||
fmt.Fprintf(o, " statistics. Requires -o.\n")
|
||||
fmt.Fprintf(o, "-c {a,b,c} Substrings for collapse mode. All fields which have the same names\n")
|
||||
fmt.Fprintf(o, " after removing substrings will be accumulated together. Please see\n")
|
||||
fmt.Fprintf(o, " examples below.\n")
|
||||
fmt.Fprintf(o, "-i Use interpolated percentiles, like R's type=7; default like type=1.\n")
|
||||
fmt.Fprintf(o, " Not sensical for string-valued fields.\n")
|
||||
fmt.Fprintf(o, "-o {name} Output field basename for -f/-r.\n")
|
||||
fmt.Fprintf(o, "-k Keep the input fields which contributed to the output statistics;\n")
|
||||
fmt.Fprintf(o, " the default is to omit them.\n")
|
||||
fmt.Fprintf(o, "\n")
|
||||
fmt.Fprintf(o, "String-valued data make sense unless arithmetic on them is required,\n")
|
||||
fmt.Fprintf(o, "e.g. for sum, mean, interpolated percentiles, etc. In case of mixed data,\n")
|
||||
fmt.Fprintf(o, "numbers are less than strings.\n")
|
||||
fmt.Fprintf(o, "\n")
|
||||
fmt.Fprintf(o, "Example input data: \"a_in_x=1,a_out_x=2,b_in_y=4,b_out_x=8\".\n")
|
||||
fmt.Fprintf(o, "Example: %s %s -a sum,count -f a_in_x,a_out_x -o foo\n", argv0, verb)
|
||||
fmt.Fprintf(o, " produces \"b_in_y=4,b_out_x=8,foo_sum=3,foo_count=2\" since \"a_in_x,a_out_x\" are\n")
|
||||
fmt.Fprintf(o, " summed over.\n")
|
||||
fmt.Fprintf(o, "Example: %s %s -a sum,count -r in_,out_ -o bar\n", argv0, verb)
|
||||
fmt.Fprintf(o, " produces \"bar_sum=15,bar_count=4\" since all four fields are summed over.\n")
|
||||
fmt.Fprintf(o, "Example: %s %s -a sum,count -c in_,out_\n", argv0, verb)
|
||||
fmt.Fprintf(o, " produces \"a_x_sum=3,a_x_count=2,b_y_sum=4,b_y_count=1,b_x_sum=8,b_x_count=1\"\n")
|
||||
fmt.Fprintf(o, " since \"a_in_x\" and \"a_out_x\" both collapse to \"a_x\", \"b_in_y\" collapses to\n")
|
||||
fmt.Fprintf(o, " \"b_y\", and \"b_out_x\" collapses to \"b_x\".\n")
|
||||
|
||||
if doExit {
|
||||
os.Exit(exitCode)
|
||||
}
|
||||
}
|
||||
|
||||
func transformerMergeFieldsParseCLI(
|
||||
pargi *int,
|
||||
argc int,
|
||||
args []string,
|
||||
_ *cliutil.TReaderOptions,
|
||||
__ *cliutil.TWriterOptions,
|
||||
) transforming.IRecordTransformer {
|
||||
|
||||
// Skip the verb name from the current spot in the mlr command line
|
||||
argi := *pargi
|
||||
verb := args[argi]
|
||||
argi++
|
||||
|
||||
accumulatorNameList := make([]string, 0)
|
||||
valueFieldNameList := make([]string, 0)
|
||||
outputFieldBasename := ""
|
||||
doWhich := e_MERGE_UNSPECIFIED
|
||||
keepInputFields := false
|
||||
doInterpolatedPercentiles := false
|
||||
|
||||
for argi < argc /* variable increment: 1 or 2 depending on flag */ {
|
||||
opt := args[argi]
|
||||
if !strings.HasPrefix(opt, "-") {
|
||||
break // No more flag options to process
|
||||
}
|
||||
argi++
|
||||
|
||||
if opt == "-h" || opt == "--help" {
|
||||
transformerMergeFieldsUsage(os.Stdout, true, 0)
|
||||
|
||||
} else if opt == "-a" {
|
||||
accumulatorNameList = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc)
|
||||
|
||||
} else if opt == "-f" {
|
||||
valueFieldNameList = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc)
|
||||
doWhich = e_MERGE_BY_NAME_LIST
|
||||
|
||||
} else if opt == "-r" {
|
||||
valueFieldNameList = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc)
|
||||
doWhich = e_MERGE_BY_NAME_REGEX
|
||||
|
||||
} else if opt == "-c" {
|
||||
valueFieldNameList = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc)
|
||||
doWhich = e_MERGE_BY_COLLAPSING
|
||||
|
||||
} else if opt == "-o" {
|
||||
outputFieldBasename = cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc)
|
||||
|
||||
} else if opt == "-k" {
|
||||
keepInputFields = true
|
||||
|
||||
} else if opt == "-i" {
|
||||
doInterpolatedPercentiles = true
|
||||
|
||||
} else if opt == "-S" {
|
||||
// No-op pass-through for backward compatibility with Miller 5
|
||||
|
||||
} else if opt == "-F" {
|
||||
// No-op pass-through for backward compatibility with Miller 5
|
||||
|
||||
} else {
|
||||
transformerMergeFieldsUsage(os.Stderr, true, 1)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: libify for use across verbs.
|
||||
if len(accumulatorNameList) == 0 {
|
||||
fmt.Fprintf(os.Stderr, "%s %s: -a option is required.\n", lib.MlrExeName(), verbNameMergeFields)
|
||||
fmt.Fprintf(os.Stderr, "Please see %s %s --help for more information.\n", lib.MlrExeName(), verbNameMergeFields)
|
||||
os.Exit(1)
|
||||
}
|
||||
if len(valueFieldNameList) == 0 {
|
||||
fmt.Fprintf(os.Stderr, "%s %s: -f option is required.\n", lib.MlrExeName(), verbNameMergeFields)
|
||||
fmt.Fprintf(os.Stderr, "Please see %s %s --help for more information.\n", lib.MlrExeName(), verbNameMergeFields)
|
||||
os.Exit(1)
|
||||
}
|
||||
if outputFieldBasename == "" {
|
||||
if doWhich == e_MERGE_BY_NAME_LIST || doWhich == e_MERGE_BY_NAME_REGEX {
|
||||
transformerMergeFieldsUsage(os.Stderr, true, 1)
|
||||
}
|
||||
}
|
||||
|
||||
transformer, _ := NewTransformerMergeFields(
|
||||
accumulatorNameList,
|
||||
valueFieldNameList,
|
||||
outputFieldBasename,
|
||||
doWhich,
|
||||
doInterpolatedPercentiles,
|
||||
keepInputFields,
|
||||
)
|
||||
|
||||
*pargi = argi
|
||||
return transformer
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Given: accumulate count,sum on values x,y group by a,b.
|
||||
//
|
||||
// Example input: Example output:
|
||||
// a b x y a b x_count x_sum y_count y_sum
|
||||
// s t 1 2 s t 2 6 2 8
|
||||
// u v 3 4 u v 1 3 1 4
|
||||
// s t 5 6 u w 1 7 1 9
|
||||
// u w 7 9
|
||||
//
|
||||
// Multilevel hashmap structure:
|
||||
// {
|
||||
// "s,t" : { <--- group-by field names
|
||||
// "x" : { <--- value field name
|
||||
// "count" : Stats1CountAccumulator object,
|
||||
// "sum" : Stats1SumAccumulator object
|
||||
// },
|
||||
// "y" : {
|
||||
// "count" : Stats1CountAccumulator object,
|
||||
// "sum" : Stats1SumAccumulator object
|
||||
// },
|
||||
// },
|
||||
// "u,v" : {
|
||||
// "x" : {
|
||||
// "count" : Stats1CountAccumulator object,
|
||||
// "sum" : Stats1SumAccumulator object
|
||||
// },
|
||||
// "y" : {
|
||||
// "count" : Stats1CountAccumulator object,
|
||||
// "sum" : Stats1SumAccumulator object
|
||||
// },
|
||||
// },
|
||||
// "u,w" : {
|
||||
// "x" : {
|
||||
// "count" : Stats1CountAccumulator object,
|
||||
// "sum" : Stats1SumAccumulator object
|
||||
// },
|
||||
// "y" : {
|
||||
// "count" : Stats1CountAccumulator object,
|
||||
// "sum" : Stats1SumAccumulator object
|
||||
// },
|
||||
// },
|
||||
// }
|
||||
|
||||
type TransformerMergeFields struct {
|
||||
// Input:
|
||||
accumulatorNameList []string
|
||||
valueFieldNameList []string
|
||||
outputFieldBasename string
|
||||
doInterpolatedPercentiles bool
|
||||
keepInputFields bool
|
||||
|
||||
// State:
|
||||
accumulatorFactory *utils.Stats1AccumulatorFactory
|
||||
valueFieldNameRegexes []*regexp.Regexp
|
||||
|
||||
// Ordered map from accumulator name to accumulator
|
||||
namedAccumulators *lib.OrderedMap
|
||||
|
||||
recordTransformerFunc transforming.RecordTransformerFunc
|
||||
}
|
||||
|
||||
func NewTransformerMergeFields(
|
||||
accumulatorNameList []string,
|
||||
valueFieldNameList []string,
|
||||
outputFieldBasename string,
|
||||
doWhich mergeByType,
|
||||
doInterpolatedPercentiles bool,
|
||||
keepInputFields bool,
|
||||
) (*TransformerMergeFields, error) {
|
||||
|
||||
for _, accumulatorName := range accumulatorNameList {
|
||||
if !utils.ValidateStats1AccumulatorName(accumulatorName) {
|
||||
return nil, errors.New(
|
||||
fmt.Sprintf(
|
||||
"%s %s: accumulator \"%s\" not found.\n",
|
||||
lib.MlrExeName(), verbNameMergeFields, accumulatorName,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
this := &TransformerMergeFields{
|
||||
accumulatorNameList: accumulatorNameList,
|
||||
valueFieldNameList: valueFieldNameList,
|
||||
outputFieldBasename: outputFieldBasename,
|
||||
doInterpolatedPercentiles: doInterpolatedPercentiles,
|
||||
keepInputFields: keepInputFields,
|
||||
accumulatorFactory: utils.NewStats1AccumulatorFactory(),
|
||||
namedAccumulators: lib.NewOrderedMap(),
|
||||
}
|
||||
|
||||
this.valueFieldNameRegexes = make([]*regexp.Regexp, len(valueFieldNameList))
|
||||
for i, regexString := range valueFieldNameList {
|
||||
// Handles "a.*b"i Miller case-insensitive-regex specification
|
||||
regex, err := lib.CompileMillerRegex(regexString)
|
||||
if err != nil {
|
||||
fmt.Fprintf(
|
||||
os.Stderr,
|
||||
"%s %s: cannot compile regex [%s]\n",
|
||||
lib.MlrExeName(), verbNameCut, regexString,
|
||||
)
|
||||
os.Exit(1)
|
||||
}
|
||||
this.valueFieldNameRegexes[i] = regex
|
||||
}
|
||||
|
||||
for _, accumulatorName := range accumulatorNameList {
|
||||
accumulator := this.accumulatorFactory.MakeNamedAccumulator(
|
||||
accumulatorName,
|
||||
"", // grouping-key used for stats1, not here
|
||||
outputFieldBasename,
|
||||
doInterpolatedPercentiles,
|
||||
)
|
||||
this.namedAccumulators.Put(accumulatorName, accumulator)
|
||||
}
|
||||
|
||||
if doWhich == e_MERGE_BY_NAME_LIST {
|
||||
this.recordTransformerFunc = this.transformByNameList
|
||||
} else if doWhich == e_MERGE_BY_NAME_REGEX {
|
||||
this.recordTransformerFunc = this.transformByNameRegex
|
||||
} else if doWhich == e_MERGE_BY_COLLAPSING {
|
||||
this.recordTransformerFunc = this.transformByCollapsing
|
||||
} else {
|
||||
lib.InternalCodingErrorIf(true)
|
||||
}
|
||||
|
||||
return this, nil
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func (this *TransformerMergeFields) Transform(
|
||||
inrecAndContext *types.RecordAndContext,
|
||||
outputChannel chan<- *types.RecordAndContext,
|
||||
) {
|
||||
this.recordTransformerFunc(inrecAndContext, outputChannel)
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func (this *TransformerMergeFields) transformByNameList(
|
||||
inrecAndContext *types.RecordAndContext,
|
||||
outputChannel chan<- *types.RecordAndContext,
|
||||
) {
|
||||
if inrecAndContext.EndOfStream {
|
||||
outputChannel <- inrecAndContext // end-of-stream marker
|
||||
return
|
||||
}
|
||||
|
||||
inrec := inrecAndContext.Record
|
||||
|
||||
for pa := this.namedAccumulators.Head; pa != nil; pa = pa.Next {
|
||||
accumulator := pa.Value.(*utils.Stats1NamedAccumulator)
|
||||
accumulator.Reset() // re-use from one record to the next
|
||||
}
|
||||
|
||||
for _, valueFieldName := range this.valueFieldNameList {
|
||||
mvalue := inrec.Get(valueFieldName)
|
||||
if mvalue == nil { // key not present
|
||||
continue
|
||||
}
|
||||
|
||||
if mvalue.IsEmpty() { // key present with empty value
|
||||
if !this.keepInputFields {
|
||||
inrec.Remove(valueFieldName)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
for pa := this.namedAccumulators.Head; pa != nil; pa = pa.Next {
|
||||
accumulator := pa.Value.(*utils.Stats1NamedAccumulator)
|
||||
accumulator.Ingest(mvalue)
|
||||
}
|
||||
|
||||
if !this.keepInputFields {
|
||||
inrec.Remove(valueFieldName)
|
||||
}
|
||||
}
|
||||
|
||||
for pa := this.namedAccumulators.Head; pa != nil; pa = pa.Next {
|
||||
accumulator := pa.Value.(*utils.Stats1NamedAccumulator)
|
||||
key, value := accumulator.Emit()
|
||||
inrec.PutReference(key, value)
|
||||
}
|
||||
|
||||
outputChannel <- inrecAndContext
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func (this *TransformerMergeFields) transformByNameRegex(
|
||||
inrecAndContext *types.RecordAndContext,
|
||||
outputChannel chan<- *types.RecordAndContext,
|
||||
) {
|
||||
if inrecAndContext.EndOfStream {
|
||||
outputChannel <- inrecAndContext // end-of-stream marker
|
||||
return
|
||||
}
|
||||
|
||||
inrec := inrecAndContext.Record
|
||||
|
||||
for pa := this.namedAccumulators.Head; pa != nil; pa = pa.Next {
|
||||
accumulator := pa.Value.(*utils.Stats1NamedAccumulator)
|
||||
accumulator.Reset() // re-use from one record to the next
|
||||
}
|
||||
|
||||
for pe := inrec.Head; pe != nil; /* increment inside loop*/ {
|
||||
valueFieldName := pe.Key
|
||||
|
||||
matched := false
|
||||
for _, valueFieldNameRegex := range this.valueFieldNameRegexes {
|
||||
if valueFieldNameRegex.MatchString(pe.Key) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
pe = pe.Next
|
||||
continue
|
||||
}
|
||||
|
||||
mvalue := inrec.Get(valueFieldName)
|
||||
|
||||
if mvalue == nil { // Key not present
|
||||
pe = pe.Next
|
||||
continue
|
||||
}
|
||||
|
||||
if mvalue.IsEmpty() { // Key present with empty value
|
||||
if !this.keepInputFields { // We are modifying the record while iterating over it.
|
||||
next := pe.Next
|
||||
inrec.Unlink(pe)
|
||||
pe = next
|
||||
} else {
|
||||
pe = pe.Next
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
for pa := this.namedAccumulators.Head; pa != nil; pa = pa.Next {
|
||||
accumulator := pa.Value.(*utils.Stats1NamedAccumulator)
|
||||
accumulator.Ingest(mvalue)
|
||||
}
|
||||
|
||||
if !this.keepInputFields { // We are modifying the record while iterating over it.
|
||||
next := pe.Next
|
||||
inrec.Unlink(pe)
|
||||
pe = next
|
||||
} else {
|
||||
pe = pe.Next
|
||||
}
|
||||
}
|
||||
|
||||
for pa := this.namedAccumulators.Head; pa != nil; pa = pa.Next {
|
||||
accumulator := pa.Value.(*utils.Stats1NamedAccumulator)
|
||||
key, value := accumulator.Emit()
|
||||
inrec.PutReference(key, value)
|
||||
}
|
||||
|
||||
outputChannel <- inrecAndContext
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// mlr merge-fields -c in_,out_ -a sum
|
||||
// a_in_x 1 a_sum_x 3
|
||||
// a_out_x 2 b_sum_y 4
|
||||
// b_in_y 4 b_sum_x 8
|
||||
// b_out_x 8
|
||||
|
||||
func (this *TransformerMergeFields) transformByCollapsing(
|
||||
inrecAndContext *types.RecordAndContext,
|
||||
outputChannel chan<- *types.RecordAndContext,
|
||||
) {
|
||||
if inrecAndContext.EndOfStream {
|
||||
outputChannel <- inrecAndContext // end-of-stream marker
|
||||
return
|
||||
}
|
||||
|
||||
inrec := inrecAndContext.Record
|
||||
this.accumulatorFactory.Reset() // discard cached percentile-keepers
|
||||
|
||||
// Ordered map from short name to accumulator name to accumulator
|
||||
collapseAccumulators := lib.NewOrderedMap()
|
||||
|
||||
for pe := inrec.Head; pe != nil; /* increment inside loop */ {
|
||||
valueFieldName := pe.Key
|
||||
|
||||
matched := false
|
||||
shortName := ""
|
||||
for _, valueFieldNameRegex := range this.valueFieldNameRegexes {
|
||||
matched = valueFieldNameRegex.MatchString(pe.Key)
|
||||
if matched {
|
||||
shortName = lib.RegexReplaceOnce(valueFieldNameRegex, valueFieldName, "")
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
pe = pe.Next
|
||||
continue
|
||||
}
|
||||
|
||||
mvalue := inrec.Get(valueFieldName)
|
||||
if mvalue == nil { // Key present
|
||||
pe = pe.Next
|
||||
continue
|
||||
}
|
||||
|
||||
if mvalue.IsEmpty() { // key present with empty value
|
||||
if !this.keepInputFields { // We are modifying the record while iterating over it.
|
||||
next := pe.Next
|
||||
inrec.Unlink(pe)
|
||||
pe = next
|
||||
} else {
|
||||
pe = pe.Next
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
var namedAccumulators *lib.OrderedMap
|
||||
iNamedAccumulators := collapseAccumulators.Get(shortName)
|
||||
if iNamedAccumulators == nil {
|
||||
namedAccumulators = lib.NewOrderedMap()
|
||||
for _, accumulatorName := range this.accumulatorNameList {
|
||||
accumulator := this.accumulatorFactory.MakeNamedAccumulator(
|
||||
accumulatorName,
|
||||
"", // grouping-key used for stats1, not here
|
||||
shortName,
|
||||
this.doInterpolatedPercentiles,
|
||||
)
|
||||
namedAccumulators.Put(accumulatorName, accumulator)
|
||||
}
|
||||
collapseAccumulators.Put(shortName, namedAccumulators)
|
||||
} else {
|
||||
namedAccumulators = iNamedAccumulators.(*lib.OrderedMap)
|
||||
}
|
||||
|
||||
for pa := namedAccumulators.Head; pa != nil; pa = pa.Next {
|
||||
accumulator := pa.Value.(*utils.Stats1NamedAccumulator)
|
||||
accumulator.Ingest(mvalue)
|
||||
}
|
||||
|
||||
if !this.keepInputFields { // We are modifying the record while iterating over it.
|
||||
next := pe.Next
|
||||
inrec.Unlink(pe)
|
||||
pe = next
|
||||
} else {
|
||||
pe = pe.Next
|
||||
}
|
||||
}
|
||||
|
||||
for ps := collapseAccumulators.Head; ps != nil; ps = ps.Next {
|
||||
namedAccumulators := ps.Value.(*lib.OrderedMap)
|
||||
for pa := namedAccumulators.Head; pa != nil; pa = pa.Next {
|
||||
accumulator := pa.Value.(*utils.Stats1NamedAccumulator)
|
||||
key, value := accumulator.Emit()
|
||||
inrec.PutReference(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
outputChannel <- inrecAndContext
|
||||
}
|
||||
|
|
@ -380,7 +380,7 @@ func (this *TransformerStats1) emitIntoOutputRecord(
|
|||
for pc := level3.Head; pc != nil; pc = pc.Next {
|
||||
namedAccumulator := pc.Value.(*utils.Stats1NamedAccumulator)
|
||||
key, value := namedAccumulator.Emit()
|
||||
outrec.PutCopy(key, &value)
|
||||
outrec.PutCopy(key, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,11 @@ func NewPercentileKeeper(
|
|||
}
|
||||
}
|
||||
|
||||
func (this *PercentileKeeper) Reset() {
|
||||
this.data = make([]*types.Mlrval, 0, 1000)
|
||||
this.sorted = false
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func (this *PercentileKeeper) Ingest(value *types.Mlrval) {
|
||||
if len(this.data) >= cap(this.data) {
|
||||
|
|
@ -259,7 +264,7 @@ func (this *PercentileKeeper) sortIfNecessary() {
|
|||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func (this *PercentileKeeper) Emit(percentile float64) types.Mlrval {
|
||||
func (this *PercentileKeeper) Emit(percentile float64) *types.Mlrval {
|
||||
if this.doInterpolatedPercentiles {
|
||||
return this.EmitLinearlyInterpolated(percentile)
|
||||
} else {
|
||||
|
|
@ -267,21 +272,21 @@ func (this *PercentileKeeper) Emit(percentile float64) types.Mlrval {
|
|||
}
|
||||
}
|
||||
|
||||
func (this *PercentileKeeper) EmitNonInterpolated(percentile float64) types.Mlrval {
|
||||
func (this *PercentileKeeper) EmitNonInterpolated(percentile float64) *types.Mlrval {
|
||||
if len(this.data) == 0 {
|
||||
return *types.MLRVAL_ABSENT
|
||||
return types.MLRVAL_VOID
|
||||
}
|
||||
this.sortIfNecessary()
|
||||
return *this.data[computeIndexNoninterpolated(int(len(this.data)), percentile)].Copy()
|
||||
return this.data[computeIndexNoninterpolated(int(len(this.data)), percentile)].Copy()
|
||||
}
|
||||
|
||||
func (this *PercentileKeeper) EmitLinearlyInterpolated(percentile float64) types.Mlrval {
|
||||
func (this *PercentileKeeper) EmitLinearlyInterpolated(percentile float64) *types.Mlrval {
|
||||
if len(this.data) == 0 {
|
||||
return *types.MLRVAL_ABSENT
|
||||
return types.MLRVAL_VOID
|
||||
}
|
||||
this.sortIfNecessary()
|
||||
output := getPercentileLinearlyInterpolated(this.data, int(len(this.data)), percentile)
|
||||
return *output.Copy()
|
||||
return output.Copy()
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -16,7 +16,8 @@ import (
|
|||
// ----------------------------------------------------------------
|
||||
type IStats1Accumulator interface {
|
||||
Ingest(value *types.Mlrval)
|
||||
Emit() types.Mlrval
|
||||
Emit() *types.Mlrval
|
||||
Reset() // for merge-fields where we reset after each record instead of replace/recreate
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
|
|
@ -124,10 +125,14 @@ func (this *Stats1NamedAccumulator) Ingest(value *types.Mlrval) {
|
|||
this.accumulator.Ingest(value)
|
||||
}
|
||||
|
||||
func (this *Stats1NamedAccumulator) Emit() (key string, value types.Mlrval) {
|
||||
func (this *Stats1NamedAccumulator) Emit() (key string, value *types.Mlrval) {
|
||||
return this.outputFieldName, this.accumulator.Emit()
|
||||
}
|
||||
|
||||
func (this *Stats1NamedAccumulator) Reset() {
|
||||
this.accumulator.Reset()
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// If we are asked for p90 and p95 on the same column, we reuse the
|
||||
// percentile-keeper object to reduce runtime memory consumption. This
|
||||
|
|
@ -195,6 +200,11 @@ func tryPercentileFromName(accumulatorName string) (float64, bool) {
|
|||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// For merge-fields wherein percentile-keepers are re-created on each record
|
||||
func (this *Stats1AccumulatorFactory) Reset() {
|
||||
this.percentileKeepers = make(map[string]map[string]*PercentileKeeper)
|
||||
}
|
||||
|
||||
func (this *Stats1AccumulatorFactory) MakeNamedAccumulator(
|
||||
accumulatorName string,
|
||||
groupingKey string,
|
||||
|
|
@ -253,7 +263,7 @@ func (this *Stats1AccumulatorFactory) MakeAccumulator(
|
|||
// To conserve memory, percentile-keeprs on the same value-field-name
|
||||
// (and grouping-key) are shared. For example, p25,p75 on field "x".
|
||||
// This means though that each datapoint must be ingested only once
|
||||
// (e.g. by the p25 accumulator) since it shares a percentile-keepr
|
||||
// (e.g. by the p25 accumulator) since it shares a percentile-keeper
|
||||
// with the p75 accumulator. We handle this by tracking the first
|
||||
// construction.
|
||||
return NewStats1PercentileAccumulator(percentileKeeper, percentile, isPrimary)
|
||||
|
|
@ -281,8 +291,11 @@ func NewStats1CountAccumulator() IStats1Accumulator {
|
|||
func (this *Stats1CountAccumulator) Ingest(value *types.Mlrval) {
|
||||
this.count++
|
||||
}
|
||||
func (this *Stats1CountAccumulator) Emit() types.Mlrval {
|
||||
return types.MlrvalFromInt(this.count)
|
||||
func (this *Stats1CountAccumulator) Emit() *types.Mlrval {
|
||||
return types.MlrvalPointerFromInt(this.count)
|
||||
}
|
||||
func (this *Stats1CountAccumulator) Reset() {
|
||||
this.count = 0
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
|
|
@ -306,9 +319,9 @@ func (this *Stats1ModeAccumulator) Ingest(value *types.Mlrval) {
|
|||
this.countsByValue.Put(key, iPrevious.(int)+1)
|
||||
}
|
||||
}
|
||||
func (this *Stats1ModeAccumulator) Emit() types.Mlrval {
|
||||
func (this *Stats1ModeAccumulator) Emit() *types.Mlrval {
|
||||
if this.countsByValue.FieldCount == 0 {
|
||||
return *types.MLRVAL_ERROR
|
||||
return types.MLRVAL_ERROR
|
||||
}
|
||||
maxValue := ""
|
||||
var maxCount = int(0)
|
||||
|
|
@ -320,7 +333,10 @@ func (this *Stats1ModeAccumulator) Emit() types.Mlrval {
|
|||
maxCount = count
|
||||
}
|
||||
}
|
||||
return types.MlrvalFromString(maxValue)
|
||||
return types.MlrvalPointerFromString(maxValue)
|
||||
}
|
||||
func (this *Stats1ModeAccumulator) Reset() {
|
||||
this.countsByValue = lib.NewOrderedMap()
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
|
|
@ -344,9 +360,9 @@ func (this *Stats1AntimodeAccumulator) Ingest(value *types.Mlrval) {
|
|||
this.countsByValue.Put(key, iPrevious.(int)+1)
|
||||
}
|
||||
}
|
||||
func (this *Stats1AntimodeAccumulator) Emit() types.Mlrval {
|
||||
func (this *Stats1AntimodeAccumulator) Emit() *types.Mlrval {
|
||||
if this.countsByValue.FieldCount == 0 {
|
||||
return *types.MLRVAL_ERROR
|
||||
return types.MLRVAL_ERROR
|
||||
}
|
||||
minValue := ""
|
||||
var minCount = int(0)
|
||||
|
|
@ -358,7 +374,10 @@ func (this *Stats1AntimodeAccumulator) Emit() types.Mlrval {
|
|||
minCount = count
|
||||
}
|
||||
}
|
||||
return types.MlrvalFromString(minValue)
|
||||
return types.MlrvalPointerFromString(minValue)
|
||||
}
|
||||
func (this *Stats1AntimodeAccumulator) Reset() {
|
||||
this.countsByValue = lib.NewOrderedMap()
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
|
|
@ -374,10 +393,11 @@ func NewStats1SumAccumulator() IStats1Accumulator {
|
|||
func (this *Stats1SumAccumulator) Ingest(value *types.Mlrval) {
|
||||
this.sum = types.MlrvalBinaryPlus(this.sum, value)
|
||||
}
|
||||
|
||||
// xxx pending output-pointer refactor
|
||||
func (this *Stats1SumAccumulator) Emit() types.Mlrval {
|
||||
return *this.sum.Copy()
|
||||
func (this *Stats1SumAccumulator) Emit() *types.Mlrval {
|
||||
return this.sum.Copy()
|
||||
}
|
||||
func (this *Stats1SumAccumulator) Reset() {
|
||||
this.sum = types.MlrvalPointerFromInt(0)
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
|
|
@ -396,16 +416,17 @@ func (this *Stats1MeanAccumulator) Ingest(value *types.Mlrval) {
|
|||
this.sum = types.MlrvalBinaryPlus(this.sum, value)
|
||||
this.count++
|
||||
}
|
||||
|
||||
// xxx pending output-pointer refactor
|
||||
func (this *Stats1MeanAccumulator) Emit() types.Mlrval {
|
||||
func (this *Stats1MeanAccumulator) Emit() *types.Mlrval {
|
||||
if this.count == 0 {
|
||||
return *types.MLRVAL_VOID
|
||||
return types.MLRVAL_VOID
|
||||
} else {
|
||||
// TODO: pointer-only refactor
|
||||
return *types.MlrvalDivide(this.sum, types.MlrvalPointerFromInt(this.count))
|
||||
return types.MlrvalDivide(this.sum, types.MlrvalPointerFromInt(this.count))
|
||||
}
|
||||
}
|
||||
func (this *Stats1MeanAccumulator) Reset() {
|
||||
this.sum = types.MlrvalPointerFromInt(0)
|
||||
this.count = 0
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
type Stats1MinAccumulator struct {
|
||||
|
|
@ -420,8 +441,15 @@ func NewStats1MinAccumulator() IStats1Accumulator {
|
|||
func (this *Stats1MinAccumulator) Ingest(value *types.Mlrval) {
|
||||
this.min = types.MlrvalBinaryMin(this.min, value)
|
||||
}
|
||||
func (this *Stats1MinAccumulator) Emit() types.Mlrval {
|
||||
return *this.min.Copy()
|
||||
func (this *Stats1MinAccumulator) Emit() *types.Mlrval {
|
||||
if this.min.IsAbsent() {
|
||||
return types.MLRVAL_VOID
|
||||
} else {
|
||||
return this.min.Copy()
|
||||
}
|
||||
}
|
||||
func (this *Stats1MinAccumulator) Reset() {
|
||||
this.min = types.MLRVAL_ABSENT
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
|
|
@ -437,8 +465,15 @@ func NewStats1MaxAccumulator() IStats1Accumulator {
|
|||
func (this *Stats1MaxAccumulator) Ingest(value *types.Mlrval) {
|
||||
this.max = types.MlrvalBinaryMax(this.max, value)
|
||||
}
|
||||
func (this *Stats1MaxAccumulator) Emit() types.Mlrval {
|
||||
return *this.max.Copy()
|
||||
func (this *Stats1MaxAccumulator) Emit() *types.Mlrval {
|
||||
if this.max.IsAbsent() {
|
||||
return types.MLRVAL_VOID
|
||||
} else {
|
||||
return this.max.Copy()
|
||||
}
|
||||
}
|
||||
func (this *Stats1MaxAccumulator) Reset() {
|
||||
this.max = types.MLRVAL_ABSENT
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
|
|
@ -461,10 +496,14 @@ func (this *Stats1VarAccumulator) Ingest(value *types.Mlrval) {
|
|||
this.sum = types.MlrvalBinaryPlus(this.sum, value)
|
||||
this.sum2 = types.MlrvalBinaryPlus(this.sum2, value2)
|
||||
}
|
||||
func (this *Stats1VarAccumulator) Emit() types.Mlrval {
|
||||
func (this *Stats1VarAccumulator) Emit() *types.Mlrval {
|
||||
mcount := types.MlrvalFromInt(this.count)
|
||||
// TODO: pointer-only refactor
|
||||
return *types.MlrvalGetVar(&mcount, this.sum, this.sum2)
|
||||
return types.MlrvalGetVar(&mcount, this.sum, this.sum2)
|
||||
}
|
||||
func (this *Stats1VarAccumulator) Reset() {
|
||||
this.count = 0
|
||||
this.sum = types.MlrvalPointerFromInt(0)
|
||||
this.sum2 = types.MlrvalPointerFromInt(0)
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
|
|
@ -487,9 +526,14 @@ func (this *Stats1StddevAccumulator) Ingest(value *types.Mlrval) {
|
|||
this.sum = types.MlrvalBinaryPlus(this.sum, value)
|
||||
this.sum2 = types.MlrvalBinaryPlus(this.sum2, value2)
|
||||
}
|
||||
func (this *Stats1StddevAccumulator) Emit() types.Mlrval {
|
||||
func (this *Stats1StddevAccumulator) Emit() *types.Mlrval {
|
||||
mcount := types.MlrvalFromInt(this.count)
|
||||
return *types.MlrvalGetStddev(&mcount, this.sum, this.sum2)
|
||||
return types.MlrvalGetStddev(&mcount, this.sum, this.sum2)
|
||||
}
|
||||
func (this *Stats1StddevAccumulator) Reset() {
|
||||
this.count = 0
|
||||
this.sum = types.MlrvalPointerFromInt(0)
|
||||
this.sum2 = types.MlrvalPointerFromInt(0)
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
|
|
@ -512,9 +556,14 @@ func (this *Stats1MeanEBAccumulator) Ingest(value *types.Mlrval) {
|
|||
this.sum = types.MlrvalBinaryPlus(this.sum, value)
|
||||
this.sum2 = types.MlrvalBinaryPlus(this.sum2, value2)
|
||||
}
|
||||
func (this *Stats1MeanEBAccumulator) Emit() types.Mlrval {
|
||||
func (this *Stats1MeanEBAccumulator) Emit() *types.Mlrval {
|
||||
mcount := types.MlrvalPointerFromInt(this.count)
|
||||
return *types.MlrvalGetMeanEB(mcount, this.sum, this.sum2)
|
||||
return types.MlrvalGetMeanEB(mcount, this.sum, this.sum2)
|
||||
}
|
||||
func (this *Stats1MeanEBAccumulator) Reset() {
|
||||
this.count = 0
|
||||
this.sum = types.MlrvalPointerFromInt(0)
|
||||
this.sum2 = types.MlrvalPointerFromInt(0)
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
|
|
@ -541,10 +590,16 @@ func (this *Stats1SkewnessAccumulator) Ingest(value *types.Mlrval) {
|
|||
this.sum2 = types.MlrvalBinaryPlus(this.sum2, value2)
|
||||
this.sum3 = types.MlrvalBinaryPlus(this.sum3, value3)
|
||||
}
|
||||
func (this *Stats1SkewnessAccumulator) Emit() types.Mlrval {
|
||||
func (this *Stats1SkewnessAccumulator) Emit() *types.Mlrval {
|
||||
mcount := types.MlrvalPointerFromInt(this.count)
|
||||
return types.MlrvalGetSkewness(mcount, this.sum, this.sum2, this.sum3)
|
||||
}
|
||||
func (this *Stats1SkewnessAccumulator) Reset() {
|
||||
this.count = 0
|
||||
this.sum = types.MlrvalPointerFromInt(0)
|
||||
this.sum2 = types.MlrvalPointerFromInt(0)
|
||||
this.sum3 = types.MlrvalPointerFromInt(0)
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
type Stats1KurtosisAccumulator struct {
|
||||
|
|
@ -574,10 +629,17 @@ func (this *Stats1KurtosisAccumulator) Ingest(value *types.Mlrval) {
|
|||
this.sum3 = types.MlrvalBinaryPlus(this.sum3, value3)
|
||||
this.sum4 = types.MlrvalBinaryPlus(this.sum4, value4)
|
||||
}
|
||||
func (this *Stats1KurtosisAccumulator) Emit() types.Mlrval {
|
||||
func (this *Stats1KurtosisAccumulator) Emit() *types.Mlrval {
|
||||
mcount := types.MlrvalPointerFromInt(this.count)
|
||||
return types.MlrvalGetKurtosis(mcount, this.sum, this.sum2, this.sum3, this.sum4)
|
||||
}
|
||||
func (this *Stats1KurtosisAccumulator) Reset() {
|
||||
this.count = 0
|
||||
this.sum = types.MlrvalPointerFromInt(0)
|
||||
this.sum2 = types.MlrvalPointerFromInt(0)
|
||||
this.sum3 = types.MlrvalPointerFromInt(0)
|
||||
this.sum4 = types.MlrvalPointerFromInt(0)
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// To conserve memory, percentile-keeprs on the same value-field-name (and
|
||||
|
|
@ -609,6 +671,12 @@ func (this *Stats1PercentileAccumulator) Ingest(value *types.Mlrval) {
|
|||
}
|
||||
}
|
||||
|
||||
func (this *Stats1PercentileAccumulator) Emit() types.Mlrval {
|
||||
func (this *Stats1PercentileAccumulator) Emit() *types.Mlrval {
|
||||
return this.percentileKeeper.Emit(this.percentile)
|
||||
}
|
||||
|
||||
func (this *Stats1PercentileAccumulator) Reset() {
|
||||
if this.isPrimary {
|
||||
this.percentileKeeper.Reset()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -86,11 +86,11 @@ func MlrvalGetMeanEB(mn, msum, msum2 *Mlrval) *Mlrval {
|
|||
// = sumx2 - n mean^2
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func MlrvalGetSkewness(mn, msum, msum2, msum3 *Mlrval) Mlrval {
|
||||
func MlrvalGetSkewness(mn, msum, msum2, msum3 *Mlrval) *Mlrval {
|
||||
n, isInt := mn.GetIntValue()
|
||||
lib.InternalCodingErrorIf(!isInt)
|
||||
if n < 2 {
|
||||
return *MLRVAL_VOID
|
||||
return MLRVAL_VOID
|
||||
}
|
||||
fn := float64(n)
|
||||
sum, isNumber := msum.GetNumericToFloatValue()
|
||||
|
|
@ -105,7 +105,7 @@ func MlrvalGetSkewness(mn, msum, msum2, msum3 *Mlrval) Mlrval {
|
|||
numerator = numerator / fn
|
||||
denominator := (sum2 - fn*mean*mean) / (fn - 1.0)
|
||||
denominator = math.Pow(denominator, 1.5)
|
||||
return MlrvalFromFloat64(numerator / denominator)
|
||||
return MlrvalPointerFromFloat64(numerator / denominator)
|
||||
}
|
||||
|
||||
// Unbiased:
|
||||
|
|
@ -123,11 +123,11 @@ func MlrvalGetSkewness(mn, msum, msum2, msum3 *Mlrval) Mlrval {
|
|||
// = sumx4 - mean*(4 sumx3 - mean*(6 sumx2 - 3 n mean^2))
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func MlrvalGetKurtosis(mn, msum, msum2, msum3, msum4 *Mlrval) Mlrval {
|
||||
func MlrvalGetKurtosis(mn, msum, msum2, msum3, msum4 *Mlrval) *Mlrval {
|
||||
n, isInt := mn.GetIntValue()
|
||||
lib.InternalCodingErrorIf(!isInt)
|
||||
if n < 2 {
|
||||
return *MLRVAL_VOID
|
||||
return MLRVAL_VOID
|
||||
}
|
||||
fn := float64(n)
|
||||
sum, isNumber := msum.GetNumericToFloatValue()
|
||||
|
|
@ -145,6 +145,6 @@ func MlrvalGetKurtosis(mn, msum, msum2, msum3, msum4 *Mlrval) Mlrval {
|
|||
numerator = numerator / fn
|
||||
denominator := (sum2 - fn*mean*mean) / fn
|
||||
denominator = denominator * denominator
|
||||
return MlrvalFromFloat64(numerator/denominator - 3.0)
|
||||
return MlrvalPointerFromFloat64(numerator/denominator - 3.0)
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,13 @@
|
|||
================================================================
|
||||
TOP OF LIST:
|
||||
|
||||
! mrpl :h (and miller --help-xxx): substring ...
|
||||
!!! mlr --from s stats1 -f x,y -a corr
|
||||
panic: runtime error: invalid memory address or nil pointer dereference
|
||||
> needs UT
|
||||
> needs _ -> err x all
|
||||
|
||||
! seqgen help saying -stop not --stop
|
||||
|
||||
! MlrvalPointerFromString -> NewMlrvalPointerFromString et al.
|
||||
|
||||
* blocker: regexes
|
||||
|
|
|
|||
23
go/u/gg
23
go/u/gg
|
|
@ -53,16 +53,17 @@ mention() {
|
|||
|
||||
# ================================================================
|
||||
|
||||
run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f x,y -b u
|
||||
run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f u,v -b x
|
||||
run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f z,w,s -b v
|
||||
run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f u,v -b nonesuch
|
||||
run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f u,nonesuch -b x
|
||||
|
||||
run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f x,y -a u
|
||||
run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f u,v -a x
|
||||
run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f z,w,s -a v
|
||||
run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f u,v -a nonesuch
|
||||
run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f u,nonesuch -a x
|
||||
mlr --c2p cat regtest/input/merge-fields-in-out.csv
|
||||
echo
|
||||
../c/mlr --csvlite merge-fields -a min,max,count,sum -c _in,_out regtest/input/merge-fields-in-out.csv
|
||||
echo
|
||||
../go/mlr --csvlite merge-fields -a min,max,count,sum -c _in,_out regtest/input/merge-fields-in-out.csv
|
||||
echo
|
||||
echo
|
||||
mlr --c2p cat regtest/input/merge-fields-in-out.csv
|
||||
echo
|
||||
../c/mlr --csvlite merge-fields -a p0,p29,p100 -c _in,_out regtest/input/merge-fields-in-out.csv
|
||||
echo
|
||||
../go/mlr --csvlite merge-fields -a p0,p29,p100 -c _in,_out regtest/input/merge-fields-in-out.csv
|
||||
|
||||
# ================================================================
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue