diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0003/expout b/go/regtest/cases-pending-go-port/verb-merge-fields/0003/expout deleted file mode 100644 index 15654f5ac..000000000 --- a/go/regtest/cases-pending-go-port/verb-merge-fields/0003/expout +++ /dev/null @@ -1,4 +0,0 @@ -a_p0 a_min a_p29 a_max a_p100 a_sum b_p0 b_min b_p29 b_max b_p100 b_sum -436.000000 436 451.660000 490 490 926 195.000000 195 267.790000 446 446 641 -320.000000 320 379.740000 526 526 846 780.000000 780 833.070000 963 963 1743 -220.000000 220 413.720000 888 888 1108 705.000000 705 741.540000 831 831 1536 diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0004/expout b/go/regtest/cases-pending-go-port/verb-merge-fields/0004/expout deleted file mode 100644 index 3bf54dc47..000000000 --- a/go/regtest/cases-pending-go-port/verb-merge-fields/0004/expout +++ /dev/null @@ -1,4 +0,0 @@ -a_in a_out b_in b_out a_p0 a_min a_p29 a_max a_p100 a_sum b_p0 b_min b_p29 b_max b_p100 b_sum -436 490 446 195 436.000000 436 451.660000 490 490 926 195.000000 195 267.790000 446 446 641 -526 320 963 780 320.000000 320 379.740000 526 526 846 780.000000 780 833.070000 963 963 1743 -220 888 705 831 220.000000 220 413.720000 888 888 1108 705.000000 705 741.540000 831 831 1536 diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0008/cmd b/go/regtest/cases-pending-go-port/verb-merge-fields/0008/cmd deleted file mode 100644 index 43d3c4116..000000000 --- a/go/regtest/cases-pending-go-port/verb-merge-fields/0008/cmd +++ /dev/null @@ -1 +0,0 @@ -mlr --oxtab merge-fields -k -a p0,min,p29,max,p100,sum,count -r in_,out_ -o bar regtest/input/merge-fields-abxy.dkvp diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0008/expout b/go/regtest/cases-pending-go-port/verb-merge-fields/0008/expout deleted file mode 100644 index e18a5ec35..000000000 --- a/go/regtest/cases-pending-go-port/verb-merge-fields/0008/expout +++ /dev/null @@ -1,227 +0,0 @@ -a_in_x 1 -a_out_x 2 -b_in_y 4 -b_out_x 8 -bar_p0 1 -bar_min 1 -bar_p29 2 -bar_max 8 -bar_p100 8 -bar_sum 15 -bar_count 4 - -z 1 -bar_p0 -bar_min -bar_p29 -bar_max -bar_p100 -bar_sum 0 -bar_count 0 - -a_on_x 1 -a_out_x 2 -b_in_y 4 -b_out_x 8 -bar_p0 2 -bar_min 2 -bar_p29 2 -bar_max 8 -bar_p100 8 -bar_sum 14 -bar_count 3 - -a_in_x 1 -a_oot_x 2 -b_in_y 4 -b_out_x 8 -bar_p0 1 -bar_min 1 -bar_p29 1 -bar_max 8 -bar_p100 8 -bar_sum 13 -bar_count 3 - -a_in_x 1 -a_out_x 2 -b_on_y 4 -b_out_x 8 -bar_p0 1 -bar_min 1 -bar_p29 1 -bar_max 8 -bar_p100 8 -bar_sum 11 -bar_count 3 - -a_in_x 1 -a_out_x 2 -b_in_y 4 -b_oot_x 8 -bar_p0 1 -bar_min 1 -bar_p29 1 -bar_max 4 -bar_p100 4 -bar_sum 7 -bar_count 3 - -z 2 -bar_p0 -bar_min -bar_p29 -bar_max -bar_p100 -bar_sum 0 -bar_count 0 - -a_on_x 1 -a_oot_x 2 -b_in_y 4 -b_out_x 8 -bar_p0 4 -bar_min 4 -bar_p29 4 -bar_max 8 -bar_p100 8 -bar_sum 12 -bar_count 2 - -a_on_x 1 -a_out_x 2 -b_on_y 4 -b_out_x 8 -bar_p0 2 -bar_min 2 -bar_p29 2 -bar_max 8 -bar_p100 8 -bar_sum 10 -bar_count 2 - -a_on_x 1 -a_out_x 2 -b_in_y 4 -b_oot_x 8 -bar_p0 2 -bar_min 2 -bar_p29 2 -bar_max 4 -bar_p100 4 -bar_sum 6 -bar_count 2 - -a_in_x 1 -a_oot_x 2 -b_on_y 4 -b_out_x 8 -bar_p0 1 -bar_min 1 -bar_p29 1 -bar_max 8 -bar_p100 8 -bar_sum 9 -bar_count 2 - -a_in_x 1 -a_oot_x 2 -b_in_y 4 -b_oot_x 8 -bar_p0 1 -bar_min 1 -bar_p29 1 -bar_max 4 -bar_p100 4 -bar_sum 5 -bar_count 2 - -a_in_x 1 -a_out_x 2 -b_on_y 4 -b_oot_x 8 -bar_p0 1 -bar_min 1 -bar_p29 1 -bar_max 2 -bar_p100 2 -bar_sum 3 -bar_count 2 - -z 3 -bar_p0 -bar_min -bar_p29 -bar_max -bar_p100 -bar_sum 0 -bar_count 0 - -a_in_x 1 -a_oot_x 2 -b_on_y 4 -b_oot_x 8 -bar_p0 1 -bar_min 1 -bar_p29 1 -bar_max 1 -bar_p100 1 -bar_sum 1 -bar_count 1 - -a_on_x 1 -a_out_x 2 -b_on_y 4 -b_oot_x 8 -bar_p0 2 -bar_min 2 -bar_p29 2 -bar_max 2 -bar_p100 2 -bar_sum 2 -bar_count 1 - -a_on_x 1 -a_oot_x 2 -b_in_y 4 -b_oot_x 8 -bar_p0 4 -bar_min 4 -bar_p29 4 -bar_max 4 -bar_p100 4 -bar_sum 4 -bar_count 1 - -a_on_x 1 -a_oot_x 2 -b_on_y 4 -b_out_x 8 -bar_p0 8 -bar_min 8 -bar_p29 8 -bar_max 8 -bar_p100 8 -bar_sum 8 -bar_count 1 - -z 4 -bar_p0 -bar_min -bar_p29 -bar_max -bar_p100 -bar_sum 0 -bar_count 0 - -a_on_x 1 -a_oot_x 2 -b_on_y 4 -b_oot_x 8 -bar_p0 -bar_min -bar_p29 -bar_max -bar_p100 -bar_sum 0 -bar_count 0 diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0011/cmd b/go/regtest/cases-pending-go-port/verb-merge-fields/0011/cmd deleted file mode 100644 index 886d040aa..000000000 --- a/go/regtest/cases-pending-go-port/verb-merge-fields/0011/cmd +++ /dev/null @@ -1 +0,0 @@ -mlr --oxtab merge-fields -i -k -a p0,min,p29,max,p100,sum,count -r in_,out_ -o bar regtest/input/merge-fields-abxy.dkvp diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0011/expout b/go/regtest/cases-pending-go-port/verb-merge-fields/0011/expout deleted file mode 100644 index 3f7a43a9a..000000000 --- a/go/regtest/cases-pending-go-port/verb-merge-fields/0011/expout +++ /dev/null @@ -1,227 +0,0 @@ -a_in_x 1 -a_out_x 2 -b_in_y 4 -b_out_x 8 -bar_p0 1.000000 -bar_min 1 -bar_p29 1.870000 -bar_max 8 -bar_p100 8 -bar_sum 15 -bar_count 4 - -z 1 -bar_p0 -bar_min -bar_p29 -bar_max -bar_p100 -bar_sum 0 -bar_count 0 - -a_on_x 1 -a_out_x 2 -b_in_y 4 -b_out_x 8 -bar_p0 2.000000 -bar_min 2 -bar_p29 3.160000 -bar_max 8 -bar_p100 8 -bar_sum 14 -bar_count 3 - -a_in_x 1 -a_oot_x 2 -b_in_y 4 -b_out_x 8 -bar_p0 1.000000 -bar_min 1 -bar_p29 2.740000 -bar_max 8 -bar_p100 8 -bar_sum 13 -bar_count 3 - -a_in_x 1 -a_out_x 2 -b_on_y 4 -b_out_x 8 -bar_p0 1.000000 -bar_min 1 -bar_p29 1.580000 -bar_max 8 -bar_p100 8 -bar_sum 11 -bar_count 3 - -a_in_x 1 -a_out_x 2 -b_in_y 4 -b_oot_x 8 -bar_p0 1.000000 -bar_min 1 -bar_p29 1.580000 -bar_max 4 -bar_p100 4 -bar_sum 7 -bar_count 3 - -z 2 -bar_p0 -bar_min -bar_p29 -bar_max -bar_p100 -bar_sum 0 -bar_count 0 - -a_on_x 1 -a_oot_x 2 -b_in_y 4 -b_out_x 8 -bar_p0 4.000000 -bar_min 4 -bar_p29 5.160000 -bar_max 8 -bar_p100 8 -bar_sum 12 -bar_count 2 - -a_on_x 1 -a_out_x 2 -b_on_y 4 -b_out_x 8 -bar_p0 2.000000 -bar_min 2 -bar_p29 3.740000 -bar_max 8 -bar_p100 8 -bar_sum 10 -bar_count 2 - -a_on_x 1 -a_out_x 2 -b_in_y 4 -b_oot_x 8 -bar_p0 2.000000 -bar_min 2 -bar_p29 2.580000 -bar_max 4 -bar_p100 4 -bar_sum 6 -bar_count 2 - -a_in_x 1 -a_oot_x 2 -b_on_y 4 -b_out_x 8 -bar_p0 1.000000 -bar_min 1 -bar_p29 3.030000 -bar_max 8 -bar_p100 8 -bar_sum 9 -bar_count 2 - -a_in_x 1 -a_oot_x 2 -b_in_y 4 -b_oot_x 8 -bar_p0 1.000000 -bar_min 1 -bar_p29 1.870000 -bar_max 4 -bar_p100 4 -bar_sum 5 -bar_count 2 - -a_in_x 1 -a_out_x 2 -b_on_y 4 -b_oot_x 8 -bar_p0 1.000000 -bar_min 1 -bar_p29 1.290000 -bar_max 2 -bar_p100 2 -bar_sum 3 -bar_count 2 - -z 3 -bar_p0 -bar_min -bar_p29 -bar_max -bar_p100 -bar_sum 0 -bar_count 0 - -a_in_x 1 -a_oot_x 2 -b_on_y 4 -b_oot_x 8 -bar_p0 1 -bar_min 1 -bar_p29 1 -bar_max 1 -bar_p100 1 -bar_sum 1 -bar_count 1 - -a_on_x 1 -a_out_x 2 -b_on_y 4 -b_oot_x 8 -bar_p0 2 -bar_min 2 -bar_p29 2 -bar_max 2 -bar_p100 2 -bar_sum 2 -bar_count 1 - -a_on_x 1 -a_oot_x 2 -b_in_y 4 -b_oot_x 8 -bar_p0 4 -bar_min 4 -bar_p29 4 -bar_max 4 -bar_p100 4 -bar_sum 4 -bar_count 1 - -a_on_x 1 -a_oot_x 2 -b_on_y 4 -b_out_x 8 -bar_p0 8 -bar_min 8 -bar_p29 8 -bar_max 8 -bar_p100 8 -bar_sum 8 -bar_count 1 - -z 4 -bar_p0 -bar_min -bar_p29 -bar_max -bar_p100 -bar_sum 0 -bar_count 0 - -a_on_x 1 -a_oot_x 2 -b_on_y 4 -b_oot_x 8 -bar_p0 -bar_min -bar_p29 -bar_max -bar_p100 -bar_sum 0 -bar_count 0 diff --git a/go/regtest/cases/cli-help/0001/expout b/go/regtest/cases/cli-help/0001/expout index 53b5877cb..0ffd204ab 100644 --- a/go/regtest/cases/cli-help/0001/expout +++ b/go/regtest/cases/cli-help/0001/expout @@ -362,6 +362,50 @@ Options: -o {name} Field name for output count. Default "count". See also "mlr most-frequent". ================================================================ +Usage: mlr merge-fields [options] +Computes univariate statistics for each input record, accumulated across +specified fields. +Options: +-a {sum,count,...} Names of accumulators. One or more of: + count Count instances of fields + mode Find most-frequently-occurring values for fields; first-found wins tie + antimode Find least-frequently-occurring values for fields; first-found wins tie + sum Compute sums of specified fields + mean Compute averages (sample means) of specified fields + var Compute sample variance of specified fields + stddev Compute sample standard deviation of specified fields + meaneb Estimate error bars for averages (assuming no sample autocorrelation) + skewness Compute sample skewness of specified fields + kurtosis Compute sample kurtosis of specified fields + min Compute minimum values of specified fields + max Compute maximum values of specified fields +-f {a,b,c} Value-field names on which to compute statistics. Requires -o. +-r {a,b,c} Regular expressions for value-field names on which to compute + statistics. Requires -o. +-c {a,b,c} Substrings for collapse mode. All fields which have the same names + after removing substrings will be accumulated together. Please see + examples below. +-i Use interpolated percentiles, like R's type=7; default like type=1. + Not sensical for string-valued fields. +-o {name} Output field basename for -f/-r. +-k Keep the input fields which contributed to the output statistics; + the default is to omit them. + +String-valued data make sense unless arithmetic on them is required, +e.g. for sum, mean, interpolated percentiles, etc. In case of mixed data, +numbers are less than strings. + +Example input data: "a_in_x=1,a_out_x=2,b_in_y=4,b_out_x=8". +Example: mlr merge-fields -a sum,count -f a_in_x,a_out_x -o foo + produces "b_in_y=4,b_out_x=8,foo_sum=3,foo_count=2" since "a_in_x,a_out_x" are + summed over. +Example: mlr merge-fields -a sum,count -r in_,out_ -o bar + produces "bar_sum=15,bar_count=4" since all four fields are summed over. +Example: mlr merge-fields -a sum,count -c in_,out_ + produces "a_x_sum=3,a_x_count=2,b_y_sum=4,b_y_count=1,b_x_sum=8,b_x_count=1" + since "a_in_x" and "a_out_x" both collapse to "a_x", "b_in_y" collapses to + "b_y", and "b_out_x" collapses to "b_x". +================================================================ Usage: mlr most-frequent [options] Shows the most frequently occurring distinct values for specified field names. The first entry is the statistical mode; the remaining are runners-up. diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0001/cmd b/go/regtest/cases/verb-merge-fields/collapse-0001/cmd similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0001/cmd rename to go/regtest/cases/verb-merge-fields/collapse-0001/cmd diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0001/experr b/go/regtest/cases/verb-merge-fields/collapse-0001/experr similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0001/experr rename to go/regtest/cases/verb-merge-fields/collapse-0001/experr diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0001/expout b/go/regtest/cases/verb-merge-fields/collapse-0001/expout similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0001/expout rename to go/regtest/cases/verb-merge-fields/collapse-0001/expout diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0002/cmd b/go/regtest/cases/verb-merge-fields/collapse-0002/cmd similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0002/cmd rename to go/regtest/cases/verb-merge-fields/collapse-0002/cmd diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0002/experr b/go/regtest/cases/verb-merge-fields/collapse-0002/experr similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0002/experr rename to go/regtest/cases/verb-merge-fields/collapse-0002/experr diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0002/expout b/go/regtest/cases/verb-merge-fields/collapse-0002/expout similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0002/expout rename to go/regtest/cases/verb-merge-fields/collapse-0002/expout diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0003/cmd b/go/regtest/cases/verb-merge-fields/collapse-0003/cmd similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0003/cmd rename to go/regtest/cases/verb-merge-fields/collapse-0003/cmd diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0003/experr b/go/regtest/cases/verb-merge-fields/collapse-0003/experr similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0003/experr rename to go/regtest/cases/verb-merge-fields/collapse-0003/experr diff --git a/go/regtest/cases/verb-merge-fields/collapse-0003/expout b/go/regtest/cases/verb-merge-fields/collapse-0003/expout new file mode 100644 index 000000000..51ed33857 --- /dev/null +++ b/go/regtest/cases/verb-merge-fields/collapse-0003/expout @@ -0,0 +1,4 @@ +a_p0 a_min a_p29 a_max a_p100 a_sum b_p0 b_min b_p29 b_max b_p100 b_sum +436 436 451.66 490 490 926 195 195 267.78999999999996 446 446 641 +320 320 379.74 526 526 846 780 780 833.0699999999999 963 963 1743 +220 220 413.72 888 888 1108 705 705 741.54 831 831 1536 diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0004/cmd b/go/regtest/cases/verb-merge-fields/collapse-0004/cmd similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0004/cmd rename to go/regtest/cases/verb-merge-fields/collapse-0004/cmd diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0004/experr b/go/regtest/cases/verb-merge-fields/collapse-0004/experr similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0004/experr rename to go/regtest/cases/verb-merge-fields/collapse-0004/experr diff --git a/go/regtest/cases/verb-merge-fields/collapse-0004/expout b/go/regtest/cases/verb-merge-fields/collapse-0004/expout new file mode 100644 index 000000000..8d8d0ffd7 --- /dev/null +++ b/go/regtest/cases/verb-merge-fields/collapse-0004/expout @@ -0,0 +1,4 @@ +a_in a_out b_in b_out a_p0 a_min a_p29 a_max a_p100 a_sum b_p0 b_min b_p29 b_max b_p100 b_sum +436 490 446 195 436 436 451.66 490 490 926 195 195 267.78999999999996 446 446 641 +526 320 963 780 320 320 379.74 526 526 846 780 780 833.0699999999999 963 963 1743 +220 888 705 831 220 220 413.72 888 888 1108 705 705 741.54 831 831 1536 diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0005/cmd b/go/regtest/cases/verb-merge-fields/collapse-0005/cmd similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0005/cmd rename to go/regtest/cases/verb-merge-fields/collapse-0005/cmd diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0005/experr b/go/regtest/cases/verb-merge-fields/collapse-0005/experr similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0005/experr rename to go/regtest/cases/verb-merge-fields/collapse-0005/experr diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0005/expout b/go/regtest/cases/verb-merge-fields/collapse-0005/expout similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0005/expout rename to go/regtest/cases/verb-merge-fields/collapse-0005/expout diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0006/cmd b/go/regtest/cases/verb-merge-fields/collapse-0006/cmd similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0006/cmd rename to go/regtest/cases/verb-merge-fields/collapse-0006/cmd diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0006/experr b/go/regtest/cases/verb-merge-fields/collapse-0006/experr similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0006/experr rename to go/regtest/cases/verb-merge-fields/collapse-0006/experr diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0006/expout b/go/regtest/cases/verb-merge-fields/collapse-0006/expout similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0006/expout rename to go/regtest/cases/verb-merge-fields/collapse-0006/expout diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0009/cmd b/go/regtest/cases/verb-merge-fields/collapse-0007/cmd similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0009/cmd rename to go/regtest/cases/verb-merge-fields/collapse-0007/cmd diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0007/experr b/go/regtest/cases/verb-merge-fields/collapse-0007/experr similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0007/experr rename to go/regtest/cases/verb-merge-fields/collapse-0007/experr diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0009/expout b/go/regtest/cases/verb-merge-fields/collapse-0007/expout similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0009/expout rename to go/regtest/cases/verb-merge-fields/collapse-0007/expout diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0012/cmd b/go/regtest/cases/verb-merge-fields/collapse-0008/cmd similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0012/cmd rename to go/regtest/cases/verb-merge-fields/collapse-0008/cmd diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0008/experr b/go/regtest/cases/verb-merge-fields/collapse-0008/experr similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0008/experr rename to go/regtest/cases/verb-merge-fields/collapse-0008/experr diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0012/expout b/go/regtest/cases/verb-merge-fields/collapse-0008/expout similarity index 95% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0012/expout rename to go/regtest/cases/verb-merge-fields/collapse-0008/expout index 3453b2196..48630e5b1 100644 --- a/go/regtest/cases-pending-go-port/verb-merge-fields/0012/expout +++ b/go/regtest/cases/verb-merge-fields/collapse-0008/expout @@ -2,9 +2,9 @@ a_in_x 1 a_out_x 2 b_in_y 4 b_out_x 8 -a_x_p0 1.000000 +a_x_p0 1 a_x_min 1 -a_x_p29 1.290000 +a_x_p29 1.29 a_x_max 2 a_x_p100 2 a_x_sum 3 @@ -82,9 +82,9 @@ a_in_x 1 a_out_x 2 b_on_y 4 b_out_x 8 -a_x_p0 1.000000 +a_x_p0 1 a_x_min 1 -a_x_p29 1.290000 +a_x_p29 1.29 a_x_max 2 a_x_p100 2 a_x_sum 3 @@ -101,9 +101,9 @@ a_in_x 1 a_out_x 2 b_in_y 4 b_oot_x 8 -a_x_p0 1.000000 +a_x_p0 1 a_x_min 1 -a_x_p29 1.290000 +a_x_p29 1.29 a_x_max 2 a_x_p100 2 a_x_sum 3 @@ -217,9 +217,9 @@ a_in_x 1 a_out_x 2 b_on_y 4 b_oot_x 8 -a_x_p0 1.000000 +a_x_p0 1 a_x_min 1 -a_x_p29 1.290000 +a_x_p29 1.29 a_x_max 2 a_x_p100 2 a_x_sum 3 diff --git a/go/regtest/cases/verb-merge-fields/name-0001/cmd b/go/regtest/cases/verb-merge-fields/name-0001/cmd new file mode 100644 index 000000000..c17796eb9 --- /dev/null +++ b/go/regtest/cases/verb-merge-fields/name-0001/cmd @@ -0,0 +1 @@ +mlr --oxtab head -n 2 then merge-fields -k -a p0,min,p29,max,p100,sum,count -f x,y -o xy regtest/input/abixy diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0009/experr b/go/regtest/cases/verb-merge-fields/name-0001/experr similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0009/experr rename to go/regtest/cases/verb-merge-fields/name-0001/experr diff --git a/go/regtest/cases/verb-merge-fields/name-0001/expout b/go/regtest/cases/verb-merge-fields/name-0001/expout new file mode 100644 index 000000000..85a2a7776 --- /dev/null +++ b/go/regtest/cases/verb-merge-fields/name-0001/expout @@ -0,0 +1,25 @@ +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +xy_p0 0.3467901443380824 +xy_min 0.3467901443380824 +xy_p29 0.3467901443380824 +xy_max 0.7268028627434533 +xy_p100 0.7268028627434533 +xy_sum 1.0735930070815356 +xy_count 2 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 +xy_p0 0.5221511083334797 +xy_min 0.5221511083334797 +xy_p29 0.5221511083334797 +xy_max 0.7586799647899636 +xy_p100 0.7586799647899636 +xy_sum 1.2808310731234434 +xy_count 2 diff --git a/go/regtest/cases/verb-merge-fields/name-0002/cmd b/go/regtest/cases/verb-merge-fields/name-0002/cmd new file mode 100644 index 000000000..6ba8a0504 --- /dev/null +++ b/go/regtest/cases/verb-merge-fields/name-0002/cmd @@ -0,0 +1 @@ +mlr --oxtab head -n 2 then merge-fields -k -a p0,min,p29,max,p100,sum,count -f x,v -o xv regtest/input/abixy diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0010/experr b/go/regtest/cases/verb-merge-fields/name-0002/experr similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0010/experr rename to go/regtest/cases/verb-merge-fields/name-0002/experr diff --git a/go/regtest/cases/verb-merge-fields/name-0002/expout b/go/regtest/cases/verb-merge-fields/name-0002/expout new file mode 100644 index 000000000..fc0e4658b --- /dev/null +++ b/go/regtest/cases/verb-merge-fields/name-0002/expout @@ -0,0 +1,25 @@ +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +xv_p0 0.3467901443380824 +xv_min 0.3467901443380824 +xv_p29 0.3467901443380824 +xv_max 0.3467901443380824 +xv_p100 0.3467901443380824 +xv_sum 0.3467901443380824 +xv_count 1 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 +xv_p0 0.7586799647899636 +xv_min 0.7586799647899636 +xv_p29 0.7586799647899636 +xv_max 0.7586799647899636 +xv_p100 0.7586799647899636 +xv_sum 0.7586799647899636 +xv_count 1 diff --git a/go/regtest/cases/verb-merge-fields/name-0003/cmd b/go/regtest/cases/verb-merge-fields/name-0003/cmd new file mode 100644 index 000000000..89d505490 --- /dev/null +++ b/go/regtest/cases/verb-merge-fields/name-0003/cmd @@ -0,0 +1 @@ +mlr --oxtab head -n 2 then merge-fields -k -a p0,min,p29,max,p100,sum,count -f u,v -o xy regtest/input/abixy diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0011/experr b/go/regtest/cases/verb-merge-fields/name-0003/experr similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0011/experr rename to go/regtest/cases/verb-merge-fields/name-0003/experr diff --git a/go/regtest/cases/verb-merge-fields/name-0003/expout b/go/regtest/cases/verb-merge-fields/name-0003/expout new file mode 100644 index 000000000..594485cc0 --- /dev/null +++ b/go/regtest/cases/verb-merge-fields/name-0003/expout @@ -0,0 +1,25 @@ +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +xy_p0 +xy_min +xy_p29 +xy_max +xy_p100 +xy_sum 0 +xy_count 0 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 +xy_p0 +xy_min +xy_p29 +xy_max +xy_p100 +xy_sum 0 +xy_count 0 diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0007/cmd b/go/regtest/cases/verb-merge-fields/name-0004/cmd similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0007/cmd rename to go/regtest/cases/verb-merge-fields/name-0004/cmd diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0012/experr b/go/regtest/cases/verb-merge-fields/name-0004/experr similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0012/experr rename to go/regtest/cases/verb-merge-fields/name-0004/experr diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0007/expout b/go/regtest/cases/verb-merge-fields/name-0004/expout similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0007/expout rename to go/regtest/cases/verb-merge-fields/name-0004/expout diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0010/cmd b/go/regtest/cases/verb-merge-fields/name-0005/cmd similarity index 100% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0010/cmd rename to go/regtest/cases/verb-merge-fields/name-0005/cmd diff --git a/go/regtest/cases/verb-merge-fields/name-0005/experr b/go/regtest/cases/verb-merge-fields/name-0005/experr new file mode 100644 index 000000000..e69de29bb diff --git a/go/regtest/cases-pending-go-port/verb-merge-fields/0010/expout b/go/regtest/cases/verb-merge-fields/name-0005/expout similarity index 93% rename from go/regtest/cases-pending-go-port/verb-merge-fields/0010/expout rename to go/regtest/cases/verb-merge-fields/name-0005/expout index 11b105358..ff94a16c7 100644 --- a/go/regtest/cases-pending-go-port/verb-merge-fields/0010/expout +++ b/go/regtest/cases/verb-merge-fields/name-0005/expout @@ -2,9 +2,9 @@ a_in_x 1 a_out_x 2 b_in_y 4 b_out_x 8 -foo_p0 1.000000 +foo_p0 1 foo_min 1 -foo_p29 1.290000 +foo_p29 1.29 foo_max 2 foo_p100 2 foo_sum 3 @@ -47,9 +47,9 @@ a_in_x 1 a_out_x 2 b_on_y 4 b_out_x 8 -foo_p0 1.000000 +foo_p0 1 foo_min 1 -foo_p29 1.290000 +foo_p29 1.29 foo_max 2 foo_p100 2 foo_sum 3 @@ -59,9 +59,9 @@ a_in_x 1 a_out_x 2 b_in_y 4 b_oot_x 8 -foo_p0 1.000000 +foo_p0 1 foo_min 1 -foo_p29 1.290000 +foo_p29 1.29 foo_max 2 foo_p100 2 foo_sum 3 @@ -140,9 +140,9 @@ a_in_x 1 a_out_x 2 b_on_y 4 b_oot_x 8 -foo_p0 1.000000 +foo_p0 1 foo_min 1 -foo_p29 1.290000 +foo_p29 1.29 foo_max 2 foo_p100 2 foo_sum 3 diff --git a/go/regtest/cases/verb-merge-fields/regex-0001/cmd b/go/regtest/cases/verb-merge-fields/regex-0001/cmd new file mode 100644 index 000000000..392e12410 --- /dev/null +++ b/go/regtest/cases/verb-merge-fields/regex-0001/cmd @@ -0,0 +1 @@ +mlr merge-fields -k -a p0,min,p29,max,p100,sum,count -r in_,out_ -o bar regtest/input/merge-fields-abxy.dkvp diff --git a/go/regtest/cases/verb-merge-fields/regex-0001/experr b/go/regtest/cases/verb-merge-fields/regex-0001/experr new file mode 100644 index 000000000..e69de29bb diff --git a/go/regtest/cases/verb-merge-fields/regex-0001/expout b/go/regtest/cases/verb-merge-fields/regex-0001/expout new file mode 100644 index 000000000..dbc0bac22 --- /dev/null +++ b/go/regtest/cases/verb-merge-fields/regex-0001/expout @@ -0,0 +1,20 @@ +a_in_x=1,a_out_x=2,b_in_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=2,bar_max=8,bar_p100=8,bar_sum=15,bar_count=4 +z=1,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0 +a_on_x=1,a_out_x=2,b_in_y=4,b_out_x=8,bar_p0=2,bar_min=2,bar_p29=2,bar_max=8,bar_p100=8,bar_sum=14,bar_count=3 +a_in_x=1,a_oot_x=2,b_in_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=8,bar_p100=8,bar_sum=13,bar_count=3 +a_in_x=1,a_out_x=2,b_on_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=8,bar_p100=8,bar_sum=11,bar_count=3 +a_in_x=1,a_out_x=2,b_in_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=4,bar_p100=4,bar_sum=7,bar_count=3 +z=2,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0 +a_on_x=1,a_oot_x=2,b_in_y=4,b_out_x=8,bar_p0=4,bar_min=4,bar_p29=4,bar_max=8,bar_p100=8,bar_sum=12,bar_count=2 +a_on_x=1,a_out_x=2,b_on_y=4,b_out_x=8,bar_p0=2,bar_min=2,bar_p29=2,bar_max=8,bar_p100=8,bar_sum=10,bar_count=2 +a_on_x=1,a_out_x=2,b_in_y=4,b_oot_x=8,bar_p0=2,bar_min=2,bar_p29=2,bar_max=4,bar_p100=4,bar_sum=6,bar_count=2 +a_in_x=1,a_oot_x=2,b_on_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=8,bar_p100=8,bar_sum=9,bar_count=2 +a_in_x=1,a_oot_x=2,b_in_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=4,bar_p100=4,bar_sum=5,bar_count=2 +a_in_x=1,a_out_x=2,b_on_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=2,bar_p100=2,bar_sum=3,bar_count=2 +z=3,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0 +a_in_x=1,a_oot_x=2,b_on_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=1,bar_p100=1,bar_sum=1,bar_count=1 +a_on_x=1,a_out_x=2,b_on_y=4,b_oot_x=8,bar_p0=2,bar_min=2,bar_p29=2,bar_max=2,bar_p100=2,bar_sum=2,bar_count=1 +a_on_x=1,a_oot_x=2,b_in_y=4,b_oot_x=8,bar_p0=4,bar_min=4,bar_p29=4,bar_max=4,bar_p100=4,bar_sum=4,bar_count=1 +a_on_x=1,a_oot_x=2,b_on_y=4,b_out_x=8,bar_p0=8,bar_min=8,bar_p29=8,bar_max=8,bar_p100=8,bar_sum=8,bar_count=1 +z=4,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0 +a_on_x=1,a_oot_x=2,b_on_y=4,b_oot_x=8,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0 diff --git a/go/regtest/cases/verb-merge-fields/regex-0002/cmd b/go/regtest/cases/verb-merge-fields/regex-0002/cmd new file mode 100644 index 000000000..1383d7e4c --- /dev/null +++ b/go/regtest/cases/verb-merge-fields/regex-0002/cmd @@ -0,0 +1 @@ +mlr merge-fields -i -k -a p0,min,p29,max,p100,sum,count -r in_,out_ -o bar regtest/input/merge-fields-abxy.dkvp diff --git a/go/regtest/cases/verb-merge-fields/regex-0002/experr b/go/regtest/cases/verb-merge-fields/regex-0002/experr new file mode 100644 index 000000000..e69de29bb diff --git a/go/regtest/cases/verb-merge-fields/regex-0002/expout b/go/regtest/cases/verb-merge-fields/regex-0002/expout new file mode 100644 index 000000000..d5750408c --- /dev/null +++ b/go/regtest/cases/verb-merge-fields/regex-0002/expout @@ -0,0 +1,20 @@ +a_in_x=1,a_out_x=2,b_in_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=1.8699999999999999,bar_max=8,bar_p100=8,bar_sum=15,bar_count=4 +z=1,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0 +a_on_x=1,a_out_x=2,b_in_y=4,b_out_x=8,bar_p0=2,bar_min=2,bar_p29=3.16,bar_max=8,bar_p100=8,bar_sum=14,bar_count=3 +a_in_x=1,a_oot_x=2,b_in_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=2.7399999999999998,bar_max=8,bar_p100=8,bar_sum=13,bar_count=3 +a_in_x=1,a_out_x=2,b_on_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=1.58,bar_max=8,bar_p100=8,bar_sum=11,bar_count=3 +a_in_x=1,a_out_x=2,b_in_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1.58,bar_max=4,bar_p100=4,bar_sum=7,bar_count=3 +z=2,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0 +a_on_x=1,a_oot_x=2,b_in_y=4,b_out_x=8,bar_p0=4,bar_min=4,bar_p29=5.16,bar_max=8,bar_p100=8,bar_sum=12,bar_count=2 +a_on_x=1,a_out_x=2,b_on_y=4,b_out_x=8,bar_p0=2,bar_min=2,bar_p29=3.7399999999999998,bar_max=8,bar_p100=8,bar_sum=10,bar_count=2 +a_on_x=1,a_out_x=2,b_in_y=4,b_oot_x=8,bar_p0=2,bar_min=2,bar_p29=2.58,bar_max=4,bar_p100=4,bar_sum=6,bar_count=2 +a_in_x=1,a_oot_x=2,b_on_y=4,b_out_x=8,bar_p0=1,bar_min=1,bar_p29=3.03,bar_max=8,bar_p100=8,bar_sum=9,bar_count=2 +a_in_x=1,a_oot_x=2,b_in_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1.8699999999999999,bar_max=4,bar_p100=4,bar_sum=5,bar_count=2 +a_in_x=1,a_out_x=2,b_on_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1.29,bar_max=2,bar_p100=2,bar_sum=3,bar_count=2 +z=3,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0 +a_in_x=1,a_oot_x=2,b_on_y=4,b_oot_x=8,bar_p0=1,bar_min=1,bar_p29=1,bar_max=1,bar_p100=1,bar_sum=1,bar_count=1 +a_on_x=1,a_out_x=2,b_on_y=4,b_oot_x=8,bar_p0=2,bar_min=2,bar_p29=2,bar_max=2,bar_p100=2,bar_sum=2,bar_count=1 +a_on_x=1,a_oot_x=2,b_in_y=4,b_oot_x=8,bar_p0=4,bar_min=4,bar_p29=4,bar_max=4,bar_p100=4,bar_sum=4,bar_count=1 +a_on_x=1,a_oot_x=2,b_on_y=4,b_out_x=8,bar_p0=8,bar_min=8,bar_p29=8,bar_max=8,bar_p100=8,bar_sum=8,bar_count=1 +z=4,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0 +a_on_x=1,a_oot_x=2,b_on_y=4,b_oot_x=8,bar_p0=,bar_min=,bar_p29=,bar_max=,bar_p100=,bar_sum=0,bar_count=0 diff --git a/go/src/cli/mlrcli_transformers.go b/go/src/cli/mlrcli_transformers.go index 56bbb0b43..70f407e2e 100644 --- a/go/src/cli/mlrcli_transformers.go +++ b/go/src/cli/mlrcli_transformers.go @@ -39,6 +39,7 @@ var MAPPER_LOOKUP_TABLE = []transforming.TransformerSetup{ transformers.JoinSetup, transformers.LabelSetup, transformers.LeastFrequentSetup, + transformers.MergeFieldsSetup, transformers.MostFrequentSetup, transformers.NestSetup, transformers.NothingSetup, diff --git a/go/src/transformers/merge-fields.go b/go/src/transformers/merge-fields.go new file mode 100644 index 000000000..fd305fd00 --- /dev/null +++ b/go/src/transformers/merge-fields.go @@ -0,0 +1,534 @@ +package transformers + +import ( + "errors" + "fmt" + "os" + "regexp" + "strings" + + "miller/src/cliutil" + "miller/src/lib" + "miller/src/transformers/utils" + "miller/src/transforming" + "miller/src/types" +) + +// ---------------------------------------------------------------- +const verbNameMergeFields = "merge-fields" + +var MergeFieldsSetup = transforming.TransformerSetup{ + Verb: verbNameMergeFields, + UsageFunc: transformerMergeFieldsUsage, + ParseCLIFunc: transformerMergeFieldsParseCLI, + IgnoresInput: false, +} + +type mergeByType int + +const ( + e_MERGE_BY_NAME_LIST mergeByType = iota + e_MERGE_BY_NAME_REGEX + e_MERGE_BY_COLLAPSING + e_MERGE_UNSPECIFIED +) + +func transformerMergeFieldsUsage( + o *os.File, + doExit bool, + exitCode int, +) { + argv0 := lib.MlrExeName() + verb := verbNameMergeFields + fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb) + fmt.Fprintf(o, "Computes univariate statistics for each input record, accumulated across\n") + fmt.Fprintf(o, "specified fields.\n") + fmt.Fprintf(o, "Options:\n") + fmt.Fprintf(o, "-a {sum,count,...} Names of accumulators. One or more of:\n") + utils.ListStats1Accumulators(o) + fmt.Fprintf(o, "-f {a,b,c} Value-field names on which to compute statistics. Requires -o.\n") + fmt.Fprintf(o, "-r {a,b,c} Regular expressions for value-field names on which to compute\n") + fmt.Fprintf(o, " statistics. Requires -o.\n") + fmt.Fprintf(o, "-c {a,b,c} Substrings for collapse mode. All fields which have the same names\n") + fmt.Fprintf(o, " after removing substrings will be accumulated together. Please see\n") + fmt.Fprintf(o, " examples below.\n") + fmt.Fprintf(o, "-i Use interpolated percentiles, like R's type=7; default like type=1.\n") + fmt.Fprintf(o, " Not sensical for string-valued fields.\n") + fmt.Fprintf(o, "-o {name} Output field basename for -f/-r.\n") + fmt.Fprintf(o, "-k Keep the input fields which contributed to the output statistics;\n") + fmt.Fprintf(o, " the default is to omit them.\n") + fmt.Fprintf(o, "\n") + fmt.Fprintf(o, "String-valued data make sense unless arithmetic on them is required,\n") + fmt.Fprintf(o, "e.g. for sum, mean, interpolated percentiles, etc. In case of mixed data,\n") + fmt.Fprintf(o, "numbers are less than strings.\n") + fmt.Fprintf(o, "\n") + fmt.Fprintf(o, "Example input data: \"a_in_x=1,a_out_x=2,b_in_y=4,b_out_x=8\".\n") + fmt.Fprintf(o, "Example: %s %s -a sum,count -f a_in_x,a_out_x -o foo\n", argv0, verb) + fmt.Fprintf(o, " produces \"b_in_y=4,b_out_x=8,foo_sum=3,foo_count=2\" since \"a_in_x,a_out_x\" are\n") + fmt.Fprintf(o, " summed over.\n") + fmt.Fprintf(o, "Example: %s %s -a sum,count -r in_,out_ -o bar\n", argv0, verb) + fmt.Fprintf(o, " produces \"bar_sum=15,bar_count=4\" since all four fields are summed over.\n") + fmt.Fprintf(o, "Example: %s %s -a sum,count -c in_,out_\n", argv0, verb) + fmt.Fprintf(o, " produces \"a_x_sum=3,a_x_count=2,b_y_sum=4,b_y_count=1,b_x_sum=8,b_x_count=1\"\n") + fmt.Fprintf(o, " since \"a_in_x\" and \"a_out_x\" both collapse to \"a_x\", \"b_in_y\" collapses to\n") + fmt.Fprintf(o, " \"b_y\", and \"b_out_x\" collapses to \"b_x\".\n") + + if doExit { + os.Exit(exitCode) + } +} + +func transformerMergeFieldsParseCLI( + pargi *int, + argc int, + args []string, + _ *cliutil.TReaderOptions, + __ *cliutil.TWriterOptions, +) transforming.IRecordTransformer { + + // Skip the verb name from the current spot in the mlr command line + argi := *pargi + verb := args[argi] + argi++ + + accumulatorNameList := make([]string, 0) + valueFieldNameList := make([]string, 0) + outputFieldBasename := "" + doWhich := e_MERGE_UNSPECIFIED + keepInputFields := false + doInterpolatedPercentiles := false + + for argi < argc /* variable increment: 1 or 2 depending on flag */ { + opt := args[argi] + if !strings.HasPrefix(opt, "-") { + break // No more flag options to process + } + argi++ + + if opt == "-h" || opt == "--help" { + transformerMergeFieldsUsage(os.Stdout, true, 0) + + } else if opt == "-a" { + accumulatorNameList = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) + + } else if opt == "-f" { + valueFieldNameList = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) + doWhich = e_MERGE_BY_NAME_LIST + + } else if opt == "-r" { + valueFieldNameList = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) + doWhich = e_MERGE_BY_NAME_REGEX + + } else if opt == "-c" { + valueFieldNameList = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) + doWhich = e_MERGE_BY_COLLAPSING + + } else if opt == "-o" { + outputFieldBasename = cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc) + + } else if opt == "-k" { + keepInputFields = true + + } else if opt == "-i" { + doInterpolatedPercentiles = true + + } else if opt == "-S" { + // No-op pass-through for backward compatibility with Miller 5 + + } else if opt == "-F" { + // No-op pass-through for backward compatibility with Miller 5 + + } else { + transformerMergeFieldsUsage(os.Stderr, true, 1) + } + } + + // TODO: libify for use across verbs. + if len(accumulatorNameList) == 0 { + fmt.Fprintf(os.Stderr, "%s %s: -a option is required.\n", lib.MlrExeName(), verbNameMergeFields) + fmt.Fprintf(os.Stderr, "Please see %s %s --help for more information.\n", lib.MlrExeName(), verbNameMergeFields) + os.Exit(1) + } + if len(valueFieldNameList) == 0 { + fmt.Fprintf(os.Stderr, "%s %s: -f option is required.\n", lib.MlrExeName(), verbNameMergeFields) + fmt.Fprintf(os.Stderr, "Please see %s %s --help for more information.\n", lib.MlrExeName(), verbNameMergeFields) + os.Exit(1) + } + if outputFieldBasename == "" { + if doWhich == e_MERGE_BY_NAME_LIST || doWhich == e_MERGE_BY_NAME_REGEX { + transformerMergeFieldsUsage(os.Stderr, true, 1) + } + } + + transformer, _ := NewTransformerMergeFields( + accumulatorNameList, + valueFieldNameList, + outputFieldBasename, + doWhich, + doInterpolatedPercentiles, + keepInputFields, + ) + + *pargi = argi + return transformer +} + +// ---------------------------------------------------------------- +// Given: accumulate count,sum on values x,y group by a,b. +// +// Example input: Example output: +// a b x y a b x_count x_sum y_count y_sum +// s t 1 2 s t 2 6 2 8 +// u v 3 4 u v 1 3 1 4 +// s t 5 6 u w 1 7 1 9 +// u w 7 9 +// +// Multilevel hashmap structure: +// { +// "s,t" : { <--- group-by field names +// "x" : { <--- value field name +// "count" : Stats1CountAccumulator object, +// "sum" : Stats1SumAccumulator object +// }, +// "y" : { +// "count" : Stats1CountAccumulator object, +// "sum" : Stats1SumAccumulator object +// }, +// }, +// "u,v" : { +// "x" : { +// "count" : Stats1CountAccumulator object, +// "sum" : Stats1SumAccumulator object +// }, +// "y" : { +// "count" : Stats1CountAccumulator object, +// "sum" : Stats1SumAccumulator object +// }, +// }, +// "u,w" : { +// "x" : { +// "count" : Stats1CountAccumulator object, +// "sum" : Stats1SumAccumulator object +// }, +// "y" : { +// "count" : Stats1CountAccumulator object, +// "sum" : Stats1SumAccumulator object +// }, +// }, +// } + +type TransformerMergeFields struct { + // Input: + accumulatorNameList []string + valueFieldNameList []string + outputFieldBasename string + doInterpolatedPercentiles bool + keepInputFields bool + + // State: + accumulatorFactory *utils.Stats1AccumulatorFactory + valueFieldNameRegexes []*regexp.Regexp + + // Ordered map from accumulator name to accumulator + namedAccumulators *lib.OrderedMap + + recordTransformerFunc transforming.RecordTransformerFunc +} + +func NewTransformerMergeFields( + accumulatorNameList []string, + valueFieldNameList []string, + outputFieldBasename string, + doWhich mergeByType, + doInterpolatedPercentiles bool, + keepInputFields bool, +) (*TransformerMergeFields, error) { + + for _, accumulatorName := range accumulatorNameList { + if !utils.ValidateStats1AccumulatorName(accumulatorName) { + return nil, errors.New( + fmt.Sprintf( + "%s %s: accumulator \"%s\" not found.\n", + lib.MlrExeName(), verbNameMergeFields, accumulatorName, + ), + ) + } + } + + this := &TransformerMergeFields{ + accumulatorNameList: accumulatorNameList, + valueFieldNameList: valueFieldNameList, + outputFieldBasename: outputFieldBasename, + doInterpolatedPercentiles: doInterpolatedPercentiles, + keepInputFields: keepInputFields, + accumulatorFactory: utils.NewStats1AccumulatorFactory(), + namedAccumulators: lib.NewOrderedMap(), + } + + this.valueFieldNameRegexes = make([]*regexp.Regexp, len(valueFieldNameList)) + for i, regexString := range valueFieldNameList { + // Handles "a.*b"i Miller case-insensitive-regex specification + regex, err := lib.CompileMillerRegex(regexString) + if err != nil { + fmt.Fprintf( + os.Stderr, + "%s %s: cannot compile regex [%s]\n", + lib.MlrExeName(), verbNameCut, regexString, + ) + os.Exit(1) + } + this.valueFieldNameRegexes[i] = regex + } + + for _, accumulatorName := range accumulatorNameList { + accumulator := this.accumulatorFactory.MakeNamedAccumulator( + accumulatorName, + "", // grouping-key used for stats1, not here + outputFieldBasename, + doInterpolatedPercentiles, + ) + this.namedAccumulators.Put(accumulatorName, accumulator) + } + + if doWhich == e_MERGE_BY_NAME_LIST { + this.recordTransformerFunc = this.transformByNameList + } else if doWhich == e_MERGE_BY_NAME_REGEX { + this.recordTransformerFunc = this.transformByNameRegex + } else if doWhich == e_MERGE_BY_COLLAPSING { + this.recordTransformerFunc = this.transformByCollapsing + } else { + lib.InternalCodingErrorIf(true) + } + + return this, nil +} + +// ---------------------------------------------------------------- +func (this *TransformerMergeFields) Transform( + inrecAndContext *types.RecordAndContext, + outputChannel chan<- *types.RecordAndContext, +) { + this.recordTransformerFunc(inrecAndContext, outputChannel) +} + +// ---------------------------------------------------------------- +func (this *TransformerMergeFields) transformByNameList( + inrecAndContext *types.RecordAndContext, + outputChannel chan<- *types.RecordAndContext, +) { + if inrecAndContext.EndOfStream { + outputChannel <- inrecAndContext // end-of-stream marker + return + } + + inrec := inrecAndContext.Record + + for pa := this.namedAccumulators.Head; pa != nil; pa = pa.Next { + accumulator := pa.Value.(*utils.Stats1NamedAccumulator) + accumulator.Reset() // re-use from one record to the next + } + + for _, valueFieldName := range this.valueFieldNameList { + mvalue := inrec.Get(valueFieldName) + if mvalue == nil { // key not present + continue + } + + if mvalue.IsEmpty() { // key present with empty value + if !this.keepInputFields { + inrec.Remove(valueFieldName) + } + continue + } + + for pa := this.namedAccumulators.Head; pa != nil; pa = pa.Next { + accumulator := pa.Value.(*utils.Stats1NamedAccumulator) + accumulator.Ingest(mvalue) + } + + if !this.keepInputFields { + inrec.Remove(valueFieldName) + } + } + + for pa := this.namedAccumulators.Head; pa != nil; pa = pa.Next { + accumulator := pa.Value.(*utils.Stats1NamedAccumulator) + key, value := accumulator.Emit() + inrec.PutReference(key, value) + } + + outputChannel <- inrecAndContext +} + +// ---------------------------------------------------------------- +func (this *TransformerMergeFields) transformByNameRegex( + inrecAndContext *types.RecordAndContext, + outputChannel chan<- *types.RecordAndContext, +) { + if inrecAndContext.EndOfStream { + outputChannel <- inrecAndContext // end-of-stream marker + return + } + + inrec := inrecAndContext.Record + + for pa := this.namedAccumulators.Head; pa != nil; pa = pa.Next { + accumulator := pa.Value.(*utils.Stats1NamedAccumulator) + accumulator.Reset() // re-use from one record to the next + } + + for pe := inrec.Head; pe != nil; /* increment inside loop*/ { + valueFieldName := pe.Key + + matched := false + for _, valueFieldNameRegex := range this.valueFieldNameRegexes { + if valueFieldNameRegex.MatchString(pe.Key) { + matched = true + break + } + } + if !matched { + pe = pe.Next + continue + } + + mvalue := inrec.Get(valueFieldName) + + if mvalue == nil { // Key not present + pe = pe.Next + continue + } + + if mvalue.IsEmpty() { // Key present with empty value + if !this.keepInputFields { // We are modifying the record while iterating over it. + next := pe.Next + inrec.Unlink(pe) + pe = next + } else { + pe = pe.Next + } + continue + } + + for pa := this.namedAccumulators.Head; pa != nil; pa = pa.Next { + accumulator := pa.Value.(*utils.Stats1NamedAccumulator) + accumulator.Ingest(mvalue) + } + + if !this.keepInputFields { // We are modifying the record while iterating over it. + next := pe.Next + inrec.Unlink(pe) + pe = next + } else { + pe = pe.Next + } + } + + for pa := this.namedAccumulators.Head; pa != nil; pa = pa.Next { + accumulator := pa.Value.(*utils.Stats1NamedAccumulator) + key, value := accumulator.Emit() + inrec.PutReference(key, value) + } + + outputChannel <- inrecAndContext +} + +// ---------------------------------------------------------------- +// mlr merge-fields -c in_,out_ -a sum +// a_in_x 1 a_sum_x 3 +// a_out_x 2 b_sum_y 4 +// b_in_y 4 b_sum_x 8 +// b_out_x 8 + +func (this *TransformerMergeFields) transformByCollapsing( + inrecAndContext *types.RecordAndContext, + outputChannel chan<- *types.RecordAndContext, +) { + if inrecAndContext.EndOfStream { + outputChannel <- inrecAndContext // end-of-stream marker + return + } + + inrec := inrecAndContext.Record + this.accumulatorFactory.Reset() // discard cached percentile-keepers + + // Ordered map from short name to accumulator name to accumulator + collapseAccumulators := lib.NewOrderedMap() + + for pe := inrec.Head; pe != nil; /* increment inside loop */ { + valueFieldName := pe.Key + + matched := false + shortName := "" + for _, valueFieldNameRegex := range this.valueFieldNameRegexes { + matched = valueFieldNameRegex.MatchString(pe.Key) + if matched { + shortName = lib.RegexReplaceOnce(valueFieldNameRegex, valueFieldName, "") + break + } + } + if !matched { + pe = pe.Next + continue + } + + mvalue := inrec.Get(valueFieldName) + if mvalue == nil { // Key present + pe = pe.Next + continue + } + + if mvalue.IsEmpty() { // key present with empty value + if !this.keepInputFields { // We are modifying the record while iterating over it. + next := pe.Next + inrec.Unlink(pe) + pe = next + } else { + pe = pe.Next + } + continue + } + + var namedAccumulators *lib.OrderedMap + iNamedAccumulators := collapseAccumulators.Get(shortName) + if iNamedAccumulators == nil { + namedAccumulators = lib.NewOrderedMap() + for _, accumulatorName := range this.accumulatorNameList { + accumulator := this.accumulatorFactory.MakeNamedAccumulator( + accumulatorName, + "", // grouping-key used for stats1, not here + shortName, + this.doInterpolatedPercentiles, + ) + namedAccumulators.Put(accumulatorName, accumulator) + } + collapseAccumulators.Put(shortName, namedAccumulators) + } else { + namedAccumulators = iNamedAccumulators.(*lib.OrderedMap) + } + + for pa := namedAccumulators.Head; pa != nil; pa = pa.Next { + accumulator := pa.Value.(*utils.Stats1NamedAccumulator) + accumulator.Ingest(mvalue) + } + + if !this.keepInputFields { // We are modifying the record while iterating over it. + next := pe.Next + inrec.Unlink(pe) + pe = next + } else { + pe = pe.Next + } + } + + for ps := collapseAccumulators.Head; ps != nil; ps = ps.Next { + namedAccumulators := ps.Value.(*lib.OrderedMap) + for pa := namedAccumulators.Head; pa != nil; pa = pa.Next { + accumulator := pa.Value.(*utils.Stats1NamedAccumulator) + key, value := accumulator.Emit() + inrec.PutReference(key, value) + } + } + + outputChannel <- inrecAndContext +} diff --git a/go/src/transformers/stats1.go b/go/src/transformers/stats1.go index 503eaa0dd..2fd196a0b 100644 --- a/go/src/transformers/stats1.go +++ b/go/src/transformers/stats1.go @@ -380,7 +380,7 @@ func (this *TransformerStats1) emitIntoOutputRecord( for pc := level3.Head; pc != nil; pc = pc.Next { namedAccumulator := pc.Value.(*utils.Stats1NamedAccumulator) key, value := namedAccumulator.Emit() - outrec.PutCopy(key, &value) + outrec.PutCopy(key, value) } } } diff --git a/go/src/transformers/utils/percentile-keeper.go b/go/src/transformers/utils/percentile-keeper.go index 86359a276..5f8735ebe 100644 --- a/go/src/transformers/utils/percentile-keeper.go +++ b/go/src/transformers/utils/percentile-keeper.go @@ -29,6 +29,11 @@ func NewPercentileKeeper( } } +func (this *PercentileKeeper) Reset() { + this.data = make([]*types.Mlrval, 0, 1000) + this.sorted = false +} + // ---------------------------------------------------------------- func (this *PercentileKeeper) Ingest(value *types.Mlrval) { if len(this.data) >= cap(this.data) { @@ -259,7 +264,7 @@ func (this *PercentileKeeper) sortIfNecessary() { } // ---------------------------------------------------------------- -func (this *PercentileKeeper) Emit(percentile float64) types.Mlrval { +func (this *PercentileKeeper) Emit(percentile float64) *types.Mlrval { if this.doInterpolatedPercentiles { return this.EmitLinearlyInterpolated(percentile) } else { @@ -267,21 +272,21 @@ func (this *PercentileKeeper) Emit(percentile float64) types.Mlrval { } } -func (this *PercentileKeeper) EmitNonInterpolated(percentile float64) types.Mlrval { +func (this *PercentileKeeper) EmitNonInterpolated(percentile float64) *types.Mlrval { if len(this.data) == 0 { - return *types.MLRVAL_ABSENT + return types.MLRVAL_VOID } this.sortIfNecessary() - return *this.data[computeIndexNoninterpolated(int(len(this.data)), percentile)].Copy() + return this.data[computeIndexNoninterpolated(int(len(this.data)), percentile)].Copy() } -func (this *PercentileKeeper) EmitLinearlyInterpolated(percentile float64) types.Mlrval { +func (this *PercentileKeeper) EmitLinearlyInterpolated(percentile float64) *types.Mlrval { if len(this.data) == 0 { - return *types.MLRVAL_ABSENT + return types.MLRVAL_VOID } this.sortIfNecessary() output := getPercentileLinearlyInterpolated(this.data, int(len(this.data)), percentile) - return *output.Copy() + return output.Copy() } // ---------------------------------------------------------------- diff --git a/go/src/transformers/utils/stats1-accumulators.go b/go/src/transformers/utils/stats1-accumulators.go index 3b0a645e9..135a2c084 100644 --- a/go/src/transformers/utils/stats1-accumulators.go +++ b/go/src/transformers/utils/stats1-accumulators.go @@ -16,7 +16,8 @@ import ( // ---------------------------------------------------------------- type IStats1Accumulator interface { Ingest(value *types.Mlrval) - Emit() types.Mlrval + Emit() *types.Mlrval + Reset() // for merge-fields where we reset after each record instead of replace/recreate } // ---------------------------------------------------------------- @@ -124,10 +125,14 @@ func (this *Stats1NamedAccumulator) Ingest(value *types.Mlrval) { this.accumulator.Ingest(value) } -func (this *Stats1NamedAccumulator) Emit() (key string, value types.Mlrval) { +func (this *Stats1NamedAccumulator) Emit() (key string, value *types.Mlrval) { return this.outputFieldName, this.accumulator.Emit() } +func (this *Stats1NamedAccumulator) Reset() { + this.accumulator.Reset() +} + // ---------------------------------------------------------------- // If we are asked for p90 and p95 on the same column, we reuse the // percentile-keeper object to reduce runtime memory consumption. This @@ -195,6 +200,11 @@ func tryPercentileFromName(accumulatorName string) (float64, bool) { } // ---------------------------------------------------------------- +// For merge-fields wherein percentile-keepers are re-created on each record +func (this *Stats1AccumulatorFactory) Reset() { + this.percentileKeepers = make(map[string]map[string]*PercentileKeeper) +} + func (this *Stats1AccumulatorFactory) MakeNamedAccumulator( accumulatorName string, groupingKey string, @@ -253,7 +263,7 @@ func (this *Stats1AccumulatorFactory) MakeAccumulator( // To conserve memory, percentile-keeprs on the same value-field-name // (and grouping-key) are shared. For example, p25,p75 on field "x". // This means though that each datapoint must be ingested only once - // (e.g. by the p25 accumulator) since it shares a percentile-keepr + // (e.g. by the p25 accumulator) since it shares a percentile-keeper // with the p75 accumulator. We handle this by tracking the first // construction. return NewStats1PercentileAccumulator(percentileKeeper, percentile, isPrimary) @@ -281,8 +291,11 @@ func NewStats1CountAccumulator() IStats1Accumulator { func (this *Stats1CountAccumulator) Ingest(value *types.Mlrval) { this.count++ } -func (this *Stats1CountAccumulator) Emit() types.Mlrval { - return types.MlrvalFromInt(this.count) +func (this *Stats1CountAccumulator) Emit() *types.Mlrval { + return types.MlrvalPointerFromInt(this.count) +} +func (this *Stats1CountAccumulator) Reset() { + this.count = 0 } // ---------------------------------------------------------------- @@ -306,9 +319,9 @@ func (this *Stats1ModeAccumulator) Ingest(value *types.Mlrval) { this.countsByValue.Put(key, iPrevious.(int)+1) } } -func (this *Stats1ModeAccumulator) Emit() types.Mlrval { +func (this *Stats1ModeAccumulator) Emit() *types.Mlrval { if this.countsByValue.FieldCount == 0 { - return *types.MLRVAL_ERROR + return types.MLRVAL_ERROR } maxValue := "" var maxCount = int(0) @@ -320,7 +333,10 @@ func (this *Stats1ModeAccumulator) Emit() types.Mlrval { maxCount = count } } - return types.MlrvalFromString(maxValue) + return types.MlrvalPointerFromString(maxValue) +} +func (this *Stats1ModeAccumulator) Reset() { + this.countsByValue = lib.NewOrderedMap() } // ---------------------------------------------------------------- @@ -344,9 +360,9 @@ func (this *Stats1AntimodeAccumulator) Ingest(value *types.Mlrval) { this.countsByValue.Put(key, iPrevious.(int)+1) } } -func (this *Stats1AntimodeAccumulator) Emit() types.Mlrval { +func (this *Stats1AntimodeAccumulator) Emit() *types.Mlrval { if this.countsByValue.FieldCount == 0 { - return *types.MLRVAL_ERROR + return types.MLRVAL_ERROR } minValue := "" var minCount = int(0) @@ -358,7 +374,10 @@ func (this *Stats1AntimodeAccumulator) Emit() types.Mlrval { minCount = count } } - return types.MlrvalFromString(minValue) + return types.MlrvalPointerFromString(minValue) +} +func (this *Stats1AntimodeAccumulator) Reset() { + this.countsByValue = lib.NewOrderedMap() } // ---------------------------------------------------------------- @@ -374,10 +393,11 @@ func NewStats1SumAccumulator() IStats1Accumulator { func (this *Stats1SumAccumulator) Ingest(value *types.Mlrval) { this.sum = types.MlrvalBinaryPlus(this.sum, value) } - -// xxx pending output-pointer refactor -func (this *Stats1SumAccumulator) Emit() types.Mlrval { - return *this.sum.Copy() +func (this *Stats1SumAccumulator) Emit() *types.Mlrval { + return this.sum.Copy() +} +func (this *Stats1SumAccumulator) Reset() { + this.sum = types.MlrvalPointerFromInt(0) } // ---------------------------------------------------------------- @@ -396,16 +416,17 @@ func (this *Stats1MeanAccumulator) Ingest(value *types.Mlrval) { this.sum = types.MlrvalBinaryPlus(this.sum, value) this.count++ } - -// xxx pending output-pointer refactor -func (this *Stats1MeanAccumulator) Emit() types.Mlrval { +func (this *Stats1MeanAccumulator) Emit() *types.Mlrval { if this.count == 0 { - return *types.MLRVAL_VOID + return types.MLRVAL_VOID } else { - // TODO: pointer-only refactor - return *types.MlrvalDivide(this.sum, types.MlrvalPointerFromInt(this.count)) + return types.MlrvalDivide(this.sum, types.MlrvalPointerFromInt(this.count)) } } +func (this *Stats1MeanAccumulator) Reset() { + this.sum = types.MlrvalPointerFromInt(0) + this.count = 0 +} // ---------------------------------------------------------------- type Stats1MinAccumulator struct { @@ -420,8 +441,15 @@ func NewStats1MinAccumulator() IStats1Accumulator { func (this *Stats1MinAccumulator) Ingest(value *types.Mlrval) { this.min = types.MlrvalBinaryMin(this.min, value) } -func (this *Stats1MinAccumulator) Emit() types.Mlrval { - return *this.min.Copy() +func (this *Stats1MinAccumulator) Emit() *types.Mlrval { + if this.min.IsAbsent() { + return types.MLRVAL_VOID + } else { + return this.min.Copy() + } +} +func (this *Stats1MinAccumulator) Reset() { + this.min = types.MLRVAL_ABSENT } // ---------------------------------------------------------------- @@ -437,8 +465,15 @@ func NewStats1MaxAccumulator() IStats1Accumulator { func (this *Stats1MaxAccumulator) Ingest(value *types.Mlrval) { this.max = types.MlrvalBinaryMax(this.max, value) } -func (this *Stats1MaxAccumulator) Emit() types.Mlrval { - return *this.max.Copy() +func (this *Stats1MaxAccumulator) Emit() *types.Mlrval { + if this.max.IsAbsent() { + return types.MLRVAL_VOID + } else { + return this.max.Copy() + } +} +func (this *Stats1MaxAccumulator) Reset() { + this.max = types.MLRVAL_ABSENT } // ---------------------------------------------------------------- @@ -461,10 +496,14 @@ func (this *Stats1VarAccumulator) Ingest(value *types.Mlrval) { this.sum = types.MlrvalBinaryPlus(this.sum, value) this.sum2 = types.MlrvalBinaryPlus(this.sum2, value2) } -func (this *Stats1VarAccumulator) Emit() types.Mlrval { +func (this *Stats1VarAccumulator) Emit() *types.Mlrval { mcount := types.MlrvalFromInt(this.count) - // TODO: pointer-only refactor - return *types.MlrvalGetVar(&mcount, this.sum, this.sum2) + return types.MlrvalGetVar(&mcount, this.sum, this.sum2) +} +func (this *Stats1VarAccumulator) Reset() { + this.count = 0 + this.sum = types.MlrvalPointerFromInt(0) + this.sum2 = types.MlrvalPointerFromInt(0) } // ---------------------------------------------------------------- @@ -487,9 +526,14 @@ func (this *Stats1StddevAccumulator) Ingest(value *types.Mlrval) { this.sum = types.MlrvalBinaryPlus(this.sum, value) this.sum2 = types.MlrvalBinaryPlus(this.sum2, value2) } -func (this *Stats1StddevAccumulator) Emit() types.Mlrval { +func (this *Stats1StddevAccumulator) Emit() *types.Mlrval { mcount := types.MlrvalFromInt(this.count) - return *types.MlrvalGetStddev(&mcount, this.sum, this.sum2) + return types.MlrvalGetStddev(&mcount, this.sum, this.sum2) +} +func (this *Stats1StddevAccumulator) Reset() { + this.count = 0 + this.sum = types.MlrvalPointerFromInt(0) + this.sum2 = types.MlrvalPointerFromInt(0) } // ---------------------------------------------------------------- @@ -512,9 +556,14 @@ func (this *Stats1MeanEBAccumulator) Ingest(value *types.Mlrval) { this.sum = types.MlrvalBinaryPlus(this.sum, value) this.sum2 = types.MlrvalBinaryPlus(this.sum2, value2) } -func (this *Stats1MeanEBAccumulator) Emit() types.Mlrval { +func (this *Stats1MeanEBAccumulator) Emit() *types.Mlrval { mcount := types.MlrvalPointerFromInt(this.count) - return *types.MlrvalGetMeanEB(mcount, this.sum, this.sum2) + return types.MlrvalGetMeanEB(mcount, this.sum, this.sum2) +} +func (this *Stats1MeanEBAccumulator) Reset() { + this.count = 0 + this.sum = types.MlrvalPointerFromInt(0) + this.sum2 = types.MlrvalPointerFromInt(0) } // ---------------------------------------------------------------- @@ -541,10 +590,16 @@ func (this *Stats1SkewnessAccumulator) Ingest(value *types.Mlrval) { this.sum2 = types.MlrvalBinaryPlus(this.sum2, value2) this.sum3 = types.MlrvalBinaryPlus(this.sum3, value3) } -func (this *Stats1SkewnessAccumulator) Emit() types.Mlrval { +func (this *Stats1SkewnessAccumulator) Emit() *types.Mlrval { mcount := types.MlrvalPointerFromInt(this.count) return types.MlrvalGetSkewness(mcount, this.sum, this.sum2, this.sum3) } +func (this *Stats1SkewnessAccumulator) Reset() { + this.count = 0 + this.sum = types.MlrvalPointerFromInt(0) + this.sum2 = types.MlrvalPointerFromInt(0) + this.sum3 = types.MlrvalPointerFromInt(0) +} // ---------------------------------------------------------------- type Stats1KurtosisAccumulator struct { @@ -574,10 +629,17 @@ func (this *Stats1KurtosisAccumulator) Ingest(value *types.Mlrval) { this.sum3 = types.MlrvalBinaryPlus(this.sum3, value3) this.sum4 = types.MlrvalBinaryPlus(this.sum4, value4) } -func (this *Stats1KurtosisAccumulator) Emit() types.Mlrval { +func (this *Stats1KurtosisAccumulator) Emit() *types.Mlrval { mcount := types.MlrvalPointerFromInt(this.count) return types.MlrvalGetKurtosis(mcount, this.sum, this.sum2, this.sum3, this.sum4) } +func (this *Stats1KurtosisAccumulator) Reset() { + this.count = 0 + this.sum = types.MlrvalPointerFromInt(0) + this.sum2 = types.MlrvalPointerFromInt(0) + this.sum3 = types.MlrvalPointerFromInt(0) + this.sum4 = types.MlrvalPointerFromInt(0) +} // ---------------------------------------------------------------- // To conserve memory, percentile-keeprs on the same value-field-name (and @@ -609,6 +671,12 @@ func (this *Stats1PercentileAccumulator) Ingest(value *types.Mlrval) { } } -func (this *Stats1PercentileAccumulator) Emit() types.Mlrval { +func (this *Stats1PercentileAccumulator) Emit() *types.Mlrval { return this.percentileKeeper.Emit(this.percentile) } + +func (this *Stats1PercentileAccumulator) Reset() { + if this.isPrimary { + this.percentileKeeper.Reset() + } +} diff --git a/go/src/types/mlrval_functions_stats.go b/go/src/types/mlrval_functions_stats.go index f5897d742..c6ca0f75d 100644 --- a/go/src/types/mlrval_functions_stats.go +++ b/go/src/types/mlrval_functions_stats.go @@ -86,11 +86,11 @@ func MlrvalGetMeanEB(mn, msum, msum2 *Mlrval) *Mlrval { // = sumx2 - n mean^2 // ---------------------------------------------------------------- -func MlrvalGetSkewness(mn, msum, msum2, msum3 *Mlrval) Mlrval { +func MlrvalGetSkewness(mn, msum, msum2, msum3 *Mlrval) *Mlrval { n, isInt := mn.GetIntValue() lib.InternalCodingErrorIf(!isInt) if n < 2 { - return *MLRVAL_VOID + return MLRVAL_VOID } fn := float64(n) sum, isNumber := msum.GetNumericToFloatValue() @@ -105,7 +105,7 @@ func MlrvalGetSkewness(mn, msum, msum2, msum3 *Mlrval) Mlrval { numerator = numerator / fn denominator := (sum2 - fn*mean*mean) / (fn - 1.0) denominator = math.Pow(denominator, 1.5) - return MlrvalFromFloat64(numerator / denominator) + return MlrvalPointerFromFloat64(numerator / denominator) } // Unbiased: @@ -123,11 +123,11 @@ func MlrvalGetSkewness(mn, msum, msum2, msum3 *Mlrval) Mlrval { // = sumx4 - mean*(4 sumx3 - mean*(6 sumx2 - 3 n mean^2)) // ---------------------------------------------------------------- -func MlrvalGetKurtosis(mn, msum, msum2, msum3, msum4 *Mlrval) Mlrval { +func MlrvalGetKurtosis(mn, msum, msum2, msum3, msum4 *Mlrval) *Mlrval { n, isInt := mn.GetIntValue() lib.InternalCodingErrorIf(!isInt) if n < 2 { - return *MLRVAL_VOID + return MLRVAL_VOID } fn := float64(n) sum, isNumber := msum.GetNumericToFloatValue() @@ -145,6 +145,6 @@ func MlrvalGetKurtosis(mn, msum, msum2, msum3, msum4 *Mlrval) Mlrval { numerator = numerator / fn denominator := (sum2 - fn*mean*mean) / fn denominator = denominator * denominator - return MlrvalFromFloat64(numerator/denominator - 3.0) + return MlrvalPointerFromFloat64(numerator/denominator - 3.0) } diff --git a/go/todo.txt b/go/todo.txt index c3372d128..67a3c81c6 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -1,7 +1,13 @@ ================================================================ TOP OF LIST: -! mrpl :h (and miller --help-xxx): substring ... +!!! mlr --from s stats1 -f x,y -a corr + panic: runtime error: invalid memory address or nil pointer dereference + > needs UT + > needs _ -> err x all + +! seqgen help saying -stop not --stop + ! MlrvalPointerFromString -> NewMlrvalPointerFromString et al. * blocker: regexes diff --git a/go/u/gg b/go/u/gg index 977137f31..1ed3c4e63 100755 --- a/go/u/gg +++ b/go/u/gg @@ -53,16 +53,17 @@ mention() { # ================================================================ -run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f x,y -b u -run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f u,v -b x -run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f z,w,s -b v -run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f u,v -b nonesuch -run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f u,nonesuch -b x - -run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f x,y -a u -run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f u,v -a x -run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f z,w,s -a v -run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f u,v -a nonesuch -run_mlr --from ../go/regtest/input/reorder.dkvp reorder -f u,nonesuch -a x +mlr --c2p cat regtest/input/merge-fields-in-out.csv +echo +../c/mlr --csvlite merge-fields -a min,max,count,sum -c _in,_out regtest/input/merge-fields-in-out.csv +echo +../go/mlr --csvlite merge-fields -a min,max,count,sum -c _in,_out regtest/input/merge-fields-in-out.csv +echo +echo +mlr --c2p cat regtest/input/merge-fields-in-out.csv +echo +../c/mlr --csvlite merge-fields -a p0,p29,p100 -c _in,_out regtest/input/merge-fields-in-out.csv +echo +../go/mlr --csvlite merge-fields -a p0,p29,p100 -c _in,_out regtest/input/merge-fields-in-out.csv # ================================================================