From 96deed048a8b162f05980a6b9a4cd4c4c3f1db7d Mon Sep 17 00:00:00 2001 From: John Kerl Date: Tue, 7 Jul 2026 15:49:53 -0400 Subject: [PATCH] stats1: add rank accumulator (#383) Adds `mlr stats1 -a rank` for standard competition ranking (1,2,2,4,...) on pre-sorted data, most useful with -s for a rank on every record. --- docs/src/manpage.md | 8 +++- docs/src/manpage.txt | 8 +++- docs/src/reference-verbs.md | 6 +++ man/manpage.txt | 8 +++- man/mlr.1 | 10 +++- pkg/transformers/stats1.go | 6 ++- pkg/transformers/utils/stats1_accumulators.go | 48 +++++++++++++++++++ test/cases/cli-help/0001/expout | 6 +++ test/cases/verb-stats1/0020/cmd | 1 + test/cases/verb-stats1/0020/experr | 0 test/cases/verb-stats1/0020/expout | 5 ++ test/cases/verb-stats1/0021/cmd | 1 + test/cases/verb-stats1/0021/experr | 0 test/cases/verb-stats1/0021/expout | 6 +++ test/input/rank-data-grouped.csv | 6 +++ test/input/rank-data.csv | 5 ++ 16 files changed, 118 insertions(+), 6 deletions(-) create mode 100644 test/cases/verb-stats1/0020/cmd create mode 100644 test/cases/verb-stats1/0020/experr create mode 100644 test/cases/verb-stats1/0020/expout create mode 100644 test/cases/verb-stats1/0021/cmd create mode 100644 test/cases/verb-stats1/0021/experr create mode 100644 test/cases/verb-stats1/0021/expout create mode 100644 test/input/rank-data-grouped.csv create mode 100644 test/input/rank-data.csv diff --git a/docs/src/manpage.md b/docs/src/manpage.md index 38091ec62..1f553d4f7 100644 --- a/docs/src/manpage.md +++ b/docs/src/manpage.md @@ -1626,6 +1626,7 @@ This is simply a copy of what you should see on running `man mlr` at a command p kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields + rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields @@ -2202,6 +2203,7 @@ This is simply a copy of what you should see on running `man mlr` at a command p kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields + rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape @@ -2221,6 +2223,10 @@ This is simply a copy of what you should see on running `man mlr` at a command p * count and mode allow text input; the rest require numeric input. In particular, 1 and 1.0 are distinct text for count and mode. * When there are mode ties, the first-encountered datum wins. + * rank assumes the input is already sorted (e.g. by mlr sort) on the field + being ranked, and assigns standard competition rank (1,2,2,4,...) by + comparing each record's value to the immediately preceding one. Use with + -s to get a rank on every input record. 1mstats20m Usage: mlr stats2 [options] @@ -4062,5 +4068,5 @@ This is simply a copy of what you should see on running `man mlr` at a command p MIME Type for Comma-Separated Values (CSV) Files, the Miller docsite https://miller.readthedocs.io - 2026-07-06 4mMILLER24m(1) + 2026-07-07 4mMILLER24m(1) diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt index b0d6322ca..8a7c9813b 100644 --- a/docs/src/manpage.txt +++ b/docs/src/manpage.txt @@ -1605,6 +1605,7 @@ kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields + rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields @@ -2181,6 +2182,7 @@ kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields + rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape @@ -2200,6 +2202,10 @@ * count and mode allow text input; the rest require numeric input. In particular, 1 and 1.0 are distinct text for count and mode. * When there are mode ties, the first-encountered datum wins. + * rank assumes the input is already sorted (e.g. by mlr sort) on the field + being ranked, and assigns standard competition rank (1,2,2,4,...) by + comparing each record's value to the immediately preceding one. Use with + -s to get a rank on every input record. 1mstats20m Usage: mlr stats2 [options] @@ -4041,4 +4047,4 @@ MIME Type for Comma-Separated Values (CSV) Files, the Miller docsite https://miller.readthedocs.io - 2026-07-06 4mMILLER24m(1) + 2026-07-07 4mMILLER24m(1) diff --git a/docs/src/reference-verbs.md b/docs/src/reference-verbs.md index 4a9eeb42b..2abdbb068 100644 --- a/docs/src/reference-verbs.md +++ b/docs/src/reference-verbs.md @@ -2317,6 +2317,7 @@ Accumulators for -a: kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields + rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields @@ -3568,6 +3569,7 @@ Names of accumulators for -a, one or more of: kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields + rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape @@ -3587,6 +3589,10 @@ Notes: * count and mode allow text input; the rest require numeric input. In particular, 1 and 1.0 are distinct text for count and mode. * When there are mode ties, the first-encountered datum wins. +* rank assumes the input is already sorted (e.g. by mlr sort) on the field + being ranked, and assigns standard competition rank (1,2,2,4,...) by + comparing each record's value to the immediately preceding one. Use with + -s to get a rank on every input record. These are simple univariate statistics on one or more number-valued fields diff --git a/man/manpage.txt b/man/manpage.txt index b0d6322ca..8a7c9813b 100644 --- a/man/manpage.txt +++ b/man/manpage.txt @@ -1605,6 +1605,7 @@ kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields + rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields @@ -2181,6 +2182,7 @@ kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields + rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape @@ -2200,6 +2202,10 @@ * count and mode allow text input; the rest require numeric input. In particular, 1 and 1.0 are distinct text for count and mode. * When there are mode ties, the first-encountered datum wins. + * rank assumes the input is already sorted (e.g. by mlr sort) on the field + being ranked, and assigns standard competition rank (1,2,2,4,...) by + comparing each record's value to the immediately preceding one. Use with + -s to get a rank on every input record. 1mstats20m Usage: mlr stats2 [options] @@ -4041,4 +4047,4 @@ MIME Type for Comma-Separated Values (CSV) Files, the Miller docsite https://miller.readthedocs.io - 2026-07-06 4mMILLER24m(1) + 2026-07-07 4mMILLER24m(1) diff --git a/man/mlr.1 b/man/mlr.1 index 1578a7e64..8e237dd6b 100644 --- a/man/mlr.1 +++ b/man/mlr.1 @@ -2,12 +2,12 @@ .\" Title: mlr .\" Author: [see the "AUTHOR" section] .\" Generator: ./mkman.rb -.\" Date: 2026-07-06 +.\" Date: 2026-07-07 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "MILLER" "1" "2026-07-06" "\ \&" "\ \&" +.TH "MILLER" "1" "2026-07-07" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Portability definitions .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1997,6 +1997,7 @@ Accumulators for -a: kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields + rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields @@ -2705,6 +2706,7 @@ Names of accumulators for -a, one or more of: kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields + rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape @@ -2724,6 +2726,10 @@ Notes: * count and mode allow text input; the rest require numeric input. In particular, 1 and 1.0 are distinct text for count and mode. * When there are mode ties, the first-encountered datum wins. +* rank assumes the input is already sorted (e.g. by mlr sort) on the field + being ranked, and assigns standard competition rank (1,2,2,4,...) by + comparing each record's value to the immediately preceding one. Use with + -s to get a rank on every input record. .fi .if n \{\ .RE diff --git a/pkg/transformers/stats1.go b/pkg/transformers/stats1.go index 245960bb2..86124759c 100644 --- a/pkg/transformers/stats1.go +++ b/pkg/transformers/stats1.go @@ -18,7 +18,7 @@ import ( const verbNameStats1 = "stats1" var stats1Options = []OptionSpec{ - {Flag: "-a", Arg: "{sum,count,...}", Type: "enum", Desc: "Names of accumulators: one or more of the listed values. Also accepts median (same as p50) and percentiles p{n} for n in 0..100, e.g. p10 p25.2 p50 p98 p100.", Values: []string{"count", "null_count", "distinct_count", "mode", "antimode", "sum", "mean", "mad", "var", "stddev", "meaneb", "skewness", "kurtosis", "min", "max", "minlen", "maxlen"}}, + {Flag: "-a", Arg: "{sum,count,...}", Type: "enum", Desc: "Names of accumulators: one or more of the listed values. Also accepts median (same as p50) and percentiles p{n} for n in 0..100, e.g. p10 p25.2 p50 p98 p100.", Values: []string{"count", "null_count", "distinct_count", "mode", "antimode", "sum", "mean", "mad", "var", "stddev", "meaneb", "skewness", "kurtosis", "min", "max", "rank", "minlen", "maxlen"}}, {Flag: "-f", Arg: "{a,b,c}", Type: "csv-list", Desc: "Value-field names on which to compute statistics."}, {Flag: "--fr", Arg: "{regex}", Type: "regex", Desc: "Regex for value-field names on which to compute statistics (compute statistics on values in all field names matching the regex)."}, {Flag: "--fx", Arg: "{regex}", Type: "regex", Desc: "Inverted regex for value-field names on which to compute statistics (compute statistics on values in all field names not matching the regex)."}, @@ -79,6 +79,10 @@ the input record stream. * count and mode allow text input; the rest require numeric input. In particular, 1 and 1.0 are distinct text for count and mode. * When there are mode ties, the first-encountered datum wins. +* rank assumes the input is already sorted (e.g. by mlr sort) on the field + being ranked, and assigns standard competition rank (1,2,2,4,...) by + comparing each record's value to the immediately preceding one. Use with + -s to get a rank on every input record. `) } diff --git a/pkg/transformers/utils/stats1_accumulators.go b/pkg/transformers/utils/stats1_accumulators.go index 13e32f3db..0974852d1 100644 --- a/pkg/transformers/utils/stats1_accumulators.go +++ b/pkg/transformers/utils/stats1_accumulators.go @@ -111,6 +111,12 @@ var stats1AccumulatorInfos []stats1AccumulatorInfo = []stats1AccumulatorInfo{ NewStats1MaxAccumulator, }, + { + "rank", + "Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s", + NewStats1RankAccumulator, + }, + { "minlen", "Compute minimum string-lengths of specified fields", @@ -533,6 +539,48 @@ func (acc *Stats1MeanAbsDevAccumulator) Reset() { acc.samples = make([]*mlrval.Mlrval, 0, 1000) } +// Stats1RankAccumulator implements standard competition ranking (1,2,2,4,...) +// on a sequence of values, e.g. as produced by 'mlr sort'. This assumes +// same-valued items are adjacent in the input stream: it compares each +// ingested value only to the immediately preceding one. This accumulator is +// most useful with 'stats1 -s' so that a rank is emitted for every input +// record, rather than only once at end of stream. +type Stats1RankAccumulator struct { + count int64 + rank int64 + havePreviousValue bool + previousValueString string +} + +func NewStats1RankAccumulator() IStats1Accumulator { + return &Stats1RankAccumulator{ + count: 0, + rank: 0, + havePreviousValue: false, + } +} +func (acc *Stats1RankAccumulator) Ingest(value *mlrval.Mlrval) { + acc.count++ + valueString := value.String() // 1, 1.0, and 1.000 are distinct + if !acc.havePreviousValue || valueString != acc.previousValueString { + acc.rank = acc.count + acc.previousValueString = valueString + acc.havePreviousValue = true + } +} +func (acc *Stats1RankAccumulator) Emit() *mlrval.Mlrval { + if acc.count == 0 { + return mlrval.VOID + } + return mlrval.FromInt(acc.rank) +} +func (acc *Stats1RankAccumulator) Reset() { + acc.count = 0 + acc.rank = 0 + acc.havePreviousValue = false + acc.previousValueString = "" +} + type Stats1MinAccumulator struct { min *mlrval.Mlrval } diff --git a/test/cases/cli-help/0001/expout b/test/cases/cli-help/0001/expout index 51ccd63c4..323ed32d1 100644 --- a/test/cases/cli-help/0001/expout +++ b/test/cases/cli-help/0001/expout @@ -645,6 +645,7 @@ Accumulators for -a: kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields + rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields @@ -1243,6 +1244,7 @@ Names of accumulators for -a, one or more of: kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields + rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape @@ -1262,6 +1264,10 @@ Notes: * count and mode allow text input; the rest require numeric input. In particular, 1 and 1.0 are distinct text for count and mode. * When there are mode ties, the first-encountered datum wins. +* rank assumes the input is already sorted (e.g. by mlr sort) on the field + being ranked, and assigns standard competition rank (1,2,2,4,...) by + comparing each record's value to the immediately preceding one. Use with + -s to get a rank on every input record. ================================================================ stats2 diff --git a/test/cases/verb-stats1/0020/cmd b/test/cases/verb-stats1/0020/cmd new file mode 100644 index 000000000..6ccc0eaad --- /dev/null +++ b/test/cases/verb-stats1/0020/cmd @@ -0,0 +1 @@ +mlr --icsv --ocsv stats1 -a rank -f x -s test/input/rank-data.csv diff --git a/test/cases/verb-stats1/0020/experr b/test/cases/verb-stats1/0020/experr new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/verb-stats1/0020/expout b/test/cases/verb-stats1/0020/expout new file mode 100644 index 000000000..543c233c9 --- /dev/null +++ b/test/cases/verb-stats1/0020/expout @@ -0,0 +1,5 @@ +x,x_rank +10,1 +20,2 +20,2 +30,4 diff --git a/test/cases/verb-stats1/0021/cmd b/test/cases/verb-stats1/0021/cmd new file mode 100644 index 000000000..051ee118d --- /dev/null +++ b/test/cases/verb-stats1/0021/cmd @@ -0,0 +1 @@ +mlr --icsv --ocsv stats1 -a rank -f x -g g -s test/input/rank-data-grouped.csv diff --git a/test/cases/verb-stats1/0021/experr b/test/cases/verb-stats1/0021/experr new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/verb-stats1/0021/expout b/test/cases/verb-stats1/0021/expout new file mode 100644 index 000000000..2461a6e9a --- /dev/null +++ b/test/cases/verb-stats1/0021/expout @@ -0,0 +1,6 @@ +g,x,x_rank +a,10,1 +a,10,1 +a,20,3 +b,5,1 +b,5,1 diff --git a/test/input/rank-data-grouped.csv b/test/input/rank-data-grouped.csv new file mode 100644 index 000000000..62ad5200c --- /dev/null +++ b/test/input/rank-data-grouped.csv @@ -0,0 +1,6 @@ +g,x +a,10 +a,10 +a,20 +b,5 +b,5 diff --git a/test/input/rank-data.csv b/test/input/rank-data.csv new file mode 100644 index 000000000..4617ce85c --- /dev/null +++ b/test/input/rank-data.csv @@ -0,0 +1,5 @@ +x +10 +20 +20 +30