From aa45a591feb12b0e685701146029dac16c920792 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Tue, 7 Jul 2026 16:12:00 -0400 Subject: [PATCH] stats1: make rank order-independent by default, add --rank-sorted opt-in fast path The rank accumulator previously only compared each value to the immediately preceding record, silently giving wrong ranks for non-adjacent duplicates (e.g. unsorted input, or interleaved -g groups). Default to correctly computing standard competition rank from all values seen so far (order-independent, buffers values, same approach as percentile accumulators). Add --rank-sorted for callers who can promise sorted input and want the previous O(1)-space streaming behavior instead. --- docs/src/manpage.md | 19 +++-- docs/src/manpage.txt | 19 +++-- docs/src/reference-verbs.md | 19 +++-- man/manpage.txt | 19 +++-- man/mlr.1 | 19 +++-- pkg/transformers/merge_fields.go | 2 + pkg/transformers/stats1.go | 18 +++-- pkg/transformers/utils/percentile_keeper.go | 21 ++++++ pkg/transformers/utils/stats1_accumulators.go | 70 +++++++++++++++---- test/cases/cli-help/0001/expout | 19 +++-- test/cases/verb-stats1/0022/cmd | 1 + test/cases/verb-stats1/0022/experr | 0 test/cases/verb-stats1/0022/expout | 5 ++ test/cases/verb-stats1/0023/cmd | 1 + test/cases/verb-stats1/0023/experr | 0 test/cases/verb-stats1/0023/expout | 6 ++ test/cases/verb-stats1/0024/cmd | 1 + test/cases/verb-stats1/0024/experr | 0 test/cases/verb-stats1/0024/expout | 5 ++ test/input/rank-data-grouped-unsorted.csv | 6 ++ test/input/rank-data-unsorted.csv | 5 ++ 21 files changed, 203 insertions(+), 52 deletions(-) create mode 100644 test/cases/verb-stats1/0022/cmd create mode 100644 test/cases/verb-stats1/0022/experr create mode 100644 test/cases/verb-stats1/0022/expout create mode 100644 test/cases/verb-stats1/0023/cmd create mode 100644 test/cases/verb-stats1/0023/experr create mode 100644 test/cases/verb-stats1/0023/expout create mode 100644 test/cases/verb-stats1/0024/cmd create mode 100644 test/cases/verb-stats1/0024/experr create mode 100644 test/cases/verb-stats1/0024/expout create mode 100644 test/input/rank-data-grouped-unsorted.csv create mode 100644 test/input/rank-data-unsorted.csv diff --git a/docs/src/manpage.md b/docs/src/manpage.md index 1f553d4f7..d83de46be 100644 --- a/docs/src/manpage.md +++ b/docs/src/manpage.md @@ -1626,7 +1626,7 @@ This is simply a copy of what you should see on running `man mlr` at a command p kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields - rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s + rank Compute standard competition rank (1,2,2,4,...) of specified fields; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields @@ -2178,6 +2178,14 @@ This is simply a copy of what you should see on running `man mlr` at a command p --grfx {regex} Shorthand for --gr {regex} --fx {that same regex}. -i Use interpolated percentiles, like R's type=7; default like type=1. Not sensical for string-valued fields. + --rank-sorted For the rank accumulator: promise that the input is already + sorted by the field(s) being ranked, and rank by comparing + each record's value only to the immediately preceding one. + This is faster and streams in O(1) space, but produces wrong + output on unsorted input. Without this flag, rank computes + standard competition rank from all values seen so far, + independent of input order, at the cost of buffering those + values. -s Print iterative stats. Useful in tail -f contexts, in which case please avoid pprint-format output since end of input stream will never be seen. Likewise, if input is coming from @@ -2203,7 +2211,7 @@ This is simply a copy of what you should see on running `man mlr` at a command p kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields - rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s + rank Compute standard competition rank (1,2,2,4,...) of specified fields; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape @@ -2223,10 +2231,9 @@ This is simply a copy of what you should see on running `man mlr` at a command p * count and mode allow text input; the rest require numeric input. In particular, 1 and 1.0 are distinct text for count and mode. * When there are mode ties, the first-encountered datum wins. - * rank assumes the input is already sorted (e.g. by mlr sort) on the field - being ranked, and assigns standard competition rank (1,2,2,4,...) by - comparing each record's value to the immediately preceding one. Use with - -s to get a rank on every input record. + * rank assigns standard competition rank (1,2,2,4,...), independent of + input order: the rank of a value is one plus the count of values less + than it seen so far. Use with -s to get a rank on every input record. 1mstats20m Usage: mlr stats2 [options] diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt index 8a7c9813b..02208364a 100644 --- a/docs/src/manpage.txt +++ b/docs/src/manpage.txt @@ -1605,7 +1605,7 @@ kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields - rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s + rank Compute standard competition rank (1,2,2,4,...) of specified fields; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields @@ -2157,6 +2157,14 @@ --grfx {regex} Shorthand for --gr {regex} --fx {that same regex}. -i Use interpolated percentiles, like R's type=7; default like type=1. Not sensical for string-valued fields. + --rank-sorted For the rank accumulator: promise that the input is already + sorted by the field(s) being ranked, and rank by comparing + each record's value only to the immediately preceding one. + This is faster and streams in O(1) space, but produces wrong + output on unsorted input. Without this flag, rank computes + standard competition rank from all values seen so far, + independent of input order, at the cost of buffering those + values. -s Print iterative stats. Useful in tail -f contexts, in which case please avoid pprint-format output since end of input stream will never be seen. Likewise, if input is coming from @@ -2182,7 +2190,7 @@ kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields - rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s + rank Compute standard competition rank (1,2,2,4,...) of specified fields; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape @@ -2202,10 +2210,9 @@ * count and mode allow text input; the rest require numeric input. In particular, 1 and 1.0 are distinct text for count and mode. * When there are mode ties, the first-encountered datum wins. - * rank assumes the input is already sorted (e.g. by mlr sort) on the field - being ranked, and assigns standard competition rank (1,2,2,4,...) by - comparing each record's value to the immediately preceding one. Use with - -s to get a rank on every input record. + * rank assigns standard competition rank (1,2,2,4,...), independent of + input order: the rank of a value is one plus the count of values less + than it seen so far. Use with -s to get a rank on every input record. 1mstats20m Usage: mlr stats2 [options] diff --git a/docs/src/reference-verbs.md b/docs/src/reference-verbs.md index 2abdbb068..ebe885ce8 100644 --- a/docs/src/reference-verbs.md +++ b/docs/src/reference-verbs.md @@ -2317,7 +2317,7 @@ Accumulators for -a: kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields - rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s + rank Compute standard competition rank (1,2,2,4,...) of specified fields; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields @@ -3544,6 +3544,14 @@ Options: --grfx {regex} Shorthand for --gr {regex} --fx {that same regex}. -i Use interpolated percentiles, like R's type=7; default like type=1. Not sensical for string-valued fields. +--rank-sorted For the rank accumulator: promise that the input is already + sorted by the field(s) being ranked, and rank by comparing + each record's value only to the immediately preceding one. + This is faster and streams in O(1) space, but produces wrong + output on unsorted input. Without this flag, rank computes + standard competition rank from all values seen so far, + independent of input order, at the cost of buffering those + values. -s Print iterative stats. Useful in tail -f contexts, in which case please avoid pprint-format output since end of input stream will never be seen. Likewise, if input is coming from @@ -3569,7 +3577,7 @@ Names of accumulators for -a, one or more of: kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields - rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s + rank Compute standard competition rank (1,2,2,4,...) of specified fields; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape @@ -3589,10 +3597,9 @@ Notes: * count and mode allow text input; the rest require numeric input. In particular, 1 and 1.0 are distinct text for count and mode. * When there are mode ties, the first-encountered datum wins. -* rank assumes the input is already sorted (e.g. by mlr sort) on the field - being ranked, and assigns standard competition rank (1,2,2,4,...) by - comparing each record's value to the immediately preceding one. Use with - -s to get a rank on every input record. +* rank assigns standard competition rank (1,2,2,4,...), independent of + input order: the rank of a value is one plus the count of values less + than it seen so far. Use with -s to get a rank on every input record. These are simple univariate statistics on one or more number-valued fields diff --git a/man/manpage.txt b/man/manpage.txt index 8a7c9813b..02208364a 100644 --- a/man/manpage.txt +++ b/man/manpage.txt @@ -1605,7 +1605,7 @@ kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields - rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s + rank Compute standard competition rank (1,2,2,4,...) of specified fields; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields @@ -2157,6 +2157,14 @@ --grfx {regex} Shorthand for --gr {regex} --fx {that same regex}. -i Use interpolated percentiles, like R's type=7; default like type=1. Not sensical for string-valued fields. + --rank-sorted For the rank accumulator: promise that the input is already + sorted by the field(s) being ranked, and rank by comparing + each record's value only to the immediately preceding one. + This is faster and streams in O(1) space, but produces wrong + output on unsorted input. Without this flag, rank computes + standard competition rank from all values seen so far, + independent of input order, at the cost of buffering those + values. -s Print iterative stats. Useful in tail -f contexts, in which case please avoid pprint-format output since end of input stream will never be seen. Likewise, if input is coming from @@ -2182,7 +2190,7 @@ kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields - rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s + rank Compute standard competition rank (1,2,2,4,...) of specified fields; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape @@ -2202,10 +2210,9 @@ * count and mode allow text input; the rest require numeric input. In particular, 1 and 1.0 are distinct text for count and mode. * When there are mode ties, the first-encountered datum wins. - * rank assumes the input is already sorted (e.g. by mlr sort) on the field - being ranked, and assigns standard competition rank (1,2,2,4,...) by - comparing each record's value to the immediately preceding one. Use with - -s to get a rank on every input record. + * rank assigns standard competition rank (1,2,2,4,...), independent of + input order: the rank of a value is one plus the count of values less + than it seen so far. Use with -s to get a rank on every input record. 1mstats20m Usage: mlr stats2 [options] diff --git a/man/mlr.1 b/man/mlr.1 index 8e237dd6b..f41c30c80 100644 --- a/man/mlr.1 +++ b/man/mlr.1 @@ -1997,7 +1997,7 @@ Accumulators for -a: kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields - rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s + rank Compute standard competition rank (1,2,2,4,...) of specified fields; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields @@ -2681,6 +2681,14 @@ Options: --grfx {regex} Shorthand for --gr {regex} --fx {that same regex}. -i Use interpolated percentiles, like R's type=7; default like type=1. Not sensical for string-valued fields. +--rank-sorted For the rank accumulator: promise that the input is already + sorted by the field(s) being ranked, and rank by comparing + each record's value only to the immediately preceding one. + This is faster and streams in O(1) space, but produces wrong + output on unsorted input. Without this flag, rank computes + standard competition rank from all values seen so far, + independent of input order, at the cost of buffering those + values. -s Print iterative stats. Useful in tail -f contexts, in which case please avoid pprint-format output since end of input stream will never be seen. Likewise, if input is coming from @@ -2706,7 +2714,7 @@ Names of accumulators for -a, one or more of: kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields - rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s + rank Compute standard competition rank (1,2,2,4,...) of specified fields; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape @@ -2726,10 +2734,9 @@ Notes: * count and mode allow text input; the rest require numeric input. In particular, 1 and 1.0 are distinct text for count and mode. * When there are mode ties, the first-encountered datum wins. -* rank assumes the input is already sorted (e.g. by mlr sort) on the field - being ranked, and assigns standard competition rank (1,2,2,4,...) by - comparing each record's value to the immediately preceding one. Use with - -s to get a rank on every input record. +* rank assigns standard competition rank (1,2,2,4,...), independent of + input order: the rank of a value is one plus the count of values less + than it seen so far. Use with -s to get a rank on every input record. .fi .if n \{\ .RE diff --git a/pkg/transformers/merge_fields.go b/pkg/transformers/merge_fields.go index 8e8d5928c..33c0e650d 100644 --- a/pkg/transformers/merge_fields.go +++ b/pkg/transformers/merge_fields.go @@ -298,6 +298,7 @@ func NewTransformerMergeFields( "", // grouping-key used for stats1, not here outputFieldBasename, doInterpolatedPercentiles, + false, // --rank-sorted is stats1-only ) tr.namedAccumulators.Put(accumulatorName, accumulator) } @@ -507,6 +508,7 @@ func (tr *TransformerMergeFields) transformByCollapsing( "", // grouping-key used for stats1, not here shortName, tr.doInterpolatedPercentiles, + false, // --rank-sorted is stats1-only ) namedAccumulators.Put(accumulatorName, accumulator) } diff --git a/pkg/transformers/stats1.go b/pkg/transformers/stats1.go index 86124759c..daa6e8d74 100644 --- a/pkg/transformers/stats1.go +++ b/pkg/transformers/stats1.go @@ -27,6 +27,7 @@ var stats1Options = []OptionSpec{ {Flag: "--gx", Arg: "{regex}", Type: "regex", Desc: "Inverted regex for optional group-by-field names (group by values in field names not matching the regex)."}, {Flag: "--grfx", Arg: "{regex}", Type: "regex", Desc: "Shorthand for --gr {regex} --fx {that same regex}."}, {Flag: "-i", Type: "bool", Desc: "Use interpolated percentiles, like R's type=7; default like type=1. Not sensical for string-valued fields."}, + {Flag: "--rank-sorted", Type: "bool", Desc: "For the rank accumulator: promise that the input is already sorted by the field(s) being ranked, and rank by comparing each record's value only to the immediately preceding one. This is faster and streams in O(1) space, but produces wrong output on unsorted input. Without this flag, rank computes standard competition rank from all values seen so far, independent of input order, at the cost of buffering those values."}, {Flag: "-s", Type: "bool", Desc: "Print iterative stats. Useful in tail -f contexts, in which case please avoid pprint-format output since end of input stream will never be seen. Likewise, if input is coming from `tail -f` be sure to use `--records-per-batch 1`."}, {Flag: "-S", Type: "bool", Desc: "No-op flag for backward compatibility with Miller 5."}, {Flag: "-F", Type: "bool", Desc: "No-op flag for backward compatibility with Miller 5."}, @@ -79,10 +80,9 @@ the input record stream. * count and mode allow text input; the rest require numeric input. In particular, 1 and 1.0 are distinct text for count and mode. * When there are mode ties, the first-encountered datum wins. -* rank assumes the input is already sorted (e.g. by mlr sort) on the field - being ranked, and assigns standard competition rank (1,2,2,4,...) by - comparing each record's value to the immediately preceding one. Use with - -s to get a rank on every input record. +* rank assigns standard competition rank (1,2,2,4,...), independent of + input order: the rank of a value is one plus the count of values less + than it seen so far. Use with -s to get a rank on every input record. `) } @@ -110,6 +110,7 @@ func transformerStats1ParseCLI( doInterpolatedPercentiles := false doIterativeStats := false + doAssumeSortedRank := false var err error for argi < argc /* variable increment: 1 or 2 depending on flag */ { @@ -186,6 +187,9 @@ func transformerStats1ParseCLI( case "-i": doInterpolatedPercentiles = true + case "--rank-sorted": + doAssumeSortedRank = true + case "-s": doIterativeStats = true @@ -222,6 +226,7 @@ func transformerStats1ParseCLI( doInterpolatedPercentiles, doIterativeStats, + doAssumeSortedRank, ) if err != nil { return nil, err @@ -253,6 +258,7 @@ type TransformerStats1 struct { doInterpolatedPercentiles bool doIterativeStats bool + doAssumeSortedRank bool // State: accumulatorFactory *utils.Stats1AccumulatorFactory @@ -323,6 +329,7 @@ func NewTransformerStats1( doInterpolatedPercentiles bool, doIterativeStats bool, + doAssumeSortedRank bool, ) (*TransformerStats1, error) { for _, name := range accumulatorNameList { if !utils.ValidateStats1AccumulatorName(name) { @@ -343,6 +350,7 @@ func NewTransformerStats1( doInterpolatedPercentiles: doInterpolatedPercentiles, doIterativeStats: doIterativeStats, + doAssumeSortedRank: doAssumeSortedRank, accumulatorFactory: utils.NewStats1AccumulatorFactory(), namedAccumulators: lib.NewOrderedMap[*lib.OrderedMap[*lib.OrderedMap[*utils.Stats1NamedAccumulator]]](), groupingKeysToGroupByFieldValues: make(map[string]*lib.OrderedMap[*mlrval.Mlrval]), @@ -523,6 +531,7 @@ func (tr *TransformerStats1) ingestWithoutValueFieldRegexes( groupingKey, valueFieldName, tr.doInterpolatedPercentiles, + tr.doAssumeSortedRank, ) level3.Put(accumulatorName, namedAccumulator) } @@ -568,6 +577,7 @@ func (tr *TransformerStats1) ingestWithValueFieldRegexes( groupingKey, valueFieldName, tr.doInterpolatedPercentiles, + tr.doAssumeSortedRank, ) level3.Put(accumulatorName, namedAccumulator) } diff --git a/pkg/transformers/utils/percentile_keeper.go b/pkg/transformers/utils/percentile_keeper.go index 37725e051..811a0586b 100644 --- a/pkg/transformers/utils/percentile_keeper.go +++ b/pkg/transformers/utils/percentile_keeper.go @@ -82,6 +82,27 @@ func (keeper *PercentileKeeper) EmitLinearlyInterpolated(percentile float64) *ml return bifs.GetPercentileLinearlyInterpolated(keeper.data, int(len(keeper.data)), percentile) } +// EmitRank returns the standard competition rank (1,2,2,4,...) of value +// among all values ingested so far: one plus the number of ingested values +// strictly less than it. +func (keeper *PercentileKeeper) EmitRank(value *mlrval.Mlrval) *mlrval.Mlrval { + if len(keeper.data) == 0 { + return mlrval.VOID + } + keeper.sortIfNecessary() + n := len(keeper.data) + lo, hi := 0, n + for lo < hi { + mid := (lo + hi) / 2 + if mlrval.LessThan(keeper.data[mid], value) { + lo = mid + 1 + } else { + hi = mid + } + } + return mlrval.FromInt(int64(lo + 1)) +} + // TODO: COMMENT func (keeper *PercentileKeeper) EmitNamed(name string) *mlrval.Mlrval { switch name { diff --git a/pkg/transformers/utils/stats1_accumulators.go b/pkg/transformers/utils/stats1_accumulators.go index 0974852d1..b169073a5 100644 --- a/pkg/transformers/utils/stats1_accumulators.go +++ b/pkg/transformers/utils/stats1_accumulators.go @@ -113,7 +113,7 @@ var stats1AccumulatorInfos []stats1AccumulatorInfo = []stats1AccumulatorInfo{ { "rank", - "Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s", + "Compute standard competition rank (1,2,2,4,...) of specified fields; use with -s", NewStats1RankAccumulator, }, @@ -237,6 +237,7 @@ func (fac *Stats1AccumulatorFactory) MakeNamedAccumulator( groupingKey string, valueFieldName string, doInterpolatedPercentiles bool, + doAssumeSortedRank bool, ) *Stats1NamedAccumulator { accumulator := fac.MakeAccumulator( @@ -244,6 +245,7 @@ func (fac *Stats1AccumulatorFactory) MakeNamedAccumulator( groupingKey, valueFieldName, doInterpolatedPercentiles, + doAssumeSortedRank, ) // We don't return an error here -- we fatal. The nominal case is that the stats1 verb has already // pre-validated accumulator names, and this is just a fallback. The accumulators are instantiated for @@ -266,7 +268,13 @@ func (fac *Stats1AccumulatorFactory) MakeAccumulator( groupingKey string, valueFieldName string, doInterpolatedPercentiles bool, + doAssumeSortedRank bool, ) IStats1Accumulator { + // rank has a fast/streaming variant, opted into via --rank-sorted. + if accumulatorName == "rank" && doAssumeSortedRank { + return NewStats1SortedRankAccumulator() + } + // First try percentiles, which have parameterized names. percentile, ok := tryPercentileFromName(accumulatorName) if ok { @@ -539,27 +547,65 @@ func (acc *Stats1MeanAbsDevAccumulator) Reset() { acc.samples = make([]*mlrval.Mlrval, 0, 1000) } -// Stats1RankAccumulator implements standard competition ranking (1,2,2,4,...) -// on a sequence of values, e.g. as produced by 'mlr sort'. This assumes -// same-valued items are adjacent in the input stream: it compares each -// ingested value only to the immediately preceding one. This accumulator is -// most useful with 'stats1 -s' so that a rank is emitted for every input -// record, rather than only once at end of stream. +// Stats1RankAccumulator implements standard competition ranking (1,2,2,4,...): +// the rank of the most-recently-ingested value is one plus the number of +// values strictly less than it, among all values ingested so far. This is +// independent of input order. This accumulator is most useful with +// 'stats1 -s' so that a rank is emitted for every input record, rather than +// only once at end of stream (in which case only the last-ingested value's +// rank within its whole group is emitted). type Stats1RankAccumulator struct { + keeper *PercentileKeeper + lastValue *mlrval.Mlrval + haveLastValue bool +} + +func NewStats1RankAccumulator() IStats1Accumulator { + return &Stats1RankAccumulator{ + keeper: NewPercentileKeeper(false), + haveLastValue: false, + } +} +func (acc *Stats1RankAccumulator) Ingest(value *mlrval.Mlrval) { + acc.keeper.Ingest(value) + acc.lastValue = value.Copy() + acc.haveLastValue = true +} +func (acc *Stats1RankAccumulator) Emit() *mlrval.Mlrval { + if !acc.haveLastValue { + return mlrval.VOID + } + return acc.keeper.EmitRank(acc.lastValue) +} +func (acc *Stats1RankAccumulator) Reset() { + acc.keeper.Reset() + acc.lastValue = nil + acc.haveLastValue = false +} + +// Stats1SortedRankAccumulator is the --rank-sorted opt-in variant of +// Stats1RankAccumulator: it assumes the caller has promised the input is +// already sorted by the field being ranked, so same-valued items are +// adjacent in the input stream. It computes standard competition rank +// (1,2,2,4,...) by comparing each ingested value only to the immediately +// preceding one, in O(1) space, rather than buffering all values seen so +// far. On unsorted input this produces wrong (input-order-dependent) +// results. +type Stats1SortedRankAccumulator struct { count int64 rank int64 havePreviousValue bool previousValueString string } -func NewStats1RankAccumulator() IStats1Accumulator { - return &Stats1RankAccumulator{ +func NewStats1SortedRankAccumulator() IStats1Accumulator { + return &Stats1SortedRankAccumulator{ count: 0, rank: 0, havePreviousValue: false, } } -func (acc *Stats1RankAccumulator) Ingest(value *mlrval.Mlrval) { +func (acc *Stats1SortedRankAccumulator) Ingest(value *mlrval.Mlrval) { acc.count++ valueString := value.String() // 1, 1.0, and 1.000 are distinct if !acc.havePreviousValue || valueString != acc.previousValueString { @@ -568,13 +614,13 @@ func (acc *Stats1RankAccumulator) Ingest(value *mlrval.Mlrval) { acc.havePreviousValue = true } } -func (acc *Stats1RankAccumulator) Emit() *mlrval.Mlrval { +func (acc *Stats1SortedRankAccumulator) Emit() *mlrval.Mlrval { if acc.count == 0 { return mlrval.VOID } return mlrval.FromInt(acc.rank) } -func (acc *Stats1RankAccumulator) Reset() { +func (acc *Stats1SortedRankAccumulator) Reset() { acc.count = 0 acc.rank = 0 acc.havePreviousValue = false diff --git a/test/cases/cli-help/0001/expout b/test/cases/cli-help/0001/expout index 323ed32d1..30355bc18 100644 --- a/test/cases/cli-help/0001/expout +++ b/test/cases/cli-help/0001/expout @@ -645,7 +645,7 @@ Accumulators for -a: kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields - rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s + rank Compute standard competition rank (1,2,2,4,...) of specified fields; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields @@ -1219,6 +1219,14 @@ Options: --grfx {regex} Shorthand for --gr {regex} --fx {that same regex}. -i Use interpolated percentiles, like R's type=7; default like type=1. Not sensical for string-valued fields. +--rank-sorted For the rank accumulator: promise that the input is already + sorted by the field(s) being ranked, and rank by comparing + each record's value only to the immediately preceding one. + This is faster and streams in O(1) space, but produces wrong + output on unsorted input. Without this flag, rank computes + standard competition rank from all values seen so far, + independent of input order, at the cost of buffering those + values. -s Print iterative stats. Useful in tail -f contexts, in which case please avoid pprint-format output since end of input stream will never be seen. Likewise, if input is coming from @@ -1244,7 +1252,7 @@ Names of accumulators for -a, one or more of: kurtosis Compute sample kurtosis of specified fields min Compute minimum values of specified fields max Compute maximum values of specified fields - rank Compute rank 1,2,2,4,... of specified fields, assuming input is sorted by that field; use with -s + rank Compute standard competition rank (1,2,2,4,...) of specified fields; use with -s minlen Compute minimum string-lengths of specified fields maxlen Compute maximum string-lengths of specified fields Example: mlr stats1 -a min,p10,p50,p90,max -f value -g size,shape @@ -1264,10 +1272,9 @@ Notes: * count and mode allow text input; the rest require numeric input. In particular, 1 and 1.0 are distinct text for count and mode. * When there are mode ties, the first-encountered datum wins. -* rank assumes the input is already sorted (e.g. by mlr sort) on the field - being ranked, and assigns standard competition rank (1,2,2,4,...) by - comparing each record's value to the immediately preceding one. Use with - -s to get a rank on every input record. +* rank assigns standard competition rank (1,2,2,4,...), independent of + input order: the rank of a value is one plus the count of values less + than it seen so far. Use with -s to get a rank on every input record. ================================================================ stats2 diff --git a/test/cases/verb-stats1/0022/cmd b/test/cases/verb-stats1/0022/cmd new file mode 100644 index 000000000..f08a2d4e3 --- /dev/null +++ b/test/cases/verb-stats1/0022/cmd @@ -0,0 +1 @@ +mlr --icsv --ocsv stats1 -a rank -f x -s test/input/rank-data-unsorted.csv diff --git a/test/cases/verb-stats1/0022/experr b/test/cases/verb-stats1/0022/experr new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/verb-stats1/0022/expout b/test/cases/verb-stats1/0022/expout new file mode 100644 index 000000000..8a4cb72d8 --- /dev/null +++ b/test/cases/verb-stats1/0022/expout @@ -0,0 +1,5 @@ +x,x_rank +20,1 +10,1 +20,2 +30,4 diff --git a/test/cases/verb-stats1/0023/cmd b/test/cases/verb-stats1/0023/cmd new file mode 100644 index 000000000..7ceb54b00 --- /dev/null +++ b/test/cases/verb-stats1/0023/cmd @@ -0,0 +1 @@ +mlr --icsv --ocsv stats1 -a rank -f x -g g -s test/input/rank-data-grouped-unsorted.csv diff --git a/test/cases/verb-stats1/0023/experr b/test/cases/verb-stats1/0023/experr new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/verb-stats1/0023/expout b/test/cases/verb-stats1/0023/expout new file mode 100644 index 000000000..83dbfe7e5 --- /dev/null +++ b/test/cases/verb-stats1/0023/expout @@ -0,0 +1,6 @@ +g,x,x_rank +a,20,1 +b,5,1 +a,10,1 +b,5,1 +a,20,2 diff --git a/test/cases/verb-stats1/0024/cmd b/test/cases/verb-stats1/0024/cmd new file mode 100644 index 000000000..78e5063c0 --- /dev/null +++ b/test/cases/verb-stats1/0024/cmd @@ -0,0 +1 @@ +mlr --icsv --ocsv stats1 -a rank -f x -s --rank-sorted test/input/rank-data.csv diff --git a/test/cases/verb-stats1/0024/experr b/test/cases/verb-stats1/0024/experr new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/verb-stats1/0024/expout b/test/cases/verb-stats1/0024/expout new file mode 100644 index 000000000..543c233c9 --- /dev/null +++ b/test/cases/verb-stats1/0024/expout @@ -0,0 +1,5 @@ +x,x_rank +10,1 +20,2 +20,2 +30,4 diff --git a/test/input/rank-data-grouped-unsorted.csv b/test/input/rank-data-grouped-unsorted.csv new file mode 100644 index 000000000..c4d1092ca --- /dev/null +++ b/test/input/rank-data-grouped-unsorted.csv @@ -0,0 +1,6 @@ +g,x +a,20 +b,5 +a,10 +b,5 +a,20 diff --git a/test/input/rank-data-unsorted.csv b/test/input/rank-data-unsorted.csv new file mode 100644 index 000000000..9d68f16b0 --- /dev/null +++ b/test/input/rank-data-unsorted.csv @@ -0,0 +1,5 @@ +x +20 +10 +20 +30