From 53a70efca6bbb1fd06131b681606f9e17d7d1be9 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sun, 14 Mar 2021 01:55:04 -0400 Subject: [PATCH] Port most-frequent/least-frequent verbs from C to Go --- ...nt.sh => case-verb-most-least-frequent.sh} | 0 .../case-c-verb-most-least-frequent.sh.err | 0 go/reg-test/expected/case-cli-help.sh.out | 28 ++ .../case-verb-most-least-frequent.sh.err | 1 + ...t => case-verb-most-least-frequent.sh.out} | 0 go/reg-test/run | 16 +- go/src/cli/mlrcli_transformers.go | 2 + go/src/transformers/most-or-least-frequent.go | 271 ++++++++++++++++++ .../{put_or_filter.go => put-or-filter.go} | 0 go/src/types/mlrmap_accessors.go | 1 + 10 files changed, 318 insertions(+), 1 deletion(-) rename go/reg-test/cases/{case-c-verb-most-least-frequent.sh => case-verb-most-least-frequent.sh} (100%) delete mode 100644 go/reg-test/expected/case-c-verb-most-least-frequent.sh.err create mode 100644 go/reg-test/expected/case-verb-most-least-frequent.sh.err rename go/reg-test/expected/{case-c-verb-most-least-frequent.sh.out => case-verb-most-least-frequent.sh.out} (100%) create mode 100644 go/src/transformers/most-or-least-frequent.go rename go/src/transformers/{put_or_filter.go => put-or-filter.go} (100%) diff --git a/go/reg-test/cases/case-c-verb-most-least-frequent.sh b/go/reg-test/cases/case-verb-most-least-frequent.sh similarity index 100% rename from go/reg-test/cases/case-c-verb-most-least-frequent.sh rename to go/reg-test/cases/case-verb-most-least-frequent.sh diff --git a/go/reg-test/expected/case-c-verb-most-least-frequent.sh.err b/go/reg-test/expected/case-c-verb-most-least-frequent.sh.err deleted file mode 100644 index e69de29bb..000000000 diff --git a/go/reg-test/expected/case-cli-help.sh.out b/go/reg-test/expected/case-cli-help.sh.out index a0f813a5d..27df66c7c 100644 --- a/go/reg-test/expected/case-cli-help.sh.out +++ b/go/reg-test/expected/case-cli-help.sh.out @@ -332,6 +332,26 @@ useful names to otherwise integer-indexed fields. Options: -h|--help Show this message. ================================================================ +Usage: mlr least-frequent [options] +Shows the least frequently occurring distinct values for specified field names. +The first entry is the statistical anti-mode; the remaining are runners-up. +Options: +-f {one or more comma-separated field names}. Required flag. +-n {count}. Optional flag defaulting to %!l(int=10)ld. +-b Suppress counts; show only field values. +-o {name} Field name for output count. Default "count". +See also "mlr most-frequent". +================================================================ +Usage: mlr most-frequent [options] +Shows the most frequently occurring distinct values for specified field names. +The first entry is the statistical mode; the remaining are runners-up. +Options: +-f {one or more comma-separated field names}. Required flag. +-n {count}. Optional flag defaulting to %!l(int=10)ld. +-b Suppress counts; show only field values. +-o {name} Field name for output count. Default "count". +See also "mlr least-frequent". +================================================================ Usage: mlr nothing [options] Drops all input records. Useful for testing, or after tee/print/etc. have produced other output. @@ -466,6 +486,14 @@ Options: -k {k} Required: number of records to output in total, or by group if using -g. -h|--help Show this message. ================================================================ +Usage: ../c/mlr sec2gmtdate {comma-separated list of field names} +Replaces a numeric field representing seconds since the epoch with the +corresponding GMT year-month-day timestamp; leaves non-numbers as-is. +This is nothing more than a keystroke-saver for the sec2gmtdate function: + ../c/mlr sec2gmtdate time1,time2 +is the same as + ../c/mlr put '$time1=sec2gmtdate($time1);$time2=sec2gmtdate($time2)' +================================================================ Usage: mlr sec2gmt [options] {comma-separated list of field names} Replaces a numeric field representing seconds since the epoch with the corresponding GMT timestamp; leaves non-numbers as-is. This is nothing diff --git a/go/reg-test/expected/case-verb-most-least-frequent.sh.err b/go/reg-test/expected/case-verb-most-least-frequent.sh.err new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/go/reg-test/expected/case-verb-most-least-frequent.sh.err @@ -0,0 +1 @@ + diff --git a/go/reg-test/expected/case-c-verb-most-least-frequent.sh.out b/go/reg-test/expected/case-verb-most-least-frequent.sh.out similarity index 100% rename from go/reg-test/expected/case-c-verb-most-least-frequent.sh.out rename to go/reg-test/expected/case-verb-most-least-frequent.sh.out diff --git a/go/reg-test/run b/go/reg-test/run index c0c5c0988..19e5b2bed 100755 --- a/go/reg-test/run +++ b/go/reg-test/run @@ -475,24 +475,38 @@ for case_file in $case_file_names; do width=$(stty size | awk '{print $2}') echo sdiff -w $width $diff_context_flag $expected_stdout_file $actual_stdout_file sdiff -w $width $diff_context_flag $expected_stdout_file $actual_stdout_file + ostatus=$? echo sdiff -w $width $diff_context_flag $expected_stderr_file $actual_stderr_file sdiff -w $width $diff_context_flag $expected_stderr_file $actual_stderr_file + estatus=$? else echo diff -a -I '^mlr' -I '^Miller:' -I '^cat' $diff_context_flag $expected_stdout_file $actual_stdout_file diff -a -I '^mlr' -I '^Miller:' -I '^cat' $diff_context_flag $expected_stdout_file $actual_stdout_file + ostatus=$? echo diff -a -I '^mlr' -I '^Miller:' -I '^cat' $diff_context_flag $expected_stderr_file $actual_stderr_file diff -a -I '^mlr' -I '^Miller:' -I '^cat' $diff_context_flag $expected_stderr_file $actual_stderr_file + estatus=$? fi else diff -q -a -I '^mlr' -I '^Miller:' -I '^cat' $diff_context_flag $expected_stdout_file $actual_stdout_file > /dev/null + ostatus=$? diff -q -a -I '^mlr' -I '^Miller:' -I '^cat' $diff_context_flag $expected_stderr_file $actual_stderr_file > /dev/null + estatus=$? fi status=$? - if [ $status -ne 0 ]; then + + if [ $ostatus -ne 0 ]; then if [ "$verbose" = "true" ]; then echo "Case failed due to expected output != actual output:" echo "$expected_stdout_file" echo "$actual_stdout_file" + fi + case_passed=false + fi + + if [ $estatus -ne 0 ]; then + if [ "$verbose" = "true" ]; then + echo "Case failed due to expected output != actual output:" echo "$expected_stderr_file" echo "$actual_stderr_file" fi diff --git a/go/src/cli/mlrcli_transformers.go b/go/src/cli/mlrcli_transformers.go index 8cdb23418..1bafbff9e 100644 --- a/go/src/cli/mlrcli_transformers.go +++ b/go/src/cli/mlrcli_transformers.go @@ -36,6 +36,8 @@ var MAPPER_LOOKUP_TABLE = []transforming.TransformerSetup{ transformers.JSONStringifySetup, transformers.JoinSetup, transformers.LabelSetup, + transformers.LeastFrequentSetup, + transformers.MostFrequentSetup, transformers.NothingSetup, transformers.PutSetup, transformers.RegularizeSetup, diff --git a/go/src/transformers/most-or-least-frequent.go b/go/src/transformers/most-or-least-frequent.go new file mode 100644 index 000000000..6352a1eb7 --- /dev/null +++ b/go/src/transformers/most-or-least-frequent.go @@ -0,0 +1,271 @@ +package transformers + +import ( + "fmt" + "os" + "sort" + "strings" + + "miller/src/cliutil" + "miller/src/lib" + "miller/src/transforming" + "miller/src/types" +) + +// ---------------------------------------------------------------- +const verbNameMostFrequent = "most-frequent" +const verbNameLeastFrequent = "least-frequent" + +const mostLeastFrequentDefaultMaxOutputLength = 10 +const mostLeastFrequentDefaultOutputFieldName = "count" + +var MostFrequentSetup = transforming.TransformerSetup{ + Verb: verbNameMostFrequent, + UsageFunc: transformerMostFrequentUsage, + ParseCLIFunc: transformerMostFrequentParseCLI, + IgnoresInput: false, +} + +var LeastFrequentSetup = transforming.TransformerSetup{ + Verb: verbNameLeastFrequent, + UsageFunc: transformerLeastFrequentUsage, + ParseCLIFunc: transformerLeastFrequentParseCLI, + IgnoresInput: false, +} + +func transformerMostFrequentUsage( + o *os.File, + doExit bool, + exitCode int, +) { + argv0 := lib.MlrExeName() + verb := verbNameMostFrequent + fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb) + fmt.Fprintf(o, "Shows the most frequently occurring distinct values for specified field names.\n") + fmt.Fprintf(o, "The first entry is the statistical mode; the remaining are runners-up.\n") + fmt.Fprintf(o, "Options:\n") + fmt.Fprintf(o, "-f {one or more comma-separated field names}. Required flag.\n") + fmt.Fprintf(o, "-n {count}. Optional flag defaulting to %lld.\n", mostLeastFrequentDefaultMaxOutputLength) + fmt.Fprintf(o, "-b Suppress counts; show only field values.\n") + fmt.Fprintf(o, "-o {name} Field name for output count. Default \"%s\".\n", mostLeastFrequentDefaultOutputFieldName) + fmt.Fprintf(o, "See also \"%s %s\".\n", argv0, "least-frequent") + if doExit { + os.Exit(exitCode) + } +} + +func transformerLeastFrequentUsage( + o *os.File, + doExit bool, + exitCode int, +) { + argv0 := lib.MlrExeName() + verb := verbNameLeastFrequent + fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb) + fmt.Fprintf(o, "Shows the least frequently occurring distinct values for specified field names.\n") + fmt.Fprintf(o, "The first entry is the statistical anti-mode; the remaining are runners-up.\n") + fmt.Fprintf(o, "Options:\n") + fmt.Fprintf(o, "-f {one or more comma-separated field names}. Required flag.\n") + fmt.Fprintf(o, "-n {count}. Optional flag defaulting to %lld.\n", mostLeastFrequentDefaultMaxOutputLength) + fmt.Fprintf(o, "-b Suppress counts; show only field values.\n") + fmt.Fprintf(o, "-o {name} Field name for output count. Default \"%s\".\n", mostLeastFrequentDefaultOutputFieldName) + fmt.Fprintf(o, "See also \"%s %s\".\n", argv0, "most-frequent") + if doExit { + os.Exit(exitCode) + } +} + +func transformerMostFrequentParseCLI( + pargi *int, + argc int, + args []string, + _ *cliutil.TReaderOptions, + __ *cliutil.TWriterOptions, +) transforming.IRecordTransformer { + return transformerMostOrLeastFrequentParseCLI(pargi, argc, args, true, transformerMostFrequentUsage) +} + +func transformerLeastFrequentParseCLI( + pargi *int, + argc int, + args []string, + _ *cliutil.TReaderOptions, + __ *cliutil.TWriterOptions, +) transforming.IRecordTransformer { + return transformerMostOrLeastFrequentParseCLI(pargi, argc, args, false, transformerLeastFrequentUsage) +} + +func transformerMostOrLeastFrequentParseCLI( + pargi *int, + argc int, + args []string, + descending bool, + usageFunc transforming.TransformerUsageFunc, +) transforming.IRecordTransformer { + + // Skip the verb name from the current spot in the mlr command line + argi := *pargi + verb := args[argi] + argi++ + + // Parse local flags + var groupByFieldNames []string = nil + maxOutputLength := mostLeastFrequentDefaultMaxOutputLength + showCounts := true + outputFieldName := mostLeastFrequentDefaultOutputFieldName + + for argi < argc /* variable increment: 1 or 2 depending on flag */ { + opt := args[argi] + if !strings.HasPrefix(opt, "-") { + break // No more flag options to process + } + argi++ + + if opt == "-h" || opt == "--help" { + usageFunc(os.Stdout, true, 0) + + } else if opt == "-f" { + groupByFieldNames = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) + + } else if opt == "-n" { + maxOutputLength = cliutil.VerbGetIntArgOrDie(verb, opt, args, &argi, argc) + + } else if opt == "-b" { + showCounts = false + + } else if opt == "-o" { + outputFieldName = cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc) + + } else { + usageFunc(os.Stderr, true, 1) + } + } + + if groupByFieldNames == nil { + usageFunc(os.Stderr, true, 1) + return nil + } + + transformer, _ := NewTransformerMostOrLeastFrequent( + groupByFieldNames, + maxOutputLength, + showCounts, + outputFieldName, + descending, + ) + + *pargi = argi + return transformer +} + +// ---------------------------------------------------------------- +type TransformerMostOrLeastFrequent struct { + groupByFieldNames []string + maxOutputLength int + showCounts bool + outputFieldName string + descending bool + countsByGroup map[string]int + valuesForGroup map[string][]*types.Mlrval +} + +type tMostOrLeastFrequentSortPair struct { + count int + groupingKey string +} + +// ---------------------------------------------------------------- +func NewTransformerMostOrLeastFrequent( + groupByFieldNames []string, + maxOutputLength int, + showCounts bool, + outputFieldName string, + descending bool, +) (*TransformerMostOrLeastFrequent, error) { + this := &TransformerMostOrLeastFrequent{ + groupByFieldNames: groupByFieldNames, + maxOutputLength: maxOutputLength, + showCounts: showCounts, + outputFieldName: outputFieldName, + descending: descending, + countsByGroup: make(map[string]int), + valuesForGroup: make(map[string][]*types.Mlrval), + } + + return this, nil +} + +// ---------------------------------------------------------------- +func (this *TransformerMostOrLeastFrequent) Transform( + inrecAndContext *types.RecordAndContext, + outputChannel chan<- *types.RecordAndContext, +) { + if !inrecAndContext.EndOfStream { + inrec := inrecAndContext.Record + groupingKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNames) + if !ok { + return + } + + this.countsByGroup[groupingKey]++ + if this.valuesForGroup[groupingKey] == nil { + selectedValues, _ := inrec.GetSelectedValues(this.groupByFieldNames) + this.valuesForGroup[groupingKey] = selectedValues + } + + } else { + // TODO: Use a heap so this would be m log(n) not n log(n), where m is + // the output length and n is the input length. (Each delete-max would + // be O(log n) and there would be m of them.) + + // Copy keys and counters from hashmap to array for sorting + inputLength := len(this.countsByGroup) + + sortPairs := make([]tMostOrLeastFrequentSortPair, inputLength) + i := 0 + for groupingKey, count := range this.countsByGroup { + sortPairs[i].groupingKey = groupingKey + sortPairs[i].count = count + i++ + } + + // Sort by count + // Go sort API: for ascending sort, return true if element i < element j. + if this.descending { + + sort.Slice(sortPairs, func(i, j int) bool { + return sortPairs[i].count > sortPairs[j].count + }) + + } else { + + sort.Slice(sortPairs, func(i, j int) bool { + return sortPairs[i].count < sortPairs[j].count + }) + + } + + // Emit top n + outputLength := inputLength + if inputLength > this.maxOutputLength { + outputLength = this.maxOutputLength + } + for i := 0; i < outputLength; i++ { + outrec := types.NewMlrmapAsRecord() + groupByFieldValues := this.valuesForGroup[sortPairs[i].groupingKey] + for j, _ := range this.groupByFieldNames { + outrec.PutCopy( + this.groupByFieldNames[j], + groupByFieldValues[j], + ) + } + + if this.showCounts { + outrec.PutReference(this.outputFieldName, types.MlrvalPointerFromInt(sortPairs[i].count)) + } + outputChannel <- types.NewRecordAndContext(outrec, &inrecAndContext.Context) + } + + outputChannel <- inrecAndContext // End-of-stream marker + } +} diff --git a/go/src/transformers/put_or_filter.go b/go/src/transformers/put-or-filter.go similarity index 100% rename from go/src/transformers/put_or_filter.go rename to go/src/transformers/put-or-filter.go diff --git a/go/src/types/mlrmap_accessors.go b/go/src/types/mlrmap_accessors.go index 619a37cd1..88603b7fe 100644 --- a/go/src/types/mlrmap_accessors.go +++ b/go/src/types/mlrmap_accessors.go @@ -471,6 +471,7 @@ func (this *Mlrmap) ReferenceSelectedValues(selectedFieldNames []string) ([]*Mlr return mlrvals, allFound } +// TODO: rename to CopySelectedValues // As previous but with copying. For stats1. func (this *Mlrmap) GetSelectedValues(selectedFieldNames []string) ([]*Mlrval, bool) { allFound := true