Merge pull request #449 from johnkerl/most-least-frequent

Port most-frequent/least-frequent verbs from C to Go

go/reg-test/expected/case-cli-help.sh.out

@@ -332,6 +332,26 @@ useful names to otherwise integer-indexed fields.
 Options:
 -h|--help Show this message.
 ================================================================
+Usage: mlr least-frequent [options]
+Shows the least frequently occurring distinct values for specified field names.
+The first entry is the statistical anti-mode; the remaining are runners-up.
+Options:
+-f {one or more comma-separated field names}. Required flag.
+-n {count}. Optional flag defaulting to %!l(int=10)ld.
+-b          Suppress counts; show only field values.
+-o {name}   Field name for output count. Default "count".
+See also "mlr most-frequent".
+================================================================
+Usage: mlr most-frequent [options]
+Shows the most frequently occurring distinct values for specified field names.
+The first entry is the statistical mode; the remaining are runners-up.
+Options:
+-f {one or more comma-separated field names}. Required flag.
+-n {count}. Optional flag defaulting to %!l(int=10)ld.
+-b          Suppress counts; show only field values.
+-o {name}   Field name for output count. Default "count".
+See also "mlr least-frequent".
+================================================================
 Usage: mlr nothing [options]
 Drops all input records. Useful for testing, or after tee/print/etc. have
 produced other output.
@@ -466,6 +486,14 @@ Options:
 -k {k} Required: number of records to output in total, or by group if using -g.
 -h|--help Show this message.
 ================================================================
+Usage: ../c/mlr sec2gmtdate {comma-separated list of field names}
+Replaces a numeric field representing seconds since the epoch with the
+corresponding GMT year-month-day timestamp; leaves non-numbers as-is.
+This is nothing more than a keystroke-saver for the sec2gmtdate function:
+  ../c/mlr sec2gmtdate time1,time2
+is the same as
+  ../c/mlr put '$time1=sec2gmtdate($time1);$time2=sec2gmtdate($time2)'
+================================================================
 Usage: mlr sec2gmt [options] {comma-separated list of field names}
 Replaces a numeric field representing seconds since the epoch with the
 corresponding GMT timestamp; leaves non-numbers as-is. This is nothing

go/reg-test/expected/case-verb-most-least-frequent.sh.err

@@ -0,0 +1 @@
+

go/reg-test/run

@@ -475,24 +475,38 @@ for case_file in $case_file_names; do
       width=$(stty size | awk '{print $2}')
       echo sdiff -w $width $diff_context_flag $expected_stdout_file $actual_stdout_file
       sdiff -w $width $diff_context_flag $expected_stdout_file $actual_stdout_file
+      ostatus=$?
       echo sdiff -w $width $diff_context_flag $expected_stderr_file $actual_stderr_file
       sdiff -w $width $diff_context_flag $expected_stderr_file $actual_stderr_file
+      estatus=$?
     else
       echo diff -a -I '^mlr' -I '^Miller:' -I '^cat' $diff_context_flag $expected_stdout_file $actual_stdout_file
       diff -a -I '^mlr' -I '^Miller:' -I '^cat' $diff_context_flag $expected_stdout_file $actual_stdout_file
+      ostatus=$?
       echo diff -a -I '^mlr' -I '^Miller:' -I '^cat' $diff_context_flag $expected_stderr_file $actual_stderr_file
       diff -a -I '^mlr' -I '^Miller:' -I '^cat' $diff_context_flag $expected_stderr_file $actual_stderr_file
+      estatus=$?
     fi
   else
     diff -q -a -I '^mlr' -I '^Miller:' -I '^cat' $diff_context_flag $expected_stdout_file $actual_stdout_file > /dev/null
+    ostatus=$?
     diff -q -a -I '^mlr' -I '^Miller:' -I '^cat' $diff_context_flag $expected_stderr_file $actual_stderr_file > /dev/null
+    estatus=$?
   fi
   status=$?
-  if [ $status -ne 0 ]; then
+
+  if [ $ostatus -ne 0 ]; then
     if [ "$verbose" = "true" ]; then
       echo "Case failed due to expected output != actual output:"
       echo "$expected_stdout_file"
       echo "$actual_stdout_file"
+    fi
+    case_passed=false
+  fi
+
+  if [ $estatus -ne 0 ]; then
+    if [ "$verbose" = "true" ]; then
+      echo "Case failed due to expected output != actual output:"
       echo "$expected_stderr_file"
       echo "$actual_stderr_file"
     fi

go/src/cli/mlrcli_transformers.go

@@ -36,6 +36,8 @@ var MAPPER_LOOKUP_TABLE = []transforming.TransformerSetup{
 	transformers.JSONStringifySetup,
 	transformers.JoinSetup,
 	transformers.LabelSetup,
+	transformers.LeastFrequentSetup,
+	transformers.MostFrequentSetup,
 	transformers.NothingSetup,
 	transformers.PutSetup,
 	transformers.RegularizeSetup,

go/src/transformers/most-or-least-frequent.go

@@ -0,0 +1,271 @@
+package transformers
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+
+	"miller/src/cliutil"
+	"miller/src/lib"
+	"miller/src/transforming"
+	"miller/src/types"
+)
+
+// ----------------------------------------------------------------
+const verbNameMostFrequent = "most-frequent"
+const verbNameLeastFrequent = "least-frequent"
+
+const mostLeastFrequentDefaultMaxOutputLength = 10
+const mostLeastFrequentDefaultOutputFieldName = "count"
+
+var MostFrequentSetup = transforming.TransformerSetup{
+	Verb:         verbNameMostFrequent,
+	UsageFunc:    transformerMostFrequentUsage,
+	ParseCLIFunc: transformerMostFrequentParseCLI,
+	IgnoresInput: false,
+}
+
+var LeastFrequentSetup = transforming.TransformerSetup{
+	Verb:         verbNameLeastFrequent,
+	UsageFunc:    transformerLeastFrequentUsage,
+	ParseCLIFunc: transformerLeastFrequentParseCLI,
+	IgnoresInput: false,
+}
+
+func transformerMostFrequentUsage(
+	o *os.File,
+	doExit bool,
+	exitCode int,
+) {
+	argv0 := lib.MlrExeName()
+	verb := verbNameMostFrequent
+	fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb)
+	fmt.Fprintf(o, "Shows the most frequently occurring distinct values for specified field names.\n")
+	fmt.Fprintf(o, "The first entry is the statistical mode; the remaining are runners-up.\n")
+	fmt.Fprintf(o, "Options:\n")
+	fmt.Fprintf(o, "-f {one or more comma-separated field names}. Required flag.\n")
+	fmt.Fprintf(o, "-n {count}. Optional flag defaulting to %lld.\n", mostLeastFrequentDefaultMaxOutputLength)
+	fmt.Fprintf(o, "-b          Suppress counts; show only field values.\n")
+	fmt.Fprintf(o, "-o {name}   Field name for output count. Default \"%s\".\n", mostLeastFrequentDefaultOutputFieldName)
+	fmt.Fprintf(o, "See also \"%s %s\".\n", argv0, "least-frequent")
+	if doExit {
+		os.Exit(exitCode)
+	}
+}
+
+func transformerLeastFrequentUsage(
+	o *os.File,
+	doExit bool,
+	exitCode int,
+) {
+	argv0 := lib.MlrExeName()
+	verb := verbNameLeastFrequent
+	fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb)
+	fmt.Fprintf(o, "Shows the least frequently occurring distinct values for specified field names.\n")
+	fmt.Fprintf(o, "The first entry is the statistical anti-mode; the remaining are runners-up.\n")
+	fmt.Fprintf(o, "Options:\n")
+	fmt.Fprintf(o, "-f {one or more comma-separated field names}. Required flag.\n")
+	fmt.Fprintf(o, "-n {count}. Optional flag defaulting to %lld.\n", mostLeastFrequentDefaultMaxOutputLength)
+	fmt.Fprintf(o, "-b          Suppress counts; show only field values.\n")
+	fmt.Fprintf(o, "-o {name}   Field name for output count. Default \"%s\".\n", mostLeastFrequentDefaultOutputFieldName)
+	fmt.Fprintf(o, "See also \"%s %s\".\n", argv0, "most-frequent")
+	if doExit {
+		os.Exit(exitCode)
+	}
+}
+
+func transformerMostFrequentParseCLI(
+	pargi *int,
+	argc int,
+	args []string,
+	_ *cliutil.TReaderOptions,
+	__ *cliutil.TWriterOptions,
+) transforming.IRecordTransformer {
+	return transformerMostOrLeastFrequentParseCLI(pargi, argc, args, true, transformerMostFrequentUsage)
+}
+
+func transformerLeastFrequentParseCLI(
+	pargi *int,
+	argc int,
+	args []string,
+	_ *cliutil.TReaderOptions,
+	__ *cliutil.TWriterOptions,
+) transforming.IRecordTransformer {
+	return transformerMostOrLeastFrequentParseCLI(pargi, argc, args, false, transformerLeastFrequentUsage)
+}
+
+func transformerMostOrLeastFrequentParseCLI(
+	pargi *int,
+	argc int,
+	args []string,
+	descending bool,
+	usageFunc transforming.TransformerUsageFunc,
+) transforming.IRecordTransformer {
+
+	// Skip the verb name from the current spot in the mlr command line
+	argi := *pargi
+	verb := args[argi]
+	argi++
+
+	// Parse local flags
+	var groupByFieldNames []string = nil
+	maxOutputLength := mostLeastFrequentDefaultMaxOutputLength
+	showCounts := true
+	outputFieldName := mostLeastFrequentDefaultOutputFieldName
+
+	for argi < argc /* variable increment: 1 or 2 depending on flag */ {
+		opt := args[argi]
+		if !strings.HasPrefix(opt, "-") {
+			break // No more flag options to process
+		}
+		argi++
+
+		if opt == "-h" || opt == "--help" {
+			usageFunc(os.Stdout, true, 0)
+
+		} else if opt == "-f" {
+			groupByFieldNames = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc)
+
+		} else if opt == "-n" {
+			maxOutputLength = cliutil.VerbGetIntArgOrDie(verb, opt, args, &argi, argc)
+
+		} else if opt == "-b" {
+			showCounts = false
+
+		} else if opt == "-o" {
+			outputFieldName = cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc)
+
+		} else {
+			usageFunc(os.Stderr, true, 1)
+		}
+	}
+
+	if groupByFieldNames == nil {
+		usageFunc(os.Stderr, true, 1)
+		return nil
+	}
+
+	transformer, _ := NewTransformerMostOrLeastFrequent(
+		groupByFieldNames,
+		maxOutputLength,
+		showCounts,
+		outputFieldName,
+		descending,
+	)
+
+	*pargi = argi
+	return transformer
+}
+
+// ----------------------------------------------------------------
+type TransformerMostOrLeastFrequent struct {
+	groupByFieldNames []string
+	maxOutputLength   int
+	showCounts        bool
+	outputFieldName   string
+	descending        bool
+	countsByGroup     map[string]int
+	valuesForGroup    map[string][]*types.Mlrval
+}
+
+type tMostOrLeastFrequentSortPair struct {
+	count       int
+	groupingKey string
+}
+
+// ----------------------------------------------------------------
+func NewTransformerMostOrLeastFrequent(
+	groupByFieldNames []string,
+	maxOutputLength int,
+	showCounts bool,
+	outputFieldName string,
+	descending bool,
+) (*TransformerMostOrLeastFrequent, error) {
+	this := &TransformerMostOrLeastFrequent{
+		groupByFieldNames: groupByFieldNames,
+		maxOutputLength:   maxOutputLength,
+		showCounts:        showCounts,
+		outputFieldName:   outputFieldName,
+		descending:        descending,
+		countsByGroup:     make(map[string]int),
+		valuesForGroup:    make(map[string][]*types.Mlrval),
+	}
+
+	return this, nil
+}
+
+// ----------------------------------------------------------------
+func (this *TransformerMostOrLeastFrequent) Transform(
+	inrecAndContext *types.RecordAndContext,
+	outputChannel chan<- *types.RecordAndContext,
+) {
+	if !inrecAndContext.EndOfStream {
+		inrec := inrecAndContext.Record
+		groupingKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNames)
+		if !ok {
+			return
+		}
+
+		this.countsByGroup[groupingKey]++
+		if this.valuesForGroup[groupingKey] == nil {
+			selectedValues, _ := inrec.GetSelectedValues(this.groupByFieldNames)
+			this.valuesForGroup[groupingKey] = selectedValues
+		}
+
+	} else {
+		// TODO: Use a heap so this would be m log(n) not n log(n), where m is
+		// the output length and n is the input length. (Each delete-max would
+		// be O(log n) and there would be m of them.)
+
+		// Copy keys and counters from hashmap to array for sorting
+		inputLength := len(this.countsByGroup)
+
+		sortPairs := make([]tMostOrLeastFrequentSortPair, inputLength)
+		i := 0
+		for groupingKey, count := range this.countsByGroup {
+			sortPairs[i].groupingKey = groupingKey
+			sortPairs[i].count = count
+			i++
+		}
+
+		// Sort by count
+		// Go sort API: for ascending sort, return true if element i < element j.
+		if this.descending {
+
+			sort.Slice(sortPairs, func(i, j int) bool {
+				return sortPairs[i].count > sortPairs[j].count
+			})
+
+		} else {
+
+			sort.Slice(sortPairs, func(i, j int) bool {
+				return sortPairs[i].count < sortPairs[j].count
+			})
+
+		}
+
+		// Emit top n
+		outputLength := inputLength
+		if inputLength > this.maxOutputLength {
+			outputLength = this.maxOutputLength
+		}
+		for i := 0; i < outputLength; i++ {
+			outrec := types.NewMlrmapAsRecord()
+			groupByFieldValues := this.valuesForGroup[sortPairs[i].groupingKey]
+			for j, _ := range this.groupByFieldNames {
+				outrec.PutCopy(
+					this.groupByFieldNames[j],
+					groupByFieldValues[j],
+				)
+			}
+
+			if this.showCounts {
+				outrec.PutReference(this.outputFieldName, types.MlrvalPointerFromInt(sortPairs[i].count))
+			}
+			outputChannel <- types.NewRecordAndContext(outrec, &inrecAndContext.Context)
+		}
+
+		outputChannel <- inrecAndContext // End-of-stream marker
+	}
+}

go/src/types/mlrmap_accessors.go

@@ -471,6 +471,7 @@ func (this *Mlrmap) ReferenceSelectedValues(selectedFieldNames []string) ([]*Mlr
 	return mlrvals, allFound
 }
 
+// TODO: rename to CopySelectedValues
 // As previous but with copying. For stats1.
 func (this *Mlrmap) GetSelectedValues(selectedFieldNames []string) ([]*Mlrval, bool) {
 	allFound := true