package transformers import ( "fmt" "os" "strings" "github.com/johnkerl/miller/v6/pkg/cli" "github.com/johnkerl/miller/v6/pkg/lib" "github.com/johnkerl/miller/v6/pkg/mlrval" "github.com/johnkerl/miller/v6/pkg/types" ) const verbNameCount = "count" var countOptions = []OptionSpec{ {Flag: "-g", Arg: "{a,b,c}", Type: "csv-list", Desc: "Optional group-by-field names for counts, e.g. a,b,c"}, {Flag: "-n", Type: "bool", Desc: "Show only the number of distinct values. Not interesting without -g."}, {Flag: "-o", Arg: "{name}", Type: "string", Desc: "Field name for output-count. Default \"count\"."}, } var CountSetup = TransformerSetup{ Verb: verbNameCount, UsageFunc: transformerCountUsage, ParseCLIFunc: transformerCountParseCLI, IgnoresInput: false, Options: countOptions, } func transformerCountUsage( o *os.File, ) { fmt.Fprintf(o, "Usage: %s %s [options]\n", "mlr", verbNameCount) fmt.Fprint(o, `Prints number of records, optionally grouped by distinct values for specified field names. `) WriteVerbOptions(o, countOptions) } func transformerCountParseCLI( pargi *int, argc int, args []string, _ *cli.TOptions, doConstruct bool, // false for first pass of CLI-parse, true for second pass ) (RecordTransformer, error) { // Skip the verb name from the current spot in the mlr command line argi := *pargi verb := args[argi] argi++ var groupByFieldNames []string = nil showCountsOnly := false outputFieldName := "count" var err error for argi < argc /* variable increment: 1 or 2 depending on flag */ { opt := args[argi] if !strings.HasPrefix(opt, "-") { break // No more flag options to process } if args[argi] == "--" { break // All transformers must do this so main-flags can follow verb-flags } argi++ switch opt { case "-h", "--help": transformerCountUsage(os.Stdout) return nil, cli.ErrHelpRequested case "-g": groupByFieldNames, err = cli.VerbGetStringArrayArg(verb, opt, args, &argi, argc) if err != nil { return nil, err } case "-n": showCountsOnly = true case "-o": outputFieldName, err = cli.VerbGetStringArg(verb, opt, args, &argi, argc) if err != nil { return nil, err } default: return nil, cli.VerbErrorf(verb, "option \"%s\" not recognized", opt) } } *pargi = argi if !doConstruct { // All transformers must do this for main command-line parsing return nil, nil } transformer, err := NewTransformerCount( groupByFieldNames, showCountsOnly, outputFieldName, ) if err != nil { return nil, err } return transformer, nil } type TransformerCount struct { // input groupByFieldNames []string showCountsOnly bool outputFieldName string // state recordTransformerFunc RecordTransformerFunc ungroupedCount int64 // Example: // * Suppose group-by fields are a,b. // * One record has a=foo,b=bar // * Another record has a=baz,b=quux // * Map keys are strings "foo,bar" and "baz,quux". // * groupedCounts maps "foo,bar" to 1 and "baz,quux" to 1. // * groupByValues maps "foo,bar" to ["foo", "bar"] and "baz,quux" to ["baz", "quux"]. groupedCounts *lib.OrderedMap[int64] groupingValues *lib.OrderedMap[[]*mlrval.Mlrval] } func NewTransformerCount( groupByFieldNames []string, showCountsOnly bool, outputFieldName string, ) (*TransformerCount, error) { tr := &TransformerCount{ groupByFieldNames: groupByFieldNames, showCountsOnly: showCountsOnly, outputFieldName: outputFieldName, ungroupedCount: 0, groupedCounts: lib.NewOrderedMap[int64](), groupingValues: lib.NewOrderedMap[[]*mlrval.Mlrval](), } if groupByFieldNames == nil { tr.recordTransformerFunc = tr.countUngrouped } else { tr.recordTransformerFunc = tr.countGrouped } return tr, nil } func (tr *TransformerCount) Transform( inrecAndContext *types.RecordAndContext, outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) error { HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel) return tr.recordTransformerFunc(inrecAndContext, outputRecordsAndContexts, inputDownstreamDoneChannel, outputDownstreamDoneChannel) } func (tr *TransformerCount) countUngrouped( inrecAndContext *types.RecordAndContext, outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) error { if !inrecAndContext.EndOfStream { tr.ungroupedCount++ } else { newrec := mlrval.NewMlrmapAsRecord() newrec.PutCopy(tr.outputFieldName, mlrval.FromInt(tr.ungroupedCount)) *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } return nil } func (tr *TransformerCount) countGrouped( inrecAndContext *types.RecordAndContext, outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) error { if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record groupingKey, selectedValues, ok := inrec.GetSelectedValuesAndJoined( tr.groupByFieldNames, ) if !ok { // Current record does not have specified fields; ignore return nil } if !tr.groupedCounts.Has(groupingKey) { var count int64 = 1 tr.groupedCounts.Put(groupingKey, count) tr.groupingValues.Put(groupingKey, selectedValues) } else { tr.groupedCounts.Put( groupingKey, tr.groupedCounts.Get(groupingKey)+1, ) } } else { if tr.showCountsOnly { newrec := mlrval.NewMlrmapAsRecord() newrec.PutCopy(tr.outputFieldName, mlrval.FromInt(tr.groupedCounts.FieldCount)) outrecAndContext := types.NewRecordAndContext(newrec, &inrecAndContext.Context) *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) } else { for outer := tr.groupedCounts.Head; outer != nil; outer = outer.Next { groupingKey := outer.Key newrec := mlrval.NewMlrmapAsRecord() // Example: // * Suppose group-by fields are a,b. // * Record has a=foo,b=bar // * Grouping key is "foo,bar" // * Grouping values for key is ["foo", "bar"] // Here we populate a record with "a=foo,b=bar". groupingValuesForKey := tr.groupingValues.Get(groupingKey) i := 0 for _, groupingValueForKey := range groupingValuesForKey { newrec.PutCopy(tr.groupByFieldNames[i], groupingValueForKey) i++ } countForGroup := outer.Value newrec.PutCopy(tr.outputFieldName, mlrval.FromInt(countForGroup)) outrecAndContext := types.NewRecordAndContext(newrec, &inrecAndContext.Context) *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) } } *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } return nil }