diff --git a/go/src/miller/cli/mlrcli_mappers.go b/go/src/miller/cli/mlrcli_mappers.go index c899cfe16..888a38d6a 100644 --- a/go/src/miller/cli/mlrcli_mappers.go +++ b/go/src/miller/cli/mlrcli_mappers.go @@ -13,6 +13,8 @@ import ( var MAPPER_LOOKUP_TABLE = []mapping.MapperSetup{ mappers.CatSetup, mappers.CutSetup, + mappers.GroupBySetup, + mappers.GroupLikeSetup, mappers.HeadSetup, mappers.NothingSetup, mappers.PutSetup, diff --git a/go/src/miller/mappers/group-by.go b/go/src/miller/mappers/group-by.go new file mode 100644 index 000000000..5b2f12d11 --- /dev/null +++ b/go/src/miller/mappers/group-by.go @@ -0,0 +1,140 @@ +package mappers + +import ( + "container/list" + "flag" + "fmt" + "os" + + "miller/clitypes" + "miller/lib" + "miller/mapping" + "miller/types" +) + +// ---------------------------------------------------------------- +var GroupBySetup = mapping.MapperSetup{ + Verb: "group-by", + ParseCLIFunc: mapperGroupByParseCLI, + IgnoresInput: false, +} + +func mapperGroupByParseCLI( + pargi *int, + argc int, + args []string, + errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError + _ *clitypes.TReaderOptions, + __ *clitypes.TWriterOptions, +) mapping.IRecordMapper { + + // Get the verb name from the current spot in the mlr command line + argi := *pargi + verb := args[argi] + argi++ + + // Parse local flags + flagSet := flag.NewFlagSet(verb, errorHandling) + + flagSet.Usage = func() { + ostream := os.Stderr + if errorHandling == flag.ContinueOnError { // help intentionally requested + ostream = os.Stdout + } + mapperGroupByUsage(ostream, args[0], verb, flagSet) + } + flagSet.Parse(args[argi:]) + if errorHandling == flag.ContinueOnError { // help intentioally requested + return nil + } + + // Find out how many flags were consumed by this verb and advance for the + // next verb + argi = len(args) - len(flagSet.Args()) + + // Get the group-by field names from the command line + if argi >= argc { + flagSet.Usage() + os.Exit(1) + } + groupByFieldNames := args[argi] + argi += 1 + + mapper, _ := NewMapperGroupBy( + groupByFieldNames, + ) + + *pargi = argi + return mapper +} + +func mapperGroupByUsage( + o *os.File, + argv0 string, + verb string, + flagSet *flag.FlagSet, +) { + fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb) + fmt.Fprint(o, + `Outputs records in batches having identical values at specified field names. +`) + // flagSet.PrintDefaults() doesn't let us control stdout vs stderr + flagSet.VisitAll(func(f *flag.Flag) { + fmt.Fprintf(o, " -%v (default %v) %v\n", f.Name, f.Value, f.Usage) // f.Name, f.Value + }) +} + +// ---------------------------------------------------------------- +type MapperGroupBy struct { + // input + groupByFieldNameList []string + + // state + recordListsByGroup map[string]*list.List +} + +func NewMapperGroupBy( + groupByFieldNames string, +) (*MapperGroupBy, error) { + + groupByFieldNameList := lib.SplitString(groupByFieldNames, ",") + + this := &MapperGroupBy{ + groupByFieldNameList: groupByFieldNameList, + + recordListsByGroup: make(map[string]*list.List), + } + + return this, nil +} + +// ---------------------------------------------------------------- +func (this *MapperGroupBy) Map( + inrecAndContext *types.RecordAndContext, + outputChannel chan<- *types.RecordAndContext, +) { + inrec := inrecAndContext.Record + if inrec != nil { // not end of record stream + + groupByKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNameList) + if !ok { + return + } + + recordListForGroup, present := this.recordListsByGroup[groupByKey] + if !present { // first time + recordListForGroup = list.New() + this.recordListsByGroup[groupByKey] = recordListForGroup + } + + recordListForGroup.PushBack(inrecAndContext) + + } else { + for _, recordListForGroup := range this.recordListsByGroup { + for entry := recordListForGroup.Front(); entry != nil; entry = entry.Next() { + outputChannel <- entry.Value.(*types.RecordAndContext) + } + } + outputChannel <- inrecAndContext // end-of-stream marker + } +} diff --git a/go/src/miller/mappers/group-like.go b/go/src/miller/mappers/group-like.go new file mode 100644 index 000000000..4f3a9c279 --- /dev/null +++ b/go/src/miller/mappers/group-like.go @@ -0,0 +1,116 @@ +package mappers + +import ( + "container/list" + "flag" + "fmt" + "os" + + "miller/clitypes" + "miller/mapping" + "miller/types" +) + +// ---------------------------------------------------------------- +var GroupLikeSetup = mapping.MapperSetup{ + Verb: "group-like", + ParseCLIFunc: mapperGroupLikeParseCLI, + IgnoresInput: false, +} + +func mapperGroupLikeParseCLI( + pargi *int, + argc int, + args []string, + errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError + _ *clitypes.TReaderOptions, + __ *clitypes.TWriterOptions, +) mapping.IRecordMapper { + + // Get the verb name from the current spot in the mlr command line + argi := *pargi + verb := args[argi] + argi++ + + // Parse local flags + flagSet := flag.NewFlagSet(verb, errorHandling) + + flagSet.Usage = func() { + ostream := os.Stderr + if errorHandling == flag.ContinueOnError { // help intentionally requested + ostream = os.Stdout + } + mapperGroupLikeUsage(ostream, args[0], verb, flagSet) + } + flagSet.Parse(args[argi:]) + if errorHandling == flag.ContinueOnError { // help intentioally requested + return nil + } + + // Find out how many flags were consumed by this verb and advance for the + // next verb + argi = len(args) - len(flagSet.Args()) + + mapper, _ := NewMapperGroupLike() + + *pargi = argi + return mapper +} + +func mapperGroupLikeUsage( + o *os.File, + argv0 string, + verb string, + flagSet *flag.FlagSet, +) { + fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb) + fmt.Fprint(o, + `Outputs records in batches having identical field names. +`) + // flagSet.PrintDefaults() doesn't let us control stdout vs stderr + flagSet.VisitAll(func(f *flag.Flag) { + fmt.Fprintf(o, " -%v (default %v) %v\n", f.Name, f.Value, f.Usage) // f.Name, f.Value + }) +} + +// ---------------------------------------------------------------- +type MapperGroupLike struct { + recordListsByGroup map[string]*list.List +} + +func NewMapperGroupLike() (*MapperGroupLike, error) { + + this := &MapperGroupLike{ + recordListsByGroup: make(map[string]*list.List), + } + + return this, nil +} + +// ---------------------------------------------------------------- +func (this *MapperGroupLike) Map( + inrecAndContext *types.RecordAndContext, + outputChannel chan<- *types.RecordAndContext, +) { + inrec := inrecAndContext.Record + if inrec != nil { // not end of record stream + + groupByKey := inrec.GetKeysJoined() + + recordListForGroup, present := this.recordListsByGroup[groupByKey] + if !present { // first time + recordListForGroup = list.New() + this.recordListsByGroup[groupByKey] = recordListForGroup + } + + recordListForGroup.PushBack(inrecAndContext) + + } else { + for _, recordListForGroup := range this.recordListsByGroup { + for entry := recordListForGroup.Front(); entry != nil; entry = entry.Next() { + outputChannel <- entry.Value.(*types.RecordAndContext) + } + } + outputChannel <- inrecAndContext // end-of-stream marker + } +} diff --git a/go/src/miller/types/mlrmap_accessors.go b/go/src/miller/types/mlrmap_accessors.go index 25c259ce1..3162c30f9 100644 --- a/go/src/miller/types/mlrmap_accessors.go +++ b/go/src/miller/types/mlrmap_accessors.go @@ -280,6 +280,20 @@ func (this *Mlrmap) PutIndexed(indices []*Mlrval, rvalue *Mlrval) error { return putIndexedOnMap(this, indices, rvalue) } +// ---------------------------------------------------------------- +func (this *Mlrmap) GetKeysJoined() string { + var buffer bytes.Buffer + i := 0 + for pe := this.Head; pe != nil; pe = pe.Next { + if i > 0 { + buffer.WriteString(",") + } + i++ + buffer.WriteString(*pe.Key) + } + return buffer.String() +} + // ---------------------------------------------------------------- // For group-by in several mappers. If the record is 'a=x,b=y,c=3,d=4,e=5' and // selectedFieldNames is 'a,b,c' then values are 'x,y,3'. This is returned as a diff --git a/go/todo.txt b/go/todo.txt index cae274bbc..1d1246c34 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -8,7 +8,11 @@ TOP OF LIST: > string-split -> true [] and/or ok flag > cut.go use function pointers > get-group-by - - head/tail + - head/tail/group-by/group-like + ! unordered dicts in Go, alas!! this is going to be a pain. fix it. + ? use Mlrmap? or something like it? needs interface{} keys ... + > double-check usages + > write UTs - sort o functions - M_PI / M_E in CST via math.Pi and math.E