mlr group-by and mlr group-like

go/src/miller/cli/mlrcli_mappers.go

@@ -13,6 +13,8 @@ import (
 var MAPPER_LOOKUP_TABLE = []mapping.MapperSetup{
 	mappers.CatSetup,
 	mappers.CutSetup,
+	mappers.GroupBySetup,
+	mappers.GroupLikeSetup,
 	mappers.HeadSetup,
 	mappers.NothingSetup,
 	mappers.PutSetup,

go/src/miller/mappers/group-by.go

@@ -0,0 +1,140 @@
+package mappers
+
+import (
+	"container/list"
+	"flag"
+	"fmt"
+	"os"
+
+	"miller/clitypes"
+	"miller/lib"
+	"miller/mapping"
+	"miller/types"
+)
+
+// ----------------------------------------------------------------
+var GroupBySetup = mapping.MapperSetup{
+	Verb:         "group-by",
+	ParseCLIFunc: mapperGroupByParseCLI,
+	IgnoresInput: false,
+}
+
+func mapperGroupByParseCLI(
+	pargi *int,
+	argc int,
+	args []string,
+	errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError
+	_ *clitypes.TReaderOptions,
+	__ *clitypes.TWriterOptions,
+) mapping.IRecordMapper {
+
+	// Get the verb name from the current spot in the mlr command line
+	argi := *pargi
+	verb := args[argi]
+	argi++
+
+	// Parse local flags
+	flagSet := flag.NewFlagSet(verb, errorHandling)
+
+	flagSet.Usage = func() {
+		ostream := os.Stderr
+		if errorHandling == flag.ContinueOnError { // help intentionally requested
+			ostream = os.Stdout
+		}
+		mapperGroupByUsage(ostream, args[0], verb, flagSet)
+	}
+	flagSet.Parse(args[argi:])
+	if errorHandling == flag.ContinueOnError { // help intentioally requested
+		return nil
+	}
+
+	// Find out how many flags were consumed by this verb and advance for the
+	// next verb
+	argi = len(args) - len(flagSet.Args())
+
+	// Get the group-by field names from the command line
+	if argi >= argc {
+		flagSet.Usage()
+		os.Exit(1)
+	}
+	groupByFieldNames := args[argi]
+	argi += 1
+
+	mapper, _ := NewMapperGroupBy(
+		groupByFieldNames,
+	)
+
+	*pargi = argi
+	return mapper
+}
+
+func mapperGroupByUsage(
+	o *os.File,
+	argv0 string,
+	verb string,
+	flagSet *flag.FlagSet,
+) {
+	fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb)
+	fmt.Fprint(o,
+		`Outputs records in batches having identical values at specified field names.
+`)
+	// flagSet.PrintDefaults() doesn't let us control stdout vs stderr
+	flagSet.VisitAll(func(f *flag.Flag) {
+		fmt.Fprintf(o, " -%v (default %v) %v\n", f.Name, f.Value, f.Usage) // f.Name, f.Value
+	})
+}
+
+// ----------------------------------------------------------------
+type MapperGroupBy struct {
+	// input
+	groupByFieldNameList []string
+
+	// state
+	recordListsByGroup map[string]*list.List
+}
+
+func NewMapperGroupBy(
+	groupByFieldNames string,
+) (*MapperGroupBy, error) {
+
+	groupByFieldNameList := lib.SplitString(groupByFieldNames, ",")
+
+	this := &MapperGroupBy{
+		groupByFieldNameList: groupByFieldNameList,
+
+		recordListsByGroup: make(map[string]*list.List),
+	}
+
+	return this, nil
+}
+
+// ----------------------------------------------------------------
+func (this *MapperGroupBy) Map(
+	inrecAndContext *types.RecordAndContext,
+	outputChannel chan<- *types.RecordAndContext,
+) {
+	inrec := inrecAndContext.Record
+	if inrec != nil { // not end of record stream
+
+		groupByKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNameList)
+		if !ok {
+			return
+		}
+
+		recordListForGroup, present := this.recordListsByGroup[groupByKey]
+		if !present { // first time
+			recordListForGroup = list.New()
+			this.recordListsByGroup[groupByKey] = recordListForGroup
+		}
+
+		recordListForGroup.PushBack(inrecAndContext)
+
+	} else {
+		for _, recordListForGroup := range this.recordListsByGroup {
+			for entry := recordListForGroup.Front(); entry != nil; entry = entry.Next() {
+				outputChannel <- entry.Value.(*types.RecordAndContext)
+			}
+		}
+		outputChannel <- inrecAndContext // end-of-stream marker
+	}
+}

go/src/miller/mappers/group-like.go

@@ -0,0 +1,116 @@
+package mappers
+
+import (
+	"container/list"
+	"flag"
+	"fmt"
+	"os"
+
+	"miller/clitypes"
+	"miller/mapping"
+	"miller/types"
+)
+
+// ----------------------------------------------------------------
+var GroupLikeSetup = mapping.MapperSetup{
+	Verb:         "group-like",
+	ParseCLIFunc: mapperGroupLikeParseCLI,
+	IgnoresInput: false,
+}
+
+func mapperGroupLikeParseCLI(
+	pargi *int,
+	argc int,
+	args []string,
+	errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError
+	_ *clitypes.TReaderOptions,
+	__ *clitypes.TWriterOptions,
+) mapping.IRecordMapper {
+
+	// Get the verb name from the current spot in the mlr command line
+	argi := *pargi
+	verb := args[argi]
+	argi++
+
+	// Parse local flags
+	flagSet := flag.NewFlagSet(verb, errorHandling)
+
+	flagSet.Usage = func() {
+		ostream := os.Stderr
+		if errorHandling == flag.ContinueOnError { // help intentionally requested
+			ostream = os.Stdout
+		}
+		mapperGroupLikeUsage(ostream, args[0], verb, flagSet)
+	}
+	flagSet.Parse(args[argi:])
+	if errorHandling == flag.ContinueOnError { // help intentioally requested
+		return nil
+	}
+
+	// Find out how many flags were consumed by this verb and advance for the
+	// next verb
+	argi = len(args) - len(flagSet.Args())
+
+	mapper, _ := NewMapperGroupLike()
+
+	*pargi = argi
+	return mapper
+}
+
+func mapperGroupLikeUsage(
+	o *os.File,
+	argv0 string,
+	verb string,
+	flagSet *flag.FlagSet,
+) {
+	fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb)
+	fmt.Fprint(o,
+		`Outputs records in batches having identical field names.
+`)
+	// flagSet.PrintDefaults() doesn't let us control stdout vs stderr
+	flagSet.VisitAll(func(f *flag.Flag) {
+		fmt.Fprintf(o, " -%v (default %v) %v\n", f.Name, f.Value, f.Usage) // f.Name, f.Value
+	})
+}
+
+// ----------------------------------------------------------------
+type MapperGroupLike struct {
+	recordListsByGroup map[string]*list.List
+}
+
+func NewMapperGroupLike() (*MapperGroupLike, error) {
+
+	this := &MapperGroupLike{
+		recordListsByGroup: make(map[string]*list.List),
+	}
+
+	return this, nil
+}
+
+// ----------------------------------------------------------------
+func (this *MapperGroupLike) Map(
+	inrecAndContext *types.RecordAndContext,
+	outputChannel chan<- *types.RecordAndContext,
+) {
+	inrec := inrecAndContext.Record
+	if inrec != nil { // not end of record stream
+
+		groupByKey := inrec.GetKeysJoined()
+
+		recordListForGroup, present := this.recordListsByGroup[groupByKey]
+		if !present { // first time
+			recordListForGroup = list.New()
+			this.recordListsByGroup[groupByKey] = recordListForGroup
+		}
+
+		recordListForGroup.PushBack(inrecAndContext)
+
+	} else {
+		for _, recordListForGroup := range this.recordListsByGroup {
+			for entry := recordListForGroup.Front(); entry != nil; entry = entry.Next() {
+				outputChannel <- entry.Value.(*types.RecordAndContext)
+			}
+		}
+		outputChannel <- inrecAndContext // end-of-stream marker
+	}
+}

go/src/miller/types/mlrmap_accessors.go

@@ -280,6 +280,20 @@ func (this *Mlrmap) PutIndexed(indices []*Mlrval, rvalue *Mlrval) error {
 	return putIndexedOnMap(this, indices, rvalue)
 }
 
+// ----------------------------------------------------------------
+func (this *Mlrmap) GetKeysJoined() string {
+	var buffer bytes.Buffer
+	i := 0
+	for pe := this.Head; pe != nil; pe = pe.Next {
+		if i > 0 {
+			buffer.WriteString(",")
+		}
+		i++
+		buffer.WriteString(*pe.Key)
+	}
+	return buffer.String()
+}
+
 // ----------------------------------------------------------------
 // For group-by in several mappers.  If the record is 'a=x,b=y,c=3,d=4,e=5' and
 // selectedFieldNames is 'a,b,c' then values are 'x,y,3'. This is returned as a

go/todo.txt

@@ -8,7 +8,11 @@ TOP OF LIST:
       > string-split -> true [] and/or ok flag
       > cut.go use function pointers
       > get-group-by
-    - head/tail
+    - head/tail/group-by/group-like
+      ! unordered dicts in Go, alas!! this is going to be a pain. fix it.
+        ? use Mlrmap? or something like it? needs interface{} keys ...
+      > double-check usages
+      > write UTs
     - sort
   o functions
     - M_PI / M_E in CST via math.Pi and math.E