mlr regularize

go/src/miller/cli/mlrcli_mappers.go

@@ -19,6 +19,7 @@ var MAPPER_LOOKUP_TABLE = []mapping.MapperSetup{
 	mappers.LabelSetup,
 	mappers.NothingSetup,
 	mappers.PutSetup,
+	mappers.RegularizeSetup,
 	mappers.RenameSetup,
 	mappers.SortSetup,
 	mappers.SortWithinRecordsSetup,

go/src/miller/lib/util.go

@@ -1,6 +1,7 @@
 package lib
 
 import (
+	"sort"
 	"strconv"
 	"strings"
 )
@@ -32,6 +33,25 @@ func StringListToSet(stringList []string) map[string]bool {
 	return stringSet
 }
 
+func SortStrings(strings []string) {
+	// Go sort API: for ascending sort, return true if element i < element j.
+	sort.Slice(strings, func(i, j int) bool {
+		return strings[i] < strings[j]
+	})
+}
+
+func SortedStrings(strings []string) []string {
+	copy := make([]string, len(strings))
+	for i, s := range(strings) {
+		copy[i] = s
+	}
+	// Go sort API: for ascending sort, return true if element i < element j.
+	sort.Slice(copy, func(i, j int) bool {
+		return copy[i] < copy[j]
+	})
+	return copy
+}
+
 func IntMin2(a, b int) int {
 	if a < b {
 		return a

go/src/miller/mappers/regularize.go

@@ -0,0 +1,115 @@
+package mappers
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"miller/clitypes"
+	"miller/lib"
+	"miller/mapping"
+	"miller/types"
+)
+
+// ----------------------------------------------------------------
+var RegularizeSetup = mapping.MapperSetup{
+	Verb:         "regularize",
+	ParseCLIFunc: mapperRegularizeParseCLI,
+	IgnoresInput: false,
+}
+
+func mapperRegularizeParseCLI(
+	pargi *int,
+	argc int,
+	args []string,
+	errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError
+	_ *clitypes.TReaderOptions,
+	__ *clitypes.TWriterOptions,
+) mapping.IRecordMapper {
+
+	// Get the verb name from the current spot in the mlr command line
+	argi := *pargi
+	verb := args[argi]
+	argi++
+
+	// Parse local flags
+	flagSet := flag.NewFlagSet(verb, errorHandling)
+
+	flagSet.Usage = func() {
+		ostream := os.Stderr
+		if errorHandling == flag.ContinueOnError { // help intentionally requested
+			ostream = os.Stdout
+		}
+		mapperRegularizeUsage(ostream, args[0], verb, flagSet)
+	}
+	flagSet.Parse(args[argi:])
+	if errorHandling == flag.ContinueOnError { // help intentionally requested
+		return nil
+	}
+
+	// Find out how many flags were consumed by this verb and advance for the
+	// next verb
+	argi = len(args) - len(flagSet.Args())
+
+	mapper, _ := NewMapperRegularize()
+
+	*pargi = argi
+	return mapper
+}
+
+func mapperRegularizeUsage(
+	o *os.File,
+	argv0 string,
+	verb string,
+	flagSet *flag.FlagSet,
+) {
+	fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb)
+	fmt.Fprint(o,
+		`Outputs records sorted lexically ascending by keys.
+`)
+	// flagSet.PrintDefaults() doesn't let us control stdout vs stderr
+	flagSet.VisitAll(func(f *flag.Flag) {
+		fmt.Fprintf(o, " -%v (default %v) %v\n", f.Name, f.Value, f.Usage) // f.Name, f.Value
+	})
+}
+
+// ----------------------------------------------------------------
+type MapperRegularize struct {
+	// map from string to []string
+	sortedToOriginal map[string][]string
+}
+
+func NewMapperRegularize() (*MapperRegularize, error) {
+	this := &MapperRegularize{
+		make(map[string][]string),
+	}
+	return this, nil
+}
+
+// ----------------------------------------------------------------
+func (this *MapperRegularize) Map(
+	inrecAndContext *types.RecordAndContext,
+	outputChannel chan<- *types.RecordAndContext,
+) {
+	inrec := inrecAndContext.Record
+	if inrec != nil { // not end of record stream
+		currentFieldNames := inrec.GetKeys()
+		currentSortedFieldNames := lib.SortedStrings(currentFieldNames)
+		currentSortedFieldNamesJoined := strings.Join(currentSortedFieldNames, ",")
+		previousSortedFieldNames := this.sortedToOriginal[currentSortedFieldNamesJoined]
+		if previousSortedFieldNames == nil {
+			this.sortedToOriginal[currentSortedFieldNamesJoined] = currentFieldNames
+			outputChannel <- inrecAndContext
+		} else {
+			outrec := types.NewMlrmapAsRecord()
+			for _, fieldName := range previousSortedFieldNames {
+				outrec.PutReference(&fieldName, inrec.Get(&fieldName)) // inrec will be GC'ed
+			}
+			outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context)
+			outputChannel <- outrecAndContext
+		}
+	} else {
+		outputChannel <- inrecAndContext // end-of-stream marker
+	}
+}

go/src/miller/mappers/sort-within-records.go

@@ -91,10 +91,7 @@ func (this *MapperSortWithinRecords) Map(
 ) {
 	inrec := inrecAndContext.Record
 	if inrec != nil { // not end of record stream
-
 		inrec.SortByKey()
-
-	} else {
 	}
 	outputChannel <- inrecAndContext // end-of-stream marker
 }

go/src/miller/types/mlrmap_accessors.go

@@ -3,7 +3,6 @@ package types
 import (
 	"bytes"
 	"errors"
-	"sort"
 	"strconv"
 
 	"miller/lib"
@@ -352,10 +351,7 @@ func (this *Mlrmap) Label(newNames []string) {
 func (this *Mlrmap) SortByKey() {
 	keys := this.GetKeys()
 
-	// Go sort API: for ascending sort, return true if element i < element j.
-	sort.Slice(keys, func(i, j int) bool {
-		return keys[i] < keys[j]
-	})
+	lib.SortStrings(keys)
 
 	that := NewMlrmapAsRecord()
 

go/todo.txt

@@ -1,9 +1,6 @@
 ----------------------------------------------------------------
 TOP OF LIST:
 
-* sort-within-records
-* regularize
-
 ! try Lvalue/Rvalue coalesce
   > to make condish work & unhack filter
   > better error messages in the CST builder anyway, so it's a win

go/u/needs-regularize.json

@@ -0,0 +1,16 @@
+{
+  "b": 2,
+  "c": 3,
+  "a": 1
+}
+{
+  "b": 2,
+  "a": 1,
+  "c": 3
+}
+{
+  "d": 4,
+  "a": 1,
+  "b": 2,
+  "c": 3
+}

go/u/try-help.out

@@ -53,6 +53,10 @@ TODO: put detailed on-line help here.
     full transparency on the precedence and associativity rules of
     Miller's grammar, to stdout.
 
+================================================================
+Usage: ./mlr regularize [options]
+Outputs records sorted lexically ascending by keys.
+
 ================================================================
 Usage: ./mlr rename [options] {old1,new1,old2,new2,...}
 Renames specified fields.

go/u/try-verbs

@@ -50,3 +50,4 @@ run_mlr --opprint --from u/s.dkvp sort -f b -n i
 run_mlr --opprint --from u/s.dkvp sort -f b -nr i
 
 run_mlr --json --from u/needs-sorting.json sort-within-records
+run_mlr --json --from u/needs-regularize.json regularize