From dd9f065243d1f08dcc746fdc68591dfd03abdc3b Mon Sep 17 00:00:00 2001 From: John Kerl Date: Tue, 22 Sep 2020 19:48:23 -0400 Subject: [PATCH] mlr regularize --- go/src/miller/cli/mlrcli_mappers.go | 1 + go/src/miller/lib/util.go | 20 ++++ go/src/miller/mappers/regularize.go | 115 +++++++++++++++++++ go/src/miller/mappers/sort-within-records.go | 3 - go/src/miller/types/mlrmap_accessors.go | 6 +- go/todo.txt | 3 - go/u/needs-regularize.json | 16 +++ go/u/try-help.out | 4 + go/u/try-verbs | 1 + 9 files changed, 158 insertions(+), 11 deletions(-) create mode 100644 go/src/miller/mappers/regularize.go create mode 100644 go/u/needs-regularize.json diff --git a/go/src/miller/cli/mlrcli_mappers.go b/go/src/miller/cli/mlrcli_mappers.go index d1e0d894a..3daea5c1f 100644 --- a/go/src/miller/cli/mlrcli_mappers.go +++ b/go/src/miller/cli/mlrcli_mappers.go @@ -19,6 +19,7 @@ var MAPPER_LOOKUP_TABLE = []mapping.MapperSetup{ mappers.LabelSetup, mappers.NothingSetup, mappers.PutSetup, + mappers.RegularizeSetup, mappers.RenameSetup, mappers.SortSetup, mappers.SortWithinRecordsSetup, diff --git a/go/src/miller/lib/util.go b/go/src/miller/lib/util.go index 3f9277a0d..3b7f8c58a 100644 --- a/go/src/miller/lib/util.go +++ b/go/src/miller/lib/util.go @@ -1,6 +1,7 @@ package lib import ( + "sort" "strconv" "strings" ) @@ -32,6 +33,25 @@ func StringListToSet(stringList []string) map[string]bool { return stringSet } +func SortStrings(strings []string) { + // Go sort API: for ascending sort, return true if element i < element j. + sort.Slice(strings, func(i, j int) bool { + return strings[i] < strings[j] + }) +} + +func SortedStrings(strings []string) []string { + copy := make([]string, len(strings)) + for i, s := range(strings) { + copy[i] = s + } + // Go sort API: for ascending sort, return true if element i < element j. + sort.Slice(copy, func(i, j int) bool { + return copy[i] < copy[j] + }) + return copy +} + func IntMin2(a, b int) int { if a < b { return a diff --git a/go/src/miller/mappers/regularize.go b/go/src/miller/mappers/regularize.go new file mode 100644 index 000000000..caf319422 --- /dev/null +++ b/go/src/miller/mappers/regularize.go @@ -0,0 +1,115 @@ +package mappers + +import ( + "flag" + "fmt" + "os" + "strings" + + "miller/clitypes" + "miller/lib" + "miller/mapping" + "miller/types" +) + +// ---------------------------------------------------------------- +var RegularizeSetup = mapping.MapperSetup{ + Verb: "regularize", + ParseCLIFunc: mapperRegularizeParseCLI, + IgnoresInput: false, +} + +func mapperRegularizeParseCLI( + pargi *int, + argc int, + args []string, + errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError + _ *clitypes.TReaderOptions, + __ *clitypes.TWriterOptions, +) mapping.IRecordMapper { + + // Get the verb name from the current spot in the mlr command line + argi := *pargi + verb := args[argi] + argi++ + + // Parse local flags + flagSet := flag.NewFlagSet(verb, errorHandling) + + flagSet.Usage = func() { + ostream := os.Stderr + if errorHandling == flag.ContinueOnError { // help intentionally requested + ostream = os.Stdout + } + mapperRegularizeUsage(ostream, args[0], verb, flagSet) + } + flagSet.Parse(args[argi:]) + if errorHandling == flag.ContinueOnError { // help intentionally requested + return nil + } + + // Find out how many flags were consumed by this verb and advance for the + // next verb + argi = len(args) - len(flagSet.Args()) + + mapper, _ := NewMapperRegularize() + + *pargi = argi + return mapper +} + +func mapperRegularizeUsage( + o *os.File, + argv0 string, + verb string, + flagSet *flag.FlagSet, +) { + fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb) + fmt.Fprint(o, + `Outputs records sorted lexically ascending by keys. +`) + // flagSet.PrintDefaults() doesn't let us control stdout vs stderr + flagSet.VisitAll(func(f *flag.Flag) { + fmt.Fprintf(o, " -%v (default %v) %v\n", f.Name, f.Value, f.Usage) // f.Name, f.Value + }) +} + +// ---------------------------------------------------------------- +type MapperRegularize struct { + // map from string to []string + sortedToOriginal map[string][]string +} + +func NewMapperRegularize() (*MapperRegularize, error) { + this := &MapperRegularize{ + make(map[string][]string), + } + return this, nil +} + +// ---------------------------------------------------------------- +func (this *MapperRegularize) Map( + inrecAndContext *types.RecordAndContext, + outputChannel chan<- *types.RecordAndContext, +) { + inrec := inrecAndContext.Record + if inrec != nil { // not end of record stream + currentFieldNames := inrec.GetKeys() + currentSortedFieldNames := lib.SortedStrings(currentFieldNames) + currentSortedFieldNamesJoined := strings.Join(currentSortedFieldNames, ",") + previousSortedFieldNames := this.sortedToOriginal[currentSortedFieldNamesJoined] + if previousSortedFieldNames == nil { + this.sortedToOriginal[currentSortedFieldNamesJoined] = currentFieldNames + outputChannel <- inrecAndContext + } else { + outrec := types.NewMlrmapAsRecord() + for _, fieldName := range previousSortedFieldNames { + outrec.PutReference(&fieldName, inrec.Get(&fieldName)) // inrec will be GC'ed + } + outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context) + outputChannel <- outrecAndContext + } + } else { + outputChannel <- inrecAndContext // end-of-stream marker + } +} diff --git a/go/src/miller/mappers/sort-within-records.go b/go/src/miller/mappers/sort-within-records.go index d3ce805fa..baae583f7 100644 --- a/go/src/miller/mappers/sort-within-records.go +++ b/go/src/miller/mappers/sort-within-records.go @@ -91,10 +91,7 @@ func (this *MapperSortWithinRecords) Map( ) { inrec := inrecAndContext.Record if inrec != nil { // not end of record stream - inrec.SortByKey() - - } else { } outputChannel <- inrecAndContext // end-of-stream marker } diff --git a/go/src/miller/types/mlrmap_accessors.go b/go/src/miller/types/mlrmap_accessors.go index 52d5d5535..3bfec3928 100644 --- a/go/src/miller/types/mlrmap_accessors.go +++ b/go/src/miller/types/mlrmap_accessors.go @@ -3,7 +3,6 @@ package types import ( "bytes" "errors" - "sort" "strconv" "miller/lib" @@ -352,10 +351,7 @@ func (this *Mlrmap) Label(newNames []string) { func (this *Mlrmap) SortByKey() { keys := this.GetKeys() - // Go sort API: for ascending sort, return true if element i < element j. - sort.Slice(keys, func(i, j int) bool { - return keys[i] < keys[j] - }) + lib.SortStrings(keys) that := NewMlrmapAsRecord() diff --git a/go/todo.txt b/go/todo.txt index 22f3f00e1..4669f9e3c 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -1,9 +1,6 @@ ---------------------------------------------------------------- TOP OF LIST: -* sort-within-records -* regularize - ! try Lvalue/Rvalue coalesce > to make condish work & unhack filter > better error messages in the CST builder anyway, so it's a win diff --git a/go/u/needs-regularize.json b/go/u/needs-regularize.json new file mode 100644 index 000000000..7c9e0be01 --- /dev/null +++ b/go/u/needs-regularize.json @@ -0,0 +1,16 @@ +{ + "b": 2, + "c": 3, + "a": 1 +} +{ + "b": 2, + "a": 1, + "c": 3 +} +{ + "d": 4, + "a": 1, + "b": 2, + "c": 3 +} diff --git a/go/u/try-help.out b/go/u/try-help.out index 01d314665..8b57c8889 100644 --- a/go/u/try-help.out +++ b/go/u/try-help.out @@ -53,6 +53,10 @@ TODO: put detailed on-line help here. full transparency on the precedence and associativity rules of Miller's grammar, to stdout. +================================================================ +Usage: ./mlr regularize [options] +Outputs records sorted lexically ascending by keys. + ================================================================ Usage: ./mlr rename [options] {old1,new1,old2,new2,...} Renames specified fields. diff --git a/go/u/try-verbs b/go/u/try-verbs index e50bce60d..b1ac245b3 100755 --- a/go/u/try-verbs +++ b/go/u/try-verbs @@ -50,3 +50,4 @@ run_mlr --opprint --from u/s.dkvp sort -f b -n i run_mlr --opprint --from u/s.dkvp sort -f b -nr i run_mlr --json --from u/needs-sorting.json sort-within-records +run_mlr --json --from u/needs-regularize.json regularize