diff --git a/go/src/miller/mappers/cat.go b/go/src/miller/mappers/cat.go index a81610cae..e2025bbf1 100644 --- a/go/src/miller/mappers/cat.go +++ b/go/src/miller/mappers/cat.go @@ -54,40 +54,11 @@ func mapperCatParseCLI( mapperCatUsage(ostream, args[0], verb, flagSet) } flagSet.Parse(args[argi:]) - if errorHandling == flag.ContinueOnError { // help intentioally requested + if errorHandling == flag.ContinueOnError { // help intentionally requested return nil } - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // xxx to be ported: - // - // char* default_counter_field_name = DEFAULT_COUNTER_FIELD_NAME; - // char* counter_field_name = nil; - // int do_counters = FALSE; - // int verbose = FALSE; - // slls_t* pgroup_by_field_names = slls_alloc(); - // - // ap_state_t* pstate = ap_alloc(); - // ap_define_true_flag(pstate, "-n", &do_counters); - // ap_define_true_flag(pstate, "-v", &verbose); - // ap_define_string_flag(pstate, "-N", &counter_field_name); - // ap_define_string_list_flag(pstate, "-g", &pgroup_by_field_names); - // - // if (!ap_parse(pstate, verb, pargi, argc, args)) { - // mapper_cat_usage(stderr, args[0], verb); - // return nil; - // } - // - // if (counter_field_name != nil) { - // do_counters = TRUE; - // } else if (do_counters) { - // counter_field_name = default_counter_field_name; - // } - // - // mapper_t* pmapper = mapper_cat_alloc(pstate, do_counters, verbose, counter_field_name, pgroup_by_field_names); - // return pmapper; - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // xxx to port: // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // fmt.Fprintf(o, "Options:\n"); // fmt.Fprintf(o, "-n Prepend field \"%s\" to each record with record-counter starting at 1\n", diff --git a/go/src/miller/mappers/cut.go b/go/src/miller/mappers/cut.go index 19c68b033..20eb934b6 100644 --- a/go/src/miller/mappers/cut.go +++ b/go/src/miller/mappers/cut.go @@ -74,7 +74,7 @@ Default is to retain them in the order found in the input data.`, mapperCutUsage(ostream, args[0], verb, flagSet) } flagSet.Parse(args[argi:]) - if errorHandling == flag.ContinueOnError { // help intentioally requested + if errorHandling == flag.ContinueOnError { // help intentionally requested return nil } @@ -156,6 +156,7 @@ func NewMapperCut( return this, nil } +// xxx to port: // if (!do_regexes) { // pstate->pfield_name_list = pfield_name_list; // slls_reverse(pstate->pfield_name_list); @@ -248,6 +249,7 @@ func (this *MapperCut) exclude( outputChannel <- inrecAndContext } +// xxx to port: //// ---------------------------------------------------------------- //static sllv_t* mapper_cut_process_with_regexes(lrec_t* pinrec, context_t* pctx, void* pvstate) { // if (pinrec != NULL) { diff --git a/go/src/miller/mappers/group-by.go b/go/src/miller/mappers/group-by.go index ef4dae506..5cb6cbb56 100644 --- a/go/src/miller/mappers/group-by.go +++ b/go/src/miller/mappers/group-by.go @@ -44,7 +44,7 @@ func mapperGroupByParseCLI( mapperGroupByUsage(ostream, args[0], verb, flagSet) } flagSet.Parse(args[argi:]) - if errorHandling == flag.ContinueOnError { // help intentioally requested + if errorHandling == flag.ContinueOnError { // help intentionally requested return nil } @@ -117,15 +117,15 @@ func (this *MapperGroupBy) Map( inrec := inrecAndContext.Record if inrec != nil { // not end of record stream - groupByKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNameList) + groupingKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNameList) if !ok { return } - recordListForGroup := this.recordListsByGroup.Get(groupByKey) + recordListForGroup := this.recordListsByGroup.Get(groupingKey) if recordListForGroup == nil { recordListForGroup = list.New() - this.recordListsByGroup.Put(groupByKey, recordListForGroup) + this.recordListsByGroup.Put(groupingKey, recordListForGroup) } recordListForGroup.(*list.List).PushBack(inrecAndContext) diff --git a/go/src/miller/mappers/group-like.go b/go/src/miller/mappers/group-like.go index 9e4e4652b..c1f077c9b 100644 --- a/go/src/miller/mappers/group-like.go +++ b/go/src/miller/mappers/group-like.go @@ -44,7 +44,7 @@ func mapperGroupLikeParseCLI( mapperGroupLikeUsage(ostream, args[0], verb, flagSet) } flagSet.Parse(args[argi:]) - if errorHandling == flag.ContinueOnError { // help intentioally requested + if errorHandling == flag.ContinueOnError { // help intentionally requested return nil } @@ -97,12 +97,12 @@ func (this *MapperGroupLike) Map( inrec := inrecAndContext.Record if inrec != nil { // not end of record stream - groupByKey := inrec.GetKeysJoined() + groupingKey := inrec.GetKeysJoined() - recordListForGroup := this.recordListsByGroup.Get(groupByKey) + recordListForGroup := this.recordListsByGroup.Get(groupingKey) if recordListForGroup == nil { // first time recordListForGroup = list.New() - this.recordListsByGroup.Put(groupByKey, recordListForGroup) + this.recordListsByGroup.Put(groupingKey, recordListForGroup) } recordListForGroup.(*list.List).PushBack(inrecAndContext) diff --git a/go/src/miller/mappers/head.go b/go/src/miller/mappers/head.go index fbc9fc994..62b730133 100644 --- a/go/src/miller/mappers/head.go +++ b/go/src/miller/mappers/head.go @@ -59,7 +59,7 @@ func mapperHeadParseCLI( mapperHeadUsage(ostream, args[0], verb, flagSet) } flagSet.Parse(args[argi:]) - if errorHandling == flag.ContinueOnError { // help intentioally requested + if errorHandling == flag.ContinueOnError { // help intentionally requested return nil } @@ -164,17 +164,17 @@ func (this *MapperHead) mapKeyed( inrec := inrecAndContext.Record if inrec != nil { // not end of record stream - groupByKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNameList) + groupingKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNameList) if !ok { return } - count, present := this.keyedRecordCounts[groupByKey] + count, present := this.keyedRecordCounts[groupingKey] if !present { // first time - this.keyedRecordCounts[groupByKey] = 1 + this.keyedRecordCounts[groupingKey] = 1 count = 1 } else { - this.keyedRecordCounts[groupByKey] += 1 + this.keyedRecordCounts[groupingKey] += 1 count += 1 } diff --git a/go/src/miller/mappers/label.go b/go/src/miller/mappers/label.go index 032b2ea4d..813ff9515 100644 --- a/go/src/miller/mappers/label.go +++ b/go/src/miller/mappers/label.go @@ -43,7 +43,7 @@ func mapperLabelParseCLI( mapperLabelUsage(ostream, args[0], verb, flagSet) } flagSet.Parse(args[argi:]) - if errorHandling == flag.ContinueOnError { // help intentioally requested + if errorHandling == flag.ContinueOnError { // help intentionally requested return nil } diff --git a/go/src/miller/mappers/nothing.go b/go/src/miller/mappers/nothing.go index 94749a3c8..dc828c311 100644 --- a/go/src/miller/mappers/nothing.go +++ b/go/src/miller/mappers/nothing.go @@ -41,7 +41,7 @@ func mapperNothingParseCLI( mapperNothingUsage(ostream, args[0], verb, flagSet) } flagSet.Parse(args[argi:]) - if errorHandling == flag.ContinueOnError { // help intentioally requested + if errorHandling == flag.ContinueOnError { // help intentionally requested return nil } diff --git a/go/src/miller/mappers/put.go b/go/src/miller/mappers/put.go index cc56d4ddf..64b00bb2b 100644 --- a/go/src/miller/mappers/put.go +++ b/go/src/miller/mappers/put.go @@ -64,7 +64,7 @@ func mapperPutParseCLI( mapperPutUsage(ostream, args[0], verb, flagSet) } flagSet.Parse(args[argi:]) - if errorHandling == flag.ContinueOnError { // help intentioally requested + if errorHandling == flag.ContinueOnError { // help intentionally requested return nil } diff --git a/go/src/miller/mappers/rename.go b/go/src/miller/mappers/rename.go index 97d344f4d..1c6d901f8 100644 --- a/go/src/miller/mappers/rename.go +++ b/go/src/miller/mappers/rename.go @@ -44,7 +44,7 @@ func mapperRenameParseCLI( mapperRenameUsage(ostream, args[0], verb, flagSet) } flagSet.Parse(args[argi:]) - if errorHandling == flag.ContinueOnError { // help intentioally requested + if errorHandling == flag.ContinueOnError { // help intentionally requested return nil } diff --git a/go/src/miller/mappers/sort.go b/go/src/miller/mappers/sort.go index ff33cc49b..fff5a9f23 100644 --- a/go/src/miller/mappers/sort.go +++ b/go/src/miller/mappers/sort.go @@ -19,17 +19,17 @@ package mappers // E.g. the list ["red", "1.0"] maps to the array ["red", 1.0]. // // * The pairing of parsed-value array the linked list of same-key-value records -// is called a *bucket*. E.g the records +// is called a *bucket* or a *group*. E.g the records // {"a":"red","b":"circle","x":"1.0","y":"3.9"} // {"a":"red","b":"square","x":"1.0","z":"5.7", "q":"even"} -// would both land in the ["red","1.0"] bucket. +// would both land in the ["red","1.0"] group. // -// * Buckets are retained in a hash map: the key is the string-list of the form +// * Groups are retained in a hash map: the key is the string-list of the form // ["red","1.0"] and the value is the pairing of parsed-value array ["red",1.0] // and linked list of records. // // * Once all the input records are ingested into this hash map, we copy the -// bucket-pointers into an array and sort it: this being the pairing of +// group-pointers into an array and sort it: this being the pairing of // parsed-value array and linked list of records. The comparator callback for // the sort walks through the parsed-value arrays one slot at a time, // looking at the first difference, e.g. if one has "a"="red" and the other @@ -37,7 +37,7 @@ package mappers // second field, and so on. // // * Note in particular that string keys ["a":"red","x":"1"] and -// ["a":"red","x":"1.0"] map to different buckets, but will sort equally. +// ["a":"red","x":"1.0"] map to different groups, but will sort equally. // // ================================================================ @@ -75,55 +75,129 @@ func mapperSortParseCLI( verb := args[argi] argi++ - // Parse local flags - flagSet := flag.NewFlagSet(verb, errorHandling) + // Unlike other mappers, we can't use flagSet here. The syntax of 'mlr + // sort' is it needs to take things like 'mlr sort -f a -n b -n c', i.e. + // first sort lexically on field a, then numerically on field b, then + // lexically on field c. The flagSet API would let the '-f c' clobber the + // '-f a', while we want both. - flagSet.Usage = func() { - ostream := os.Stderr - if errorHandling == flag.ContinueOnError { // help intentionally requested - ostream = os.Stdout + groupByFieldNameList := make([]string, 0) + comparatorFuncs := make([]types.ComparatorFunc, 0) + + for argi < argc /* variable increment: 1 or 2 depending on flag */ { + if args[argi][0] != '-' { + break // No more flag options to process + } else if args[argi] == "-h" || args[argi] == "--help" { + mapperSortUsage(os.Stdout, 0, errorHandling, args[0], verb) + argi += 1 + + } else if args[argi] == "-f" { + subList := lib.SplitString(args[argi+1], ",") + for _, item := range subList { + groupByFieldNameList = append(groupByFieldNameList, item) + comparatorFuncs = append(comparatorFuncs, types.LexicalAscendingComparatorfunc) + } + // xxx append sorters lexical ascending + argi += 2 + } else if args[argi] == "-r" { + subList := lib.SplitString(args[argi+1], ",") + for _, item := range subList { + groupByFieldNameList = append(groupByFieldNameList, item) + comparatorFuncs = append(comparatorFuncs, types.LexicalDescendingComparatorfunc) + } + // xxx append sorters lexical scending + argi += 2 + } else if args[argi] == "-n" { + subList := lib.SplitString(args[argi+1], ",") + for _, item := range subList { + groupByFieldNameList = append(groupByFieldNameList, item) + comparatorFuncs = append(comparatorFuncs, types.NumericAscendingComparatorfunc) + } + // xxx append sorters numeric ascending + argi += 2 + } else if args[argi] == "-nf" { + subList := lib.SplitString(args[argi+1], ",") + for _, item := range subList { + groupByFieldNameList = append(groupByFieldNameList, item) + comparatorFuncs = append(comparatorFuncs, types.NumericAscendingComparatorfunc) + } + // xxx append sorters numeric ascending + argi += 2 + } else if args[argi] == "-nr" { + subList := lib.SplitString(args[argi+1], ",") + for _, item := range subList { + groupByFieldNameList = append(groupByFieldNameList, item) + comparatorFuncs = append(comparatorFuncs, types.NumericDescendingComparatorfunc) + } + // xxx append sorters numeric descending + argi += 2 + + } else { + mapperSortUsage(os.Stderr, 1, flag.ExitOnError, args[0], verb) } - mapperSortUsage(ostream, args[0], verb, flagSet) } - flagSet.Parse(args[argi:]) - if errorHandling == flag.ContinueOnError { // help intentioally requested + + // xxx + if errorHandling == flag.ContinueOnError { // help intentionally requested return nil } // Find out how many flags were consumed by this verb and advance for the // next verb - argi = len(args) - len(flagSet.Args()) // Get the sort field names from the command line if argi >= argc { - flagSet.Usage() + mapperSortUsage(os.Stderr, 1, flag.ExitOnError, args[0], verb) os.Exit(1) } - groupByFieldNames := args[argi] - argi += 1 mapper, _ := NewMapperSort( - groupByFieldNames, + groupByFieldNameList, + comparatorFuncs, ) *pargi = argi return mapper } +// For flags with values, e.g. ["-n" "10"], while we're looking at the "-n" +// this let us see if the "10" slot exists. +func checkArgCount(args []string, argi int, argc int, n int) { + if (argc - argi) < n { + fmt.Fprintf(os.Stderr, "%s: option \"%s\" missing argument(s).\n", args[0], args[argi]) + mapperSortUsage(os.Stderr, 1, flag.ExitOnError, os.Args[0], "sort") + os.Exit(1) + } +} + func mapperSortUsage( o *os.File, + exitCode int, + errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError argv0 string, verb string, - flagSet *flag.FlagSet, ) { - fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb) - fmt.Fprint(o, - `Outputs records in batches having identical values at specified field names. -`) - // flagSet.PrintDefaults() doesn't let us control stdout vs stderr - flagSet.VisitAll(func(f *flag.Flag) { - fmt.Fprintf(o, " -%v (default %v) %v\n", f.Name, f.Value, f.Usage) // f.Name, f.Value - }) + fmt.Fprintf(o, "Usage: %s %s {flags}\n", argv0, verb) + fmt.Fprintf(o, "Sorts records primarily by the first specified field, secondarily by the second\n") + fmt.Fprintf(o, "field, and so on. (Any records not having all specified sort keys will appear\n") + fmt.Fprintf(o, "at the end of the output, in the order they were encountered, regardless of the\n") + fmt.Fprintf(o, "specified sort order.) The sort is stable: records that compare equal will sort\n") + fmt.Fprintf(o, "in the order they were encountered in the input record stream.\n") + fmt.Fprintf(o, "\n") + fmt.Fprintf(o, "Flags:\n") + fmt.Fprintf(o, " -f {comma-separated field names} Lexical ascending\n") + fmt.Fprintf(o, " -n {comma-separated field names} Numerical ascending; nulls sort last\n") + fmt.Fprintf(o, " -nf {comma-separated field names} Same as -n\n") + fmt.Fprintf(o, " -r {comma-separated field names} Lexical descending\n") + fmt.Fprintf(o, " -nr {comma-separated field names} Numerical descending; nulls sort first\n") + fmt.Fprintf(o, "\n") + fmt.Fprintf(o, "Example:\n") + fmt.Fprintf(o, " %s %s -f a,b -nr x,y,z\n", argv0, verb) + fmt.Fprintf(o, "which is the same as:\n") + fmt.Fprintf(o, " %s %s -f a -f b -nr x -nr y -nr z\n", argv0, verb) + if errorHandling == flag.ExitOnError { + os.Exit(exitCode) + } } // ---------------------------------------------------------------- @@ -132,39 +206,40 @@ func mapperSortUsage( // * group-by field-name list is "a,i" // * input record 'a=pan,b=pan,i=1,x=0.3467,y=0.7268' // o values at a,i are "pan",1 -// o grouping key for the ordered map from string to record-bucket is the string "pan,1" +// o grouping key for the ordered map from string to record-group is the string "pan,1" // o we also need a map from "pan,1" to the array of mlrvals ["pan", 1]. // * next input record 'a=eks,b=pan,i=2,x=0.7586,y=0.5221' // o values at a,i are "eks",2 -// o grouping key for the ordered map from string to record-bucket is the string "eks,2" +// o grouping key for the ordered map from string to record-group is the string "eks,2" // o we also need a map from "eks,2" to the array of mlrvals ["eks", 2]. -// * what gets sorted are the bucket-heading arrays of mlrvals: +// * what gets sorted are the group-heading arrays of mlrvals: // o make an array [ ("pan,1", ["pan", 1]), ("eks,2", ["eks", 2]) // o sort that -// * output is simply for each slot in the array, emit each record in the bucket +// * output is simply for each slot in the array, emit each record in the group type MapperSort struct { - // Input + // -- Input groupByFieldNameList []string + comparatorFuncs []types.ComparatorFunc - // State + // -- State // Map from string to *list.List: recordListsByGroup *lib.OrderedMap // Map from string to []*lib.Mlrval: - bucketHeads *lib.OrderedMap + groupHeads *lib.OrderedMap } func NewMapperSort( - groupByFieldNames string, + groupByFieldNameList []string, + comparatorFuncs []types.ComparatorFunc, ) (*MapperSort, error) { - groupByFieldNameList := lib.SplitString(groupByFieldNames, ",") - this := &MapperSort{ groupByFieldNameList: groupByFieldNameList, + comparatorFuncs: comparatorFuncs, recordListsByGroup: lib.NewOrderedMap(), - bucketHeads: lib.NewOrderedMap(), + groupHeads: lib.NewOrderedMap(), } return this, nil @@ -183,27 +258,26 @@ func (this *MapperSort) Map( inrec := inrecAndContext.Record if inrec != nil { // Not end of record stream - // xxx need to factor out the joined and non-joined -- this verb needs both - groupByKey, selectedValues, ok := inrec.GetSelectedValuesAndJoined( + groupingKey, selectedValues, ok := inrec.GetSelectedValuesAndJoined( this.groupByFieldNameList, ) if !ok { - // xxx need a spill-bucket + // xxx need a spill-group return } - recordListForGroup := this.recordListsByGroup.Get(groupByKey) + recordListForGroup := this.recordListsByGroup.Get(groupingKey) if recordListForGroup == nil { recordListForGroup = list.New() - this.recordListsByGroup.Put(groupByKey, recordListForGroup) - this.bucketHeads.Put(groupByKey, selectedValues) + this.recordListsByGroup.Put(groupingKey, recordListForGroup) + this.groupHeads.Put(groupingKey, selectedValues) } recordListForGroup.(*list.List).PushBack(inrecAndContext) } else { // End of record stream - // At this point, in the above example, bucketHeads is: + // At this point, in the above example, groupHeads is: // // { // "pan,1" : ["pan", 1], @@ -217,23 +291,29 @@ func (this *MapperSort) Map( // [ "eks,2", ["eks', 2] // ] - groupingKeysAndMlrvals := bucketHeadsToArray(this.bucketHeads) + groupingKeysAndMlrvals := groupHeadsToArray(this.groupHeads) - // Next sort that array - // xxx this is a stub: - // xxx make a helper method + // Go sort API: for ascending sort, return true if element i < element j. sort.Slice(groupingKeysAndMlrvals, func(i, j int) bool { - a, _ := groupingKeysAndMlrvals[i].mlrvals[0].GetFloatValue() - b, _ := groupingKeysAndMlrvals[j].mlrvals[0].GetFloatValue() - return a < b - //return groupingKeysAndMlrvals[i].mlrvals[0] > groupingKeysAndMlrvals[j].mlrvals[0] + for k, comparator := range(this.comparatorFuncs) { + result := comparator( + &groupingKeysAndMlrvals[i].mlrvals[k], + &groupingKeysAndMlrvals[j].mlrvals[k], + ) + if result < 0 { + return true + } else if result > 0 { + return false + } + } + return false }) - // Now output the buckets + // Now output the groups for _, groupingKeyAndMlrvals := range groupingKeysAndMlrvals { - iRecordsInBucket := this.recordListsByGroup.Get(groupingKeyAndMlrvals.groupingKey) - recordsInBucket := iRecordsInBucket.(*list.List) - for iRecord := recordsInBucket.Front(); iRecord != nil; iRecord = iRecord.Next() { + iRecordsInGroup := this.recordListsByGroup.Get(groupingKeyAndMlrvals.groupingKey) + recordsInGroup := iRecordsInGroup.(*list.List) + for iRecord := recordsInGroup.Front(); iRecord != nil; iRecord = iRecord.Next() { outputChannel <- iRecord.Value.(*types.RecordAndContext) } } @@ -242,11 +322,11 @@ func (this *MapperSort) Map( } } -func bucketHeadsToArray(bucketHeads *lib.OrderedMap) []GroupingKeysAndMlrvals { - retval := make([]GroupingKeysAndMlrvals, bucketHeads.FieldCount) +func groupHeadsToArray(groupHeads *lib.OrderedMap) []GroupingKeysAndMlrvals { + retval := make([]GroupingKeysAndMlrvals, groupHeads.FieldCount) i := 0 - for entry := bucketHeads.Head; entry != nil; entry = entry.Next { + for entry := groupHeads.Head; entry != nil; entry = entry.Next { retval[i] = GroupingKeysAndMlrvals{ groupingKey: entry.Key, mlrvals: entry.Value.([]types.Mlrval), @@ -256,252 +336,3 @@ func bucketHeadsToArray(bucketHeads *lib.OrderedMap) []GroupingKeysAndMlrvals { return retval } - -//type GroupingKeysAndMlrvals struct { -// groupingKey string -// mlrvals []*types.Mlrval -//} - -//#define SORT_NUMERIC 0x80 -//#define SORT_DESCENDING 0x40 - -//typedef struct _mapper_sort_state_t { -// // Input parameters -// slls_t* pkey_field_names; // Fields to sort on -// int* sort_params; // Lexical/numeric; ascending/descending -// // Sort state: buckets of like records. -// lhmslv_t* pbuckets_by_key_field_values; -// sllv_t* precords_missing_sort_keys; -//} mapper_sort_state_t; - -//typedef struct _sort_bucket_t { -// typed_sort_key_t* typed_sort_keys; -// sllv_t* precords; -//} sort_bucket_t; - -//// ---------------------------------------------------------------- -//static void mapper_sort_usage(FILE* o, char* argv0, char* verb) { -// fprintf(o, "Usage: %s %s {flags}\n", argv0, verb); -// fprintf(o, "Flags:\n"); -// fprintf(o, " -f {comma-separated field names} Lexical ascending\n"); -// fprintf(o, " -n {comma-separated field names} Numerical ascending; nulls sort last\n"); -// fprintf(o, " -nf {comma-separated field names} Same as -n\n"); -// fprintf(o, " -r {comma-separated field names} Lexical descending\n"); -// fprintf(o, " -nr {comma-separated field names} Numerical descending; nulls sort first\n"); -// fprintf(o, "Sorts records primarily by the first specified field, secondarily by the second\n"); -// fprintf(o, "field, and so on. (Any records not having all specified sort keys will appear\n"); -// fprintf(o, "at the end of the output, in the order they were encountered, regardless of the\n"); -// fprintf(o, "specified sort order.) The sort is stable: records that compare equal will sort\n"); -// fprintf(o, "in the order they were encountered in the input record stream.\n"); -// fprintf(o, "\n"); -// fprintf(o, "Example:\n"); -// fprintf(o, " %s %s -f a,b -nr x,y,z\n", argv0, verb); -// fprintf(o, "which is the same as:\n"); -// fprintf(o, " %s %s -f a -f b -nr x -nr y -nr z\n", argv0, verb); -//} - -//static mapper_t* mapper_sort_parse_cli(int* pargi, int argc, char** argv, -// cli_reader_opts_t* _, cli_writer_opts_t* __) -//{ -// if ((argc - *pargi) < 3) { -// mapper_sort_usage(stderr, argv[0], argv[*pargi]); -// return NULL; -// } -// char* verb = argv[*pargi]; -// *pargi += 1; -// slls_t* pnames = slls_alloc(); -// slls_t* pflags = slls_alloc(); -// -// while ((argc - *pargi) >= 1 && argv[*pargi][0] == '-') { -// if ((argc - *pargi) < 2) -// mapper_sort_usage(stderr, argv[0], verb); -// char* flag = argv[*pargi]; -// char* value = argv[*pargi+1]; -// *pargi += 2; -// -// if (streq(flag, "-f")) { -// } else if (streq(flag, "-n")) { -// } else if (streq(flag, "-nf")) { -// } else if (streq(flag, "-r")) { -// } else if (streq(flag, "-nr")) { -// } else { -// mapper_sort_usage(stderr, argv[0], verb); -// } -// slls_t* pnames_for_flag = slls_from_line(value, ',', FALSE); -// // E.g. with "-nr a,b,c", replicate the "-nr" flag three times. -// for (sllse_t* pe = pnames_for_flag->phead; pe != NULL; pe = pe->pnext) { -// slls_append_no_free(pnames, pe->value); -// slls_append_no_free(pflags, flag); -// } -// slls_free(pnames_for_flag); -// } -// -// if (pnames->length < 1) -// mapper_sort_usage(stderr, argv[0], verb); -// -// // Convert the list such as ["-nf","-nf","-r","-r","-r"] into an array of -// // bit-flags, one per sort-key field. -// int* opt_array = mlr_malloc_or_die(pnames->length * sizeof(int)); -// sllse_t* pe; -// int di; -// for (pe = pflags->phead, di = 0; pe != NULL; pe = pe->pnext, di++) { -// char* flag = pe->value; -// int opt = -// streq(flag, "-nf") ? SORT_NUMERIC : -// streq(flag, "-n") ? SORT_NUMERIC : -// streq(flag, "-r") ? SORT_DESCENDING : -// streq(flag, "-nr") ? SORT_NUMERIC|SORT_DESCENDING : -// 0; -// opt_array[di] =opt; -// } -// slls_free(pflags); -// -// return mapper_sort_alloc(pnames, opt_array, TRUE); -//} - -//// ---------------------------------------------------------------- -//static mapper_t* mapper_sort_alloc(slls_t* pkey_field_names, int* sort_params, int do_sort) { -// mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t)); -// -// mapper_sort_state_t* pstate = mlr_malloc_or_die(sizeof(mapper_sort_state_t)); -// -// pstate->pkey_field_names = pkey_field_names; -// pstate->sort_params = sort_params; -// pstate->pbuckets_by_key_field_values = lhmslv_alloc(); -// pstate->precords_missing_sort_keys = sllv_alloc(); -// pstate->do_sort = do_sort; -// -// pmapper->pvstate = pstate; -// pmapper->pprocess_func = mapper_sort_process; -// pmapper->pfree_func = mapper_sort_free; -// -// return pmapper; -//} -// -//// ---------------------------------------------------------------- -//static void mapper_sort_free(mapper_t* pmapper, context_t* _) { -// mapper_sort_state_t* pstate = pmapper->pvstate; -// if (pstate->pkey_field_names != NULL) -// slls_free(pstate->pkey_field_names); -// // lhmslv_free will free the hashmap keys; we need to free the void-star hashmap values. -// for (lhmslve_t* pa = pstate->pbuckets_by_key_field_values->phead; pa != NULL; pa = pa->pnext) { -// sort_bucket_t* pbucket = pa->pvvalue; -// free(pbucket->typed_sort_keys); -// free(pbucket); -// // precords freed in emitter -// } -// lhmslv_free(pstate->pbuckets_by_key_field_values); -// sllv_free(pstate->precords_missing_sort_keys); -// free(pstate->sort_params); -// free(pstate); -// free(pmapper); -//} -// -//// ---------------------------------------------------------------- -//static sllv_t* mapper_sort_process(lrec_t* pinrec, context_t* pctx, void* pvstate) { -// mapper_sort_state_t* pstate = pvstate; -// if (pinrec != NULL) { -// // Consume another input record. -// slls_t* pkey_field_values = mlr_reference_selected_values_from_record(pinrec, pstate->pkey_field_names); -// if (pkey_field_values == NULL) { -// sllv_append(pstate->precords_missing_sort_keys, pinrec); -// } else { -// sort_bucket_t* pbucket = lhmslv_get(pstate->pbuckets_by_key_field_values, pkey_field_values); -// if (pbucket == NULL) { // New key-field-value: new bucket and hash-map entry -// slls_t* pkey_field_values_copy = slls_copy(pkey_field_values); -// sort_bucket_t* pbucket = mlr_malloc_or_die(sizeof(sort_bucket_t)); -// pbucket->typed_sort_keys = parse_sort_keys(pkey_field_values_copy, pstate->sort_params, pctx); -// pbucket->precords = sllv_alloc(); -// sllv_append(pbucket->precords, pinrec); -// lhmslv_put(pstate->pbuckets_by_key_field_values, pkey_field_values_copy, pbucket, -// FREE_ENTRY_KEY); -// } else { // Previously seen key-field-value: append record to bucket -// sllv_append(pbucket->precords, pinrec); -// } -// slls_free(pkey_field_values); -// } -// return NULL; -// } else { -// // End of input stream: sort bucket labels -// int num_buckets = pstate->pbuckets_by_key_field_values->num_occupied; -// sort_bucket_t** pbucket_array = mlr_malloc_or_die(num_buckets * sizeof(sort_bucket_t*)); -// -// // Copy bucket-pointers to an array for qsort -// int i = 0; -// for (lhmslve_t* pe = pstate->pbuckets_by_key_field_values->phead; pe != NULL; pe = pe->pnext, i++) { -// pbucket_array[i] = pe->pvvalue; -// } -// -// pcmp_sort_params = pstate->sort_params; -// cmp_params_length = pstate->pkey_field_names->length; -// -// qsort(pbucket_array, num_buckets, sizeof(sort_bucket_t*), pbucket_comparator); -// -// pcmp_sort_params = NULL; -// cmp_params_length = 0; -// -// // Emit each bucket's record -// sllv_t* poutput = sllv_alloc(); -// for (i = 0; i < num_buckets; i++) { -// sllv_t* plist = pbucket_array[i]->precords; -// sllv_transfer(poutput, plist); -// sllv_free(plist); -// } -// sllv_transfer(poutput, pstate->precords_missing_sort_keys); -// free(pbucket_array); -// sllv_append(poutput, NULL); // Signal end of output-record stream. -// return poutput; -// } -//} -// -//static int pbucket_comparator(const void* pva, const void* pvb) { -// // We are sorting an array of sort_bucket_t*. -// const sort_bucket_t** pba = (const sort_bucket_t**)pva; -// const sort_bucket_t** pbb = (const sort_bucket_t**)pvb; -// typed_sort_key_t* akeys = (*pba)->typed_sort_keys; -// typed_sort_key_t* bkeys = (*pbb)->typed_sort_keys; -// for (int i = 0; i < cmp_params_length; i++) { -// int sort_param = pcmp_sort_params[i]; -// if (sort_param & SORT_NUMERIC) { -// double a = akeys[i].u.d; -// double b = bkeys[i].u.d; -// if (isnan(a)) { // null input value -// if (!isnan(b)) { -// return (sort_param & SORT_DESCENDING) ? -1 : 1; -// } -// } else if (isnan(b)) { -// return (sort_param & SORT_DESCENDING) ? 1 : -1; -// } else { -// double d = a - b; -// int s = (d < 0) ? -1 : (d > 0) ? 1 : 0; -// if (s != 0) -// return (sort_param & SORT_DESCENDING) ? -s : s; -// } -// } else { -// int s = strcmp(akeys[i].u.s, bkeys[i].u.s); -// if (s != 0) -// return (sort_param & SORT_DESCENDING) ? -s : s; -// } -// } -// return 0; -//} -// -//// E.g. parse the list ["red","1.0"] into the array ["red",1.0]. -//static typed_sort_key_t* parse_sort_keys(slls_t* pkey_field_values, int* sort_params, context_t* pctx) { -// typed_sort_key_t* typed_sort_keys = mlr_malloc_or_die(pkey_field_values->length * sizeof(typed_sort_key_t)); -// int i = 0; -// for (sllse_t* pe = pkey_field_values->phead; pe != NULL; pe = pe->pnext, i++) { -// if (sort_params[i] & SORT_NUMERIC) { -// if (*pe->value == 0) { // null input value -// typed_sort_keys[i].u.d = nan(""); -// } else if (!mlr_try_float_from_string(pe->value, &typed_sort_keys[i].u.d)) { -// fprintf(stderr, "%s: couldn't parse \"%s\" as number in file \"%s\" record %lld.\n", -// MLR_GLOBALS.bargv0, pe->value, pctx->filename, pctx->fnr); -// exit(1); -// } -// } else { -// typed_sort_keys[i].u.s = pe->value; -// } -// } -// return typed_sort_keys; -//} diff --git a/go/src/miller/mappers/tac.go b/go/src/miller/mappers/tac.go index 272383f86..df2e1bac7 100644 --- a/go/src/miller/mappers/tac.go +++ b/go/src/miller/mappers/tac.go @@ -42,7 +42,7 @@ func mapperTacParseCLI( mapperTacUsage(ostream, args[0], verb, flagSet) } flagSet.Parse(args[argi:]) - if errorHandling == flag.ContinueOnError { // help intentioally requested + if errorHandling == flag.ContinueOnError { // help intentionally requested return nil } diff --git a/go/src/miller/mappers/tail.go b/go/src/miller/mappers/tail.go index 418d40913..c1d63850a 100644 --- a/go/src/miller/mappers/tail.go +++ b/go/src/miller/mappers/tail.go @@ -60,7 +60,7 @@ func mapperTailParseCLI( mapperTailUsage(ostream, args[0], verb, flagSet) } flagSet.Parse(args[argi:]) - if errorHandling == flag.ContinueOnError { // help intentioally requested + if errorHandling == flag.ContinueOnError { // help intentionally requested return nil } @@ -129,15 +129,15 @@ func (this *MapperTail) Map( inrec := inrecAndContext.Record if inrec != nil { // not end of record stream - groupByKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNameList) + groupingKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNameList) if !ok { return } - irecordListForGroup := this.recordListsByGroup.Get(groupByKey) + irecordListForGroup := this.recordListsByGroup.Get(groupingKey) if irecordListForGroup == nil { // first time irecordListForGroup = list.New() - this.recordListsByGroup.Put(groupByKey, irecordListForGroup) + this.recordListsByGroup.Put(groupingKey, irecordListForGroup) } recordListForGroup := irecordListForGroup.(*list.List) diff --git a/go/src/miller/types/mlrval_functions.go b/go/src/miller/types/mlrval_functions.go index 1cc9f5692..41be2ae9e 100644 --- a/go/src/miller/types/mlrval_functions.go +++ b/go/src/miller/types/mlrval_functions.go @@ -74,6 +74,9 @@ type TernaryFunc func(*Mlrval, *Mlrval, *Mlrval) Mlrval // Function-pointer type for variadic functions. type VariadicFunc func([]*Mlrval) Mlrval +// Function-pointer type for sorting. Returns < 0 if a < b, 0 if a == b, > 0 if a > b. +type ComparatorFunc func(*Mlrval, *Mlrval) int + // ================================================================ // The following are frequently used in disposition matrices for various // operators and are defined here for re-use. The names are VERY short, @@ -1557,3 +1560,172 @@ func MlrvalVariadicMax(mlrvals []*Mlrval) Mlrval { return retval } } + +// ================================================================ +// For sorting + +// Lexical sort: just stringify everything. +func LexicalAscendingComparatorfunc(ma *Mlrval, mb *Mlrval) int { + sa := ma.String() + sb := mb.String() + if sa < sb { + return -1 + } else if sa > sb { + return 1 + } else { + return 0 + } +} +func LexicalDescendingComparatorfunc(ma *Mlrval, mb *Mlrval) int { + return LexicalAscendingComparatorfunc(mb, ma) +} + +// ---------------------------------------------------------------- +// Sort rules (same for min, max, and comparator): +// * NUMERICS < BOOL < STRINGS < ERROR < ABSENT +// * error == error (singleton type) +// * absent == absent (singleton type) +// * string compares on strings +// * numeric compares on numbers +// * false < true + +func _neg1(ma, mb *Mlrval) int { + return -1 +} +func _zero(ma, mb *Mlrval) int { + return 0 +} +func _pos1(ma, mb *Mlrval) int { + return 1 +} + +func _scmp(ma, mb *Mlrval) int { + if ma.printrep < mb.printrep { + return -1 + } else if ma.printrep > mb.printrep { + return 1 + } else { + return 0 + } +} + +func iicmp(ma, mb *Mlrval) int { + ca := ma.intval + cb := mb.intval + if ca < cb { + return -1 + } else if ca > cb { + return 1 + } else { + return 0 + } +} +func ifcmp(ma, mb *Mlrval) int { + ca := float64(ma.intval) + cb := mb.floatval + if ca < cb { + return -1 + } else if ca > cb { + return 1 + } else { + return 0 + } +} +func ficmp(ma, mb *Mlrval) int { + ca := ma.floatval + cb := float64(mb.intval) + if ca < cb { + return -1 + } else if ca > cb { + return 1 + } else { + return 0 + } +} +func ffcmp(ma, mb *Mlrval) int { + ca := ma.floatval + cb := mb.floatval + if ca < cb { + return -1 + } else if ca > cb { + return 1 + } else { + return 0 + } +} + +func bbcmp(ma, mb *Mlrval) int { + a := ma.boolval + b := mb.boolval + if a == false { + if b == false { + return 0 + } else { + return -1 + } + } else { + if b == false { + return 1 + } else { + return 0 + } + } +} + +// ---------------------------------------------------------------- +// Sort rules (same for min, max, and comparator): +// * NUMERICS < BOOL < STRINGS < ERROR < ABSENT +// * error == error (singleton type) +// * absent == absent (singleton type) +// * string compares on strings +// * numeric compares on numbers +// * false < true + +var num_cmp_dispositions = [MT_DIM][MT_DIM]ComparatorFunc{ + // . ERROR ABSENT EMPTY STRING INT FLOAT BOOL ARRAY MAP + /*ERROR */ {_zero, _neg1, _pos1, _pos1, _pos1, _pos1, _pos1, _zero, _zero}, + /*ABSENT */ {_pos1, _zero, _pos1, _pos1, _pos1, _pos1, _pos1, _zero, _zero}, + /*EMPTY */ {_neg1, _neg1, _scmp, _scmp, _pos1, _pos1, _pos1, _zero, _zero}, + /*STRING */ {_neg1, _neg1, _scmp, _scmp, _pos1, _pos1, _pos1, _zero, _zero}, + /*INT */ {_neg1, _neg1, _neg1, _neg1, iicmp, ifcmp, _neg1, _zero, _zero}, + /*FLOAT */ {_neg1, _neg1, _neg1, _neg1, ficmp, ffcmp, _neg1, _zero, _zero}, + /*BOOL */ {_neg1, _neg1, _neg1, _neg1, _pos1, _pos1, bbcmp, _zero, _zero}, + /*ARRAY */ {_zero, _zero, _zero, _zero, _zero, _zero, _zero, _zero, _zero}, + /*MAP */ {_zero, _zero, _zero, _zero, _zero, _zero, _zero, _zero, _zero}, +} + +func NumericAscendingComparatorfunc(ma *Mlrval, mb *Mlrval) int { + return num_cmp_dispositions[ma.mvtype][mb.mvtype](ma, mb) +} +func NumericDescendingComparatorfunc(ma *Mlrval, mb *Mlrval) int { + return NumericAscendingComparatorfunc(mb, ma) +} + +//static int mv_cmp_eq(const mv_t* pa, const mv_t* pb) { return 0; } +//static int mv_cmp_lt(const mv_t* pa, const mv_t* pb) { return -1; } +//static int mv_cmp_gt(const mv_t* pa, const mv_t* pb) { return 1; } +// +//static int mv_bb_comparator(const mv_t* pa, const mv_t* pb) { +// int d = pa->u.boolv - pb->u.boolv; +// return (d < 0) ? -1 : (d > 0) ? 1 : 0; +//} +//static int mv_ss_cmp(const mv_t* pa, const mv_t* pb) { +// return strcmp(pa->u.strv, pb->u.strv); +//} + +//static mv_i_xx_comparator_func_t* mv_xx_comparator_dispositions[MT_DIM][MT_DIM] = { +// // ERROR ABSENT EMPTY STRING INT FLOAT BOOL +// /*ERROR*/ {mv_cmp_eq, mv_cmp_lt, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt}, +// /*ABSENT*/ {mv_cmp_gt, mv_cmp_eq, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt}, +// /*EMPTY*/ {mv_cmp_lt, mv_cmp_lt, mv_cmp_eq, mv_ss_cmp, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt}, +// /*STRING*/ {mv_cmp_lt, mv_cmp_lt, mv_ss_cmp, mv_ss_cmp, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt}, +// /*INT*/ {mv_cmp_lt, mv_cmp_lt, mv_cmp_lt, mv_cmp_lt, mv_ii_cmp, mv_if_cmp, mv_cmp_lt}, +// /*FLOAT*/ {mv_cmp_lt, mv_cmp_lt, mv_cmp_lt, mv_cmp_lt, mv_fi_cmp, mv_ff_cmp, mv_cmp_lt}, +// /*BOOL*/ {mv_cmp_lt, mv_cmp_lt, mv_cmp_lt, mv_cmp_lt, mv_cmp_gt, mv_cmp_gt, mv_bb_comparator}, +// }; +// +//int mv_xx_comparator(const void* pva, const void* pvb) { +// const mv_t* pa = pva; +// const mv_t* pb = pvb; +// return mv_xx_comparator_dispositions[pa->type][pb->type](pa, pb); +//} diff --git a/go/todo.txt b/go/todo.txt index de68c534c..b58601853 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -2,7 +2,12 @@ TOP OF LIST: * go-try: + o groupByKey -> groupingKey x all mappers + + o rename: bulk-edit for b,i,x vs x,i,b cases + ! sort + - UT per se for 'NUMERICS < BOOL < STRINGS < ERROR < ABSENT' ! filter ! emit; print/dump @@ -15,6 +20,8 @@ TOP OF LIST: o clean up mlr --help to not advertise things it doesn't do yet o ASV? +! all dispo matrices -- check new MT_DIM -- ! + * widen verb coverage o count o sort diff --git a/go/u/try-help.out b/go/u/try-help.out index 25c4e344c..aac69822e 100644 --- a/go/u/try-help.out +++ b/go/u/try-help.out @@ -58,8 +58,24 @@ Usage: mlr rename [options] {old1,new1,old2,new2,...} Renames specified fields. ================================================================ -Usage: mlr sort [options] -Outputs records in batches having identical values at specified field names. +Usage: mlr sort {flags} +Sorts records primarily by the first specified field, secondarily by the second +field, and so on. (Any records not having all specified sort keys will appear +at the end of the output, in the order they were encountered, regardless of the +specified sort order.) The sort is stable: records that compare equal will sort +in the order they were encountered in the input record stream. + +Flags: + -f {comma-separated field names} Lexical ascending + -n {comma-separated field names} Numerical ascending; nulls sort last + -nf {comma-separated field names} Same as -n + -r {comma-separated field names} Lexical descending + -nr {comma-separated field names} Numerical descending; nulls sort first + +Example: + mlr sort -f a,b -nr x,y,z +which is the same as: + mlr sort -f a -f b -nr x -nr y -nr z ================================================================ Usage: mlr tac