Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-08-04 05:23:20 +00:00
Code Issues Projects Releases Wiki Activity

mlr sort iterate

Browse source
This commit is contained in:
John Kerl 2020-09-17 18:27:26 -04:00
parent 897bb4a811
commit 2e75273de3
15 changed files with 365 additions and 366 deletions
Download patch file Download diff file Expand all files Collapse all files

33
go/src/miller/mappers/cat.go
View file

@ -54,40 +54,11 @@ func mapperCatParseCLI(
mapperCatUsage(ostream, args[0], verb, flagSet)
}
flagSet.Parse(args[argi:])
if errorHandling == flag.ContinueOnError { // help intentioally requested
if errorHandling == flag.ContinueOnError { // help intentionally requested
return nil
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// xxx to be ported:
//
// char* default_counter_field_name = DEFAULT_COUNTER_FIELD_NAME;
// char* counter_field_name = nil;
// int do_counters = FALSE;
// int verbose = FALSE;
// slls_t* pgroup_by_field_names = slls_alloc();
//
// ap_state_t* pstate = ap_alloc();
// ap_define_true_flag(pstate, "-n", &do_counters);
// ap_define_true_flag(pstate, "-v", &verbose);
// ap_define_string_flag(pstate, "-N", &counter_field_name);
// ap_define_string_list_flag(pstate, "-g", &pgroup_by_field_names);
//
// if (!ap_parse(pstate, verb, pargi, argc, args)) {
// mapper_cat_usage(stderr, args[0], verb);
// return nil;
// }
//
// if (counter_field_name != nil) {
// do_counters = TRUE;
// } else if (do_counters) {
// counter_field_name = default_counter_field_name;
// }
//
// mapper_t* pmapper = mapper_cat_alloc(pstate, do_counters, verbose, counter_field_name, pgroup_by_field_names);
// return pmapper;
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// xxx to port:
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// fmt.Fprintf(o, "Options:\n");
// fmt.Fprintf(o, "-n Prepend field \"%s\" to each record with record-counter starting at 1\n",

4
go/src/miller/mappers/cut.go
View file

@ -74,7 +74,7 @@ Default is to retain them in the order found in the input data.`,
mapperCutUsage(ostream, args[0], verb, flagSet)
}
flagSet.Parse(args[argi:])
if errorHandling == flag.ContinueOnError { // help intentioally requested
if errorHandling == flag.ContinueOnError { // help intentionally requested
return nil
}
@ -156,6 +156,7 @@ func NewMapperCut(
return this, nil
}
// xxx to port:
// if (!do_regexes) {
// pstate->pfield_name_list = pfield_name_list;
// slls_reverse(pstate->pfield_name_list);
@ -248,6 +249,7 @@ func (this *MapperCut) exclude(
outputChannel <- inrecAndContext
}
// xxx to port:
//// ----------------------------------------------------------------
//static sllv_t* mapper_cut_process_with_regexes(lrec_t* pinrec, context_t* pctx, void* pvstate) {
// if (pinrec != NULL) {

8
go/src/miller/mappers/group-by.go
View file

@ -44,7 +44,7 @@ func mapperGroupByParseCLI(
mapperGroupByUsage(ostream, args[0], verb, flagSet)
}
flagSet.Parse(args[argi:])
if errorHandling == flag.ContinueOnError { // help intentioally requested
if errorHandling == flag.ContinueOnError { // help intentionally requested
return nil
}
@ -117,15 +117,15 @@ func (this *MapperGroupBy) Map(
inrec := inrecAndContext.Record
if inrec != nil { // not end of record stream
groupByKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNameList)
groupingKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNameList)
if !ok {
return
}
recordListForGroup := this.recordListsByGroup.Get(groupByKey)
recordListForGroup := this.recordListsByGroup.Get(groupingKey)
if recordListForGroup == nil {
recordListForGroup = list.New()
this.recordListsByGroup.Put(groupByKey, recordListForGroup)
this.recordListsByGroup.Put(groupingKey, recordListForGroup)
}
recordListForGroup.(*list.List).PushBack(inrecAndContext)

8
go/src/miller/mappers/group-like.go
View file

@ -44,7 +44,7 @@ func mapperGroupLikeParseCLI(
mapperGroupLikeUsage(ostream, args[0], verb, flagSet)
}
flagSet.Parse(args[argi:])
if errorHandling == flag.ContinueOnError { // help intentioally requested
if errorHandling == flag.ContinueOnError { // help intentionally requested
return nil
}
@ -97,12 +97,12 @@ func (this *MapperGroupLike) Map(
inrec := inrecAndContext.Record
if inrec != nil { // not end of record stream
groupByKey := inrec.GetKeysJoined()
groupingKey := inrec.GetKeysJoined()
recordListForGroup := this.recordListsByGroup.Get(groupByKey)
recordListForGroup := this.recordListsByGroup.Get(groupingKey)
if recordListForGroup == nil { // first time
recordListForGroup = list.New()
this.recordListsByGroup.Put(groupByKey, recordListForGroup)
this.recordListsByGroup.Put(groupingKey, recordListForGroup)
}
recordListForGroup.(*list.List).PushBack(inrecAndContext)

10
go/src/miller/mappers/head.go
View file

@ -59,7 +59,7 @@ func mapperHeadParseCLI(
mapperHeadUsage(ostream, args[0], verb, flagSet)
}
flagSet.Parse(args[argi:])
if errorHandling == flag.ContinueOnError { // help intentioally requested
if errorHandling == flag.ContinueOnError { // help intentionally requested
return nil
}
@ -164,17 +164,17 @@ func (this *MapperHead) mapKeyed(
inrec := inrecAndContext.Record
if inrec != nil { // not end of record stream
groupByKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNameList)
groupingKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNameList)
if !ok {
return
}
count, present := this.keyedRecordCounts[groupByKey]
count, present := this.keyedRecordCounts[groupingKey]
if !present { // first time
this.keyedRecordCounts[groupByKey] = 1
this.keyedRecordCounts[groupingKey] = 1
count = 1
} else {
this.keyedRecordCounts[groupByKey] += 1
this.keyedRecordCounts[groupingKey] += 1
count += 1
}

2
go/src/miller/mappers/label.go
View file

@ -43,7 +43,7 @@ func mapperLabelParseCLI(
mapperLabelUsage(ostream, args[0], verb, flagSet)
}
flagSet.Parse(args[argi:])
if errorHandling == flag.ContinueOnError { // help intentioally requested
if errorHandling == flag.ContinueOnError { // help intentionally requested
return nil
}

2
go/src/miller/mappers/nothing.go
View file

@ -41,7 +41,7 @@ func mapperNothingParseCLI(
mapperNothingUsage(ostream, args[0], verb, flagSet)
}
flagSet.Parse(args[argi:])
if errorHandling == flag.ContinueOnError { // help intentioally requested
if errorHandling == flag.ContinueOnError { // help intentionally requested
return nil
}

2
go/src/miller/mappers/put.go
View file

@ -64,7 +64,7 @@ func mapperPutParseCLI(
mapperPutUsage(ostream, args[0], verb, flagSet)
}
flagSet.Parse(args[argi:])
if errorHandling == flag.ContinueOnError { // help intentioally requested
if errorHandling == flag.ContinueOnError { // help intentionally requested
return nil
}

2
go/src/miller/mappers/rename.go
View file

@ -44,7 +44,7 @@ func mapperRenameParseCLI(
mapperRenameUsage(ostream, args[0], verb, flagSet)
}
flagSet.Parse(args[argi:])
if errorHandling == flag.ContinueOnError { // help intentioally requested
if errorHandling == flag.ContinueOnError { // help intentionally requested
return nil
}

451
go/src/miller/mappers/sort.go
View file

@ -19,17 +19,17 @@ package mappers
// E.g. the list ["red", "1.0"] maps to the array ["red", 1.0].
//
// * The pairing of parsed-value array the linked list of same-key-value records
// is called a *bucket*. E.g the records
// is called a *bucket* or a *group*. E.g the records
// {"a":"red","b":"circle","x":"1.0","y":"3.9"}
// {"a":"red","b":"square","x":"1.0","z":"5.7", "q":"even"}
// would both land in the ["red","1.0"] bucket.
// would both land in the ["red","1.0"] group.
//
// * Buckets are retained in a hash map: the key is the string-list of the form
// * Groups are retained in a hash map: the key is the string-list of the form
// ["red","1.0"] and the value is the pairing of parsed-value array ["red",1.0]
// and linked list of records.
//
// * Once all the input records are ingested into this hash map, we copy the
// bucket-pointers into an array and sort it: this being the pairing of
// group-pointers into an array and sort it: this being the pairing of
// parsed-value array and linked list of records. The comparator callback for
// the sort walks through the parsed-value arrays one slot at a time,
// looking at the first difference, e.g. if one has "a"="red" and the other
@ -37,7 +37,7 @@ package mappers
// second field, and so on.
//
// * Note in particular that string keys ["a":"red","x":"1"] and
// ["a":"red","x":"1.0"] map to different buckets, but will sort equally.
// ["a":"red","x":"1.0"] map to different groups, but will sort equally.
//
// ================================================================
@ -75,55 +75,129 @@ func mapperSortParseCLI(
verb := args[argi]
argi++
// Parse local flags
flagSet := flag.NewFlagSet(verb, errorHandling)
// Unlike other mappers, we can't use flagSet here. The syntax of 'mlr
// sort' is it needs to take things like 'mlr sort -f a -n b -n c', i.e.
// first sort lexically on field a, then numerically on field b, then
// lexically on field c. The flagSet API would let the '-f c' clobber the
// '-f a', while we want both.
flagSet.Usage = func() {
ostream := os.Stderr
if errorHandling == flag.ContinueOnError { // help intentionally requested
ostream = os.Stdout
groupByFieldNameList := make([]string, 0)
comparatorFuncs := make([]types.ComparatorFunc, 0)
for argi < argc /* variable increment: 1 or 2 depending on flag */ {
if args[argi][0] != '-' {
break // No more flag options to process
} else if args[argi] == "-h" || args[argi] == "--help" {
mapperSortUsage(os.Stdout, 0, errorHandling, args[0], verb)
argi += 1
} else if args[argi] == "-f" {
subList := lib.SplitString(args[argi+1], ",")
for _, item := range subList {
groupByFieldNameList = append(groupByFieldNameList, item)
comparatorFuncs = append(comparatorFuncs, types.LexicalAscendingComparatorfunc)
}
// xxx append sorters lexical ascending
argi += 2
} else if args[argi] == "-r" {
subList := lib.SplitString(args[argi+1], ",")
for _, item := range subList {
groupByFieldNameList = append(groupByFieldNameList, item)
comparatorFuncs = append(comparatorFuncs, types.LexicalDescendingComparatorfunc)
}
// xxx append sorters lexical scending
argi += 2
} else if args[argi] == "-n" {
subList := lib.SplitString(args[argi+1], ",")
for _, item := range subList {
groupByFieldNameList = append(groupByFieldNameList, item)
comparatorFuncs = append(comparatorFuncs, types.NumericAscendingComparatorfunc)
}
// xxx append sorters numeric ascending
argi += 2
} else if args[argi] == "-nf" {
subList := lib.SplitString(args[argi+1], ",")
for _, item := range subList {
groupByFieldNameList = append(groupByFieldNameList, item)
comparatorFuncs = append(comparatorFuncs, types.NumericAscendingComparatorfunc)
}
// xxx append sorters numeric ascending
argi += 2
} else if args[argi] == "-nr" {
subList := lib.SplitString(args[argi+1], ",")
for _, item := range subList {
groupByFieldNameList = append(groupByFieldNameList, item)
comparatorFuncs = append(comparatorFuncs, types.NumericDescendingComparatorfunc)
}
// xxx append sorters numeric descending
argi += 2
} else {
mapperSortUsage(os.Stderr, 1, flag.ExitOnError, args[0], verb)
}
mapperSortUsage(ostream, args[0], verb, flagSet)
}
flagSet.Parse(args[argi:])
if errorHandling == flag.ContinueOnError { // help intentioally requested
// xxx
if errorHandling == flag.ContinueOnError { // help intentionally requested
return nil
}
// Find out how many flags were consumed by this verb and advance for the
// next verb
argi = len(args) - len(flagSet.Args())
// Get the sort field names from the command line
if argi >= argc {
flagSet.Usage()
mapperSortUsage(os.Stderr, 1, flag.ExitOnError, args[0], verb)
os.Exit(1)
}
groupByFieldNames := args[argi]
argi += 1
mapper, _ := NewMapperSort(
groupByFieldNames,
groupByFieldNameList,
comparatorFuncs,
)
*pargi = argi
return mapper
}
// For flags with values, e.g. ["-n" "10"], while we're looking at the "-n"
// this let us see if the "10" slot exists.
func checkArgCount(args []string, argi int, argc int, n int) {
if (argc - argi) < n {
fmt.Fprintf(os.Stderr, "%s: option \"%s\" missing argument(s).\n", args[0], args[argi])
mapperSortUsage(os.Stderr, 1, flag.ExitOnError, os.Args[0], "sort")
os.Exit(1)
}
}
func mapperSortUsage(
o *os.File,
exitCode int,
errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError
argv0 string,
verb string,
flagSet *flag.FlagSet,
) {
fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb)
fmt.Fprint(o,
`Outputs records in batches having identical values at specified field names.
`)
// flagSet.PrintDefaults() doesn't let us control stdout vs stderr
flagSet.VisitAll(func(f *flag.Flag) {
fmt.Fprintf(o, " -%v (default %v) %v\n", f.Name, f.Value, f.Usage) // f.Name, f.Value
})
fmt.Fprintf(o, "Usage: %s %s {flags}\n", argv0, verb)
fmt.Fprintf(o, "Sorts records primarily by the first specified field, secondarily by the second\n")
fmt.Fprintf(o, "field, and so on. (Any records not having all specified sort keys will appear\n")
fmt.Fprintf(o, "at the end of the output, in the order they were encountered, regardless of the\n")
fmt.Fprintf(o, "specified sort order.) The sort is stable: records that compare equal will sort\n")
fmt.Fprintf(o, "in the order they were encountered in the input record stream.\n")
fmt.Fprintf(o, "\n")
fmt.Fprintf(o, "Flags:\n")
fmt.Fprintf(o, " -f {comma-separated field names} Lexical ascending\n")
fmt.Fprintf(o, " -n {comma-separated field names} Numerical ascending; nulls sort last\n")
fmt.Fprintf(o, " -nf {comma-separated field names} Same as -n\n")
fmt.Fprintf(o, " -r {comma-separated field names} Lexical descending\n")
fmt.Fprintf(o, " -nr {comma-separated field names} Numerical descending; nulls sort first\n")
fmt.Fprintf(o, "\n")
fmt.Fprintf(o, "Example:\n")
fmt.Fprintf(o, " %s %s -f a,b -nr x,y,z\n", argv0, verb)
fmt.Fprintf(o, "which is the same as:\n")
fmt.Fprintf(o, " %s %s -f a -f b -nr x -nr y -nr z\n", argv0, verb)
if errorHandling == flag.ExitOnError {
os.Exit(exitCode)
}
}
// ----------------------------------------------------------------
@ -132,39 +206,40 @@ func mapperSortUsage(
// * group-by field-name list is "a,i"
// * input record 'a=pan,b=pan,i=1,x=0.3467,y=0.7268'
// o values at a,i are "pan",1
// o grouping key for the ordered map from string to record-bucket is the string "pan,1"
// o grouping key for the ordered map from string to record-group is the string "pan,1"
// o we also need a map from "pan,1" to the array of mlrvals ["pan", 1].
// * next input record 'a=eks,b=pan,i=2,x=0.7586,y=0.5221'
// o values at a,i are "eks",2
// o grouping key for the ordered map from string to record-bucket is the string "eks,2"
// o grouping key for the ordered map from string to record-group is the string "eks,2"
// o we also need a map from "eks,2" to the array of mlrvals ["eks", 2].
// * what gets sorted are the bucket-heading arrays of mlrvals:
// * what gets sorted are the group-heading arrays of mlrvals:
// o make an array [ ("pan,1", ["pan", 1]), ("eks,2", ["eks", 2])
// o sort that
// * output is simply for each slot in the array, emit each record in the bucket
// * output is simply for each slot in the array, emit each record in the group
type MapperSort struct {
// Input
// -- Input
groupByFieldNameList []string
comparatorFuncs []types.ComparatorFunc
// State
// -- State
// Map from string to *list.List:
recordListsByGroup *lib.OrderedMap
// Map from string to []*lib.Mlrval:
bucketHeads *lib.OrderedMap
groupHeads *lib.OrderedMap
}
func NewMapperSort(
groupByFieldNames string,
groupByFieldNameList []string,
comparatorFuncs []types.ComparatorFunc,
) (*MapperSort, error) {
groupByFieldNameList := lib.SplitString(groupByFieldNames, ",")
this := &MapperSort{
groupByFieldNameList: groupByFieldNameList,
comparatorFuncs: comparatorFuncs,
recordListsByGroup: lib.NewOrderedMap(),
bucketHeads: lib.NewOrderedMap(),
groupHeads: lib.NewOrderedMap(),
}
return this, nil
@ -183,27 +258,26 @@ func (this *MapperSort) Map(
inrec := inrecAndContext.Record
if inrec != nil { // Not end of record stream
// xxx need to factor out the joined and non-joined -- this verb needs both
groupByKey, selectedValues, ok := inrec.GetSelectedValuesAndJoined(
groupingKey, selectedValues, ok := inrec.GetSelectedValuesAndJoined(
this.groupByFieldNameList,
)
if !ok {
// xxx need a spill-bucket
// xxx need a spill-group
return
}
recordListForGroup := this.recordListsByGroup.Get(groupByKey)
recordListForGroup := this.recordListsByGroup.Get(groupingKey)
if recordListForGroup == nil {
recordListForGroup = list.New()
this.recordListsByGroup.Put(groupByKey, recordListForGroup)
this.bucketHeads.Put(groupByKey, selectedValues)
this.recordListsByGroup.Put(groupingKey, recordListForGroup)
this.groupHeads.Put(groupingKey, selectedValues)
}
recordListForGroup.(*list.List).PushBack(inrecAndContext)
} else { // End of record stream
// At this point, in the above example, bucketHeads is:
// At this point, in the above example, groupHeads is:
//
// {
// "pan,1" : ["pan", 1],
@ -217,23 +291,29 @@ func (this *MapperSort) Map(
// [ "eks,2", ["eks', 2]
// ]
groupingKeysAndMlrvals := bucketHeadsToArray(this.bucketHeads)
groupingKeysAndMlrvals := groupHeadsToArray(this.groupHeads)
// Next sort that array
// xxx this is a stub:
// xxx make a helper method
// Go sort API: for ascending sort, return true if element i < element j.
sort.Slice(groupingKeysAndMlrvals, func(i, j int) bool {
a, _ := groupingKeysAndMlrvals[i].mlrvals[0].GetFloatValue()
b, _ := groupingKeysAndMlrvals[j].mlrvals[0].GetFloatValue()
return a < b
//return groupingKeysAndMlrvals[i].mlrvals[0] > groupingKeysAndMlrvals[j].mlrvals[0]
for k, comparator := range(this.comparatorFuncs) {
result := comparator(
&groupingKeysAndMlrvals[i].mlrvals[k],
&groupingKeysAndMlrvals[j].mlrvals[k],
)
if result < 0 {
return true
} else if result > 0 {
return false
}
}
return false
})
// Now output the buckets
// Now output the groups
for _, groupingKeyAndMlrvals := range groupingKeysAndMlrvals {
iRecordsInBucket := this.recordListsByGroup.Get(groupingKeyAndMlrvals.groupingKey)
recordsInBucket := iRecordsInBucket.(*list.List)
for iRecord := recordsInBucket.Front(); iRecord != nil; iRecord = iRecord.Next() {
iRecordsInGroup := this.recordListsByGroup.Get(groupingKeyAndMlrvals.groupingKey)
recordsInGroup := iRecordsInGroup.(*list.List)
for iRecord := recordsInGroup.Front(); iRecord != nil; iRecord = iRecord.Next() {
outputChannel <- iRecord.Value.(*types.RecordAndContext)
}
}
@ -242,11 +322,11 @@ func (this *MapperSort) Map(
}
}
func bucketHeadsToArray(bucketHeads *lib.OrderedMap) []GroupingKeysAndMlrvals {
retval := make([]GroupingKeysAndMlrvals, bucketHeads.FieldCount)
func groupHeadsToArray(groupHeads *lib.OrderedMap) []GroupingKeysAndMlrvals {
retval := make([]GroupingKeysAndMlrvals, groupHeads.FieldCount)
i := 0
for entry := bucketHeads.Head; entry != nil; entry = entry.Next {
for entry := groupHeads.Head; entry != nil; entry = entry.Next {
retval[i] = GroupingKeysAndMlrvals{
groupingKey: entry.Key,
mlrvals: entry.Value.([]types.Mlrval),
@ -256,252 +336,3 @@ func bucketHeadsToArray(bucketHeads *lib.OrderedMap) []GroupingKeysAndMlrvals {
return retval
}
//type GroupingKeysAndMlrvals struct {
// groupingKey string
// mlrvals []*types.Mlrval
//}
//#define SORT_NUMERIC 0x80
//#define SORT_DESCENDING 0x40
//typedef struct _mapper_sort_state_t {
// // Input parameters
// slls_t* pkey_field_names; // Fields to sort on
// int* sort_params; // Lexical/numeric; ascending/descending
// // Sort state: buckets of like records.
// lhmslv_t* pbuckets_by_key_field_values;
// sllv_t* precords_missing_sort_keys;
//} mapper_sort_state_t;
//typedef struct _sort_bucket_t {
// typed_sort_key_t* typed_sort_keys;
// sllv_t* precords;
//} sort_bucket_t;
//// ----------------------------------------------------------------
//static void mapper_sort_usage(FILE* o, char* argv0, char* verb) {
// fprintf(o, "Usage: %s %s {flags}\n", argv0, verb);
// fprintf(o, "Flags:\n");
// fprintf(o, " -f {comma-separated field names} Lexical ascending\n");
// fprintf(o, " -n {comma-separated field names} Numerical ascending; nulls sort last\n");
// fprintf(o, " -nf {comma-separated field names} Same as -n\n");
// fprintf(o, " -r {comma-separated field names} Lexical descending\n");
// fprintf(o, " -nr {comma-separated field names} Numerical descending; nulls sort first\n");
// fprintf(o, "Sorts records primarily by the first specified field, secondarily by the second\n");
// fprintf(o, "field, and so on. (Any records not having all specified sort keys will appear\n");
// fprintf(o, "at the end of the output, in the order they were encountered, regardless of the\n");
// fprintf(o, "specified sort order.) The sort is stable: records that compare equal will sort\n");
// fprintf(o, "in the order they were encountered in the input record stream.\n");
// fprintf(o, "\n");
// fprintf(o, "Example:\n");
// fprintf(o, " %s %s -f a,b -nr x,y,z\n", argv0, verb);
// fprintf(o, "which is the same as:\n");
// fprintf(o, " %s %s -f a -f b -nr x -nr y -nr z\n", argv0, verb);
//}
//static mapper_t* mapper_sort_parse_cli(int* pargi, int argc, char** argv,
// cli_reader_opts_t* _, cli_writer_opts_t* __)
//{
// if ((argc - *pargi) < 3) {
// mapper_sort_usage(stderr, argv[0], argv[*pargi]);
// return NULL;
// }
// char* verb = argv[*pargi];
// *pargi += 1;
// slls_t* pnames = slls_alloc();
// slls_t* pflags = slls_alloc();
//
// while ((argc - *pargi) >= 1 && argv[*pargi][0] == '-') {
// if ((argc - *pargi) < 2)
// mapper_sort_usage(stderr, argv[0], verb);
// char* flag = argv[*pargi];
// char* value = argv[*pargi+1];
// *pargi += 2;
//
// if (streq(flag, "-f")) {
// } else if (streq(flag, "-n")) {
// } else if (streq(flag, "-nf")) {
// } else if (streq(flag, "-r")) {
// } else if (streq(flag, "-nr")) {
// } else {
// mapper_sort_usage(stderr, argv[0], verb);
// }
// slls_t* pnames_for_flag = slls_from_line(value, ',', FALSE);
// // E.g. with "-nr a,b,c", replicate the "-nr" flag three times.
// for (sllse_t* pe = pnames_for_flag->phead; pe != NULL; pe = pe->pnext) {
// slls_append_no_free(pnames, pe->value);
// slls_append_no_free(pflags, flag);
// }
// slls_free(pnames_for_flag);
// }
//
// if (pnames->length < 1)
// mapper_sort_usage(stderr, argv[0], verb);
//
// // Convert the list such as ["-nf","-nf","-r","-r","-r"] into an array of
// // bit-flags, one per sort-key field.
// int* opt_array = mlr_malloc_or_die(pnames->length * sizeof(int));
// sllse_t* pe;
// int di;
// for (pe = pflags->phead, di = 0; pe != NULL; pe = pe->pnext, di++) {
// char* flag = pe->value;
// int opt =
// streq(flag, "-nf") ? SORT_NUMERIC :
// streq(flag, "-n") ? SORT_NUMERIC :
// streq(flag, "-r") ? SORT_DESCENDING :
// streq(flag, "-nr") ? SORT_NUMERIC|SORT_DESCENDING :
// 0;
// opt_array[di] =opt;
// }
// slls_free(pflags);
//
// return mapper_sort_alloc(pnames, opt_array, TRUE);
//}
//// ----------------------------------------------------------------
//static mapper_t* mapper_sort_alloc(slls_t* pkey_field_names, int* sort_params, int do_sort) {
// mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t));
//
// mapper_sort_state_t* pstate = mlr_malloc_or_die(sizeof(mapper_sort_state_t));
//
// pstate->pkey_field_names = pkey_field_names;
// pstate->sort_params = sort_params;
// pstate->pbuckets_by_key_field_values = lhmslv_alloc();
// pstate->precords_missing_sort_keys = sllv_alloc();
// pstate->do_sort = do_sort;
//
// pmapper->pvstate = pstate;
// pmapper->pprocess_func = mapper_sort_process;
// pmapper->pfree_func = mapper_sort_free;
//
// return pmapper;
//}
//
//// ----------------------------------------------------------------
//static void mapper_sort_free(mapper_t* pmapper, context_t* _) {
// mapper_sort_state_t* pstate = pmapper->pvstate;
// if (pstate->pkey_field_names != NULL)
// slls_free(pstate->pkey_field_names);
// // lhmslv_free will free the hashmap keys; we need to free the void-star hashmap values.
// for (lhmslve_t* pa = pstate->pbuckets_by_key_field_values->phead; pa != NULL; pa = pa->pnext) {
// sort_bucket_t* pbucket = pa->pvvalue;
// free(pbucket->typed_sort_keys);
// free(pbucket);
// // precords freed in emitter
// }
// lhmslv_free(pstate->pbuckets_by_key_field_values);
// sllv_free(pstate->precords_missing_sort_keys);
// free(pstate->sort_params);
// free(pstate);
// free(pmapper);
//}
//
//// ----------------------------------------------------------------
//static sllv_t* mapper_sort_process(lrec_t* pinrec, context_t* pctx, void* pvstate) {
// mapper_sort_state_t* pstate = pvstate;
// if (pinrec != NULL) {
// // Consume another input record.
// slls_t* pkey_field_values = mlr_reference_selected_values_from_record(pinrec, pstate->pkey_field_names);
// if (pkey_field_values == NULL) {
// sllv_append(pstate->precords_missing_sort_keys, pinrec);
// } else {
// sort_bucket_t* pbucket = lhmslv_get(pstate->pbuckets_by_key_field_values, pkey_field_values);
// if (pbucket == NULL) { // New key-field-value: new bucket and hash-map entry
// slls_t* pkey_field_values_copy = slls_copy(pkey_field_values);
// sort_bucket_t* pbucket = mlr_malloc_or_die(sizeof(sort_bucket_t));
// pbucket->typed_sort_keys = parse_sort_keys(pkey_field_values_copy, pstate->sort_params, pctx);
// pbucket->precords = sllv_alloc();
// sllv_append(pbucket->precords, pinrec);
// lhmslv_put(pstate->pbuckets_by_key_field_values, pkey_field_values_copy, pbucket,
// FREE_ENTRY_KEY);
// } else { // Previously seen key-field-value: append record to bucket
// sllv_append(pbucket->precords, pinrec);
// }
// slls_free(pkey_field_values);
// }
// return NULL;
// } else {
// // End of input stream: sort bucket labels
// int num_buckets = pstate->pbuckets_by_key_field_values->num_occupied;
// sort_bucket_t** pbucket_array = mlr_malloc_or_die(num_buckets * sizeof(sort_bucket_t*));
//
// // Copy bucket-pointers to an array for qsort
// int i = 0;
// for (lhmslve_t* pe = pstate->pbuckets_by_key_field_values->phead; pe != NULL; pe = pe->pnext, i++) {
// pbucket_array[i] = pe->pvvalue;
// }
//
// pcmp_sort_params = pstate->sort_params;
// cmp_params_length = pstate->pkey_field_names->length;
//
// qsort(pbucket_array, num_buckets, sizeof(sort_bucket_t*), pbucket_comparator);
//
// pcmp_sort_params = NULL;
// cmp_params_length = 0;
//
// // Emit each bucket's record
// sllv_t* poutput = sllv_alloc();
// for (i = 0; i < num_buckets; i++) {
// sllv_t* plist = pbucket_array[i]->precords;
// sllv_transfer(poutput, plist);
// sllv_free(plist);
// }
// sllv_transfer(poutput, pstate->precords_missing_sort_keys);
// free(pbucket_array);
// sllv_append(poutput, NULL); // Signal end of output-record stream.
// return poutput;
// }
//}
//
//static int pbucket_comparator(const void* pva, const void* pvb) {
// // We are sorting an array of sort_bucket_t*.
// const sort_bucket_t** pba = (const sort_bucket_t**)pva;
// const sort_bucket_t** pbb = (const sort_bucket_t**)pvb;
// typed_sort_key_t* akeys = (*pba)->typed_sort_keys;
// typed_sort_key_t* bkeys = (*pbb)->typed_sort_keys;
// for (int i = 0; i < cmp_params_length; i++) {
// int sort_param = pcmp_sort_params[i];
// if (sort_param & SORT_NUMERIC) {
// double a = akeys[i].u.d;
// double b = bkeys[i].u.d;
// if (isnan(a)) { // null input value
// if (!isnan(b)) {
// return (sort_param & SORT_DESCENDING) ? -1 : 1;
// }
// } else if (isnan(b)) {
// return (sort_param & SORT_DESCENDING) ? 1 : -1;
// } else {
// double d = a - b;
// int s = (d < 0) ? -1 : (d > 0) ? 1 : 0;
// if (s != 0)
// return (sort_param & SORT_DESCENDING) ? -s : s;
// }
// } else {
// int s = strcmp(akeys[i].u.s, bkeys[i].u.s);
// if (s != 0)
// return (sort_param & SORT_DESCENDING) ? -s : s;
// }
// }
// return 0;
//}
//
//// E.g. parse the list ["red","1.0"] into the array ["red",1.0].
//static typed_sort_key_t* parse_sort_keys(slls_t* pkey_field_values, int* sort_params, context_t* pctx) {
// typed_sort_key_t* typed_sort_keys = mlr_malloc_or_die(pkey_field_values->length * sizeof(typed_sort_key_t));
// int i = 0;
// for (sllse_t* pe = pkey_field_values->phead; pe != NULL; pe = pe->pnext, i++) {
// if (sort_params[i] & SORT_NUMERIC) {
// if (*pe->value == 0) { // null input value
// typed_sort_keys[i].u.d = nan("");
// } else if (!mlr_try_float_from_string(pe->value, &typed_sort_keys[i].u.d)) {
// fprintf(stderr, "%s: couldn't parse \"%s\" as number in file \"%s\" record %lld.\n",
// MLR_GLOBALS.bargv0, pe->value, pctx->filename, pctx->fnr);
// exit(1);
// }
// } else {
// typed_sort_keys[i].u.s = pe->value;
// }
// }
// return typed_sort_keys;
//}

2
go/src/miller/mappers/tac.go
View file

@ -42,7 +42,7 @@ func mapperTacParseCLI(
mapperTacUsage(ostream, args[0], verb, flagSet)
}
flagSet.Parse(args[argi:])
if errorHandling == flag.ContinueOnError { // help intentioally requested
if errorHandling == flag.ContinueOnError { // help intentionally requested
return nil
}

8
go/src/miller/mappers/tail.go
View file

@ -60,7 +60,7 @@ func mapperTailParseCLI(
mapperTailUsage(ostream, args[0], verb, flagSet)
}
flagSet.Parse(args[argi:])
if errorHandling == flag.ContinueOnError { // help intentioally requested
if errorHandling == flag.ContinueOnError { // help intentionally requested
return nil
}
@ -129,15 +129,15 @@ func (this *MapperTail) Map(
inrec := inrecAndContext.Record
if inrec != nil { // not end of record stream
groupByKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNameList)
groupingKey, ok := inrec.GetSelectedValuesJoined(this.groupByFieldNameList)
if !ok {
return
}
irecordListForGroup := this.recordListsByGroup.Get(groupByKey)
irecordListForGroup := this.recordListsByGroup.Get(groupingKey)
if irecordListForGroup == nil { // first time
irecordListForGroup = list.New()
this.recordListsByGroup.Put(groupByKey, irecordListForGroup)
this.recordListsByGroup.Put(groupingKey, irecordListForGroup)
}
recordListForGroup := irecordListForGroup.(*list.List)

172
go/src/miller/types/mlrval_functions.go
View file

@ -74,6 +74,9 @@ type TernaryFunc func(*Mlrval, *Mlrval, *Mlrval) Mlrval
// Function-pointer type for variadic functions.
type VariadicFunc func([]*Mlrval) Mlrval
// Function-pointer type for sorting. Returns < 0 if a < b, 0 if a == b, > 0 if a > b.
type ComparatorFunc func(*Mlrval, *Mlrval) int
// ================================================================
// The following are frequently used in disposition matrices for various
// operators and are defined here for re-use. The names are VERY short,
@ -1557,3 +1560,172 @@ func MlrvalVariadicMax(mlrvals []*Mlrval) Mlrval {
return retval
}
}
// ================================================================
// For sorting
// Lexical sort: just stringify everything.
func LexicalAscendingComparatorfunc(ma *Mlrval, mb *Mlrval) int {
sa := ma.String()
sb := mb.String()
if sa < sb {
return -1
} else if sa > sb {
return 1
} else {
return 0
}
}
func LexicalDescendingComparatorfunc(ma *Mlrval, mb *Mlrval) int {
return LexicalAscendingComparatorfunc(mb, ma)
}
// ----------------------------------------------------------------
// Sort rules (same for min, max, and comparator):
// * NUMERICS < BOOL < STRINGS < ERROR < ABSENT
// * error == error (singleton type)
// * absent == absent (singleton type)
// * string compares on strings
// * numeric compares on numbers
// * false < true
func _neg1(ma, mb *Mlrval) int {
return -1
}
func _zero(ma, mb *Mlrval) int {
return 0
}
func _pos1(ma, mb *Mlrval) int {
return 1
}
func _scmp(ma, mb *Mlrval) int {
if ma.printrep < mb.printrep {
return -1
} else if ma.printrep > mb.printrep {
return 1
} else {
return 0
}
}
func iicmp(ma, mb *Mlrval) int {
ca := ma.intval
cb := mb.intval
if ca < cb {
return -1
} else if ca > cb {
return 1
} else {
return 0
}
}
func ifcmp(ma, mb *Mlrval) int {
ca := float64(ma.intval)
cb := mb.floatval
if ca < cb {
return -1
} else if ca > cb {
return 1
} else {
return 0
}
}
func ficmp(ma, mb *Mlrval) int {
ca := ma.floatval
cb := float64(mb.intval)
if ca < cb {
return -1
} else if ca > cb {
return 1
} else {
return 0
}
}
func ffcmp(ma, mb *Mlrval) int {
ca := ma.floatval
cb := mb.floatval
if ca < cb {
return -1
} else if ca > cb {
return 1
} else {
return 0
}
}
func bbcmp(ma, mb *Mlrval) int {
a := ma.boolval
b := mb.boolval
if a == false {
if b == false {
return 0
} else {
return -1
}
} else {
if b == false {
return 1
} else {
return 0
}
}
}
// ----------------------------------------------------------------
// Sort rules (same for min, max, and comparator):
// * NUMERICS < BOOL < STRINGS < ERROR < ABSENT
// * error == error (singleton type)
// * absent == absent (singleton type)
// * string compares on strings
// * numeric compares on numbers
// * false < true
var num_cmp_dispositions = [MT_DIM][MT_DIM]ComparatorFunc{
// . ERROR ABSENT EMPTY STRING INT FLOAT BOOL ARRAY MAP
/*ERROR */ {_zero, _neg1, _pos1, _pos1, _pos1, _pos1, _pos1, _zero, _zero},
/*ABSENT */ {_pos1, _zero, _pos1, _pos1, _pos1, _pos1, _pos1, _zero, _zero},
/*EMPTY */ {_neg1, _neg1, _scmp, _scmp, _pos1, _pos1, _pos1, _zero, _zero},
/*STRING */ {_neg1, _neg1, _scmp, _scmp, _pos1, _pos1, _pos1, _zero, _zero},
/*INT */ {_neg1, _neg1, _neg1, _neg1, iicmp, ifcmp, _neg1, _zero, _zero},
/*FLOAT */ {_neg1, _neg1, _neg1, _neg1, ficmp, ffcmp, _neg1, _zero, _zero},
/*BOOL */ {_neg1, _neg1, _neg1, _neg1, _pos1, _pos1, bbcmp, _zero, _zero},
/*ARRAY */ {_zero, _zero, _zero, _zero, _zero, _zero, _zero, _zero, _zero},
/*MAP */ {_zero, _zero, _zero, _zero, _zero, _zero, _zero, _zero, _zero},
}
func NumericAscendingComparatorfunc(ma *Mlrval, mb *Mlrval) int {
return num_cmp_dispositions[ma.mvtype][mb.mvtype](ma, mb)
}
func NumericDescendingComparatorfunc(ma *Mlrval, mb *Mlrval) int {
return NumericAscendingComparatorfunc(mb, ma)
}
//static int mv_cmp_eq(const mv_t* pa, const mv_t* pb) { return 0; }
//static int mv_cmp_lt(const mv_t* pa, const mv_t* pb) { return -1; }
//static int mv_cmp_gt(const mv_t* pa, const mv_t* pb) { return 1; }
//
//static int mv_bb_comparator(const mv_t* pa, const mv_t* pb) {
// int d = pa->u.boolv - pb->u.boolv;
// return (d < 0) ? -1 : (d > 0) ? 1 : 0;
//}
//static int mv_ss_cmp(const mv_t* pa, const mv_t* pb) {
// return strcmp(pa->u.strv, pb->u.strv);
//}
//static mv_i_xx_comparator_func_t* mv_xx_comparator_dispositions[MT_DIM][MT_DIM] = {
// // ERROR ABSENT EMPTY STRING INT FLOAT BOOL
// /*ERROR*/ {mv_cmp_eq, mv_cmp_lt, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt},
// /*ABSENT*/ {mv_cmp_gt, mv_cmp_eq, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt},
// /*EMPTY*/ {mv_cmp_lt, mv_cmp_lt, mv_cmp_eq, mv_ss_cmp, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt},
// /*STRING*/ {mv_cmp_lt, mv_cmp_lt, mv_ss_cmp, mv_ss_cmp, mv_cmp_gt, mv_cmp_gt, mv_cmp_gt},
// /*INT*/ {mv_cmp_lt, mv_cmp_lt, mv_cmp_lt, mv_cmp_lt, mv_ii_cmp, mv_if_cmp, mv_cmp_lt},
// /*FLOAT*/ {mv_cmp_lt, mv_cmp_lt, mv_cmp_lt, mv_cmp_lt, mv_fi_cmp, mv_ff_cmp, mv_cmp_lt},
// /*BOOL*/ {mv_cmp_lt, mv_cmp_lt, mv_cmp_lt, mv_cmp_lt, mv_cmp_gt, mv_cmp_gt, mv_bb_comparator},
// };
//
//int mv_xx_comparator(const void* pva, const void* pvb) {
// const mv_t* pa = pva;
// const mv_t* pb = pvb;
// return mv_xx_comparator_dispositions[pa->type][pb->type](pa, pb);
//}

7
go/todo.txt
View file

@ -2,7 +2,12 @@
TOP OF LIST:
* go-try:
o groupByKey -> groupingKey x all mappers
o rename: bulk-edit for b,i,x vs x,i,b cases
! sort
- UT per se for 'NUMERICS < BOOL < STRINGS < ERROR < ABSENT'
! filter
! emit; print/dump
@ -15,6 +20,8 @@ TOP OF LIST:
o clean up mlr --help to not advertise things it doesn't do yet
o ASV?
! all dispo matrices -- check new MT_DIM -- !
* widen verb coverage
o count
o sort

20
go/u/try-help.out
View file

@ -58,8 +58,24 @@ Usage: mlr rename [options] {old1,new1,old2,new2,...}
Renames specified fields.
================================================================
Usage: mlr sort [options]
Outputs records in batches having identical values at specified field names.
Usage: mlr sort {flags}
Sorts records primarily by the first specified field, secondarily by the second
field, and so on. (Any records not having all specified sort keys will appear
at the end of the output, in the order they were encountered, regardless of the
specified sort order.) The sort is stable: records that compare equal will sort
in the order they were encountered in the input record stream.
Flags:
-f {comma-separated field names} Lexical ascending
-n {comma-separated field names} Numerical ascending; nulls sort last
-nf {comma-separated field names} Same as -n
-r {comma-separated field names} Lexical descending
-nr {comma-separated field names} Numerical descending; nulls sort first
Example:
mlr sort -f a,b -nr x,y,z
which is the same as:
mlr sort -f a -f b -nr x -nr y -nr z
================================================================
Usage: mlr tac