Add regexed field-selection to sort-within-records (#1964)

* BROKEN

* add a test case

* fix

* fix natural-sorting issue

* fix silent-drop issue

* test case

* doc update

docs/src/reference-verbs.md

@@ -3073,8 +3073,13 @@ a=3
 Usage: mlr sort-within-records [options]
 Outputs records sorted lexically ascending by keys.
 Options:
--r        Recursively sort subobjects/submaps, e.g. for JSON input.
--h|--help Show this message.
+-f {names}   Sort only these keys; others preserve record order.
+-r {regex}   Sort only keys matching this regex; others preserve record order.
+             Example: -r '^[xy]' sorts keys starting with x or y.
+             With no regex argument, -r recursively sorts subobjects/submaps
+             (e.g. for JSON input), or combines with -f to treat names as regex.
+-n           Sort field names naturally (e.g. 2 before 12). Combines with -f/-r.
+-h|--help    Show this message.
 </pre>
 
 <pre class="pre-highlight-in-pair">

pkg/transformers/sort_within_records.go

@@ -3,9 +3,15 @@ package transformers
 import (
 	"fmt"
 	"os"
+	"regexp"
+	"slices"
+	"sort"
 	"strings"
 
+	"github.com/facette/natsort"
 	"github.com/johnkerl/miller/v6/pkg/cli"
+	"github.com/johnkerl/miller/v6/pkg/lib"
+	"github.com/johnkerl/miller/v6/pkg/mlrval"
 	"github.com/johnkerl/miller/v6/pkg/types"
 )
 
@@ -24,8 +30,13 @@ func transformerSortWithinRecordsUsage(
 	fmt.Fprintf(o, "Usage: %s %s [options]\n", "mlr", verbNameSortWithinRecords)
 	fmt.Fprintln(o, "Outputs records sorted lexically ascending by keys.")
 	fmt.Fprintf(o, "Options:\n")
-	fmt.Fprintf(o, "-r        Recursively sort subobjects/submaps, e.g. for JSON input.\n")
-	fmt.Fprintf(o, "-h|--help Show this message.\n")
+	fmt.Fprintf(o, "-f {names}   Sort only these keys; others preserve record order.\n")
+	fmt.Fprintf(o, "-r {regex}   Sort only keys matching this regex; others preserve record order.\n")
+	fmt.Fprintf(o, "             Example: -r '^[xy]' sorts keys starting with x or y.\n")
+	fmt.Fprintf(o, "             With no regex argument, -r recursively sorts subobjects/submaps\n")
+	fmt.Fprintf(o, "             (e.g. for JSON input), or combines with -f to treat names as regex.\n")
+	fmt.Fprintf(o, "-n           Sort field names naturally (e.g. 2 before 12). Combines with -f/-r.\n")
+	fmt.Fprintf(o, "-h|--help    Show this message.\n")
 }
 
 func transformerSortWithinRecordsParseCLI(
@@ -38,8 +49,12 @@ func transformerSortWithinRecordsParseCLI(
 
 	// Skip the verb name from the current spot in the mlr command line
 	argi := *pargi
+	verb := args[argi]
 	argi++
 	doRecurse := false
+	var fieldNames []string = nil
+	doRegexes := false
+	doNatural := false
 
 	for argi < argc /* variable increment: 1 or 2 depending on flag */ {
 		opt := args[argi]
@@ -56,22 +71,46 @@ func transformerSortWithinRecordsParseCLI(
 			return nil, cli.ErrHelpRequested
 
 		} else if opt == "-r" {
-			doRecurse = true
+			// If the next token exists and isn't another flag, consume it as
+			// the regex pattern. Otherwise -r is arity-0: combined with a
+			// preceding -f it means regex mode; standalone it means recursive.
+			if argi < argc && !strings.HasPrefix(args[argi], "-") {
+				fieldNames = []string{args[argi]}
+				argi++
+				doRegexes = true
+			} else if fieldNames != nil {
+				doRegexes = true
+			} else {
+				doRecurse = true
+			}
+
+		} else if opt == "-f" {
+			names, err := cli.VerbGetStringArrayArg(verb, opt, args, &argi, argc)
+			if err != nil {
+				return nil, err
+			}
+			fieldNames = names
+
+		} else if opt == "-n" {
+			doNatural = true
 
 		} else {
 			return nil, cli.VerbErrorf(verbNameSortWithinRecords, "option \"%s\" not recognized", opt)
 		}
 	}
 
-	// TODO: allow sort by key or value?
-	// TODO: allow sort ascendending/descending?
+	// -r with -f means regex; -r without -f means recurse
+	if fieldNames != nil && doRecurse {
+		doRegexes = true
+		doRecurse = false
+	}
 
 	*pargi = argi
 	if !doConstruct { // All transformers must do this for main command-line parsing
 		return nil, nil
 	}
 
-	transformer, err := NewTransformerSortWithinRecords(doRecurse)
+	transformer, err := NewTransformerSortWithinRecords(doRecurse, fieldNames, doRegexes, doNatural)
 	if err != nil {
 		return nil, err
 	}
@@ -80,15 +119,61 @@ func transformerSortWithinRecordsParseCLI(
 }
 
 type TransformerSortWithinRecords struct {
+	doRecurse   bool
+	doNatural   bool
+	fieldNames  []string
+	fieldSet    map[string]bool
+	regex       *regexp.Regexp
+	doRegexes   bool
 	recordTransformerFunc RecordTransformerFunc
 }
 
 func NewTransformerSortWithinRecords(
 	doRecurse bool,
+	fieldNames []string,
+	doRegexes bool,
+	doNatural bool,
 ) (*TransformerSortWithinRecords, error) {
 
-	tr := &TransformerSortWithinRecords{}
-	if doRecurse {
+	tr := &TransformerSortWithinRecords{
+		doRecurse: doRecurse,
+		doNatural: doNatural,
+		fieldNames: fieldNames,
+		doRegexes: doRegexes,
+	}
+
+	if fieldNames != nil {
+		if doRegexes {
+			if len(fieldNames) > 1 {
+				return nil, cli.VerbErrorf(
+					verbNameSortWithinRecords,
+					"regex mode takes a single pattern; got %d names: %s. "+
+						"Use alternation in the regex (e.g. 'a|b') instead of a comma-list.",
+					len(fieldNames), strings.Join(fieldNames, ","),
+				)
+			}
+			// Handles "a.*b"i Miller case-insensitive-regex specification
+			regexString := fieldNames[0]
+			regex, err := lib.CompileMillerRegex(regexString)
+			if err != nil {
+				return nil, cli.VerbErrorf(
+					verbNameSortWithinRecords,
+					"cannot compile regex [%s]", regexString,
+				)
+			}
+			tr.regex = regex
+		} else {
+			tr.fieldSet = lib.StringListToSet(fieldNames)
+		}
+	}
+
+	if fieldNames != nil {
+		if doRecurse {
+			tr.recordTransformerFunc = tr.transformSelectiveRecursively
+		} else {
+			tr.recordTransformerFunc = tr.transformSelective
+		}
+	} else if doRecurse {
 		tr.recordTransformerFunc = tr.transformRecursively
 	} else {
 		tr.recordTransformerFunc = tr.transformNonrecursively
@@ -107,6 +192,124 @@ func (tr *TransformerSortWithinRecords) Transform(
 	tr.recordTransformerFunc(inrecAndContext, outputRecordsAndContexts, inputDownstreamDoneChannel, outputDownstreamDoneChannel)
 }
 
+// ----------------------------------------------------------------
+func (tr *TransformerSortWithinRecords) keyMatches(key string) bool {
+	if tr.doRegexes {
+		return tr.regex.MatchString(key)
+	}
+	return tr.fieldSet[key]
+}
+
+func (tr *TransformerSortWithinRecords) sortKeys(keys []string) {
+	if tr.doNatural {
+		sort.SliceStable(keys, func(i, j int) bool {
+			return natsort.Compare(keys[i], keys[j])
+		})
+	} else {
+		slices.Sort(keys)
+	}
+}
+
+func (tr *TransformerSortWithinRecords) sortMapByKey(m *mlrval.Mlrmap) {
+	keys := m.GetKeys()
+	tr.sortKeys(keys)
+	other := mlrval.NewMlrmapAsRecord()
+	for _, key := range keys {
+		other.PutReference(key, m.Get(key))
+	}
+	*m = *other
+}
+
+func (tr *TransformerSortWithinRecords) sortMapByKeyRecursively(m *mlrval.Mlrmap) {
+	keys := m.GetKeys()
+	tr.sortKeys(keys)
+	other := mlrval.NewMlrmapAsRecord()
+	for _, key := range keys {
+		val := m.Get(key)
+		if val != nil {
+			if sub := val.GetMap(); sub != nil {
+				tr.sortMapByKeyRecursively(sub)
+			}
+		}
+		other.PutReference(key, val)
+	}
+	*m = *other
+}
+
+// ----------------------------------------------------------------
+func (tr *TransformerSortWithinRecords) transformSelective(
+	inrecAndContext *types.RecordAndContext,
+	outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
+	inputDownstreamDoneChannel <-chan bool,
+	outputDownstreamDoneChannel chan<- bool,
+) {
+	if !inrecAndContext.EndOfStream {
+		inrec := inrecAndContext.Record
+		var matchingKeys []string
+		var restEntries []*mlrval.MlrmapEntry
+		for pe := inrec.Head; pe != nil; pe = pe.Next {
+			if tr.keyMatches(pe.Key) {
+				matchingKeys = append(matchingKeys, pe.Key)
+			} else {
+				restEntries = append(restEntries, pe)
+			}
+		}
+		tr.sortKeys(matchingKeys)
+		other := mlrval.NewMlrmapAsRecord()
+		for _, key := range matchingKeys {
+			other.PutReference(key, inrec.Get(key))
+		}
+		for _, pe := range restEntries {
+			other.PutReference(pe.Key, pe.Value)
+		}
+		*inrec = *other
+	}
+	*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
+}
+
+// ----------------------------------------------------------------
+func (tr *TransformerSortWithinRecords) transformSelectiveRecursively(
+	inrecAndContext *types.RecordAndContext,
+	outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
+	inputDownstreamDoneChannel <-chan bool,
+	outputDownstreamDoneChannel chan<- bool,
+) {
+	if !inrecAndContext.EndOfStream {
+		inrec := inrecAndContext.Record
+		var matchingKeys []string
+		var restEntries []*mlrval.MlrmapEntry
+		for pe := inrec.Head; pe != nil; pe = pe.Next {
+			if tr.keyMatches(pe.Key) {
+				matchingKeys = append(matchingKeys, pe.Key)
+			} else {
+				restEntries = append(restEntries, pe)
+			}
+		}
+		tr.sortKeys(matchingKeys)
+		other := mlrval.NewMlrmapAsRecord()
+		for _, key := range matchingKeys {
+			val := inrec.Get(key)
+			if val != nil {
+				if m := val.GetMap(); m != nil {
+					tr.sortMapByKeyRecursively(m)
+				}
+			}
+			other.PutReference(key, val)
+		}
+		for _, pe := range restEntries {
+			if pe.Value != nil {
+				if m := pe.Value.GetMap(); m != nil {
+					tr.sortMapByKeyRecursively(m)
+				}
+			}
+			other.PutReference(pe.Key, pe.Value)
+		}
+		*inrec = *other
+	}
+	*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
+}
+
+// ----------------------------------------------------------------
 func (tr *TransformerSortWithinRecords) transformNonrecursively(
 	inrecAndContext *types.RecordAndContext,
 	outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
@@ -115,7 +318,11 @@ func (tr *TransformerSortWithinRecords) transformNonrecursively(
 ) {
 	if !inrecAndContext.EndOfStream {
 		inrec := inrecAndContext.Record
-		inrec.SortByKey()
+		if tr.doNatural {
+			tr.sortMapByKey(inrec)
+		} else {
+			inrec.SortByKey()
+		}
 	}
 	*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker
 }
@@ -128,7 +335,11 @@ func (tr *TransformerSortWithinRecords) transformRecursively(
 ) {
 	if !inrecAndContext.EndOfStream {
 		inrec := inrecAndContext.Record
-		inrec.SortByKeyRecursively()
+		if tr.doNatural {
+			tr.sortMapByKeyRecursively(inrec)
+		} else {
+			inrec.SortByKeyRecursively()
+		}
 	}
 	*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker
 }

test/cases/cli-help/0001/expout

@@ -1004,8 +1004,13 @@ sort-within-records
 Usage: mlr sort-within-records [options]
 Outputs records sorted lexically ascending by keys.
 Options:
--r        Recursively sort subobjects/submaps, e.g. for JSON input.
--h|--help Show this message.
+-f {names}   Sort only these keys; others preserve record order.
+-r {regex}   Sort only keys matching this regex; others preserve record order.
+             Example: -r '^[xy]' sorts keys starting with x or y.
+             With no regex argument, -r recursively sorts subobjects/submaps
+             (e.g. for JSON input), or combines with -f to treat names as regex.
+-n           Sort field names naturally (e.g. 2 before 12). Combines with -f/-r.
+-h|--help    Show this message.
 
 ================================================================
 sparsify

test/cases/verb-sort-within-records/0004-regex/cmd

@@ -0,0 +1 @@
+mlr --from test/input/sort-within-records.dkvp sort-within-records -f '^[ai]' -r

test/cases/verb-sort-within-records/0004-regex/expout

@@ -0,0 +1,10 @@
+a=pan,i=1,b=pan,x=0.34679014,y=0.72680286
+a=eks,i=2,x=0.75867996,y=0.52215111,b=pan
+a=wye,i=3,y=0.33831853,b=wye,x=0.20460331
+a=eks,i=4,x=0.38139939,b=wye,y=0.13418874
+a=wye,i=5,y=0.86362447,x=0.57328892,b=pan
+a=zee,i=6,y=0.49322129,b=pan,x=0.52712616
+a=eks,i=7,b=zee,x=0.61178406,y=0.18788492
+a=zee,i=8,x=0.59855401,b=wye,y=0.97618139
+a=hat,i=9,b=wye,x=0.03144188,y=0.74955076
+a=pan,i=10,y=0.95261836,x=0.50262601,b=wye

test/cases/verb-sort-within-records/0005-f-explicit/cmd

@@ -0,0 +1 @@
+mlr --from test/input/sort-within-records.dkvp sort-within-records -f a,i

test/cases/verb-sort-within-records/0005-f-explicit/expout

@@ -0,0 +1,10 @@
+a=pan,i=1,b=pan,x=0.34679014,y=0.72680286
+a=eks,i=2,x=0.75867996,y=0.52215111,b=pan
+a=wye,i=3,y=0.33831853,b=wye,x=0.20460331
+a=eks,i=4,x=0.38139939,b=wye,y=0.13418874
+a=wye,i=5,y=0.86362447,x=0.57328892,b=pan
+a=zee,i=6,y=0.49322129,b=pan,x=0.52712616
+a=eks,i=7,b=zee,x=0.61178406,y=0.18788492
+a=zee,i=8,x=0.59855401,b=wye,y=0.97618139
+a=hat,i=9,b=wye,x=0.03144188,y=0.74955076
+a=pan,i=10,y=0.95261836,x=0.50262601,b=wye

test/cases/verb-sort-within-records/0006-issue-1645-r-regex/cmd

@@ -0,0 +1 @@
+mlr --csv sort-within-records -r "[0-9]+" test/input/issue-1645.csv

test/cases/verb-sort-within-records/0006-issue-1645-r-regex/expout

@@ -0,0 +1,3 @@
+12,2,a
+x,c,z
+5,t,4

test/cases/verb-sort-within-records/0007-natural/cmd

@@ -0,0 +1 @@
+mlr --csv sort-within-records -n test/input/issue-1645.csv

test/cases/verb-sort-within-records/0007-natural/expout

@@ -0,0 +1,3 @@
+2,12,a
+c,x,z
+t,5,4

test/cases/verb-sort-within-records/0008-natural-filelike/cmd

@@ -0,0 +1 @@
+mlr --idkvp --ojsonl --from test/input/natural-within-records.dkvp sort-within-records -n

test/cases/verb-sort-within-records/0008-natural-filelike/expout

@@ -0,0 +1,2 @@
+{"file1": "c", "file2": "b", "file10": "a", "other": "d"}
+{"v3": "y", "v20": "x", "v100": "z"}

test/cases/verb-sort-within-records/0009-regex-rejects-list/cmd

@@ -0,0 +1 @@
+mlr --from test/input/sort-within-records.dkvp sort-within-records -f 'a,b' -r

test/cases/verb-sort-within-records/0009-regex-rejects-list/experr

@@ -0,0 +1 @@
+mlr sort-within-records: regex mode takes a single pattern; got 2 names: a,b. Use alternation in the regex (e.g. 'a|b') instead of a comma-list.

test/cases/verb-sort-within-records/0010-bad-regex/cmd

@@ -0,0 +1 @@
+mlr --from test/input/sort-within-records.dkvp sort-within-records -r '(bad[regex'

test/cases/verb-sort-within-records/0010-bad-regex/experr

@@ -0,0 +1 @@
+mlr sort-within-records: cannot compile regex [(bad[regex]

test/input/issue-1645.csv

@@ -0,0 +1,3 @@
+a,12,2
+z,x,c
+4,5,t

test/input/natural-within-records.dkvp

@@ -0,0 +1,2 @@
+file10=a,file2=b,file1=c,other=d
+v20=x,v3=y,v100=z