diff --git a/docs/src/reference-verbs.md b/docs/src/reference-verbs.md index ab8e0ba1b..724f5bcd9 100644 --- a/docs/src/reference-verbs.md +++ b/docs/src/reference-verbs.md @@ -3073,8 +3073,13 @@ a=3 Usage: mlr sort-within-records [options] Outputs records sorted lexically ascending by keys. Options: --r Recursively sort subobjects/submaps, e.g. for JSON input. --h|--help Show this message. +-f {names} Sort only these keys; others preserve record order. +-r {regex} Sort only keys matching this regex; others preserve record order. + Example: -r '^[xy]' sorts keys starting with x or y. + With no regex argument, -r recursively sorts subobjects/submaps + (e.g. for JSON input), or combines with -f to treat names as regex. +-n Sort field names naturally (e.g. 2 before 12). Combines with -f/-r. +-h|--help Show this message.
diff --git a/pkg/transformers/sort_within_records.go b/pkg/transformers/sort_within_records.go
index d9a27ddfe..080b78c59 100644
--- a/pkg/transformers/sort_within_records.go
+++ b/pkg/transformers/sort_within_records.go
@@ -3,9 +3,15 @@ package transformers
 import (
 	"fmt"
 	"os"
+	"regexp"
+	"slices"
+	"sort"
 	"strings"
 
+	"github.com/facette/natsort"
 	"github.com/johnkerl/miller/v6/pkg/cli"
+	"github.com/johnkerl/miller/v6/pkg/lib"
+	"github.com/johnkerl/miller/v6/pkg/mlrval"
 	"github.com/johnkerl/miller/v6/pkg/types"
 )
 
@@ -24,8 +30,13 @@ func transformerSortWithinRecordsUsage(
 	fmt.Fprintf(o, "Usage: %s %s [options]\n", "mlr", verbNameSortWithinRecords)
 	fmt.Fprintln(o, "Outputs records sorted lexically ascending by keys.")
 	fmt.Fprintf(o, "Options:\n")
-	fmt.Fprintf(o, "-r        Recursively sort subobjects/submaps, e.g. for JSON input.\n")
-	fmt.Fprintf(o, "-h|--help Show this message.\n")
+	fmt.Fprintf(o, "-f {names}   Sort only these keys; others preserve record order.\n")
+	fmt.Fprintf(o, "-r {regex}   Sort only keys matching this regex; others preserve record order.\n")
+	fmt.Fprintf(o, "             Example: -r '^[xy]' sorts keys starting with x or y.\n")
+	fmt.Fprintf(o, "             With no regex argument, -r recursively sorts subobjects/submaps\n")
+	fmt.Fprintf(o, "             (e.g. for JSON input), or combines with -f to treat names as regex.\n")
+	fmt.Fprintf(o, "-n           Sort field names naturally (e.g. 2 before 12). Combines with -f/-r.\n")
+	fmt.Fprintf(o, "-h|--help    Show this message.\n")
 }
 
 func transformerSortWithinRecordsParseCLI(
@@ -38,8 +49,12 @@ func transformerSortWithinRecordsParseCLI(
 
 	// Skip the verb name from the current spot in the mlr command line
 	argi := *pargi
+	verb := args[argi]
 	argi++
 	doRecurse := false
+	var fieldNames []string = nil
+	doRegexes := false
+	doNatural := false
 
 	for argi < argc /* variable increment: 1 or 2 depending on flag */ {
 		opt := args[argi]
@@ -56,22 +71,46 @@ func transformerSortWithinRecordsParseCLI(
 			return nil, cli.ErrHelpRequested
 
 		} else if opt == "-r" {
-			doRecurse = true
+			// If the next token exists and isn't another flag, consume it as
+			// the regex pattern. Otherwise -r is arity-0: combined with a
+			// preceding -f it means regex mode; standalone it means recursive.
+			if argi < argc && !strings.HasPrefix(args[argi], "-") {
+				fieldNames = []string{args[argi]}
+				argi++
+				doRegexes = true
+			} else if fieldNames != nil {
+				doRegexes = true
+			} else {
+				doRecurse = true
+			}
+
+		} else if opt == "-f" {
+			names, err := cli.VerbGetStringArrayArg(verb, opt, args, &argi, argc)
+			if err != nil {
+				return nil, err
+			}
+			fieldNames = names
+
+		} else if opt == "-n" {
+			doNatural = true
 
 		} else {
 			return nil, cli.VerbErrorf(verbNameSortWithinRecords, "option \"%s\" not recognized", opt)
 		}
 	}
 
-	// TODO: allow sort by key or value?
-	// TODO: allow sort ascendending/descending?
+	// -r with -f means regex; -r without -f means recurse
+	if fieldNames != nil && doRecurse {
+		doRegexes = true
+		doRecurse = false
+	}
 
 	*pargi = argi
 	if !doConstruct { // All transformers must do this for main command-line parsing
 		return nil, nil
 	}
 
-	transformer, err := NewTransformerSortWithinRecords(doRecurse)
+	transformer, err := NewTransformerSortWithinRecords(doRecurse, fieldNames, doRegexes, doNatural)
 	if err != nil {
 		return nil, err
 	}
@@ -80,15 +119,61 @@ func transformerSortWithinRecordsParseCLI(
 }
 
 type TransformerSortWithinRecords struct {
+	doRecurse   bool
+	doNatural   bool
+	fieldNames  []string
+	fieldSet    map[string]bool
+	regex       *regexp.Regexp
+	doRegexes   bool
 	recordTransformerFunc RecordTransformerFunc
 }
 
 func NewTransformerSortWithinRecords(
 	doRecurse bool,
+	fieldNames []string,
+	doRegexes bool,
+	doNatural bool,
 ) (*TransformerSortWithinRecords, error) {
 
-	tr := &TransformerSortWithinRecords{}
-	if doRecurse {
+	tr := &TransformerSortWithinRecords{
+		doRecurse: doRecurse,
+		doNatural: doNatural,
+		fieldNames: fieldNames,
+		doRegexes: doRegexes,
+	}
+
+	if fieldNames != nil {
+		if doRegexes {
+			if len(fieldNames) > 1 {
+				return nil, cli.VerbErrorf(
+					verbNameSortWithinRecords,
+					"regex mode takes a single pattern; got %d names: %s. "+
+						"Use alternation in the regex (e.g. 'a|b') instead of a comma-list.",
+					len(fieldNames), strings.Join(fieldNames, ","),
+				)
+			}
+			// Handles "a.*b"i Miller case-insensitive-regex specification
+			regexString := fieldNames[0]
+			regex, err := lib.CompileMillerRegex(regexString)
+			if err != nil {
+				return nil, cli.VerbErrorf(
+					verbNameSortWithinRecords,
+					"cannot compile regex [%s]", regexString,
+				)
+			}
+			tr.regex = regex
+		} else {
+			tr.fieldSet = lib.StringListToSet(fieldNames)
+		}
+	}
+
+	if fieldNames != nil {
+		if doRecurse {
+			tr.recordTransformerFunc = tr.transformSelectiveRecursively
+		} else {
+			tr.recordTransformerFunc = tr.transformSelective
+		}
+	} else if doRecurse {
 		tr.recordTransformerFunc = tr.transformRecursively
 	} else {
 		tr.recordTransformerFunc = tr.transformNonrecursively
@@ -107,6 +192,124 @@ func (tr *TransformerSortWithinRecords) Transform(
 	tr.recordTransformerFunc(inrecAndContext, outputRecordsAndContexts, inputDownstreamDoneChannel, outputDownstreamDoneChannel)
 }
 
+// ----------------------------------------------------------------
+func (tr *TransformerSortWithinRecords) keyMatches(key string) bool {
+	if tr.doRegexes {
+		return tr.regex.MatchString(key)
+	}
+	return tr.fieldSet[key]
+}
+
+func (tr *TransformerSortWithinRecords) sortKeys(keys []string) {
+	if tr.doNatural {
+		sort.SliceStable(keys, func(i, j int) bool {
+			return natsort.Compare(keys[i], keys[j])
+		})
+	} else {
+		slices.Sort(keys)
+	}
+}
+
+func (tr *TransformerSortWithinRecords) sortMapByKey(m *mlrval.Mlrmap) {
+	keys := m.GetKeys()
+	tr.sortKeys(keys)
+	other := mlrval.NewMlrmapAsRecord()
+	for _, key := range keys {
+		other.PutReference(key, m.Get(key))
+	}
+	*m = *other
+}
+
+func (tr *TransformerSortWithinRecords) sortMapByKeyRecursively(m *mlrval.Mlrmap) {
+	keys := m.GetKeys()
+	tr.sortKeys(keys)
+	other := mlrval.NewMlrmapAsRecord()
+	for _, key := range keys {
+		val := m.Get(key)
+		if val != nil {
+			if sub := val.GetMap(); sub != nil {
+				tr.sortMapByKeyRecursively(sub)
+			}
+		}
+		other.PutReference(key, val)
+	}
+	*m = *other
+}
+
+// ----------------------------------------------------------------
+func (tr *TransformerSortWithinRecords) transformSelective(
+	inrecAndContext *types.RecordAndContext,
+	outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
+	inputDownstreamDoneChannel <-chan bool,
+	outputDownstreamDoneChannel chan<- bool,
+) {
+	if !inrecAndContext.EndOfStream {
+		inrec := inrecAndContext.Record
+		var matchingKeys []string
+		var restEntries []*mlrval.MlrmapEntry
+		for pe := inrec.Head; pe != nil; pe = pe.Next {
+			if tr.keyMatches(pe.Key) {
+				matchingKeys = append(matchingKeys, pe.Key)
+			} else {
+				restEntries = append(restEntries, pe)
+			}
+		}
+		tr.sortKeys(matchingKeys)
+		other := mlrval.NewMlrmapAsRecord()
+		for _, key := range matchingKeys {
+			other.PutReference(key, inrec.Get(key))
+		}
+		for _, pe := range restEntries {
+			other.PutReference(pe.Key, pe.Value)
+		}
+		*inrec = *other
+	}
+	*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
+}
+
+// ----------------------------------------------------------------
+func (tr *TransformerSortWithinRecords) transformSelectiveRecursively(
+	inrecAndContext *types.RecordAndContext,
+	outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
+	inputDownstreamDoneChannel <-chan bool,
+	outputDownstreamDoneChannel chan<- bool,
+) {
+	if !inrecAndContext.EndOfStream {
+		inrec := inrecAndContext.Record
+		var matchingKeys []string
+		var restEntries []*mlrval.MlrmapEntry
+		for pe := inrec.Head; pe != nil; pe = pe.Next {
+			if tr.keyMatches(pe.Key) {
+				matchingKeys = append(matchingKeys, pe.Key)
+			} else {
+				restEntries = append(restEntries, pe)
+			}
+		}
+		tr.sortKeys(matchingKeys)
+		other := mlrval.NewMlrmapAsRecord()
+		for _, key := range matchingKeys {
+			val := inrec.Get(key)
+			if val != nil {
+				if m := val.GetMap(); m != nil {
+					tr.sortMapByKeyRecursively(m)
+				}
+			}
+			other.PutReference(key, val)
+		}
+		for _, pe := range restEntries {
+			if pe.Value != nil {
+				if m := pe.Value.GetMap(); m != nil {
+					tr.sortMapByKeyRecursively(m)
+				}
+			}
+			other.PutReference(pe.Key, pe.Value)
+		}
+		*inrec = *other
+	}
+	*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
+}
+
+// ----------------------------------------------------------------
 func (tr *TransformerSortWithinRecords) transformNonrecursively(
 	inrecAndContext *types.RecordAndContext,
 	outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
@@ -115,7 +318,11 @@ func (tr *TransformerSortWithinRecords) transformNonrecursively(
 ) {
 	if !inrecAndContext.EndOfStream {
 		inrec := inrecAndContext.Record
-		inrec.SortByKey()
+		if tr.doNatural {
+			tr.sortMapByKey(inrec)
+		} else {
+			inrec.SortByKey()
+		}
 	}
 	*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker
 }
@@ -128,7 +335,11 @@ func (tr *TransformerSortWithinRecords) transformRecursively(
 ) {
 	if !inrecAndContext.EndOfStream {
 		inrec := inrecAndContext.Record
-		inrec.SortByKeyRecursively()
+		if tr.doNatural {
+			tr.sortMapByKeyRecursively(inrec)
+		} else {
+			inrec.SortByKeyRecursively()
+		}
 	}
 	*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker
 }
diff --git a/test/cases/cli-help/0001/expout b/test/cases/cli-help/0001/expout
index af9c48619..0e8565a95 100644
--- a/test/cases/cli-help/0001/expout
+++ b/test/cases/cli-help/0001/expout
@@ -1004,8 +1004,13 @@ sort-within-records
 Usage: mlr sort-within-records [options]
 Outputs records sorted lexically ascending by keys.
 Options:
--r        Recursively sort subobjects/submaps, e.g. for JSON input.
--h|--help Show this message.
+-f {names}   Sort only these keys; others preserve record order.
+-r {regex}   Sort only keys matching this regex; others preserve record order.
+             Example: -r '^[xy]' sorts keys starting with x or y.
+             With no regex argument, -r recursively sorts subobjects/submaps
+             (e.g. for JSON input), or combines with -f to treat names as regex.
+-n           Sort field names naturally (e.g. 2 before 12). Combines with -f/-r.
+-h|--help    Show this message.
 
 ================================================================
 sparsify
diff --git a/test/cases/verb-sort-within-records/0004-regex/cmd b/test/cases/verb-sort-within-records/0004-regex/cmd
new file mode 100644
index 000000000..7b5d76372
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0004-regex/cmd
@@ -0,0 +1 @@
+mlr --from test/input/sort-within-records.dkvp sort-within-records -f '^[ai]' -r
diff --git a/test/cases/verb-sort-within-records/0004-regex/experr b/test/cases/verb-sort-within-records/0004-regex/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0004-regex/expout b/test/cases/verb-sort-within-records/0004-regex/expout
new file mode 100644
index 000000000..a6ceab9f6
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0004-regex/expout
@@ -0,0 +1,10 @@
+a=pan,i=1,b=pan,x=0.34679014,y=0.72680286
+a=eks,i=2,x=0.75867996,y=0.52215111,b=pan
+a=wye,i=3,y=0.33831853,b=wye,x=0.20460331
+a=eks,i=4,x=0.38139939,b=wye,y=0.13418874
+a=wye,i=5,y=0.86362447,x=0.57328892,b=pan
+a=zee,i=6,y=0.49322129,b=pan,x=0.52712616
+a=eks,i=7,b=zee,x=0.61178406,y=0.18788492
+a=zee,i=8,x=0.59855401,b=wye,y=0.97618139
+a=hat,i=9,b=wye,x=0.03144188,y=0.74955076
+a=pan,i=10,y=0.95261836,x=0.50262601,b=wye
diff --git a/test/cases/verb-sort-within-records/0005-f-explicit/cmd b/test/cases/verb-sort-within-records/0005-f-explicit/cmd
new file mode 100644
index 000000000..d98a6358c
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0005-f-explicit/cmd
@@ -0,0 +1 @@
+mlr --from test/input/sort-within-records.dkvp sort-within-records -f a,i
diff --git a/test/cases/verb-sort-within-records/0005-f-explicit/experr b/test/cases/verb-sort-within-records/0005-f-explicit/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0005-f-explicit/expout b/test/cases/verb-sort-within-records/0005-f-explicit/expout
new file mode 100644
index 000000000..a6ceab9f6
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0005-f-explicit/expout
@@ -0,0 +1,10 @@
+a=pan,i=1,b=pan,x=0.34679014,y=0.72680286
+a=eks,i=2,x=0.75867996,y=0.52215111,b=pan
+a=wye,i=3,y=0.33831853,b=wye,x=0.20460331
+a=eks,i=4,x=0.38139939,b=wye,y=0.13418874
+a=wye,i=5,y=0.86362447,x=0.57328892,b=pan
+a=zee,i=6,y=0.49322129,b=pan,x=0.52712616
+a=eks,i=7,b=zee,x=0.61178406,y=0.18788492
+a=zee,i=8,x=0.59855401,b=wye,y=0.97618139
+a=hat,i=9,b=wye,x=0.03144188,y=0.74955076
+a=pan,i=10,y=0.95261836,x=0.50262601,b=wye
diff --git a/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/cmd b/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/cmd
new file mode 100644
index 000000000..c7b8daf6c
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/cmd
@@ -0,0 +1 @@
+mlr --csv sort-within-records -r "[0-9]+" test/input/issue-1645.csv
diff --git a/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/experr b/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/expout b/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/expout
new file mode 100644
index 000000000..0845e21d2
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/expout
@@ -0,0 +1,3 @@
+12,2,a
+x,c,z
+5,t,4
diff --git a/test/cases/verb-sort-within-records/0007-natural/cmd b/test/cases/verb-sort-within-records/0007-natural/cmd
new file mode 100644
index 000000000..706093782
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0007-natural/cmd
@@ -0,0 +1 @@
+mlr --csv sort-within-records -n test/input/issue-1645.csv
diff --git a/test/cases/verb-sort-within-records/0007-natural/experr b/test/cases/verb-sort-within-records/0007-natural/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0007-natural/expout b/test/cases/verb-sort-within-records/0007-natural/expout
new file mode 100644
index 000000000..3e2254344
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0007-natural/expout
@@ -0,0 +1,3 @@
+2,12,a
+c,x,z
+t,5,4
diff --git a/test/cases/verb-sort-within-records/0008-natural-filelike/cmd b/test/cases/verb-sort-within-records/0008-natural-filelike/cmd
new file mode 100644
index 000000000..38a4b45db
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0008-natural-filelike/cmd
@@ -0,0 +1 @@
+mlr --idkvp --ojsonl --from test/input/natural-within-records.dkvp sort-within-records -n
diff --git a/test/cases/verb-sort-within-records/0008-natural-filelike/experr b/test/cases/verb-sort-within-records/0008-natural-filelike/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0008-natural-filelike/expout b/test/cases/verb-sort-within-records/0008-natural-filelike/expout
new file mode 100644
index 000000000..3d3951692
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0008-natural-filelike/expout
@@ -0,0 +1,2 @@
+{"file1": "c", "file2": "b", "file10": "a", "other": "d"}
+{"v3": "y", "v20": "x", "v100": "z"}
diff --git a/test/cases/verb-sort-within-records/0009-regex-rejects-list/cmd b/test/cases/verb-sort-within-records/0009-regex-rejects-list/cmd
new file mode 100644
index 000000000..878adfb6f
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0009-regex-rejects-list/cmd
@@ -0,0 +1 @@
+mlr --from test/input/sort-within-records.dkvp sort-within-records -f 'a,b' -r
diff --git a/test/cases/verb-sort-within-records/0009-regex-rejects-list/experr b/test/cases/verb-sort-within-records/0009-regex-rejects-list/experr
new file mode 100644
index 000000000..6939a4abf
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0009-regex-rejects-list/experr
@@ -0,0 +1 @@
+mlr sort-within-records: regex mode takes a single pattern; got 2 names: a,b. Use alternation in the regex (e.g. 'a|b') instead of a comma-list.
diff --git a/test/cases/verb-sort-within-records/0009-regex-rejects-list/expout b/test/cases/verb-sort-within-records/0009-regex-rejects-list/expout
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0009-regex-rejects-list/should-fail b/test/cases/verb-sort-within-records/0009-regex-rejects-list/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0010-bad-regex/cmd b/test/cases/verb-sort-within-records/0010-bad-regex/cmd
new file mode 100644
index 000000000..8ac0343ec
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0010-bad-regex/cmd
@@ -0,0 +1 @@
+mlr --from test/input/sort-within-records.dkvp sort-within-records -r '(bad[regex'
diff --git a/test/cases/verb-sort-within-records/0010-bad-regex/experr b/test/cases/verb-sort-within-records/0010-bad-regex/experr
new file mode 100644
index 000000000..f957a55d1
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0010-bad-regex/experr
@@ -0,0 +1 @@
+mlr sort-within-records: cannot compile regex [(bad[regex]
diff --git a/test/cases/verb-sort-within-records/0010-bad-regex/expout b/test/cases/verb-sort-within-records/0010-bad-regex/expout
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0010-bad-regex/should-fail b/test/cases/verb-sort-within-records/0010-bad-regex/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/input/issue-1645.csv b/test/input/issue-1645.csv
new file mode 100644
index 000000000..20f1f82b0
--- /dev/null
+++ b/test/input/issue-1645.csv
@@ -0,0 +1,3 @@
+a,12,2
+z,x,c
+4,5,t
diff --git a/test/input/natural-within-records.dkvp b/test/input/natural-within-records.dkvp
new file mode 100644
index 000000000..c63ed5336
--- /dev/null
+++ b/test/input/natural-within-records.dkvp
@@ -0,0 +1,2 @@
+file10=a,file2=b,file1=c,other=d
+v20=x,v3=y,v100=z