diff --git a/docs/src/reference-verbs.md b/docs/src/reference-verbs.md index ab8e0ba1b..724f5bcd9 100644 --- a/docs/src/reference-verbs.md +++ b/docs/src/reference-verbs.md @@ -3073,8 +3073,13 @@ a=3 Usage: mlr sort-within-records [options] Outputs records sorted lexically ascending by keys. Options: --r Recursively sort subobjects/submaps, e.g. for JSON input. --h|--help Show this message. +-f {names} Sort only these keys; others preserve record order. +-r {regex} Sort only keys matching this regex; others preserve record order. + Example: -r '^[xy]' sorts keys starting with x or y. + With no regex argument, -r recursively sorts subobjects/submaps + (e.g. for JSON input), or combines with -f to treat names as regex. +-n Sort field names naturally (e.g. 2 before 12). Combines with -f/-r. +-h|--help Show this message.
diff --git a/pkg/transformers/sort_within_records.go b/pkg/transformers/sort_within_records.go
index d9a27ddfe..080b78c59 100644
--- a/pkg/transformers/sort_within_records.go
+++ b/pkg/transformers/sort_within_records.go
@@ -3,9 +3,15 @@ package transformers
import (
"fmt"
"os"
+ "regexp"
+ "slices"
+ "sort"
"strings"
+ "github.com/facette/natsort"
"github.com/johnkerl/miller/v6/pkg/cli"
+ "github.com/johnkerl/miller/v6/pkg/lib"
+ "github.com/johnkerl/miller/v6/pkg/mlrval"
"github.com/johnkerl/miller/v6/pkg/types"
)
@@ -24,8 +30,13 @@ func transformerSortWithinRecordsUsage(
fmt.Fprintf(o, "Usage: %s %s [options]\n", "mlr", verbNameSortWithinRecords)
fmt.Fprintln(o, "Outputs records sorted lexically ascending by keys.")
fmt.Fprintf(o, "Options:\n")
- fmt.Fprintf(o, "-r Recursively sort subobjects/submaps, e.g. for JSON input.\n")
- fmt.Fprintf(o, "-h|--help Show this message.\n")
+ fmt.Fprintf(o, "-f {names} Sort only these keys; others preserve record order.\n")
+ fmt.Fprintf(o, "-r {regex} Sort only keys matching this regex; others preserve record order.\n")
+ fmt.Fprintf(o, " Example: -r '^[xy]' sorts keys starting with x or y.\n")
+ fmt.Fprintf(o, " With no regex argument, -r recursively sorts subobjects/submaps\n")
+ fmt.Fprintf(o, " (e.g. for JSON input), or combines with -f to treat names as regex.\n")
+ fmt.Fprintf(o, "-n Sort field names naturally (e.g. 2 before 12). Combines with -f/-r.\n")
+ fmt.Fprintf(o, "-h|--help Show this message.\n")
}
func transformerSortWithinRecordsParseCLI(
@@ -38,8 +49,12 @@ func transformerSortWithinRecordsParseCLI(
// Skip the verb name from the current spot in the mlr command line
argi := *pargi
+ verb := args[argi]
argi++
doRecurse := false
+ var fieldNames []string = nil
+ doRegexes := false
+ doNatural := false
for argi < argc /* variable increment: 1 or 2 depending on flag */ {
opt := args[argi]
@@ -56,22 +71,46 @@ func transformerSortWithinRecordsParseCLI(
return nil, cli.ErrHelpRequested
} else if opt == "-r" {
- doRecurse = true
+ // If the next token exists and isn't another flag, consume it as
+ // the regex pattern. Otherwise -r is arity-0: combined with a
+ // preceding -f it means regex mode; standalone it means recursive.
+ if argi < argc && !strings.HasPrefix(args[argi], "-") {
+ fieldNames = []string{args[argi]}
+ argi++
+ doRegexes = true
+ } else if fieldNames != nil {
+ doRegexes = true
+ } else {
+ doRecurse = true
+ }
+
+ } else if opt == "-f" {
+ names, err := cli.VerbGetStringArrayArg(verb, opt, args, &argi, argc)
+ if err != nil {
+ return nil, err
+ }
+ fieldNames = names
+
+ } else if opt == "-n" {
+ doNatural = true
} else {
return nil, cli.VerbErrorf(verbNameSortWithinRecords, "option \"%s\" not recognized", opt)
}
}
- // TODO: allow sort by key or value?
- // TODO: allow sort ascendending/descending?
+ // -r with -f means regex; -r without -f means recurse
+ if fieldNames != nil && doRecurse {
+ doRegexes = true
+ doRecurse = false
+ }
*pargi = argi
if !doConstruct { // All transformers must do this for main command-line parsing
return nil, nil
}
- transformer, err := NewTransformerSortWithinRecords(doRecurse)
+ transformer, err := NewTransformerSortWithinRecords(doRecurse, fieldNames, doRegexes, doNatural)
if err != nil {
return nil, err
}
@@ -80,15 +119,61 @@ func transformerSortWithinRecordsParseCLI(
}
type TransformerSortWithinRecords struct {
+ doRecurse bool
+ doNatural bool
+ fieldNames []string
+ fieldSet map[string]bool
+ regex *regexp.Regexp
+ doRegexes bool
recordTransformerFunc RecordTransformerFunc
}
func NewTransformerSortWithinRecords(
doRecurse bool,
+ fieldNames []string,
+ doRegexes bool,
+ doNatural bool,
) (*TransformerSortWithinRecords, error) {
- tr := &TransformerSortWithinRecords{}
- if doRecurse {
+ tr := &TransformerSortWithinRecords{
+ doRecurse: doRecurse,
+ doNatural: doNatural,
+ fieldNames: fieldNames,
+ doRegexes: doRegexes,
+ }
+
+ if fieldNames != nil {
+ if doRegexes {
+ if len(fieldNames) > 1 {
+ return nil, cli.VerbErrorf(
+ verbNameSortWithinRecords,
+ "regex mode takes a single pattern; got %d names: %s. "+
+ "Use alternation in the regex (e.g. 'a|b') instead of a comma-list.",
+ len(fieldNames), strings.Join(fieldNames, ","),
+ )
+ }
+ // Handles "a.*b"i Miller case-insensitive-regex specification
+ regexString := fieldNames[0]
+ regex, err := lib.CompileMillerRegex(regexString)
+ if err != nil {
+ return nil, cli.VerbErrorf(
+ verbNameSortWithinRecords,
+ "cannot compile regex [%s]", regexString,
+ )
+ }
+ tr.regex = regex
+ } else {
+ tr.fieldSet = lib.StringListToSet(fieldNames)
+ }
+ }
+
+ if fieldNames != nil {
+ if doRecurse {
+ tr.recordTransformerFunc = tr.transformSelectiveRecursively
+ } else {
+ tr.recordTransformerFunc = tr.transformSelective
+ }
+ } else if doRecurse {
tr.recordTransformerFunc = tr.transformRecursively
} else {
tr.recordTransformerFunc = tr.transformNonrecursively
@@ -107,6 +192,124 @@ func (tr *TransformerSortWithinRecords) Transform(
tr.recordTransformerFunc(inrecAndContext, outputRecordsAndContexts, inputDownstreamDoneChannel, outputDownstreamDoneChannel)
}
+// ----------------------------------------------------------------
+func (tr *TransformerSortWithinRecords) keyMatches(key string) bool {
+ if tr.doRegexes {
+ return tr.regex.MatchString(key)
+ }
+ return tr.fieldSet[key]
+}
+
+func (tr *TransformerSortWithinRecords) sortKeys(keys []string) {
+ if tr.doNatural {
+ sort.SliceStable(keys, func(i, j int) bool {
+ return natsort.Compare(keys[i], keys[j])
+ })
+ } else {
+ slices.Sort(keys)
+ }
+}
+
+func (tr *TransformerSortWithinRecords) sortMapByKey(m *mlrval.Mlrmap) {
+ keys := m.GetKeys()
+ tr.sortKeys(keys)
+ other := mlrval.NewMlrmapAsRecord()
+ for _, key := range keys {
+ other.PutReference(key, m.Get(key))
+ }
+ *m = *other
+}
+
+func (tr *TransformerSortWithinRecords) sortMapByKeyRecursively(m *mlrval.Mlrmap) {
+ keys := m.GetKeys()
+ tr.sortKeys(keys)
+ other := mlrval.NewMlrmapAsRecord()
+ for _, key := range keys {
+ val := m.Get(key)
+ if val != nil {
+ if sub := val.GetMap(); sub != nil {
+ tr.sortMapByKeyRecursively(sub)
+ }
+ }
+ other.PutReference(key, val)
+ }
+ *m = *other
+}
+
+// ----------------------------------------------------------------
+func (tr *TransformerSortWithinRecords) transformSelective(
+ inrecAndContext *types.RecordAndContext,
+ outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
+ inputDownstreamDoneChannel <-chan bool,
+ outputDownstreamDoneChannel chan<- bool,
+) {
+ if !inrecAndContext.EndOfStream {
+ inrec := inrecAndContext.Record
+ var matchingKeys []string
+ var restEntries []*mlrval.MlrmapEntry
+ for pe := inrec.Head; pe != nil; pe = pe.Next {
+ if tr.keyMatches(pe.Key) {
+ matchingKeys = append(matchingKeys, pe.Key)
+ } else {
+ restEntries = append(restEntries, pe)
+ }
+ }
+ tr.sortKeys(matchingKeys)
+ other := mlrval.NewMlrmapAsRecord()
+ for _, key := range matchingKeys {
+ other.PutReference(key, inrec.Get(key))
+ }
+ for _, pe := range restEntries {
+ other.PutReference(pe.Key, pe.Value)
+ }
+ *inrec = *other
+ }
+ *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
+}
+
+// ----------------------------------------------------------------
+func (tr *TransformerSortWithinRecords) transformSelectiveRecursively(
+ inrecAndContext *types.RecordAndContext,
+ outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
+ inputDownstreamDoneChannel <-chan bool,
+ outputDownstreamDoneChannel chan<- bool,
+) {
+ if !inrecAndContext.EndOfStream {
+ inrec := inrecAndContext.Record
+ var matchingKeys []string
+ var restEntries []*mlrval.MlrmapEntry
+ for pe := inrec.Head; pe != nil; pe = pe.Next {
+ if tr.keyMatches(pe.Key) {
+ matchingKeys = append(matchingKeys, pe.Key)
+ } else {
+ restEntries = append(restEntries, pe)
+ }
+ }
+ tr.sortKeys(matchingKeys)
+ other := mlrval.NewMlrmapAsRecord()
+ for _, key := range matchingKeys {
+ val := inrec.Get(key)
+ if val != nil {
+ if m := val.GetMap(); m != nil {
+ tr.sortMapByKeyRecursively(m)
+ }
+ }
+ other.PutReference(key, val)
+ }
+ for _, pe := range restEntries {
+ if pe.Value != nil {
+ if m := pe.Value.GetMap(); m != nil {
+ tr.sortMapByKeyRecursively(m)
+ }
+ }
+ other.PutReference(pe.Key, pe.Value)
+ }
+ *inrec = *other
+ }
+ *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
+}
+
+// ----------------------------------------------------------------
func (tr *TransformerSortWithinRecords) transformNonrecursively(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
@@ -115,7 +318,11 @@ func (tr *TransformerSortWithinRecords) transformNonrecursively(
) {
if !inrecAndContext.EndOfStream {
inrec := inrecAndContext.Record
- inrec.SortByKey()
+ if tr.doNatural {
+ tr.sortMapByKey(inrec)
+ } else {
+ inrec.SortByKey()
+ }
}
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker
}
@@ -128,7 +335,11 @@ func (tr *TransformerSortWithinRecords) transformRecursively(
) {
if !inrecAndContext.EndOfStream {
inrec := inrecAndContext.Record
- inrec.SortByKeyRecursively()
+ if tr.doNatural {
+ tr.sortMapByKeyRecursively(inrec)
+ } else {
+ inrec.SortByKeyRecursively()
+ }
}
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker
}
diff --git a/test/cases/cli-help/0001/expout b/test/cases/cli-help/0001/expout
index af9c48619..0e8565a95 100644
--- a/test/cases/cli-help/0001/expout
+++ b/test/cases/cli-help/0001/expout
@@ -1004,8 +1004,13 @@ sort-within-records
Usage: mlr sort-within-records [options]
Outputs records sorted lexically ascending by keys.
Options:
--r Recursively sort subobjects/submaps, e.g. for JSON input.
--h|--help Show this message.
+-f {names} Sort only these keys; others preserve record order.
+-r {regex} Sort only keys matching this regex; others preserve record order.
+ Example: -r '^[xy]' sorts keys starting with x or y.
+ With no regex argument, -r recursively sorts subobjects/submaps
+ (e.g. for JSON input), or combines with -f to treat names as regex.
+-n Sort field names naturally (e.g. 2 before 12). Combines with -f/-r.
+-h|--help Show this message.
================================================================
sparsify
diff --git a/test/cases/verb-sort-within-records/0004-regex/cmd b/test/cases/verb-sort-within-records/0004-regex/cmd
new file mode 100644
index 000000000..7b5d76372
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0004-regex/cmd
@@ -0,0 +1 @@
+mlr --from test/input/sort-within-records.dkvp sort-within-records -f '^[ai]' -r
diff --git a/test/cases/verb-sort-within-records/0004-regex/experr b/test/cases/verb-sort-within-records/0004-regex/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0004-regex/expout b/test/cases/verb-sort-within-records/0004-regex/expout
new file mode 100644
index 000000000..a6ceab9f6
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0004-regex/expout
@@ -0,0 +1,10 @@
+a=pan,i=1,b=pan,x=0.34679014,y=0.72680286
+a=eks,i=2,x=0.75867996,y=0.52215111,b=pan
+a=wye,i=3,y=0.33831853,b=wye,x=0.20460331
+a=eks,i=4,x=0.38139939,b=wye,y=0.13418874
+a=wye,i=5,y=0.86362447,x=0.57328892,b=pan
+a=zee,i=6,y=0.49322129,b=pan,x=0.52712616
+a=eks,i=7,b=zee,x=0.61178406,y=0.18788492
+a=zee,i=8,x=0.59855401,b=wye,y=0.97618139
+a=hat,i=9,b=wye,x=0.03144188,y=0.74955076
+a=pan,i=10,y=0.95261836,x=0.50262601,b=wye
diff --git a/test/cases/verb-sort-within-records/0005-f-explicit/cmd b/test/cases/verb-sort-within-records/0005-f-explicit/cmd
new file mode 100644
index 000000000..d98a6358c
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0005-f-explicit/cmd
@@ -0,0 +1 @@
+mlr --from test/input/sort-within-records.dkvp sort-within-records -f a,i
diff --git a/test/cases/verb-sort-within-records/0005-f-explicit/experr b/test/cases/verb-sort-within-records/0005-f-explicit/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0005-f-explicit/expout b/test/cases/verb-sort-within-records/0005-f-explicit/expout
new file mode 100644
index 000000000..a6ceab9f6
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0005-f-explicit/expout
@@ -0,0 +1,10 @@
+a=pan,i=1,b=pan,x=0.34679014,y=0.72680286
+a=eks,i=2,x=0.75867996,y=0.52215111,b=pan
+a=wye,i=3,y=0.33831853,b=wye,x=0.20460331
+a=eks,i=4,x=0.38139939,b=wye,y=0.13418874
+a=wye,i=5,y=0.86362447,x=0.57328892,b=pan
+a=zee,i=6,y=0.49322129,b=pan,x=0.52712616
+a=eks,i=7,b=zee,x=0.61178406,y=0.18788492
+a=zee,i=8,x=0.59855401,b=wye,y=0.97618139
+a=hat,i=9,b=wye,x=0.03144188,y=0.74955076
+a=pan,i=10,y=0.95261836,x=0.50262601,b=wye
diff --git a/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/cmd b/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/cmd
new file mode 100644
index 000000000..c7b8daf6c
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/cmd
@@ -0,0 +1 @@
+mlr --csv sort-within-records -r "[0-9]+" test/input/issue-1645.csv
diff --git a/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/experr b/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/expout b/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/expout
new file mode 100644
index 000000000..0845e21d2
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0006-issue-1645-r-regex/expout
@@ -0,0 +1,3 @@
+12,2,a
+x,c,z
+5,t,4
diff --git a/test/cases/verb-sort-within-records/0007-natural/cmd b/test/cases/verb-sort-within-records/0007-natural/cmd
new file mode 100644
index 000000000..706093782
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0007-natural/cmd
@@ -0,0 +1 @@
+mlr --csv sort-within-records -n test/input/issue-1645.csv
diff --git a/test/cases/verb-sort-within-records/0007-natural/experr b/test/cases/verb-sort-within-records/0007-natural/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0007-natural/expout b/test/cases/verb-sort-within-records/0007-natural/expout
new file mode 100644
index 000000000..3e2254344
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0007-natural/expout
@@ -0,0 +1,3 @@
+2,12,a
+c,x,z
+t,5,4
diff --git a/test/cases/verb-sort-within-records/0008-natural-filelike/cmd b/test/cases/verb-sort-within-records/0008-natural-filelike/cmd
new file mode 100644
index 000000000..38a4b45db
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0008-natural-filelike/cmd
@@ -0,0 +1 @@
+mlr --idkvp --ojsonl --from test/input/natural-within-records.dkvp sort-within-records -n
diff --git a/test/cases/verb-sort-within-records/0008-natural-filelike/experr b/test/cases/verb-sort-within-records/0008-natural-filelike/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0008-natural-filelike/expout b/test/cases/verb-sort-within-records/0008-natural-filelike/expout
new file mode 100644
index 000000000..3d3951692
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0008-natural-filelike/expout
@@ -0,0 +1,2 @@
+{"file1": "c", "file2": "b", "file10": "a", "other": "d"}
+{"v3": "y", "v20": "x", "v100": "z"}
diff --git a/test/cases/verb-sort-within-records/0009-regex-rejects-list/cmd b/test/cases/verb-sort-within-records/0009-regex-rejects-list/cmd
new file mode 100644
index 000000000..878adfb6f
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0009-regex-rejects-list/cmd
@@ -0,0 +1 @@
+mlr --from test/input/sort-within-records.dkvp sort-within-records -f 'a,b' -r
diff --git a/test/cases/verb-sort-within-records/0009-regex-rejects-list/experr b/test/cases/verb-sort-within-records/0009-regex-rejects-list/experr
new file mode 100644
index 000000000..6939a4abf
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0009-regex-rejects-list/experr
@@ -0,0 +1 @@
+mlr sort-within-records: regex mode takes a single pattern; got 2 names: a,b. Use alternation in the regex (e.g. 'a|b') instead of a comma-list.
diff --git a/test/cases/verb-sort-within-records/0009-regex-rejects-list/expout b/test/cases/verb-sort-within-records/0009-regex-rejects-list/expout
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0009-regex-rejects-list/should-fail b/test/cases/verb-sort-within-records/0009-regex-rejects-list/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0010-bad-regex/cmd b/test/cases/verb-sort-within-records/0010-bad-regex/cmd
new file mode 100644
index 000000000..8ac0343ec
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0010-bad-regex/cmd
@@ -0,0 +1 @@
+mlr --from test/input/sort-within-records.dkvp sort-within-records -r '(bad[regex'
diff --git a/test/cases/verb-sort-within-records/0010-bad-regex/experr b/test/cases/verb-sort-within-records/0010-bad-regex/experr
new file mode 100644
index 000000000..f957a55d1
--- /dev/null
+++ b/test/cases/verb-sort-within-records/0010-bad-regex/experr
@@ -0,0 +1 @@
+mlr sort-within-records: cannot compile regex [(bad[regex]
diff --git a/test/cases/verb-sort-within-records/0010-bad-regex/expout b/test/cases/verb-sort-within-records/0010-bad-regex/expout
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-sort-within-records/0010-bad-regex/should-fail b/test/cases/verb-sort-within-records/0010-bad-regex/should-fail
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/input/issue-1645.csv b/test/input/issue-1645.csv
new file mode 100644
index 000000000..20f1f82b0
--- /dev/null
+++ b/test/input/issue-1645.csv
@@ -0,0 +1,3 @@
+a,12,2
+z,x,c
+4,5,t
diff --git a/test/input/natural-within-records.dkvp b/test/input/natural-within-records.dkvp
new file mode 100644
index 000000000..c63ed5336
--- /dev/null
+++ b/test/input/natural-within-records.dkvp
@@ -0,0 +1,2 @@
+file10=a,file2=b,file1=c,other=d
+v20=x,v3=y,v100=z