Add -r (for regex field-name matching) to mlr nest (#1961)

* initial attempt

* fix up cli flag

* make dev

docs/src/manpage.md

@@ -1498,6 +1498,8 @@ This is simply a copy of what you should see on running `man mlr` at a command p
          --values,--pairs      One is required.
          --across-records,--across-fields One is required.
          -f {field name}       Required.
+         -r {field names}      Like -f but treat arguments as a regular expression. Match all
+                               field names and operate on each in record order. Example: `-r '^[xy]$`'.
          --nested-fs {string}  Defaults to ";". Field separator for nested values.
          --nested-ps {string}  Defaults to ":". Pair separator for nested key-value pairs.
          --evar {string}       Shorthand for --explode --values --across-records --nested-fs {string}

docs/src/manpage.txt

@@ -1477,6 +1477,8 @@
          --values,--pairs      One is required.
          --across-records,--across-fields One is required.
          -f {field name}       Required.
+         -r {field names}      Like -f but treat arguments as a regular expression. Match all
+                               field names and operate on each in record order. Example: `-r '^[xy]$`'.
          --nested-fs {string}  Defaults to ";". Field separator for nested values.
          --nested-ps {string}  Defaults to ":". Pair separator for nested key-value pairs.
          --evar {string}       Shorthand for --explode --values --across-records --nested-fs {string}

docs/src/reference-verbs.md

@@ -2245,6 +2245,8 @@ Options:
   --values,--pairs      One is required.
   --across-records,--across-fields One is required.
   -f {field name}       Required.
+  -r {field names}      Like -f but treat arguments as a regular expression. Match all
+                        field names and operate on each in record order. Example: `-r '^[xy]$`'.
   --nested-fs {string}  Defaults to ";". Field separator for nested values.
   --nested-ps {string}  Defaults to ":". Pair separator for nested key-value pairs.
   --evar {string}       Shorthand for --explode --values --across-records --nested-fs {string}

man/manpage.txt

@@ -1477,6 +1477,8 @@
          --values,--pairs      One is required.
          --across-records,--across-fields One is required.
          -f {field name}       Required.
+         -r {field names}      Like -f but treat arguments as a regular expression. Match all
+                               field names and operate on each in record order. Example: `-r '^[xy]$`'.
          --nested-fs {string}  Defaults to ";". Field separator for nested values.
          --nested-ps {string}  Defaults to ":". Pair separator for nested key-value pairs.
          --evar {string}       Shorthand for --explode --values --across-records --nested-fs {string}

man/mlr.1

@@ -1868,6 +1868,8 @@ Options:
   --values,--pairs      One is required.
   --across-records,--across-fields One is required.
   -f {field name}       Required.
+  -r {field names}      Like -f but treat arguments as a regular expression. Match all
+                        field names and operate on each in record order. Example: `-r '^[xy]$`'.
   --nested-fs {string}  Defaults to ";". Field separator for nested values.
   --nested-ps {string}  Defaults to ":". Pair separator for nested key-value pairs.
   --evar {string}       Shorthand for --explode --values --across-records --nested-fs {string}

pkg/transformers/nest.go

@@ -37,6 +37,8 @@ func transformerNestUsage(
 	fmt.Fprintf(o, "  --values,--pairs      One is required.\n")
 	fmt.Fprintf(o, "  --across-records,--across-fields One is required.\n")
 	fmt.Fprintf(o, "  -f {field name}       Required.\n")
+	fmt.Fprintf(o, "  -r {field names}      Like -f but treat arguments as a regular expression. Match all\n")
+	fmt.Fprintf(o, "                        field names and operate on each in record order. Example: `-r '^[xy]$`'.\n")
 	fmt.Fprintf(o, "  --nested-fs {string}  Defaults to \";\". Field separator for nested values.\n")
 	fmt.Fprintf(o, "  --nested-ps {string}  Defaults to \":\". Pair separator for nested key-value pairs.\n")
 	fmt.Fprintf(o, "  --evar {string}       Shorthand for --explode --values --across-records --nested-fs {string}\n")
@@ -102,6 +104,7 @@ func transformerNestParseCLI(
 
 	// Parse local flags
 	fieldName := ""
+	doRegexes := false
 	nestedFS := ";"
 	nestedPS := ":"
 	doExplode := true
@@ -130,6 +133,10 @@ func transformerNestParseCLI(
 		} else if opt == "-f" {
 			fieldName = cli.VerbGetStringArgOrDie(verb, opt, args, &argi, argc)
 
+		} else if opt == "-r" {
+			doRegexes = true
+			fieldName = cli.VerbGetStringArgOrDie(verb, opt, args, &argi, argc)
+
 		} else if opt == "--explode" || opt == "-e" {
 			doExplode = true
 			doExplodeSpecified = true
@@ -213,6 +220,10 @@ func transformerNestParseCLI(
 		transformerNestUsage(os.Stderr)
 		os.Exit(1)
 	}
+	if doRegexes && !doExplode {
+		fmt.Fprintf(os.Stderr, "mlr nest: -r is only supported with --explode, not --implode.\n")
+		os.Exit(1)
+	}
 
 	*pargi = argi
 	if !doConstruct { // All transformers must do this for main command-line parsing
@@ -221,6 +232,7 @@ func transformerNestParseCLI(
 
 	transformer, err := NewTransformerNest(
 		fieldName,
+		doRegexes,
 		nestedFS,
 		nestedPS,
 		doExplode,
@@ -241,7 +253,10 @@ type TransformerNest struct {
 	nestedFS  string
 	nestedPS  string
 
-	// For implode across fields
+	doRegexes  bool
+	fieldRegex *regexp.Regexp // when doRegexes, for matching field names
+
+	// For implode across fields (when !doRegexes)
 	regex *regexp.Regexp
 
 	// For implode across records
@@ -253,6 +268,7 @@ type TransformerNest struct {
 // ----------------------------------------------------------------
 func NewTransformerNest(
 	fieldName string,
+	doRegexes bool,
 	nestedFS string,
 	nestedPS string,
 	doExplode bool,
@@ -262,22 +278,38 @@ func NewTransformerNest(
 
 	tr := &TransformerNest{
 		fieldName: fieldName,
+		doRegexes: doRegexes,
 		nestedFS:  cli.SeparatorFromArg(nestedFS), // "pipe" -> "|", etc
 		nestedPS:  cli.SeparatorFromArg(nestedPS),
 	}
 
-	// For implode across fields
-	regexString := "^" + fieldName + "_[0-9]+$"
-	regex, err := lib.CompileMillerRegex(regexString)
-	if err != nil {
-		fmt.Fprintf(
-			os.Stderr,
-			"%s %s: cannot compile regex [%s]\n",
-			"mlr", verbNameNest, regexString,
-		)
-		os.Exit(1)
+	// For implode across fields: regex to match exploded form (e.g. x_1, x_2)
+	if doRegexes {
+		fieldRegex, err := lib.CompileMillerRegex(fieldName)
+		if err != nil {
+			fmt.Fprintf(
+				os.Stderr,
+				"%s %s: cannot compile regex [%s]\n",
+				"mlr", verbNameNest, fieldName,
+			)
+			os.Exit(1)
+		}
+		tr.fieldRegex = fieldRegex
+		// implode uses fieldRegex directly when doRegexes
+		tr.regex = fieldRegex
+	} else {
+		regexString := "^" + fieldName + "_[0-9]+$"
+		regex, err := lib.CompileMillerRegex(regexString)
+		if err != nil {
+			fmt.Fprintf(
+				os.Stderr,
+				"%s %s: cannot compile regex [%s]\n",
+				"mlr", verbNameNest, regexString,
+			)
+			os.Exit(1)
+		}
+		tr.regex = regex
 	}
-	tr.regex = regex
 
 	// For implode across records
 	tr.otherKeysToOtherValuesToBuckets = lib.NewOrderedMap[*lib.OrderedMap[*tNestBucket]]()
@@ -324,6 +356,25 @@ func (tr *TransformerNest) Transform(
 	tr.recordTransformerFunc(inrecAndContext, outputRecordsAndContexts, inputDownstreamDoneChannel, outputDownstreamDoneChannel)
 }
 
+// ----------------------------------------------------------------
+// getMatchingFieldNames returns field names matching tr.fieldRegex in record order.
+// When !tr.doRegexes, returns [tr.fieldName] if present, else [].
+func (tr *TransformerNest) getMatchingFieldNames(inrec *mlrval.Mlrmap) []string {
+	if !tr.doRegexes {
+		if inrec.Get(tr.fieldName) != nil {
+			return []string{tr.fieldName}
+		}
+		return nil
+	}
+	var names []string
+	for pe := inrec.Head; pe != nil; pe = pe.Next {
+		if tr.fieldRegex.MatchString(pe.Key) {
+			names = append(names, pe.Key)
+		}
+	}
+	return names
+}
+
 // ----------------------------------------------------------------
 func (tr *TransformerNest) explodeValuesAcrossFields(
 	inrecAndContext *types.RecordAndContext,
@@ -334,27 +385,34 @@ func (tr *TransformerNest) explodeValuesAcrossFields(
 	if !inrecAndContext.EndOfStream {
 
 		inrec := inrecAndContext.Record
-		originalEntry := inrec.GetEntry(tr.fieldName)
-		if originalEntry == nil {
+		fieldNames := tr.getMatchingFieldNames(inrec)
+		if len(fieldNames) == 0 {
 			*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
 			return
 		}
 
-		recordEntry := originalEntry
-		mvalue := originalEntry.Value
-		svalue := mvalue.String()
+		for _, fieldName := range fieldNames {
+			originalEntry := inrec.GetEntry(fieldName)
+			if originalEntry == nil {
+				continue
+			}
 
-		// Not lib.SplitString so 'x=' will map to 'x_1=', rather than no field at all
-		pieces := strings.Split(svalue, tr.nestedFS)
-		i := 1
-		for _, piece := range pieces {
-			key := tr.fieldName + "_" + strconv.Itoa(i)
-			value := mlrval.FromString(piece)
-			recordEntry = inrec.PutReferenceAfter(recordEntry, key, value)
-			i++
+			recordEntry := originalEntry
+			mvalue := originalEntry.Value
+			svalue := mvalue.String()
+
+			// Not lib.SplitString so 'x=' will map to 'x_1=', rather than no field at all
+			pieces := strings.Split(svalue, tr.nestedFS)
+			i := 1
+			for _, piece := range pieces {
+				key := fieldName + "_" + strconv.Itoa(i)
+				value := mlrval.FromString(piece)
+				recordEntry = inrec.PutReferenceAfter(recordEntry, key, value)
+				i++
+			}
+
+			inrec.Unlink(originalEntry)
 		}
-
-		inrec.Unlink(originalEntry)
 		*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
 
 	} else {
@@ -371,7 +429,14 @@ func (tr *TransformerNest) explodeValuesAcrossRecords(
 ) {
 	if !inrecAndContext.EndOfStream {
 		inrec := inrecAndContext.Record
-		mvalue := inrec.Get(tr.fieldName)
+		fieldNames := tr.getMatchingFieldNames(inrec)
+		if len(fieldNames) == 0 {
+			*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
+			return
+		}
+		fieldName := fieldNames[0]
+
+		mvalue := inrec.Get(fieldName)
 		if mvalue == nil {
 			*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
 			return
@@ -382,7 +447,7 @@ func (tr *TransformerNest) explodeValuesAcrossRecords(
 		pieces := strings.Split(svalue, tr.nestedFS)
 		for _, piece := range pieces {
 			outrec := inrec.Copy()
-			outrec.PutReference(tr.fieldName, mlrval.FromString(piece))
+			outrec.PutReference(fieldName, mlrval.FromString(piece))
 			*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
 		}
 
@@ -401,35 +466,42 @@ func (tr *TransformerNest) explodePairsAcrossFields(
 	if !inrecAndContext.EndOfStream {
 
 		inrec := inrecAndContext.Record
-		originalEntry := inrec.GetEntry(tr.fieldName)
-		if originalEntry == nil {
+		fieldNames := tr.getMatchingFieldNames(inrec)
+		if len(fieldNames) == 0 {
 			*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
 			return
 		}
 
-		mvalue := originalEntry.Value
-		svalue := mvalue.String()
-
-		recordEntry := originalEntry
-		pieces := lib.SplitString(svalue, tr.nestedFS)
-		for _, piece := range pieces {
-			pair := strings.SplitN(piece, tr.nestedPS, 2)
-			if len(pair) == 2 { // there is a pair
-				recordEntry = inrec.PutReferenceAfter(
-					recordEntry,
-					pair[0],
-					mlrval.FromString(pair[1]),
-				)
-			} else { // there is not a pair
-				recordEntry = inrec.PutReferenceAfter(
-					recordEntry,
-					tr.fieldName,
-					mlrval.FromString(piece),
-				)
+		for _, fieldName := range fieldNames {
+			originalEntry := inrec.GetEntry(fieldName)
+			if originalEntry == nil {
+				continue
 			}
-		}
 
-		inrec.Unlink(originalEntry)
+			mvalue := originalEntry.Value
+			svalue := mvalue.String()
+
+			recordEntry := originalEntry
+			pieces := lib.SplitString(svalue, tr.nestedFS)
+			for _, piece := range pieces {
+				pair := strings.SplitN(piece, tr.nestedPS, 2)
+				if len(pair) == 2 { // there is a pair
+					recordEntry = inrec.PutReferenceAfter(
+						recordEntry,
+						pair[0],
+						mlrval.FromString(pair[1]),
+					)
+				} else { // there is not a pair
+					recordEntry = inrec.PutReferenceAfter(
+						recordEntry,
+						fieldName,
+						mlrval.FromString(piece),
+					)
+				}
+			}
+
+			inrec.Unlink(originalEntry)
+		}
 		*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
 
 	} else {
@@ -446,7 +518,14 @@ func (tr *TransformerNest) explodePairsAcrossRecords(
 ) {
 	if !inrecAndContext.EndOfStream {
 		inrec := inrecAndContext.Record
-		mvalue := inrec.Get(tr.fieldName)
+		fieldNames := tr.getMatchingFieldNames(inrec)
+		if len(fieldNames) == 0 {
+			*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
+			return
+		}
+		fieldName := fieldNames[0]
+
+		mvalue := inrec.Get(fieldName)
 		if mvalue == nil {
 			*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
 			return
@@ -457,7 +536,7 @@ func (tr *TransformerNest) explodePairsAcrossRecords(
 		for _, piece := range pieces {
 			outrec := inrec.Copy()
 
-			originalEntry := outrec.GetEntry(tr.fieldName)
+			originalEntry := outrec.GetEntry(fieldName)
 
 			// Put the new field where the old one was -- unless there's already a field with the new
 			// name, in which case replace its value.
@@ -465,7 +544,7 @@ func (tr *TransformerNest) explodePairsAcrossRecords(
 			if len(pair) == 2 { // there is a pair
 				outrec.PutReferenceAfter(originalEntry, pair[0], mlrval.FromString(pair[1]))
 			} else { // there is not a pair
-				outrec.PutReferenceAfter(originalEntry, tr.fieldName, mlrval.FromString(piece))
+				outrec.PutReferenceAfter(originalEntry, fieldName, mlrval.FromString(piece))
 			}
 
 			outrec.Unlink(originalEntry)

test/cases/cli-help/0001/expout

@@ -615,6 +615,8 @@ Options:
   --values,--pairs      One is required.
   --across-records,--across-fields One is required.
   -f {field name}       Required.
+  -r {field names}      Like -f but treat arguments as a regular expression. Match all
+                        field names and operate on each in record order. Example: `-r '^[xy]$`'.
   --nested-fs {string}  Defaults to ";". Field separator for nested values.
   --nested-ps {string}  Defaults to ":". Pair separator for nested key-value pairs.
   --evar {string}       Shorthand for --explode --values --across-records --nested-fs {string}

test/cases/verb-nest/epaf-regex/cmd

@@ -0,0 +1 @@
+mlr nest --explode --pairs --across-fields -r '^x$' test/input/nest-explode.dkvp

test/cases/verb-nest/epaf-regex/expout

@@ -0,0 +1,4 @@
+a=1,b=2,c=3,y=d:40
+y=d:50
+u=100,y=d:60
+a=4,b=5,y=d:70

test/cases/verb-nest/epar-regex/cmd

@@ -0,0 +1 @@
+mlr nest --explode --pairs --across-records -r '^x$' test/input/nest-explode.dkvp

test/cases/verb-nest/epar-regex/expout

@@ -0,0 +1,6 @@
+a=1,y=d:40
+b=2,y=d:40
+c=3,y=d:40
+u=100,y=d:60
+a=4,y=d:70
+b=5,y=d:70

test/cases/verb-nest/evaf-regex/cmd

@@ -0,0 +1 @@
+mlr nest --explode --values --across-fields -r '^[xy]$' test/input/nest-explode-regex.dkvp

test/cases/verb-nest/evaf-regex/expout

@@ -0,0 +1,2 @@
+x_1=a,x_2=b,x_3=c,y_1=d,y_2=e,y_3=f,z=z
+x_1=1,x_2=2,y_1=3,z=other

test/cases/verb-nest/evar-regex/cmd

@@ -0,0 +1 @@
+mlr nest --explode --values --across-records -r '^x$' test/input/nest-explode.dkvp

test/cases/verb-nest/evar-regex/expout

@@ -0,0 +1,7 @@
+x=a:1,y=d:40
+x=b:2,y=d:40
+x=c:3,y=d:40
+x=,y=d:50
+u=100,y=d:60
+x=a:4,y=d:70
+x=b:5,y=d:70

test/input/nest-explode-regex.dkvp

@@ -0,0 +1,2 @@
+x=a;b;c,y=d;e;f,z=z
+x=1;2,y=3,z=other