diff --git a/docs/src/manpage.md b/docs/src/manpage.md index a2d32b89b..a6d2e1658 100644 --- a/docs/src/manpage.md +++ b/docs/src/manpage.md @@ -1498,6 +1498,8 @@ This is simply a copy of what you should see on running `man mlr` at a command p --values,--pairs One is required. --across-records,--across-fields One is required. -f {field name} Required. + -r {field names} Like -f but treat arguments as a regular expression. Match all + field names and operate on each in record order. Example: `-r '^[xy]$`'. --nested-fs {string} Defaults to ";". Field separator for nested values. --nested-ps {string} Defaults to ":". Pair separator for nested key-value pairs. --evar {string} Shorthand for --explode --values --across-records --nested-fs {string} diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt index 6d080c0f3..e08c1125a 100644 --- a/docs/src/manpage.txt +++ b/docs/src/manpage.txt @@ -1477,6 +1477,8 @@ --values,--pairs One is required. --across-records,--across-fields One is required. -f {field name} Required. + -r {field names} Like -f but treat arguments as a regular expression. Match all + field names and operate on each in record order. Example: `-r '^[xy]$`'. --nested-fs {string} Defaults to ";". Field separator for nested values. --nested-ps {string} Defaults to ":". Pair separator for nested key-value pairs. --evar {string} Shorthand for --explode --values --across-records --nested-fs {string} diff --git a/docs/src/reference-verbs.md b/docs/src/reference-verbs.md index 0b69d4031..37ea61dd3 100644 --- a/docs/src/reference-verbs.md +++ b/docs/src/reference-verbs.md @@ -2245,6 +2245,8 @@ Options: --values,--pairs One is required. --across-records,--across-fields One is required. -f {field name} Required. + -r {field names} Like -f but treat arguments as a regular expression. Match all + field names and operate on each in record order. Example: `-r '^[xy]$`'. --nested-fs {string} Defaults to ";". Field separator for nested values. --nested-ps {string} Defaults to ":". Pair separator for nested key-value pairs. --evar {string} Shorthand for --explode --values --across-records --nested-fs {string} diff --git a/man/manpage.txt b/man/manpage.txt index 6d080c0f3..e08c1125a 100644 --- a/man/manpage.txt +++ b/man/manpage.txt @@ -1477,6 +1477,8 @@ --values,--pairs One is required. --across-records,--across-fields One is required. -f {field name} Required. + -r {field names} Like -f but treat arguments as a regular expression. Match all + field names and operate on each in record order. Example: `-r '^[xy]$`'. --nested-fs {string} Defaults to ";". Field separator for nested values. --nested-ps {string} Defaults to ":". Pair separator for nested key-value pairs. --evar {string} Shorthand for --explode --values --across-records --nested-fs {string} diff --git a/man/mlr.1 b/man/mlr.1 index 791a2343e..526eaa010 100644 --- a/man/mlr.1 +++ b/man/mlr.1 @@ -1868,6 +1868,8 @@ Options: --values,--pairs One is required. --across-records,--across-fields One is required. -f {field name} Required. + -r {field names} Like -f but treat arguments as a regular expression. Match all + field names and operate on each in record order. Example: `-r '^[xy]$`'. --nested-fs {string} Defaults to ";". Field separator for nested values. --nested-ps {string} Defaults to ":". Pair separator for nested key-value pairs. --evar {string} Shorthand for --explode --values --across-records --nested-fs {string} diff --git a/pkg/transformers/nest.go b/pkg/transformers/nest.go index 2cefda50d..e765eb2b5 100644 --- a/pkg/transformers/nest.go +++ b/pkg/transformers/nest.go @@ -37,6 +37,8 @@ func transformerNestUsage( fmt.Fprintf(o, " --values,--pairs One is required.\n") fmt.Fprintf(o, " --across-records,--across-fields One is required.\n") fmt.Fprintf(o, " -f {field name} Required.\n") + fmt.Fprintf(o, " -r {field names} Like -f but treat arguments as a regular expression. Match all\n") + fmt.Fprintf(o, " field names and operate on each in record order. Example: `-r '^[xy]$`'.\n") fmt.Fprintf(o, " --nested-fs {string} Defaults to \";\". Field separator for nested values.\n") fmt.Fprintf(o, " --nested-ps {string} Defaults to \":\". Pair separator for nested key-value pairs.\n") fmt.Fprintf(o, " --evar {string} Shorthand for --explode --values --across-records --nested-fs {string}\n") @@ -102,6 +104,7 @@ func transformerNestParseCLI( // Parse local flags fieldName := "" + doRegexes := false nestedFS := ";" nestedPS := ":" doExplode := true @@ -130,6 +133,10 @@ func transformerNestParseCLI( } else if opt == "-f" { fieldName = cli.VerbGetStringArgOrDie(verb, opt, args, &argi, argc) + } else if opt == "-r" { + doRegexes = true + fieldName = cli.VerbGetStringArgOrDie(verb, opt, args, &argi, argc) + } else if opt == "--explode" || opt == "-e" { doExplode = true doExplodeSpecified = true @@ -213,6 +220,10 @@ func transformerNestParseCLI( transformerNestUsage(os.Stderr) os.Exit(1) } + if doRegexes && !doExplode { + fmt.Fprintf(os.Stderr, "mlr nest: -r is only supported with --explode, not --implode.\n") + os.Exit(1) + } *pargi = argi if !doConstruct { // All transformers must do this for main command-line parsing @@ -221,6 +232,7 @@ func transformerNestParseCLI( transformer, err := NewTransformerNest( fieldName, + doRegexes, nestedFS, nestedPS, doExplode, @@ -241,7 +253,10 @@ type TransformerNest struct { nestedFS string nestedPS string - // For implode across fields + doRegexes bool + fieldRegex *regexp.Regexp // when doRegexes, for matching field names + + // For implode across fields (when !doRegexes) regex *regexp.Regexp // For implode across records @@ -253,6 +268,7 @@ type TransformerNest struct { // ---------------------------------------------------------------- func NewTransformerNest( fieldName string, + doRegexes bool, nestedFS string, nestedPS string, doExplode bool, @@ -262,22 +278,38 @@ func NewTransformerNest( tr := &TransformerNest{ fieldName: fieldName, + doRegexes: doRegexes, nestedFS: cli.SeparatorFromArg(nestedFS), // "pipe" -> "|", etc nestedPS: cli.SeparatorFromArg(nestedPS), } - // For implode across fields - regexString := "^" + fieldName + "_[0-9]+$" - regex, err := lib.CompileMillerRegex(regexString) - if err != nil { - fmt.Fprintf( - os.Stderr, - "%s %s: cannot compile regex [%s]\n", - "mlr", verbNameNest, regexString, - ) - os.Exit(1) + // For implode across fields: regex to match exploded form (e.g. x_1, x_2) + if doRegexes { + fieldRegex, err := lib.CompileMillerRegex(fieldName) + if err != nil { + fmt.Fprintf( + os.Stderr, + "%s %s: cannot compile regex [%s]\n", + "mlr", verbNameNest, fieldName, + ) + os.Exit(1) + } + tr.fieldRegex = fieldRegex + // implode uses fieldRegex directly when doRegexes + tr.regex = fieldRegex + } else { + regexString := "^" + fieldName + "_[0-9]+$" + regex, err := lib.CompileMillerRegex(regexString) + if err != nil { + fmt.Fprintf( + os.Stderr, + "%s %s: cannot compile regex [%s]\n", + "mlr", verbNameNest, regexString, + ) + os.Exit(1) + } + tr.regex = regex } - tr.regex = regex // For implode across records tr.otherKeysToOtherValuesToBuckets = lib.NewOrderedMap[*lib.OrderedMap[*tNestBucket]]() @@ -324,6 +356,25 @@ func (tr *TransformerNest) Transform( tr.recordTransformerFunc(inrecAndContext, outputRecordsAndContexts, inputDownstreamDoneChannel, outputDownstreamDoneChannel) } +// ---------------------------------------------------------------- +// getMatchingFieldNames returns field names matching tr.fieldRegex in record order. +// When !tr.doRegexes, returns [tr.fieldName] if present, else []. +func (tr *TransformerNest) getMatchingFieldNames(inrec *mlrval.Mlrmap) []string { + if !tr.doRegexes { + if inrec.Get(tr.fieldName) != nil { + return []string{tr.fieldName} + } + return nil + } + var names []string + for pe := inrec.Head; pe != nil; pe = pe.Next { + if tr.fieldRegex.MatchString(pe.Key) { + names = append(names, pe.Key) + } + } + return names +} + // ---------------------------------------------------------------- func (tr *TransformerNest) explodeValuesAcrossFields( inrecAndContext *types.RecordAndContext, @@ -334,27 +385,34 @@ func (tr *TransformerNest) explodeValuesAcrossFields( if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - originalEntry := inrec.GetEntry(tr.fieldName) - if originalEntry == nil { + fieldNames := tr.getMatchingFieldNames(inrec) + if len(fieldNames) == 0 { *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } - recordEntry := originalEntry - mvalue := originalEntry.Value - svalue := mvalue.String() + for _, fieldName := range fieldNames { + originalEntry := inrec.GetEntry(fieldName) + if originalEntry == nil { + continue + } - // Not lib.SplitString so 'x=' will map to 'x_1=', rather than no field at all - pieces := strings.Split(svalue, tr.nestedFS) - i := 1 - for _, piece := range pieces { - key := tr.fieldName + "_" + strconv.Itoa(i) - value := mlrval.FromString(piece) - recordEntry = inrec.PutReferenceAfter(recordEntry, key, value) - i++ + recordEntry := originalEntry + mvalue := originalEntry.Value + svalue := mvalue.String() + + // Not lib.SplitString so 'x=' will map to 'x_1=', rather than no field at all + pieces := strings.Split(svalue, tr.nestedFS) + i := 1 + for _, piece := range pieces { + key := fieldName + "_" + strconv.Itoa(i) + value := mlrval.FromString(piece) + recordEntry = inrec.PutReferenceAfter(recordEntry, key, value) + i++ + } + + inrec.Unlink(originalEntry) } - - inrec.Unlink(originalEntry) *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { @@ -371,7 +429,14 @@ func (tr *TransformerNest) explodeValuesAcrossRecords( ) { if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - mvalue := inrec.Get(tr.fieldName) + fieldNames := tr.getMatchingFieldNames(inrec) + if len(fieldNames) == 0 { + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) + return + } + fieldName := fieldNames[0] + + mvalue := inrec.Get(fieldName) if mvalue == nil { *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return @@ -382,7 +447,7 @@ func (tr *TransformerNest) explodeValuesAcrossRecords( pieces := strings.Split(svalue, tr.nestedFS) for _, piece := range pieces { outrec := inrec.Copy() - outrec.PutReference(tr.fieldName, mlrval.FromString(piece)) + outrec.PutReference(fieldName, mlrval.FromString(piece)) *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } @@ -401,35 +466,42 @@ func (tr *TransformerNest) explodePairsAcrossFields( if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - originalEntry := inrec.GetEntry(tr.fieldName) - if originalEntry == nil { + fieldNames := tr.getMatchingFieldNames(inrec) + if len(fieldNames) == 0 { *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } - mvalue := originalEntry.Value - svalue := mvalue.String() - - recordEntry := originalEntry - pieces := lib.SplitString(svalue, tr.nestedFS) - for _, piece := range pieces { - pair := strings.SplitN(piece, tr.nestedPS, 2) - if len(pair) == 2 { // there is a pair - recordEntry = inrec.PutReferenceAfter( - recordEntry, - pair[0], - mlrval.FromString(pair[1]), - ) - } else { // there is not a pair - recordEntry = inrec.PutReferenceAfter( - recordEntry, - tr.fieldName, - mlrval.FromString(piece), - ) + for _, fieldName := range fieldNames { + originalEntry := inrec.GetEntry(fieldName) + if originalEntry == nil { + continue } - } - inrec.Unlink(originalEntry) + mvalue := originalEntry.Value + svalue := mvalue.String() + + recordEntry := originalEntry + pieces := lib.SplitString(svalue, tr.nestedFS) + for _, piece := range pieces { + pair := strings.SplitN(piece, tr.nestedPS, 2) + if len(pair) == 2 { // there is a pair + recordEntry = inrec.PutReferenceAfter( + recordEntry, + pair[0], + mlrval.FromString(pair[1]), + ) + } else { // there is not a pair + recordEntry = inrec.PutReferenceAfter( + recordEntry, + fieldName, + mlrval.FromString(piece), + ) + } + } + + inrec.Unlink(originalEntry) + } *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { @@ -446,7 +518,14 @@ func (tr *TransformerNest) explodePairsAcrossRecords( ) { if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - mvalue := inrec.Get(tr.fieldName) + fieldNames := tr.getMatchingFieldNames(inrec) + if len(fieldNames) == 0 { + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) + return + } + fieldName := fieldNames[0] + + mvalue := inrec.Get(fieldName) if mvalue == nil { *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return @@ -457,7 +536,7 @@ func (tr *TransformerNest) explodePairsAcrossRecords( for _, piece := range pieces { outrec := inrec.Copy() - originalEntry := outrec.GetEntry(tr.fieldName) + originalEntry := outrec.GetEntry(fieldName) // Put the new field where the old one was -- unless there's already a field with the new // name, in which case replace its value. @@ -465,7 +544,7 @@ func (tr *TransformerNest) explodePairsAcrossRecords( if len(pair) == 2 { // there is a pair outrec.PutReferenceAfter(originalEntry, pair[0], mlrval.FromString(pair[1])) } else { // there is not a pair - outrec.PutReferenceAfter(originalEntry, tr.fieldName, mlrval.FromString(piece)) + outrec.PutReferenceAfter(originalEntry, fieldName, mlrval.FromString(piece)) } outrec.Unlink(originalEntry) diff --git a/test/cases/cli-help/0001/expout b/test/cases/cli-help/0001/expout index 9b10758f2..72af8cd07 100644 --- a/test/cases/cli-help/0001/expout +++ b/test/cases/cli-help/0001/expout @@ -615,6 +615,8 @@ Options: --values,--pairs One is required. --across-records,--across-fields One is required. -f {field name} Required. + -r {field names} Like -f but treat arguments as a regular expression. Match all + field names and operate on each in record order. Example: `-r '^[xy]$`'. --nested-fs {string} Defaults to ";". Field separator for nested values. --nested-ps {string} Defaults to ":". Pair separator for nested key-value pairs. --evar {string} Shorthand for --explode --values --across-records --nested-fs {string} diff --git a/test/cases/verb-nest/epaf-regex/cmd b/test/cases/verb-nest/epaf-regex/cmd new file mode 100644 index 000000000..37573e91a --- /dev/null +++ b/test/cases/verb-nest/epaf-regex/cmd @@ -0,0 +1 @@ +mlr nest --explode --pairs --across-fields -r '^x$' test/input/nest-explode.dkvp diff --git a/test/cases/verb-nest/epaf-regex/experr b/test/cases/verb-nest/epaf-regex/experr new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/verb-nest/epaf-regex/expout b/test/cases/verb-nest/epaf-regex/expout new file mode 100644 index 000000000..24d5905f7 --- /dev/null +++ b/test/cases/verb-nest/epaf-regex/expout @@ -0,0 +1,4 @@ +a=1,b=2,c=3,y=d:40 +y=d:50 +u=100,y=d:60 +a=4,b=5,y=d:70 diff --git a/test/cases/verb-nest/epar-regex/cmd b/test/cases/verb-nest/epar-regex/cmd new file mode 100644 index 000000000..e9de6a568 --- /dev/null +++ b/test/cases/verb-nest/epar-regex/cmd @@ -0,0 +1 @@ +mlr nest --explode --pairs --across-records -r '^x$' test/input/nest-explode.dkvp diff --git a/test/cases/verb-nest/epar-regex/experr b/test/cases/verb-nest/epar-regex/experr new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/verb-nest/epar-regex/expout b/test/cases/verb-nest/epar-regex/expout new file mode 100644 index 000000000..24fc64ca2 --- /dev/null +++ b/test/cases/verb-nest/epar-regex/expout @@ -0,0 +1,6 @@ +a=1,y=d:40 +b=2,y=d:40 +c=3,y=d:40 +u=100,y=d:60 +a=4,y=d:70 +b=5,y=d:70 diff --git a/test/cases/verb-nest/evaf-regex/cmd b/test/cases/verb-nest/evaf-regex/cmd new file mode 100644 index 000000000..d430cb9c0 --- /dev/null +++ b/test/cases/verb-nest/evaf-regex/cmd @@ -0,0 +1 @@ +mlr nest --explode --values --across-fields -r '^[xy]$' test/input/nest-explode-regex.dkvp diff --git a/test/cases/verb-nest/evaf-regex/experr b/test/cases/verb-nest/evaf-regex/experr new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/verb-nest/evaf-regex/expout b/test/cases/verb-nest/evaf-regex/expout new file mode 100644 index 000000000..07c39ab49 --- /dev/null +++ b/test/cases/verb-nest/evaf-regex/expout @@ -0,0 +1,2 @@ +x_1=a,x_2=b,x_3=c,y_1=d,y_2=e,y_3=f,z=z +x_1=1,x_2=2,y_1=3,z=other diff --git a/test/cases/verb-nest/evar-regex/cmd b/test/cases/verb-nest/evar-regex/cmd new file mode 100644 index 000000000..b6a67f303 --- /dev/null +++ b/test/cases/verb-nest/evar-regex/cmd @@ -0,0 +1 @@ +mlr nest --explode --values --across-records -r '^x$' test/input/nest-explode.dkvp diff --git a/test/cases/verb-nest/evar-regex/experr b/test/cases/verb-nest/evar-regex/experr new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/verb-nest/evar-regex/expout b/test/cases/verb-nest/evar-regex/expout new file mode 100644 index 000000000..6c6ab0ad6 --- /dev/null +++ b/test/cases/verb-nest/evar-regex/expout @@ -0,0 +1,7 @@ +x=a:1,y=d:40 +x=b:2,y=d:40 +x=c:3,y=d:40 +x=,y=d:50 +u=100,y=d:60 +x=a:4,y=d:70 +x=b:5,y=d:70 diff --git a/test/input/nest-explode-regex.dkvp b/test/input/nest-explode-regex.dkvp new file mode 100644 index 000000000..f33e782fd --- /dev/null +++ b/test/input/nest-explode-regex.dkvp @@ -0,0 +1,2 @@ +x=a;b;c,y=d;e;f,z=z +x=1;2,y=3,z=other