Unify the case verb, and add options (#1306)

* temp

* Unified `case` verb

* unit-test cases

* doc-page update

* make-dev artifacts
This commit is contained in:
John Kerl 2023-06-04 14:38:22 -04:00 committed by GitHub
parent 394681c4c1
commit 59e63272ae
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
102 changed files with 450 additions and 400 deletions

2
docs/src/data/cases.csv Normal file
View file

@ -0,0 +1,2 @@
apple,ball,cat,dog,the quick brown fox
sun,tree,urchin,violet,jumped over the lazy dog
1 apple ball cat dog the quick brown fox
2 sun tree urchin violet jumped over the lazy dog

View file

@ -192,15 +192,14 @@ MILLER(1) MILLER(1)
for all things with "map" in their names.
1mVERB LIST0m
altkv bar bootstrap cat check clean-whitespace count-distinct count
count-similar cut decimate downcase fill-down fill-empty filter flatten
format-values fraction gap grep group-by group-like having-fields head
histogram json-parse json-stringify join label latin1-to-utf8 least-frequent
merge-fields most-frequent nest nothing put regularize remove-empty-columns
rename reorder repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle
skip-trivial-records sort sort-within-records split stats1 stats2 step summary
tac tail tee template top utf8-to-latin1 unflatten uniq unspace unsparsify
upcase
altkv bar bootstrap case cat check clean-whitespace count-distinct count
count-similar cut decimate fill-down fill-empty filter flatten format-values
fraction gap grep group-by group-like having-fields head histogram json-parse
json-stringify join label latin1-to-utf8 least-frequent merge-fields
most-frequent nest nothing put regularize remove-empty-columns rename reorder
repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle skip-trivial-records
sort sort-within-records split stats1 stats2 step summary tac tail tee
template top utf8-to-latin1 unflatten uniq unspace unsparsify
1mFUNCTION LIST0m
abs acos acosh any append apply arrayify asin asinh asserting_absent
@ -909,6 +908,19 @@ MILLER(1) MILLER(1)
Must be non-negative.
-h|--help Show this message.
1mcase0m
Usage: mlr case [options]
Uppercases strings in record keys and/or values.
Options:
-k Case only keys, not keys and values.
-v Case only values, not keys and values.
-f {a,b,c} Specify which field names to case (default: all)
-u Convert to uppercase
-l Convert to lowercase
-s Convert to sentence case (capitalize first letter)
-t Convert to title case (capitalize words)
-h|--help Show this message.
1mcat0m
Usage: mlr cat [options]
Passes input records directly to output. Most useful for format conversion.
@ -1006,14 +1018,6 @@ MILLER(1) MILLER(1)
-n {n} Decimation factor (default 10).
-h|--help Show this message.
1mdowncase0m
Usage: mlr downcase [options]
Lowercases strings in record keys and/or values.
Options:
-k Downcase only keys, not keys and values.
-v Downcase only values, not keys and values.
-h|--help Show this message.
1mfill-down0m
Usage: mlr fill-down [options]
If a given record has a missing value for a given field, fill that from
@ -2127,14 +2131,6 @@ MILLER(1) MILLER(1)
being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and
'a=,b=3,c=4'.
1mupcase0m
Usage: mlr upcase [options]
Uppercases strings in record keys and/or values.
Options:
-k Upcase only keys, not keys and values.
-v Upcase only values, not keys and values.
-h|--help Show this message.
1mFUNCTIONS FOR FILTER/PUT0m
1mabs0m
(class=math #args=1) Absolute value.
@ -3358,5 +3354,5 @@ MILLER(1) MILLER(1)
2023-06-03 MILLER(1)
2023-06-04 MILLER(1)
</pre>

View file

@ -171,15 +171,14 @@ MILLER(1) MILLER(1)
for all things with "map" in their names.
1mVERB LIST0m
altkv bar bootstrap cat check clean-whitespace count-distinct count
count-similar cut decimate downcase fill-down fill-empty filter flatten
format-values fraction gap grep group-by group-like having-fields head
histogram json-parse json-stringify join label latin1-to-utf8 least-frequent
merge-fields most-frequent nest nothing put regularize remove-empty-columns
rename reorder repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle
skip-trivial-records sort sort-within-records split stats1 stats2 step summary
tac tail tee template top utf8-to-latin1 unflatten uniq unspace unsparsify
upcase
altkv bar bootstrap case cat check clean-whitespace count-distinct count
count-similar cut decimate fill-down fill-empty filter flatten format-values
fraction gap grep group-by group-like having-fields head histogram json-parse
json-stringify join label latin1-to-utf8 least-frequent merge-fields
most-frequent nest nothing put regularize remove-empty-columns rename reorder
repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle skip-trivial-records
sort sort-within-records split stats1 stats2 step summary tac tail tee
template top utf8-to-latin1 unflatten uniq unspace unsparsify
1mFUNCTION LIST0m
abs acos acosh any append apply arrayify asin asinh asserting_absent
@ -888,6 +887,19 @@ MILLER(1) MILLER(1)
Must be non-negative.
-h|--help Show this message.
1mcase0m
Usage: mlr case [options]
Uppercases strings in record keys and/or values.
Options:
-k Case only keys, not keys and values.
-v Case only values, not keys and values.
-f {a,b,c} Specify which field names to case (default: all)
-u Convert to uppercase
-l Convert to lowercase
-s Convert to sentence case (capitalize first letter)
-t Convert to title case (capitalize words)
-h|--help Show this message.
1mcat0m
Usage: mlr cat [options]
Passes input records directly to output. Most useful for format conversion.
@ -985,14 +997,6 @@ MILLER(1) MILLER(1)
-n {n} Decimation factor (default 10).
-h|--help Show this message.
1mdowncase0m
Usage: mlr downcase [options]
Lowercases strings in record keys and/or values.
Options:
-k Downcase only keys, not keys and values.
-v Downcase only values, not keys and values.
-h|--help Show this message.
1mfill-down0m
Usage: mlr fill-down [options]
If a given record has a missing value for a given field, fill that from
@ -2106,14 +2110,6 @@ MILLER(1) MILLER(1)
being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and
'a=,b=3,c=4'.
1mupcase0m
Usage: mlr upcase [options]
Uppercases strings in record keys and/or values.
Options:
-k Upcase only keys, not keys and values.
-v Upcase only values, not keys and values.
-h|--help Show this message.
1mFUNCTIONS FOR FILTER/PUT0m
1mabs0m
(class=math #args=1) Absolute value.
@ -3337,4 +3333,4 @@ MILLER(1) MILLER(1)
2023-06-03 MILLER(1)
2023-06-04 MILLER(1)

View file

@ -233,6 +233,45 @@ orange 0.4802164827586204 290
green 0.5129018241860459 1075
</pre>
## case
<pre class="pre-highlight-in-pair">
<b>mlr case --help</b>
</pre>
<pre class="pre-non-highlight-in-pair">
Usage: mlr case [options]
Uppercases strings in record keys and/or values.
Options:
-k Case only keys, not keys and values.
-v Case only values, not keys and values.
-f {a,b,c} Specify which field names to case (default: all)
-u Convert to uppercase
-l Convert to lowercase
-s Convert to sentence case (capitalize first letter)
-t Convert to title case (capitalize words)
-h|--help Show this message.
</pre>
<pre class="pre-highlight-non-pair">
<b>mlr --from test/input/cases.csv --icsv --ojson case -u</b>
</pre>
<pre class="pre-highlight-non-pair">
<b>mlr --from test/input/cases.csv --icsv --ojson case -u -k</b>
</pre>
<pre class="pre-highlight-non-pair">
<b>mlr --from test/input/cases.csv --icsv --ojson case -u -v</b>
</pre>
<pre class="pre-highlight-non-pair">
<b>mlr --from test/input/cases.csv --icsv --ojson case -k -t then case -v -s</b>
</pre>
<pre class="pre-highlight-non-pair">
<b>mlr --from test/input/cases.csv --icsv --ojson case -u -f apple,ball then case -l -f cat,dog</b>
</pre>
## cat
Most useful for format conversions (see [File Formats](file-formats.md)) and concatenating multiple same-schema CSV files to have the same header:
@ -824,20 +863,6 @@ Options:
-h|--help Show this message.
</pre>
## downcase
<pre class="pre-highlight-in-pair">
<b>mlr downcase --help</b>
</pre>
<pre class="pre-non-highlight-in-pair">
Usage: mlr downcase [options]
Lowercases strings in record keys and/or values.
Options:
-k Downcase only keys, not keys and values.
-v Downcase only values, not keys and values.
-h|--help Show this message.
</pre>
## fill-down
<pre class="pre-highlight-in-pair">
@ -4288,17 +4313,3 @@ a b v u w x
1 - 2 - - 3
- - 1 - 2 -
</pre>
## upcase
<pre class="pre-highlight-in-pair">
<b>mlr upcase --help</b>
</pre>
<pre class="pre-non-highlight-in-pair">
Usage: mlr upcase [options]
Uppercases strings in record keys and/or values.
Options:
-k Upcase only keys, not keys and values.
-v Upcase only values, not keys and values.
-h|--help Show this message.
</pre>

View file

@ -127,6 +127,32 @@ orange 0.4802164827586204 290
green 0.5129018241860459 1075
GENMD-EOF
## case
GENMD-RUN-COMMAND
mlr case --help
GENMD-EOF
GENMD-CARDIFY-HIGHLIGHT-ONE
mlr --from test/input/cases.csv --icsv --ojson case -u
GENMD-EOF
GENMD-CARDIFY-HIGHLIGHT-ONE
mlr --from test/input/cases.csv --icsv --ojson case -u -k
GENMD-EOF
GENMD-CARDIFY-HIGHLIGHT-ONE
mlr --from test/input/cases.csv --icsv --ojson case -u -v
GENMD-EOF
GENMD-CARDIFY-HIGHLIGHT-ONE
mlr --from test/input/cases.csv --icsv --ojson case -k -t then case -v -s
GENMD-EOF
GENMD-CARDIFY-HIGHLIGHT-ONE
mlr --from test/input/cases.csv --icsv --ojson case -u -f apple,ball then case -l -f cat,dog
GENMD-EOF
## cat
Most useful for format conversions (see [File Formats](file-formats.md)) and concatenating multiple same-schema CSV files to have the same header:
@ -297,12 +323,6 @@ GENMD-RUN-COMMAND
mlr decimate --help
GENMD-EOF
## downcase
GENMD-RUN-COMMAND
mlr downcase --help
GENMD-EOF
## fill-down
GENMD-RUN-COMMAND
@ -1294,9 +1314,3 @@ GENMD-EOF
GENMD-RUN-COMMAND
mlr --ijson --opprint unsparsify -f a,b,u,v,w,x then regularize data/sparse.json
GENMD-EOF
## upcase
GENMD-RUN-COMMAND
mlr upcase --help
GENMD-EOF

1
go.mod
View file

@ -35,5 +35,6 @@ require (
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/text v0.9.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

2
go.sum
View file

@ -42,6 +42,8 @@ golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View file

@ -14,6 +14,7 @@ var TRANSFORMER_LOOKUP_TABLE = []TransformerSetup{
AltkvSetup,
BarSetup,
BootstrapSetup,
CaseSetup,
CatSetup,
CheckSetup,
CleanWhitespaceSetup,
@ -22,7 +23,6 @@ var TRANSFORMER_LOOKUP_TABLE = []TransformerSetup{
CountSimilarSetup,
CutSetup,
DecimateSetup,
DowncaseSetup,
FillDownSetup,
FillEmptySetup,
FilterSetup,
@ -76,7 +76,6 @@ var TRANSFORMER_LOOKUP_TABLE = []TransformerSetup{
UniqSetup,
UnspaceSetup,
UnsparsifySetup,
UpcaseSetup,
}
func ShowHelpForTransformer(verb string) bool {

View file

@ -6,33 +6,50 @@ import (
"os"
"strings"
"golang.org/x/text/cases"
"golang.org/x/text/language"
"github.com/johnkerl/miller/internal/pkg/cli"
"github.com/johnkerl/miller/internal/pkg/lib"
"github.com/johnkerl/miller/internal/pkg/mlrval"
"github.com/johnkerl/miller/internal/pkg/types"
)
// ----------------------------------------------------------------
const verbNameUpcase = "upcase"
const verbNameCase = "case"
var UpcaseSetup = TransformerSetup{
Verb: verbNameUpcase,
UsageFunc: transformerUpcaseUsage,
ParseCLIFunc: transformerUpcaseParseCLI,
var CaseSetup = TransformerSetup{
Verb: verbNameCase,
UsageFunc: transformerCaseUsage,
ParseCLIFunc: transformerCaseParseCLI,
IgnoresInput: false,
}
func transformerUpcaseUsage(
const (
e_UNSPECIFIED_CASE = iota
e_UPPER_CASE
e_LOWER_CASE
e_SENTENCE_CASE
e_TITLE_CASE
)
func transformerCaseUsage(
o *os.File,
) {
fmt.Fprintf(o, "Usage: %s %s [options]\n", "mlr", verbNameUpcase)
fmt.Fprintf(o, "Usage: %s %s [options]\n", "mlr", verbNameCase)
fmt.Fprintf(o, "Uppercases strings in record keys and/or values.\n")
fmt.Fprintf(o, "Options:\n")
fmt.Fprintf(o, "-k Upcase only keys, not keys and values.\n")
fmt.Fprintf(o, "-v Upcase only values, not keys and values.\n")
fmt.Fprintf(o, "-k Case only keys, not keys and values.\n")
fmt.Fprintf(o, "-v Case only values, not keys and values.\n")
fmt.Fprintf(o, "-f {a,b,c} Specify which field names to case (default: all)\n")
fmt.Fprintf(o, "-u Convert to uppercase\n")
fmt.Fprintf(o, "-l Convert to lowercase\n")
fmt.Fprintf(o, "-s Convert to sentence case (capitalize first letter)\n")
fmt.Fprintf(o, "-t Convert to title case (capitalize words)\n")
fmt.Fprintf(o, "-h|--help Show this message.\n")
}
func transformerUpcaseParseCLI(
func transformerCaseParseCLI(
pargi *int,
argc int,
args []string,
@ -42,9 +59,12 @@ func transformerUpcaseParseCLI(
// Skip the verb name from the current spot in the mlr command line
argi := *pargi
verb := args[argi]
argi++
which := "keys_and_values"
style := e_UNSPECIFIED_CASE
var fieldNames []string = nil
for argi < argc /* variable increment: 1 or 2 depending on flag */ {
opt := args[argi]
@ -57,7 +77,7 @@ func transformerUpcaseParseCLI(
argi++
if opt == "-h" || opt == "--help" {
transformerUpcaseUsage(os.Stdout)
transformerCaseUsage(os.Stdout)
os.Exit(0)
} else if opt == "-k" {
@ -66,8 +86,20 @@ func transformerUpcaseParseCLI(
} else if opt == "-v" {
which = "values_only"
} else if opt == "-f" {
fieldNames = cli.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc)
} else if opt == "-u" {
style = e_UPPER_CASE
} else if opt == "-l" {
style = e_LOWER_CASE
} else if opt == "-s" {
style = e_SENTENCE_CASE
} else if opt == "-t" {
style = e_TITLE_CASE
} else {
transformerUpcaseUsage(os.Stderr)
transformerCaseUsage(os.Stderr)
os.Exit(1)
}
}
@ -77,7 +109,7 @@ func transformerUpcaseParseCLI(
return nil
}
transformer, err := NewTransformerUpcase(which)
transformer, err := NewTransformerCase(which, fieldNames, style)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
@ -87,14 +119,21 @@ func transformerUpcaseParseCLI(
}
// ----------------------------------------------------------------
type TransformerUpcase struct {
type caserFuncT func(input string) string
type TransformerCase struct {
recordTransformerFunc RecordTransformerFunc
fieldNameSet map[string]bool
caserFunc caserFuncT
}
func NewTransformerUpcase(
func NewTransformerCase(
which string,
) (*TransformerUpcase, error) {
tr := &TransformerUpcase{}
fieldNames []string,
style int,
) (*TransformerCase, error) {
tr := &TransformerCase{}
if which == "keys_only" {
tr.recordTransformerFunc = tr.transformKeysOnly
} else if which == "values_only" {
@ -102,10 +141,41 @@ func NewTransformerUpcase(
} else {
tr.recordTransformerFunc = tr.transformKeysAndValues
}
if fieldNames != nil {
tr.fieldNameSet = lib.StringListToSet(fieldNames)
}
switch style {
case e_UPPER_CASE:
tr.caserFunc = cases.Upper(language.Und).String
case e_LOWER_CASE:
tr.caserFunc = cases.Lower(language.Und).String
case e_SENTENCE_CASE:
tr.caserFunc = caseSentenceFunc
case e_TITLE_CASE:
tr.caserFunc = cases.Title(language.Und).String
default:
return nil, fmt.Errorf(
"mlr %s: case option must be specified using one of -u, -l, -s, -t.",
verbNameCase,
)
}
return tr, nil
}
func (tr *TransformerUpcase) Transform(
func caseSentenceFunc(input string) string {
runes := []rune(input)
if len(runes) == 0 {
return input
}
first := string(runes[0])
rest := string(runes[1:])
return strings.ToUpper(first) + strings.ToLower(rest)
}
func (tr *TransformerCase) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
@ -124,7 +194,7 @@ func (tr *TransformerUpcase) Transform(
}
}
func (tr *TransformerUpcase) transformKeysOnly(
func (tr *TransformerCase) transformKeysOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
_ <-chan bool,
@ -133,14 +203,18 @@ func (tr *TransformerUpcase) transformKeysOnly(
inrec := inrecAndContext.Record
newrec := mlrval.NewMlrmapAsRecord()
for pe := inrec.Head; pe != nil; pe = pe.Next {
newkey := strings.ToUpper(pe.Key)
// Reference not copy since this is ownership transfer of the value from the now-abandoned inrec
newrec.PutReference(newkey, pe.Value)
if tr.fieldNameSet == nil || tr.fieldNameSet[pe.Key] {
newkey := tr.caserFunc(pe.Key)
// Reference not copy since this is ownership transfer of the value from the now-abandoned inrec
newrec.PutReference(newkey, pe.Value)
} else {
newrec.PutReference(pe.Key, pe.Value)
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
}
func (tr *TransformerUpcase) transformValuesOnly(
func (tr *TransformerCase) transformValuesOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
_ <-chan bool,
@ -148,15 +222,17 @@ func (tr *TransformerUpcase) transformValuesOnly(
) {
inrec := inrecAndContext.Record
for pe := inrec.Head; pe != nil; pe = pe.Next {
stringval, ok := pe.Value.GetStringValue()
if ok {
pe.Value = mlrval.FromString(strings.ToUpper(stringval))
if tr.fieldNameSet == nil || tr.fieldNameSet[pe.Key] {
stringval, ok := pe.Value.GetStringValue()
if ok {
pe.Value = mlrval.FromString(tr.caserFunc(stringval))
}
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(inrec, &inrecAndContext.Context))
}
func (tr *TransformerUpcase) transformKeysAndValues(
func (tr *TransformerCase) transformKeysAndValues(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
_ <-chan bool,
@ -165,13 +241,17 @@ func (tr *TransformerUpcase) transformKeysAndValues(
inrec := inrecAndContext.Record
newrec := mlrval.NewMlrmapAsRecord()
for pe := inrec.Head; pe != nil; pe = pe.Next {
newkey := strings.ToUpper(pe.Key)
stringval, ok := pe.Value.GetStringValue()
if ok {
stringval = strings.ToUpper(stringval)
newrec.PutReference(newkey, mlrval.FromString(stringval))
if tr.fieldNameSet == nil || tr.fieldNameSet[pe.Key] {
newkey := tr.caserFunc(pe.Key)
stringval, ok := pe.Value.GetStringValue()
if ok {
stringval = tr.caserFunc(stringval)
newrec.PutReference(newkey, mlrval.FromString(stringval))
} else {
newrec.PutReference(newkey, pe.Value)
}
} else {
newrec.PutReference(newkey, pe.Value)
newrec.PutReference(pe.Key, pe.Value)
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))

View file

@ -1,178 +0,0 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
"github.com/johnkerl/miller/internal/pkg/cli"
"github.com/johnkerl/miller/internal/pkg/mlrval"
"github.com/johnkerl/miller/internal/pkg/types"
)
// ----------------------------------------------------------------
const verbNameDowncase = "downcase"
var DowncaseSetup = TransformerSetup{
Verb: verbNameDowncase,
UsageFunc: transformerDowncaseUsage,
ParseCLIFunc: transformerDowncaseParseCLI,
IgnoresInput: false,
}
func transformerDowncaseUsage(
o *os.File,
) {
fmt.Fprintf(o, "Usage: %s %s [options]\n", "mlr", verbNameDowncase)
fmt.Fprintf(o, "Lowercases strings in record keys and/or values.\n")
fmt.Fprintf(o, "Options:\n")
fmt.Fprintf(o, "-k Downcase only keys, not keys and values.\n")
fmt.Fprintf(o, "-v Downcase only values, not keys and values.\n")
fmt.Fprintf(o, "-h|--help Show this message.\n")
}
func transformerDowncaseParseCLI(
pargi *int,
argc int,
args []string,
_ *cli.TOptions,
doConstruct bool, // false for first pass of CLI-parse, true for second pass
) IRecordTransformer {
// Skip the verb name from the current spot in the mlr command line
argi := *pargi
argi++
which := "keys_and_values"
for argi < argc /* variable increment: 1 or 2 depending on flag */ {
opt := args[argi]
if !strings.HasPrefix(opt, "-") {
break // No more flag options to process
}
if args[argi] == "--" {
break // All transformers must do this so main-flags can follow verb-flags
}
argi++
if opt == "-h" || opt == "--help" {
transformerDowncaseUsage(os.Stdout)
os.Exit(0)
} else if opt == "-k" {
which = "keys_only"
} else if opt == "-v" {
which = "values_only"
} else {
transformerDowncaseUsage(os.Stderr)
os.Exit(1)
}
}
*pargi = argi
if !doConstruct { // All transformers must do this for main command-line parsing
return nil
}
transformer, err := NewTransformerDowncase(which)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
return transformer
}
// ----------------------------------------------------------------
type TransformerDowncase struct {
recordTransformerFunc RecordTransformerFunc
}
func NewTransformerDowncase(
which string,
) (*TransformerDowncase, error) {
tr := &TransformerDowncase{}
if which == "keys_only" {
tr.recordTransformerFunc = tr.transformKeysOnly
} else if which == "values_only" {
tr.recordTransformerFunc = tr.transformValuesOnly
} else {
tr.recordTransformerFunc = tr.transformKeysAndValues
}
return tr, nil
}
func (tr *TransformerDowncase) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel)
if !inrecAndContext.EndOfStream {
tr.recordTransformerFunc(
inrecAndContext,
outputRecordsAndContexts,
inputDownstreamDoneChannel,
outputDownstreamDoneChannel,
)
} else { // end of record stream
outputRecordsAndContexts.PushBack(inrecAndContext)
}
}
func (tr *TransformerDowncase) transformKeysOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
_ <-chan bool,
__ chan<- bool,
) {
inrec := inrecAndContext.Record
newrec := mlrval.NewMlrmapAsRecord()
for pe := inrec.Head; pe != nil; pe = pe.Next {
newkey := strings.ToLower(pe.Key)
// Reference not copy since this is ownership transfer of the value from the now-abandoned inrec
newrec.PutReference(newkey, pe.Value)
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
}
func (tr *TransformerDowncase) transformValuesOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
_ <-chan bool,
__ chan<- bool,
) {
inrec := inrecAndContext.Record
for pe := inrec.Head; pe != nil; pe = pe.Next {
stringval, ok := pe.Value.GetStringValue()
if ok {
pe.Value = mlrval.FromString(strings.ToLower(stringval))
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(inrec, &inrecAndContext.Context))
}
func (tr *TransformerDowncase) transformKeysAndValues(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
_ <-chan bool,
__ chan<- bool,
) {
inrec := inrecAndContext.Record
newrec := mlrval.NewMlrmapAsRecord()
for pe := inrec.Head; pe != nil; pe = pe.Next {
newkey := strings.ToLower(pe.Key)
stringval, ok := pe.Value.GetStringValue()
if ok {
stringval = strings.ToLower(stringval)
newrec.PutReference(newkey, mlrval.FromString(stringval))
} else {
newrec.PutReference(newkey, pe.Value)
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
}

View file

@ -171,15 +171,14 @@ MILLER(1) MILLER(1)
for all things with "map" in their names.
1mVERB LIST0m
altkv bar bootstrap cat check clean-whitespace count-distinct count
count-similar cut decimate downcase fill-down fill-empty filter flatten
format-values fraction gap grep group-by group-like having-fields head
histogram json-parse json-stringify join label latin1-to-utf8 least-frequent
merge-fields most-frequent nest nothing put regularize remove-empty-columns
rename reorder repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle
skip-trivial-records sort sort-within-records split stats1 stats2 step summary
tac tail tee template top utf8-to-latin1 unflatten uniq unspace unsparsify
upcase
altkv bar bootstrap case cat check clean-whitespace count-distinct count
count-similar cut decimate fill-down fill-empty filter flatten format-values
fraction gap grep group-by group-like having-fields head histogram json-parse
json-stringify join label latin1-to-utf8 least-frequent merge-fields
most-frequent nest nothing put regularize remove-empty-columns rename reorder
repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle skip-trivial-records
sort sort-within-records split stats1 stats2 step summary tac tail tee
template top utf8-to-latin1 unflatten uniq unspace unsparsify
1mFUNCTION LIST0m
abs acos acosh any append apply arrayify asin asinh asserting_absent
@ -888,6 +887,19 @@ MILLER(1) MILLER(1)
Must be non-negative.
-h|--help Show this message.
1mcase0m
Usage: mlr case [options]
Uppercases strings in record keys and/or values.
Options:
-k Case only keys, not keys and values.
-v Case only values, not keys and values.
-f {a,b,c} Specify which field names to case (default: all)
-u Convert to uppercase
-l Convert to lowercase
-s Convert to sentence case (capitalize first letter)
-t Convert to title case (capitalize words)
-h|--help Show this message.
1mcat0m
Usage: mlr cat [options]
Passes input records directly to output. Most useful for format conversion.
@ -985,14 +997,6 @@ MILLER(1) MILLER(1)
-n {n} Decimation factor (default 10).
-h|--help Show this message.
1mdowncase0m
Usage: mlr downcase [options]
Lowercases strings in record keys and/or values.
Options:
-k Downcase only keys, not keys and values.
-v Downcase only values, not keys and values.
-h|--help Show this message.
1mfill-down0m
Usage: mlr fill-down [options]
If a given record has a missing value for a given field, fill that from
@ -2106,14 +2110,6 @@ MILLER(1) MILLER(1)
being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and
'a=,b=3,c=4'.
1mupcase0m
Usage: mlr upcase [options]
Uppercases strings in record keys and/or values.
Options:
-k Upcase only keys, not keys and values.
-v Upcase only values, not keys and values.
-h|--help Show this message.
1mFUNCTIONS FOR FILTER/PUT0m
1mabs0m
(class=math #args=1) Absolute value.
@ -3337,4 +3333,4 @@ MILLER(1) MILLER(1)
2023-06-03 MILLER(1)
2023-06-04 MILLER(1)

View file

@ -2,12 +2,12 @@
.\" Title: mlr
.\" Author: [see the "AUTHOR" section]
.\" Generator: ./mkman.rb
.\" Date: 2023-06-03
.\" Date: 2023-06-04
.\" Manual: \ \&
.\" Source: \ \&
.\" Language: English
.\"
.TH "MILLER" "1" "2023-06-03" "\ \&" "\ \&"
.TH "MILLER" "1" "2023-06-04" "\ \&" "\ \&"
.\" -----------------------------------------------------------------
.\" * Portability definitions
.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -212,15 +212,14 @@ for all things with "map" in their names.
.RS 0
.\}
.nf
altkv bar bootstrap cat check clean-whitespace count-distinct count
count-similar cut decimate downcase fill-down fill-empty filter flatten
format-values fraction gap grep group-by group-like having-fields head
histogram json-parse json-stringify join label latin1-to-utf8 least-frequent
merge-fields most-frequent nest nothing put regularize remove-empty-columns
rename reorder repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle
skip-trivial-records sort sort-within-records split stats1 stats2 step summary
tac tail tee template top utf8-to-latin1 unflatten uniq unspace unsparsify
upcase
altkv bar bootstrap case cat check clean-whitespace count-distinct count
count-similar cut decimate fill-down fill-empty filter flatten format-values
fraction gap grep group-by group-like having-fields head histogram json-parse
json-stringify join label latin1-to-utf8 least-frequent merge-fields
most-frequent nest nothing put regularize remove-empty-columns rename reorder
repeat reshape sample sec2gmtdate sec2gmt seqgen shuffle skip-trivial-records
sort sort-within-records split stats1 stats2 step summary tac tail tee
template top utf8-to-latin1 unflatten uniq unspace unsparsify
.fi
.if n \{\
.RE
@ -1079,6 +1078,25 @@ Options:
.fi
.if n \{\
.RE
.SS "case"
.if n \{\
.RS 0
.\}
.nf
Usage: mlr case [options]
Uppercases strings in record keys and/or values.
Options:
-k Case only keys, not keys and values.
-v Case only values, not keys and values.
-f {a,b,c} Specify which field names to case (default: all)
-u Convert to uppercase
-l Convert to lowercase
-s Convert to sentence case (capitalize first letter)
-t Convert to title case (capitalize words)
-h|--help Show this message.
.fi
.if n \{\
.RE
.SS "cat"
.if n \{\
.RS 0
@ -1224,20 +1242,6 @@ Options:
.fi
.if n \{\
.RE
.SS "downcase"
.if n \{\
.RS 0
.\}
.nf
Usage: mlr downcase [options]
Lowercases strings in record keys and/or values.
Options:
-k Downcase only keys, not keys and values.
-v Downcase only values, not keys and values.
-h|--help Show this message.
.fi
.if n \{\
.RE
.SS "fill-down"
.if n \{\
.RS 0
@ -2669,20 +2673,6 @@ being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and
.fi
.if n \{\
.RE
.SS "upcase"
.if n \{\
.RS 0
.\}
.nf
Usage: mlr upcase [options]
Uppercases strings in record keys and/or values.
Options:
-k Upcase only keys, not keys and values.
-v Upcase only values, not keys and values.
-h|--help Show this message.
.fi
.if n \{\
.RE
.SH "FUNCTIONS FOR FILTER/PUT"
.sp

View file

@ -34,6 +34,20 @@ Options:
Must be non-negative.
-h|--help Show this message.
================================================================
case
Usage: mlr case [options]
Uppercases strings in record keys and/or values.
Options:
-k Case only keys, not keys and values.
-v Case only values, not keys and values.
-f {a,b,c} Specify which field names to case (default: all)
-u Convert to uppercase
-l Convert to lowercase
-s Convert to sentence case (capitalize first letter)
-t Convert to title case (capitalize words)
-h|--help Show this message.
================================================================
cat
Usage: mlr cat [options]
@ -139,15 +153,6 @@ Options:
-n {n} Decimation factor (default 10).
-h|--help Show this message.
================================================================
downcase
Usage: mlr downcase [options]
Lowercases strings in record keys and/or values.
Options:
-k Downcase only keys, not keys and values.
-v Downcase only values, not keys and values.
-h|--help Show this message.
================================================================
fill-down
Usage: mlr fill-down [options]
@ -1313,13 +1318,4 @@ Options:
Example: if the input is two records, one being 'a=1,b=2' and the other
being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and
'a=,b=3,c=4'.
================================================================
upcase
Usage: mlr upcase [options]
Uppercases strings in record keys and/or values.
Options:
-k Upcase only keys, not keys and values.
-v Upcase only values, not keys and values.
-h|--help Show this message.
================================================================

View file

@ -0,0 +1 @@
mlr case -u test/input/abixy

View file

@ -0,0 +1 @@
mlr case -u -k test/input/abixy

View file

@ -0,0 +1 @@
mlr case -u -v test/input/abixy

View file

@ -0,0 +1 @@
mlr case -u test/input/abixy-upper

View file

@ -0,0 +1 @@
mlr case -u -k test/input/abixy-upper

View file

@ -0,0 +1 @@
mlr case -u -v test/input/abixy-upper

View file

@ -0,0 +1 @@
mlr case -l test/input/abixy

View file

@ -0,0 +1 @@
mlr case -l -k test/input/abixy

View file

@ -0,0 +1 @@
mlr case -l -v test/input/abixy

View file

@ -0,0 +1 @@
mlr case -l test/input/abixy-upper

View file

@ -0,0 +1 @@
mlr case -l -k test/input/abixy-upper

View file

@ -0,0 +1 @@
mlr case -l -v test/input/abixy-upper

View file

@ -0,0 +1 @@
mlr --from test/input/cases.csv --c2j case -u

View file

View file

@ -0,0 +1,9 @@
[
{
"APPLE": "SUN",
"BALL": "TREE",
"CAT": "URCHIN",
"DOG": "VIOLET",
"THE QUICK BROWN FOX": "JUMPED OVER THE LAZY DOG"
}
]

View file

@ -0,0 +1 @@
mlr --from test/input/cases.csv --c2j case -l

View file

View file

@ -0,0 +1,9 @@
[
{
"apple": "sun",
"ball": "tree",
"cat": "urchin",
"dog": "violet",
"the quick brown fox": "jumped over the lazy dog"
}
]

View file

@ -0,0 +1 @@
mlr --from test/input/cases.csv --c2j case -s

View file

View file

@ -0,0 +1,9 @@
[
{
"Apple": "Sun",
"Ball": "Tree",
"Cat": "Urchin",
"Dog": "Violet",
"The quick brown fox": "Jumped over the lazy dog"
}
]

View file

@ -0,0 +1 @@
mlr --from test/input/cases.csv --c2j case -t

View file

View file

@ -0,0 +1,9 @@
[
{
"Apple": "Sun",
"Ball": "Tree",
"Cat": "Urchin",
"Dog": "Violet",
"The Quick Brown Fox": "Jumped Over The Lazy Dog"
}
]

View file

@ -0,0 +1 @@
mlr --from test/input/cases.csv --c2j case -k -u

View file

View file

@ -0,0 +1,9 @@
[
{
"APPLE": "sun",
"BALL": "tree",
"CAT": "urchin",
"DOG": "violet",
"THE QUICK BROWN FOX": "jumped over the lazy dog"
}
]

View file

@ -0,0 +1 @@
mlr --from test/input/cases.csv --c2j case -k -l

View file

View file

@ -0,0 +1,9 @@
[
{
"apple": "sun",
"ball": "tree",
"cat": "urchin",
"dog": "violet",
"the quick brown fox": "jumped over the lazy dog"
}
]

View file

@ -0,0 +1 @@
mlr --from test/input/cases.csv --c2j case -k -s

View file

View file

@ -0,0 +1,9 @@
[
{
"Apple": "sun",
"Ball": "tree",
"Cat": "urchin",
"Dog": "violet",
"The quick brown fox": "jumped over the lazy dog"
}
]

View file

@ -0,0 +1 @@
mlr --from test/input/cases.csv --c2j case -k -t

View file

View file

@ -0,0 +1,9 @@
[
{
"Apple": "sun",
"Ball": "tree",
"Cat": "urchin",
"Dog": "violet",
"The Quick Brown Fox": "jumped over the lazy dog"
}
]

View file

@ -0,0 +1 @@
mlr --from test/input/cases.csv --c2j case -v -u

View file

View file

@ -0,0 +1,9 @@
[
{
"apple": "SUN",
"ball": "TREE",
"cat": "URCHIN",
"dog": "VIOLET",
"the quick brown fox": "JUMPED OVER THE LAZY DOG"
}
]

View file

@ -0,0 +1 @@
mlr --from test/input/cases.csv --c2j case -v -l

View file

View file

@ -0,0 +1,9 @@
[
{
"apple": "sun",
"ball": "tree",
"cat": "urchin",
"dog": "violet",
"the quick brown fox": "jumped over the lazy dog"
}
]

View file

@ -0,0 +1 @@
mlr --from test/input/cases.csv --c2j case -v -s

View file

View file

@ -0,0 +1,9 @@
[
{
"apple": "Sun",
"ball": "Tree",
"cat": "Urchin",
"dog": "Violet",
"the quick brown fox": "Jumped over the lazy dog"
}
]

View file

@ -0,0 +1 @@
mlr --from test/input/cases.csv --c2j case -v -t

View file

View file

@ -0,0 +1,9 @@
[
{
"apple": "Sun",
"ball": "Tree",
"cat": "Urchin",
"dog": "Violet",
"the quick brown fox": "Jumped Over The Lazy Dog"
}
]

View file

@ -0,0 +1 @@
mlr --from test/input/cases.csv --c2j case -u -f apple,ball then case -l -f cat,dog

View file

View file

@ -0,0 +1,9 @@
[
{
"APPLE": "SUN",
"BALL": "TREE",
"cat": "urchin",
"dog": "violet",
"the quick brown fox": "jumped over the lazy dog"
}
]

13
test/cases/verb-case/x Normal file
View file

@ -0,0 +1,13 @@
mkdir 0020; echo mlr --from test/input.cases-csv --c2j case -u > 0020/cmd
mkdir 0021; echo mlr --from test/input.cases-csv --c2j case -l > 0021/cmd
mkdir 0022; echo mlr --from test/input.cases-csv --c2j case -s > 0022/cmd
mkdir 0023; echo mlr --from test/input.cases-csv --c2j case -t > 0023/cmd
mkdir 0024; echo mlr --from test/input.cases-csv --c2j case -k -u > 0024/cmd
mkdir 0025; echo mlr --from test/input.cases-csv --c2j case -k -l > 0025/cmd
mkdir 0026; echo mlr --from test/input.cases-csv --c2j case -k -s > 0026/cmd
mkdir 0027; echo mlr --from test/input.cases-csv --c2j case -k -t > 0027/cmd
mkdir 0028; echo mlr --from test/input.cases-csv --c2j case -v -u > 0028/cmd
mkdir 0029; echo mlr --from test/input.cases-csv --c2j case -v -l > 0029/cmd
mkdir 0030; echo mlr --from test/input.cases-csv --c2j case -v -s > 0030/cmd
mkdir 0031; echo mlr --from test/input.cases-csv --c2j case -v -t > 0031/cmd
mkdir 0032; echo mlr --from test/input.cases-csv --c2j case -u apple,ball then case -l cat,dog > 0032/cmd

View file

@ -1 +0,0 @@
mlr downcase test/input/abixy

View file

@ -1 +0,0 @@
mlr downcase -k test/input/abixy

View file

@ -1 +0,0 @@
mlr downcase -v test/input/abixy

View file

@ -1 +0,0 @@
mlr downcase test/input/abixy-upper

View file

@ -1 +0,0 @@
mlr downcase -k test/input/abixy-upper

View file

@ -1 +0,0 @@
mlr downcase -v test/input/abixy-upper

View file

@ -1 +0,0 @@
mlr upcase test/input/abixy

View file

@ -1 +0,0 @@
mlr upcase -k test/input/abixy

View file

@ -1 +0,0 @@
mlr upcase -v test/input/abixy

View file

@ -1 +0,0 @@
mlr upcase test/input/abixy-upper

View file

@ -1 +0,0 @@
mlr upcase -k test/input/abixy-upper

Some files were not shown because too many files have changed in this diff Show more