Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-18 00:45:47 +00:00
Code Issues Projects Releases Wiki Activity

Split separators from context (#771)

Browse source 
* Fix mlr -O to return strings for all octal numbers along with 0* numbers like 07 and 08

* Factor separators out of Context struct
This commit is contained in:
John Kerl 2021-12-06 22:47:58 -05:00 • committed by GitHub
parent 8e975c9d39
commit 4c3e141df8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 42 additions and 97 deletions
Download patch file Download diff file Expand all files Collapse all files

2
docs/src/manpage.md
View file

@ -2994,5 +2994,5 @@ SEE ALSO
2021-12-02 MILLER(1)
2021-12-07 MILLER(1)
</pre>

2
docs/src/manpage.txt
View file

@ -2973,4 +2973,4 @@ SEE ALSO
2021-12-02 MILLER(1)
2021-12-07 MILLER(1)

13
internal/pkg/auxents/repl/session.go
View file

@ -60,16 +60,9 @@ func NewRepl(
// $* is the empty map {} until/unless the user opens a file and reads records from it.
inrec := types.NewMlrmapAsRecord()
// NR is 0, etc until/unless the user opens a file and reads records from it.
context := types.NewContext(
options.ReaderOptions.IPS,
options.ReaderOptions.IFS,
options.ReaderOptions.IRS,
options.WriterOptions.OPS,
options.WriterOptions.OFS,
options.WriterOptions.ORS,
options.WriterOptions.FLATSEP,
)
runtimeState := runtime.NewEmptyState()
context := types.NewContext()
runtimeState := runtime.NewEmptyState(options)
runtimeState.Update(inrec, context)
// The filter expression for the main Miller DSL is any non-assignment
// statment like 'true' or '$x > 0.5' etc. For the REPL, we re-use this for

4
internal/pkg/climain/mlrcli_parse.go
View file

@ -324,7 +324,7 @@ func parseCommandLinePassTwo(
if cli.DecideFinalFlatten(&options.WriterOptions) {
// E.g. '{"req": {"method": "GET", "path": "/api/check"}}' becomes
// req.method=GET,req.path=/api/check.
transformer, err := transformers.NewTransformerFlatten(options.WriterOptions.FLATSEP, nil)
transformer, err := transformers.NewTransformerFlatten(options.WriterOptions.FLATSEP, options, nil)
lib.InternalCodingErrorIf(err != nil)
lib.InternalCodingErrorIf(transformer == nil)
recordTransformers = append(recordTransformers, transformer)
@ -333,7 +333,7 @@ func parseCommandLinePassTwo(
if cli.DecideFinalUnflatten(options) {
// E.g. req.method=GET,req.path=/api/check becomes
// '{"req": {"method": "GET", "path": "/api/check"}}'
transformer, err := transformers.NewTransformerUnflatten(options.WriterOptions.FLATSEP, nil)
transformer, err := transformers.NewTransformerUnflatten(options.WriterOptions.FLATSEP, options, nil)
lib.InternalCodingErrorIf(err != nil)
lib.InternalCodingErrorIf(transformer == nil)
recordTransformers = append(recordTransformers, transformer)

14
internal/pkg/dsl/cst/leaves.go
View file

@ -519,7 +519,7 @@ func (root *RootNode) BuildIRSNode() *IRSNode {
func (node *IRSNode) Evaluate(
state *runtime.State,
) *types.Mlrval {
return types.MlrvalFromString(state.Context.IRS)
return types.MlrvalFromString(state.Options.ReaderOptions.IRS)
}
// ----------------------------------------------------------------
@ -532,7 +532,7 @@ func (root *RootNode) BuildIFSNode() *IFSNode {
func (node *IFSNode) Evaluate(
state *runtime.State,
) *types.Mlrval {
return types.MlrvalFromString(state.Context.IFS)
return types.MlrvalFromString(state.Options.ReaderOptions.IFS)
}
// ----------------------------------------------------------------
@ -545,7 +545,7 @@ func (root *RootNode) BuildIPSNode() *IPSNode {
func (node *IPSNode) Evaluate(
state *runtime.State,
) *types.Mlrval {
return types.MlrvalFromString(state.Context.IPS)
return types.MlrvalFromString(state.Options.ReaderOptions.IPS)
}
// ----------------------------------------------------------------
@ -558,7 +558,7 @@ func (root *RootNode) BuildORSNode() *ORSNode {
func (node *ORSNode) Evaluate(
state *runtime.State,
) *types.Mlrval {
return types.MlrvalFromString(state.Context.ORS)
return types.MlrvalFromString(state.Options.WriterOptions.ORS)
}
// ----------------------------------------------------------------
@ -571,7 +571,7 @@ func (root *RootNode) BuildOFSNode() *OFSNode {
func (node *OFSNode) Evaluate(
state *runtime.State,
) *types.Mlrval {
return types.MlrvalFromString(state.Context.OFS)
return types.MlrvalFromString(state.Options.WriterOptions.OFS)
}
// ----------------------------------------------------------------
@ -584,7 +584,7 @@ func (root *RootNode) BuildOPSNode() *OPSNode {
func (node *OPSNode) Evaluate(
state *runtime.State,
) *types.Mlrval {
return types.MlrvalFromString(state.Context.OPS)
return types.MlrvalFromString(state.Options.WriterOptions.OPS)
}
// ----------------------------------------------------------------
@ -597,7 +597,7 @@ func (root *RootNode) BuildFLATSEPNode() *FLATSEPNode {
func (node *FLATSEPNode) Evaluate(
state *runtime.State,
) *types.Mlrval {
return types.MlrvalFromString(state.Context.FLATSEP)
return types.MlrvalFromString(state.Options.WriterOptions.FLATSEP)
}
// ================================================================

5
internal/pkg/runtime/state.go
View file

@ -7,6 +7,7 @@
package runtime
import (
"github.com/johnkerl/miller/internal/pkg/cli"
"github.com/johnkerl/miller/internal/pkg/lib"
"github.com/johnkerl/miller/internal/pkg/types"
)
@ -21,9 +22,10 @@ type State struct {
// For holding "\0".."\9" between where they are set via things like
// '$x =~ "(..)_(...)"', and interpolated via things like '$y = "\2:\1"'.
RegexCaptures []string
Options *cli.TOptions
}
func NewEmptyState() *State {
func NewEmptyState(options *cli.TOptions) *State {
oosvars := types.NewMlrmap()
return &State{
Inrec: nil,
@ -36,6 +38,7 @@ func NewEmptyState() *State {
// See lib.MakeEmptyRegexCaptures for context.
RegexCaptures: lib.MakeEmptyRegexCaptures(),
Options: options,
}
}

10
internal/pkg/stream/stream.go
View file

@ -45,15 +45,7 @@ func Stream(
// Since Go is concurrent, the context struct needs to be duplicated and
// passed through the channels along with each record.
initialContext := types.NewContext(
options.ReaderOptions.IPS,
options.ReaderOptions.IFS,
options.ReaderOptions.IRS,
options.WriterOptions.OPS,
options.WriterOptions.OFS,
options.WriterOptions.ORS,
options.WriterOptions.FLATSEP,
)
initialContext := types.NewContext()
// Instantiate the record-reader
recordReader, err := input.Create(&options.ReaderOptions)

10
internal/pkg/transformers/flatten.go
View file

@ -44,7 +44,7 @@ func transformerFlattenParseCLI(
pargi *int,
argc int,
args []string,
_ *cli.TOptions,
options *cli.TOptions,
doConstruct bool, // false for first pass of CLI-parse, true for second pass
) IRecordTransformer {
@ -87,6 +87,7 @@ func transformerFlattenParseCLI(
transformer, err := NewTransformerFlatten(
oFlatSep,
options,
fieldNames,
)
if err != nil {
@ -101,6 +102,7 @@ func transformerFlattenParseCLI(
type TransformerFlatten struct {
// input
oFlatSep string
options *cli.TOptions
fieldNameSet map[string]bool
// state
@ -109,6 +111,7 @@ type TransformerFlatten struct {
func NewTransformerFlatten(
oFlatSep string,
options *cli.TOptions,
fieldNames []string,
) (*TransformerFlatten, error) {
var fieldNameSet map[string]bool = nil
@ -118,6 +121,7 @@ func NewTransformerFlatten(
retval := &TransformerFlatten{
oFlatSep: oFlatSep,
options: options,
fieldNameSet: fieldNameSet,
}
@ -152,7 +156,7 @@ func (tr *TransformerFlatten) flattenAll(
inrec := inrecAndContext.Record
oFlatSep := tr.oFlatSep
if oFlatSep == "" {
oFlatSep = inrecAndContext.Context.FLATSEP
oFlatSep = tr.options.WriterOptions.FLATSEP
}
inrec.Flatten(oFlatSep)
outputChannel <- inrecAndContext
@ -172,7 +176,7 @@ func (tr *TransformerFlatten) flattenSome(
inrec := inrecAndContext.Record
oFlatSep := tr.oFlatSep
if oFlatSep == "" {
oFlatSep = inrecAndContext.Context.FLATSEP
oFlatSep = tr.options.WriterOptions.FLATSEP
}
inrec.FlattenFields(tr.fieldNameSet, oFlatSep)
outputChannel <- inrecAndContext

2
internal/pkg/transformers/put-or-filter.go
View file

@ -435,7 +435,7 @@ func NewTransformerPut(
return nil, err
}
runtimeState := runtime.NewEmptyState()
runtimeState := runtime.NewEmptyState(options)
// E.g.
// mlr put -s sum=0

10
internal/pkg/transformers/unflatten.go
View file

@ -44,7 +44,7 @@ func transformerUnflattenParseCLI(
pargi *int,
argc int,
args []string,
_ *cli.TOptions,
options *cli.TOptions,
doConstruct bool, // false for first pass of CLI-parse, true for second pass
) IRecordTransformer {
@ -87,6 +87,7 @@ func transformerUnflattenParseCLI(
transformer, err := NewTransformerUnflatten(
oFlatSep,
options,
fieldNames,
)
if err != nil {
@ -101,6 +102,7 @@ func transformerUnflattenParseCLI(
type TransformerUnflatten struct {
// input
oFlatSep string
options *cli.TOptions
fieldNameSet map[string]bool
// state
@ -109,6 +111,7 @@ type TransformerUnflatten struct {
func NewTransformerUnflatten(
oFlatSep string,
options *cli.TOptions,
fieldNames []string,
) (*TransformerUnflatten, error) {
var fieldNameSet map[string]bool = nil
@ -118,6 +121,7 @@ func NewTransformerUnflatten(
retval := &TransformerUnflatten{
oFlatSep: oFlatSep,
options: options,
fieldNameSet: fieldNameSet,
}
retval.recordTransformerFunc = retval.unflattenAll
@ -151,7 +155,7 @@ func (tr *TransformerUnflatten) unflattenAll(
inrec := inrecAndContext.Record
oFlatSep := tr.oFlatSep
if oFlatSep == "" {
oFlatSep = inrecAndContext.Context.FLATSEP
oFlatSep = tr.options.WriterOptions.FLATSEP
}
inrec.Unflatten(oFlatSep)
outputChannel <- inrecAndContext
@ -171,7 +175,7 @@ func (tr *TransformerUnflatten) unflattenSome(
inrec := inrecAndContext.Record
oFlatSep := tr.oFlatSep
if oFlatSep == "" {
oFlatSep = inrecAndContext.Context.FLATSEP
oFlatSep = tr.options.WriterOptions.FLATSEP
}
inrec.UnflattenFields(tr.fieldNameSet, oFlatSep)
outputChannel <- inrecAndContext

61
internal/pkg/types/context.go
View file

@ -88,60 +88,19 @@ type Context struct {
// NF int
NR int
FNR int
IPS string
IFS string
IRS string
OPS string
OFS string
ORS string
FLATSEP string
}
// TODO: comment: Remember command-line values to pass along to CST evaluators.
// The options struct-pointer can be nil when invoked by non-DSL verbs such as
// join or seqgen.
func NewContext(
IPS string,
IFS string,
IRS string,
OPS string,
OFS string,
ORS string,
FLATSEP string,
) *Context {
func NewContext() *Context {
context := &Context{
FILENAME: "(stdin)",
FILENUM: 0,
NR: 0,
FNR: 0,
IPS: "=",
IFS: ",",
IRS: "\n",
OPS: "=",
OFS: ",",
ORS: "\n",
FLATSEP: ".",
NR: 0,
FNR: 0,
}
// TODO: FILENAME/FILENUM/NR/FNR should be in one struct, and the rest in
// another. The former vary per record; the latter are command-line-driven
// and do not vary per record. All they have in common is they are
// awk-like context-variables.
context.IPS = IPS
context.IFS = IFS
context.IRS = IRS
context.OPS = OPS
context.OFS = OFS
context.ORS = ORS
context.FLATSEP = FLATSEP
return context
}
@ -152,18 +111,8 @@ func NewNilContext() *Context { // TODO: rename
context := &Context{
FILENAME: "(stdin)",
FILENUM: 0,
NR: 0,
FNR: 0,
IPS: "=",
IFS: ",",
IRS: "\n",
OPS: "=",
OFS: ",",
ORS: "\n",
FLATSEP: ".",
NR: 0,
FNR: 0,
}
return context

2
man/manpage.txt
View file

@ -2973,4 +2973,4 @@ SEE ALSO
2021-12-02 MILLER(1)
2021-12-07 MILLER(1)

4
man/mlr.1
View file

@ -2,12 +2,12 @@
.\" Title: mlr
.\" Author: [see the "AUTHOR" section]
.\" Generator: ./mkman.rb
.\" Date: 2021-12-02
.\" Date: 2021-12-07
.\" Manual: \ \&
.\" Source: \ \&
.\" Language: English
.\"
.TH "MILLER" "1" "2021-12-02" "\ \&" "\ \&"
.TH "MILLER" "1" "2021-12-07" "\ \&" "\ \&"
.\" -----------------------------------------------------------------
.\" * Portability definitions
.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~