Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-08-02 04:22:59 +00:00
Code Issues Projects Releases Wiki Activity

mlr tee verb

Browse source
This commit is contained in:
John Kerl 2021-01-23 23:06:03 +00:00
parent f8b5079f07
commit f97b7ddd5c
11 changed files with 217 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

2
go/fmter
View file

@ -1,3 +1,3 @@
#!/bin/bash
for x in $(lsr \*.go); do gofmt -w $x; done
go fmt ./...

1
go/src/miller/cli/mlrcli_transformers.go
View file

@ -47,6 +47,7 @@ var MAPPER_LOOKUP_TABLE = []transforming.TransformerSetup{
transformers.Stats1Setup,
transformers.StepSetup,
transformers.TacSetup,
transformers.TeeSetup,
transformers.TailSetup,
transformers.UnflattenSetup,
transformers.UnsparsifySetup,

11
go/src/miller/dsl/cst/dump.go
View file

@ -24,6 +24,7 @@ import (
"miller/dsl"
"miller/lib"
"miller/output"
"miller/types"
)
@ -36,8 +37,8 @@ type tDumpToRedirectFunc func(
type DumpStatementNode struct {
expressionEvaluables []IEvaluable
dumpToRedirectFunc tDumpToRedirectFunc
redirectorTargetEvaluable IEvaluable // for file/pipe targets
outputHandlerManager OutputHandlerManager // for file/pipe targets
redirectorTargetEvaluable IEvaluable // for file/pipe targets
outputHandlerManager output.OutputHandlerManager // for file/pipe targets
}
// ----------------------------------------------------------------
@ -130,11 +131,11 @@ func (this *RootNode) buildDumpxStatementNode(
}
if redirectorNode.Type == dsl.NodeTypeRedirectWrite {
retval.outputHandlerManager = NewFileWritetHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewFileWritetHandlerManager(this.recordWriterOptions)
} else if redirectorNode.Type == dsl.NodeTypeRedirectAppend {
retval.outputHandlerManager = NewFileAppendHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewFileAppendHandlerManager(this.recordWriterOptions)
} else if redirectorNode.Type == dsl.NodeTypeRedirectPipe {
retval.outputHandlerManager = NewPipeWriteHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewPipeWriteHandlerManager(this.recordWriterOptions)
} else {
return nil, errors.New(
fmt.Sprintf(

13
go/src/miller/dsl/cst/emit_emitp.go
View file

@ -37,6 +37,7 @@ import (
"miller/dsl"
"miller/lib"
"miller/output"
"miller/types"
)
@ -70,7 +71,7 @@ type EmitXStatementNode struct {
redirectorTargetEvaluable IEvaluable
// For file/pipe targets: keeps track of file handles for various values of
// the redirectorTargetEvaluable expression.
outputHandlerManager OutputHandlerManager
outputHandlerManager output.OutputHandlerManager
// For code-reuse between executors.
isEmitP bool
@ -173,11 +174,11 @@ func (this *RootNode) buildEmitXStatementNode(
if redirectorTargetNode.Type == dsl.NodeTypeRedirectTargetStdout {
retval.emitToRedirectFunc = retval.emitToFileOrPipe
retval.outputHandlerManager = NewStdoutWriteHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewStdoutWriteHandlerManager(this.recordWriterOptions)
retval.redirectorTargetEvaluable = this.BuildStringLiteralNode("(stdout)")
} else if redirectorTargetNode.Type == dsl.NodeTypeRedirectTargetStderr {
retval.emitToRedirectFunc = retval.emitToFileOrPipe
retval.outputHandlerManager = NewStderrWriteHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewStderrWriteHandlerManager(this.recordWriterOptions)
retval.redirectorTargetEvaluable = this.BuildStringLiteralNode("(stderr)")
} else {
retval.emitToRedirectFunc = retval.emitToFileOrPipe
@ -188,11 +189,11 @@ func (this *RootNode) buildEmitXStatementNode(
}
if redirectorNode.Type == dsl.NodeTypeRedirectWrite {
retval.outputHandlerManager = NewFileWritetHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewFileWritetHandlerManager(this.recordWriterOptions)
} else if redirectorNode.Type == dsl.NodeTypeRedirectAppend {
retval.outputHandlerManager = NewFileAppendHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewFileAppendHandlerManager(this.recordWriterOptions)
} else if redirectorNode.Type == dsl.NodeTypeRedirectPipe {
retval.outputHandlerManager = NewPipeWriteHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewPipeWriteHandlerManager(this.recordWriterOptions)
} else {
return nil, errors.New(
fmt.Sprintf(

15
go/src/miller/dsl/cst/emitf.go
View file

@ -12,6 +12,7 @@ import (
"miller/dsl"
"miller/lib"
"miller/output"
"miller/types"
)
@ -34,8 +35,8 @@ type EmitFStatementNode struct {
emitfNames []string
emitfEvaluables []IEvaluable
emitfToRedirectFunc tEmitFToRedirectFunc
redirectorTargetEvaluable IEvaluable // for file/pipe targets
outputHandlerManager OutputHandlerManager // for file/pipe targets
redirectorTargetEvaluable IEvaluable // for file/pipe targets
outputHandlerManager output.OutputHandlerManager // for file/pipe targets
}
// ----------------------------------------------------------------
@ -97,11 +98,11 @@ func (this *RootNode) BuildEmitFStatementNode(astNode *dsl.ASTNode) (IExecutable
if redirectorTargetNode.Type == dsl.NodeTypeRedirectTargetStdout {
retval.emitfToRedirectFunc = retval.emitfToFileOrPipe
retval.outputHandlerManager = NewStdoutWriteHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewStdoutWriteHandlerManager(this.recordWriterOptions)
retval.redirectorTargetEvaluable = this.BuildStringLiteralNode("(stdout)")
} else if redirectorTargetNode.Type == dsl.NodeTypeRedirectTargetStderr {
retval.emitfToRedirectFunc = retval.emitfToFileOrPipe
retval.outputHandlerManager = NewStderrWriteHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewStderrWriteHandlerManager(this.recordWriterOptions)
retval.redirectorTargetEvaluable = this.BuildStringLiteralNode("(stderr)")
} else {
retval.emitfToRedirectFunc = retval.emitfToFileOrPipe
@ -112,11 +113,11 @@ func (this *RootNode) BuildEmitFStatementNode(astNode *dsl.ASTNode) (IExecutable
}
if redirectorNode.Type == dsl.NodeTypeRedirectWrite {
retval.outputHandlerManager = NewFileWritetHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewFileWritetHandlerManager(this.recordWriterOptions)
} else if redirectorNode.Type == dsl.NodeTypeRedirectAppend {
retval.outputHandlerManager = NewFileAppendHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewFileAppendHandlerManager(this.recordWriterOptions)
} else if redirectorNode.Type == dsl.NodeTypeRedirectPipe {
retval.outputHandlerManager = NewPipeWriteHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewPipeWriteHandlerManager(this.recordWriterOptions)
} else {
return nil, errors.New(
fmt.Sprintf(

11
go/src/miller/dsl/cst/print.go
View file

@ -12,6 +12,7 @@ import (
"miller/dsl"
"miller/lib"
"miller/output"
"miller/types"
)
@ -157,8 +158,8 @@ type PrintStatementNode struct {
expressionEvaluables []IEvaluable
terminator string
printToRedirectFunc tPrintToRedirectFunc
redirectorTargetEvaluable IEvaluable // for file/pipe targets
outputHandlerManager OutputHandlerManager // for file/pipe targets
redirectorTargetEvaluable IEvaluable // for file/pipe targets
outputHandlerManager output.OutputHandlerManager // for file/pipe targets
}
// ----------------------------------------------------------------
@ -273,11 +274,11 @@ func (this *RootNode) buildPrintxStatementNode(
}
if redirectorNode.Type == dsl.NodeTypeRedirectWrite {
retval.outputHandlerManager = NewFileWritetHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewFileWritetHandlerManager(this.recordWriterOptions)
} else if redirectorNode.Type == dsl.NodeTypeRedirectAppend {
retval.outputHandlerManager = NewFileAppendHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewFileAppendHandlerManager(this.recordWriterOptions)
} else if redirectorNode.Type == dsl.NodeTypeRedirectPipe {
retval.outputHandlerManager = NewPipeWriteHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewPipeWriteHandlerManager(this.recordWriterOptions)
} else {
return nil, errors.New(
fmt.Sprintf(

5
go/src/miller/dsl/cst/root.go
View file

@ -13,6 +13,7 @@ import (
"miller/clitypes"
"miller/dsl"
"miller/output"
"miller/types"
)
@ -221,14 +222,14 @@ func (this *RootNode) resolveSubroutineCallsites() error {
// can close all the descriptors, flush the record-output streams, etc.
func (this *RootNode) RegisterOutputHandlerManager(
outputHandlerManager OutputHandlerManager,
outputHandlerManager output.OutputHandlerManager,
) {
this.outputHandlerManagers.PushBack(outputHandlerManager)
}
func (this *RootNode) ProcessEndOfStream() {
for entry := this.outputHandlerManagers.Front(); entry != nil; entry = entry.Next() {
outputHandlerManager := entry.Value.(OutputHandlerManager)
outputHandlerManager := entry.Value.(output.OutputHandlerManager)
errs := outputHandlerManager.Close()
if len(errs) != 0 {
for _, err := range errs {

15
go/src/miller/dsl/cst/tee.go
View file

@ -11,6 +11,7 @@ import (
"miller/dsl"
"miller/lib"
"miller/output"
"miller/types"
)
@ -64,8 +65,8 @@ type tTeeToRedirectFunc func(
type TeeStatementNode struct {
expressionEvaluable IEvaluable // always the $* evaluable
teeToRedirectFunc tTeeToRedirectFunc
redirectorTargetEvaluable IEvaluable // for file/pipe targets
outputHandlerManager OutputHandlerManager // for file/pipe targets
redirectorTargetEvaluable IEvaluable // for file/pipe targets
outputHandlerManager output.OutputHandlerManager // for file/pipe targets
}
// ----------------------------------------------------------------
@ -99,11 +100,11 @@ func (this *RootNode) BuildTeeStatementNode(astNode *dsl.ASTNode) (IExecutable,
if redirectorTargetNode.Type == dsl.NodeTypeRedirectTargetStdout {
retval.teeToRedirectFunc = retval.teeToFileOrPipe
retval.outputHandlerManager = NewStdoutWriteHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewStdoutWriteHandlerManager(this.recordWriterOptions)
retval.redirectorTargetEvaluable = this.BuildStringLiteralNode("(stdout)")
} else if redirectorTargetNode.Type == dsl.NodeTypeRedirectTargetStderr {
retval.teeToRedirectFunc = retval.teeToFileOrPipe
retval.outputHandlerManager = NewStderrWriteHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewStderrWriteHandlerManager(this.recordWriterOptions)
retval.redirectorTargetEvaluable = this.BuildStringLiteralNode("(stderr)")
} else {
retval.teeToRedirectFunc = retval.teeToFileOrPipe
@ -114,11 +115,11 @@ func (this *RootNode) BuildTeeStatementNode(astNode *dsl.ASTNode) (IExecutable,
}
if redirectorNode.Type == dsl.NodeTypeRedirectWrite {
retval.outputHandlerManager = NewFileWritetHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewFileWritetHandlerManager(this.recordWriterOptions)
} else if redirectorNode.Type == dsl.NodeTypeRedirectAppend {
retval.outputHandlerManager = NewFileAppendHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewFileAppendHandlerManager(this.recordWriterOptions)
} else if redirectorNode.Type == dsl.NodeTypeRedirectPipe {
retval.outputHandlerManager = NewPipeWriteHandlerManager(this.recordWriterOptions)
retval.outputHandlerManager = output.NewPipeWriteHandlerManager(this.recordWriterOptions)
} else {
return nil, errors.New(
fmt.Sprintf(

14
go/src/miller/dsl/cst/output-handlers.go → go/src/miller/output/file-output-handlers.go
View file

@ -10,7 +10,7 @@
// ordering. (Main record-writer output is also to stdout.)
// ================================================================
package cst
package output
import (
"errors"
@ -20,7 +20,6 @@ import (
"miller/clitypes"
"miller/lib"
"miller/output"
"miller/types"
)
@ -210,7 +209,7 @@ type FileOutputHandler struct {
// called only by WriteRecrod which is called by emit and tee variants;
// print and dump variants call WriteString.
recordWriterOptions *clitypes.TWriterOptions
recordWriter output.IRecordWriter
recordWriter IRecordWriter
recordOutputChannel chan *types.RecordAndContext
recordDoneChannel chan bool
}
@ -240,7 +239,7 @@ func NewFileWriteOutputHandler(
) (*FileOutputHandler, error) {
handle, err := os.OpenFile(
filename,
os.O_CREATE|os.O_WRONLY,
os.O_CREATE|os.O_WRONLY|os.O_TRUNC,
0644, // TODO: let users parameterize this
)
if err != nil {
@ -326,9 +325,6 @@ func (this *FileOutputHandler) WriteString(outputString string) error {
}
// ----------------------------------------------------------------
// TODO: finish implementing this
// TODO: needs context
func (this *FileOutputHandler) WriteRecordAndContext(
outrecAndContext *types.RecordAndContext,
) error {
@ -349,7 +345,7 @@ func (this *FileOutputHandler) setUpRecordWriter() error {
return nil
}
recordWriter := output.Create(this.recordWriterOptions)
recordWriter := Create(this.recordWriterOptions)
if recordWriter == nil {
return errors.New(
"Output format not found: " + this.recordWriterOptions.OutputFileFormat,
@ -360,7 +356,7 @@ func (this *FileOutputHandler) setUpRecordWriter() error {
this.recordOutputChannel = make(chan *types.RecordAndContext, 1)
this.recordDoneChannel = make(chan bool, 1)
go output.ChannelWriter(
go ChannelWriter(
this.recordOutputChannel,
this.recordWriter,
this.recordDoneChannel,

168
go/src/miller/transformers/tee.go Normal file
View file

@ -0,0 +1,168 @@
package transformers
import (
"flag"
"fmt"
"os"
"strings"
"miller/clitypes"
"miller/output"
"miller/transforming"
"miller/types"
)
// ----------------------------------------------------------------
var TeeSetup = transforming.TransformerSetup{
Verb: "tee",
ParseCLIFunc: transformerTeeParseCLI,
IgnoresInput: false,
}
func transformerTeeParseCLI(
pargi *int,
argc int,
args []string,
errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError
_ *clitypes.TReaderOptions,
mainRecordWriterOptions *clitypes.TWriterOptions,
) transforming.IRecordTransformer {
// Get the verb name from the current spot in the mlr command line
argi := *pargi
verb := args[argi]
argi++
filename := ""
appending := false
// TODO: make sure this is a full nested-struct copy.
var recordWriterOptions *clitypes.TWriterOptions = nil
if mainRecordWriterOptions != nil {
copyThereof := *mainRecordWriterOptions
recordWriterOptions = &copyThereof
}
// Parse local flags.
for argi < argc /* variable increment: 1 or 2 depending on flag */ {
if !strings.HasPrefix(args[argi], "-") {
break // No more flag options to process
} else if args[argi] == "-h" || args[argi] == "--help" {
transformerTeeUsage(os.Stdout, 0, errorHandling, args[0], verb)
return nil // help intentionally requested
} else if args[argi] == "-a" {
appending = true
argi += 1
} else if clitypes.ParseWriterOptions(args, argc, &argi, recordWriterOptions) {
// This lets mlr main and mlr tee have different output formats.
// Nothing else to handle here.
} else {
transformerTeeUsage(os.Stderr, 1, flag.ExitOnError, args[0], verb)
os.Exit(1)
}
}
// Get the filename from the command line, after the flags
if argi >= argc {
transformerTeeUsage(os.Stderr, 1, flag.ExitOnError, args[0], verb)
os.Exit(1)
}
filename = args[argi]
argi += 1
transformer, err := NewTransformerTee(
appending,
filename,
recordWriterOptions,
)
if err != nil {
// Error message already printed out
os.Exit(1)
}
*pargi = argi
return transformer
}
func transformerTeeUsage(
o *os.File,
exitCode int,
errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError
argv0 string,
verb string,
) {
fmt.Fprintf(o, "Usage: %s %s [options] {filename}\n", argv0, verb)
fmt.Fprintf(o,
`-a append to existing file, if any, rather than overwriting.
Any of the output-format command-line flags (see mlr -h). Example: using
mlr --icsv --opprint put '...' then tee --ojson ./mytap.dat then stats1 ...
the input is CSV, the output is pretty-print tabular, but the tee-file output
is written in JSON format.
-h|--help Show this message.
`)
}
// ----------------------------------------------------------------
type TransformerTee struct {
filename string
fileOutputHandler *output.FileOutputHandler
}
func NewTransformerTee(
appending bool,
filename string,
recordWriterOptions *clitypes.TWriterOptions,
) (*TransformerTee, error) {
var fileOutputHandler *output.FileOutputHandler = nil
var err error = nil
if appending {
fileOutputHandler, err = output.NewFileAppendOutputHandler(filename, recordWriterOptions)
} else {
fileOutputHandler, err = output.NewFileWriteOutputHandler(filename, recordWriterOptions)
}
if err != nil {
return nil, err
}
return &TransformerTee{
filename: filename,
fileOutputHandler: fileOutputHandler,
}, nil
}
func (this *TransformerTee) Transform(
inrecAndContext *types.RecordAndContext,
outputChannel chan<- *types.RecordAndContext,
) {
if !inrecAndContext.EndOfStream {
err := this.fileOutputHandler.WriteRecordAndContext(inrecAndContext)
if err != nil {
fmt.Fprintf(
os.Stderr,
"%s: error writing to tee-file \"%s\":\n",
os.Args[0], this.filename,
)
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
outputChannel <- inrecAndContext
} else {
err := this.fileOutputHandler.Close()
if err != nil {
fmt.Fprintf(
os.Stderr,
"%s: error closing tee-file \"%s\":\n",
os.Args[0], this.filename,
)
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
outputChannel <- inrecAndContext
}
}

9
go/todo.txt
View file

@ -10,7 +10,6 @@ TOP OF LIST:
* in-place processing
* prepipe
- gzin; bz2in ...
* tee verb
* c/go both:
o https://brandur.org/logfmt is simply DKVP w/ IFS = space (need dquot though)
@ -22,9 +21,8 @@ TOP OF LIST:
bugs:
* case-c-dsl-redirects.sh
--no-fflush
tee verb missing
mlr put -q 'print | "tr \[a-z\] \[A-Z\]", "abi:".$a.$b.$i' s
k --no-fflush
o mlr put -q 'print | "tr \[a-z\] \[A-Z\]", "abi:".$a.$b.$i' s
mlr: cannot parse DSL expression.
* case-c-dsl-oosvars.sh
@ -204,7 +202,6 @@ histogram
least-frequent
most-frequent
sec2gmtdate
tee
top
uniq
@ -470,3 +467,5 @@ i https://en.wikipedia.org/wiki/Delimiter#Delimiter_collision
* doc-note: 'emit > stdout, @foo' and 'emit @foo' are not the same:
o the former allows put --oxxx
o the former is not necessarily in sync with the output record stream
* tee -p {command} -- ?