diff --git a/go/fmter b/go/fmter index 269008b05..95fcd151d 100755 --- a/go/fmter +++ b/go/fmter @@ -1,3 +1,3 @@ #!/bin/bash -for x in $(lsr \*.go); do gofmt -w $x; done +go fmt ./... diff --git a/go/src/miller/cli/mlrcli_transformers.go b/go/src/miller/cli/mlrcli_transformers.go index dca5953de..cfedf95f0 100644 --- a/go/src/miller/cli/mlrcli_transformers.go +++ b/go/src/miller/cli/mlrcli_transformers.go @@ -47,6 +47,7 @@ var MAPPER_LOOKUP_TABLE = []transforming.TransformerSetup{ transformers.Stats1Setup, transformers.StepSetup, transformers.TacSetup, + transformers.TeeSetup, transformers.TailSetup, transformers.UnflattenSetup, transformers.UnsparsifySetup, diff --git a/go/src/miller/dsl/cst/dump.go b/go/src/miller/dsl/cst/dump.go index ec8e2232d..cfd094f39 100644 --- a/go/src/miller/dsl/cst/dump.go +++ b/go/src/miller/dsl/cst/dump.go @@ -24,6 +24,7 @@ import ( "miller/dsl" "miller/lib" + "miller/output" "miller/types" ) @@ -36,8 +37,8 @@ type tDumpToRedirectFunc func( type DumpStatementNode struct { expressionEvaluables []IEvaluable dumpToRedirectFunc tDumpToRedirectFunc - redirectorTargetEvaluable IEvaluable // for file/pipe targets - outputHandlerManager OutputHandlerManager // for file/pipe targets + redirectorTargetEvaluable IEvaluable // for file/pipe targets + outputHandlerManager output.OutputHandlerManager // for file/pipe targets } // ---------------------------------------------------------------- @@ -130,11 +131,11 @@ func (this *RootNode) buildDumpxStatementNode( } if redirectorNode.Type == dsl.NodeTypeRedirectWrite { - retval.outputHandlerManager = NewFileWritetHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewFileWritetHandlerManager(this.recordWriterOptions) } else if redirectorNode.Type == dsl.NodeTypeRedirectAppend { - retval.outputHandlerManager = NewFileAppendHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewFileAppendHandlerManager(this.recordWriterOptions) } else if redirectorNode.Type == dsl.NodeTypeRedirectPipe { - retval.outputHandlerManager = NewPipeWriteHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewPipeWriteHandlerManager(this.recordWriterOptions) } else { return nil, errors.New( fmt.Sprintf( diff --git a/go/src/miller/dsl/cst/emit_emitp.go b/go/src/miller/dsl/cst/emit_emitp.go index da07ea6b7..204257af7 100644 --- a/go/src/miller/dsl/cst/emit_emitp.go +++ b/go/src/miller/dsl/cst/emit_emitp.go @@ -37,6 +37,7 @@ import ( "miller/dsl" "miller/lib" + "miller/output" "miller/types" ) @@ -70,7 +71,7 @@ type EmitXStatementNode struct { redirectorTargetEvaluable IEvaluable // For file/pipe targets: keeps track of file handles for various values of // the redirectorTargetEvaluable expression. - outputHandlerManager OutputHandlerManager + outputHandlerManager output.OutputHandlerManager // For code-reuse between executors. isEmitP bool @@ -173,11 +174,11 @@ func (this *RootNode) buildEmitXStatementNode( if redirectorTargetNode.Type == dsl.NodeTypeRedirectTargetStdout { retval.emitToRedirectFunc = retval.emitToFileOrPipe - retval.outputHandlerManager = NewStdoutWriteHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewStdoutWriteHandlerManager(this.recordWriterOptions) retval.redirectorTargetEvaluable = this.BuildStringLiteralNode("(stdout)") } else if redirectorTargetNode.Type == dsl.NodeTypeRedirectTargetStderr { retval.emitToRedirectFunc = retval.emitToFileOrPipe - retval.outputHandlerManager = NewStderrWriteHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewStderrWriteHandlerManager(this.recordWriterOptions) retval.redirectorTargetEvaluable = this.BuildStringLiteralNode("(stderr)") } else { retval.emitToRedirectFunc = retval.emitToFileOrPipe @@ -188,11 +189,11 @@ func (this *RootNode) buildEmitXStatementNode( } if redirectorNode.Type == dsl.NodeTypeRedirectWrite { - retval.outputHandlerManager = NewFileWritetHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewFileWritetHandlerManager(this.recordWriterOptions) } else if redirectorNode.Type == dsl.NodeTypeRedirectAppend { - retval.outputHandlerManager = NewFileAppendHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewFileAppendHandlerManager(this.recordWriterOptions) } else if redirectorNode.Type == dsl.NodeTypeRedirectPipe { - retval.outputHandlerManager = NewPipeWriteHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewPipeWriteHandlerManager(this.recordWriterOptions) } else { return nil, errors.New( fmt.Sprintf( diff --git a/go/src/miller/dsl/cst/emitf.go b/go/src/miller/dsl/cst/emitf.go index 6820f50f1..6b982695f 100644 --- a/go/src/miller/dsl/cst/emitf.go +++ b/go/src/miller/dsl/cst/emitf.go @@ -12,6 +12,7 @@ import ( "miller/dsl" "miller/lib" + "miller/output" "miller/types" ) @@ -34,8 +35,8 @@ type EmitFStatementNode struct { emitfNames []string emitfEvaluables []IEvaluable emitfToRedirectFunc tEmitFToRedirectFunc - redirectorTargetEvaluable IEvaluable // for file/pipe targets - outputHandlerManager OutputHandlerManager // for file/pipe targets + redirectorTargetEvaluable IEvaluable // for file/pipe targets + outputHandlerManager output.OutputHandlerManager // for file/pipe targets } // ---------------------------------------------------------------- @@ -97,11 +98,11 @@ func (this *RootNode) BuildEmitFStatementNode(astNode *dsl.ASTNode) (IExecutable if redirectorTargetNode.Type == dsl.NodeTypeRedirectTargetStdout { retval.emitfToRedirectFunc = retval.emitfToFileOrPipe - retval.outputHandlerManager = NewStdoutWriteHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewStdoutWriteHandlerManager(this.recordWriterOptions) retval.redirectorTargetEvaluable = this.BuildStringLiteralNode("(stdout)") } else if redirectorTargetNode.Type == dsl.NodeTypeRedirectTargetStderr { retval.emitfToRedirectFunc = retval.emitfToFileOrPipe - retval.outputHandlerManager = NewStderrWriteHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewStderrWriteHandlerManager(this.recordWriterOptions) retval.redirectorTargetEvaluable = this.BuildStringLiteralNode("(stderr)") } else { retval.emitfToRedirectFunc = retval.emitfToFileOrPipe @@ -112,11 +113,11 @@ func (this *RootNode) BuildEmitFStatementNode(astNode *dsl.ASTNode) (IExecutable } if redirectorNode.Type == dsl.NodeTypeRedirectWrite { - retval.outputHandlerManager = NewFileWritetHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewFileWritetHandlerManager(this.recordWriterOptions) } else if redirectorNode.Type == dsl.NodeTypeRedirectAppend { - retval.outputHandlerManager = NewFileAppendHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewFileAppendHandlerManager(this.recordWriterOptions) } else if redirectorNode.Type == dsl.NodeTypeRedirectPipe { - retval.outputHandlerManager = NewPipeWriteHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewPipeWriteHandlerManager(this.recordWriterOptions) } else { return nil, errors.New( fmt.Sprintf( diff --git a/go/src/miller/dsl/cst/print.go b/go/src/miller/dsl/cst/print.go index 9488cb7c2..20a2f495a 100644 --- a/go/src/miller/dsl/cst/print.go +++ b/go/src/miller/dsl/cst/print.go @@ -12,6 +12,7 @@ import ( "miller/dsl" "miller/lib" + "miller/output" "miller/types" ) @@ -157,8 +158,8 @@ type PrintStatementNode struct { expressionEvaluables []IEvaluable terminator string printToRedirectFunc tPrintToRedirectFunc - redirectorTargetEvaluable IEvaluable // for file/pipe targets - outputHandlerManager OutputHandlerManager // for file/pipe targets + redirectorTargetEvaluable IEvaluable // for file/pipe targets + outputHandlerManager output.OutputHandlerManager // for file/pipe targets } // ---------------------------------------------------------------- @@ -273,11 +274,11 @@ func (this *RootNode) buildPrintxStatementNode( } if redirectorNode.Type == dsl.NodeTypeRedirectWrite { - retval.outputHandlerManager = NewFileWritetHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewFileWritetHandlerManager(this.recordWriterOptions) } else if redirectorNode.Type == dsl.NodeTypeRedirectAppend { - retval.outputHandlerManager = NewFileAppendHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewFileAppendHandlerManager(this.recordWriterOptions) } else if redirectorNode.Type == dsl.NodeTypeRedirectPipe { - retval.outputHandlerManager = NewPipeWriteHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewPipeWriteHandlerManager(this.recordWriterOptions) } else { return nil, errors.New( fmt.Sprintf( diff --git a/go/src/miller/dsl/cst/root.go b/go/src/miller/dsl/cst/root.go index 276848e90..702707595 100644 --- a/go/src/miller/dsl/cst/root.go +++ b/go/src/miller/dsl/cst/root.go @@ -13,6 +13,7 @@ import ( "miller/clitypes" "miller/dsl" + "miller/output" "miller/types" ) @@ -221,14 +222,14 @@ func (this *RootNode) resolveSubroutineCallsites() error { // can close all the descriptors, flush the record-output streams, etc. func (this *RootNode) RegisterOutputHandlerManager( - outputHandlerManager OutputHandlerManager, + outputHandlerManager output.OutputHandlerManager, ) { this.outputHandlerManagers.PushBack(outputHandlerManager) } func (this *RootNode) ProcessEndOfStream() { for entry := this.outputHandlerManagers.Front(); entry != nil; entry = entry.Next() { - outputHandlerManager := entry.Value.(OutputHandlerManager) + outputHandlerManager := entry.Value.(output.OutputHandlerManager) errs := outputHandlerManager.Close() if len(errs) != 0 { for _, err := range errs { diff --git a/go/src/miller/dsl/cst/tee.go b/go/src/miller/dsl/cst/tee.go index 40f0878d1..87b589b71 100644 --- a/go/src/miller/dsl/cst/tee.go +++ b/go/src/miller/dsl/cst/tee.go @@ -11,6 +11,7 @@ import ( "miller/dsl" "miller/lib" + "miller/output" "miller/types" ) @@ -64,8 +65,8 @@ type tTeeToRedirectFunc func( type TeeStatementNode struct { expressionEvaluable IEvaluable // always the $* evaluable teeToRedirectFunc tTeeToRedirectFunc - redirectorTargetEvaluable IEvaluable // for file/pipe targets - outputHandlerManager OutputHandlerManager // for file/pipe targets + redirectorTargetEvaluable IEvaluable // for file/pipe targets + outputHandlerManager output.OutputHandlerManager // for file/pipe targets } // ---------------------------------------------------------------- @@ -99,11 +100,11 @@ func (this *RootNode) BuildTeeStatementNode(astNode *dsl.ASTNode) (IExecutable, if redirectorTargetNode.Type == dsl.NodeTypeRedirectTargetStdout { retval.teeToRedirectFunc = retval.teeToFileOrPipe - retval.outputHandlerManager = NewStdoutWriteHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewStdoutWriteHandlerManager(this.recordWriterOptions) retval.redirectorTargetEvaluable = this.BuildStringLiteralNode("(stdout)") } else if redirectorTargetNode.Type == dsl.NodeTypeRedirectTargetStderr { retval.teeToRedirectFunc = retval.teeToFileOrPipe - retval.outputHandlerManager = NewStderrWriteHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewStderrWriteHandlerManager(this.recordWriterOptions) retval.redirectorTargetEvaluable = this.BuildStringLiteralNode("(stderr)") } else { retval.teeToRedirectFunc = retval.teeToFileOrPipe @@ -114,11 +115,11 @@ func (this *RootNode) BuildTeeStatementNode(astNode *dsl.ASTNode) (IExecutable, } if redirectorNode.Type == dsl.NodeTypeRedirectWrite { - retval.outputHandlerManager = NewFileWritetHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewFileWritetHandlerManager(this.recordWriterOptions) } else if redirectorNode.Type == dsl.NodeTypeRedirectAppend { - retval.outputHandlerManager = NewFileAppendHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewFileAppendHandlerManager(this.recordWriterOptions) } else if redirectorNode.Type == dsl.NodeTypeRedirectPipe { - retval.outputHandlerManager = NewPipeWriteHandlerManager(this.recordWriterOptions) + retval.outputHandlerManager = output.NewPipeWriteHandlerManager(this.recordWriterOptions) } else { return nil, errors.New( fmt.Sprintf( diff --git a/go/src/miller/dsl/cst/output-handlers.go b/go/src/miller/output/file-output-handlers.go similarity index 97% rename from go/src/miller/dsl/cst/output-handlers.go rename to go/src/miller/output/file-output-handlers.go index bbacfabd2..dd0a70cf4 100644 --- a/go/src/miller/dsl/cst/output-handlers.go +++ b/go/src/miller/output/file-output-handlers.go @@ -10,7 +10,7 @@ // ordering. (Main record-writer output is also to stdout.) // ================================================================ -package cst +package output import ( "errors" @@ -20,7 +20,6 @@ import ( "miller/clitypes" "miller/lib" - "miller/output" "miller/types" ) @@ -210,7 +209,7 @@ type FileOutputHandler struct { // called only by WriteRecrod which is called by emit and tee variants; // print and dump variants call WriteString. recordWriterOptions *clitypes.TWriterOptions - recordWriter output.IRecordWriter + recordWriter IRecordWriter recordOutputChannel chan *types.RecordAndContext recordDoneChannel chan bool } @@ -240,7 +239,7 @@ func NewFileWriteOutputHandler( ) (*FileOutputHandler, error) { handle, err := os.OpenFile( filename, - os.O_CREATE|os.O_WRONLY, + os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644, // TODO: let users parameterize this ) if err != nil { @@ -326,9 +325,6 @@ func (this *FileOutputHandler) WriteString(outputString string) error { } // ---------------------------------------------------------------- -// TODO: finish implementing this -// TODO: needs context - func (this *FileOutputHandler) WriteRecordAndContext( outrecAndContext *types.RecordAndContext, ) error { @@ -349,7 +345,7 @@ func (this *FileOutputHandler) setUpRecordWriter() error { return nil } - recordWriter := output.Create(this.recordWriterOptions) + recordWriter := Create(this.recordWriterOptions) if recordWriter == nil { return errors.New( "Output format not found: " + this.recordWriterOptions.OutputFileFormat, @@ -360,7 +356,7 @@ func (this *FileOutputHandler) setUpRecordWriter() error { this.recordOutputChannel = make(chan *types.RecordAndContext, 1) this.recordDoneChannel = make(chan bool, 1) - go output.ChannelWriter( + go ChannelWriter( this.recordOutputChannel, this.recordWriter, this.recordDoneChannel, diff --git a/go/src/miller/transformers/tee.go b/go/src/miller/transformers/tee.go new file mode 100644 index 000000000..0c25a8c8b --- /dev/null +++ b/go/src/miller/transformers/tee.go @@ -0,0 +1,168 @@ +package transformers + +import ( + "flag" + "fmt" + "os" + "strings" + + "miller/clitypes" + "miller/output" + "miller/transforming" + "miller/types" +) + +// ---------------------------------------------------------------- +var TeeSetup = transforming.TransformerSetup{ + Verb: "tee", + ParseCLIFunc: transformerTeeParseCLI, + IgnoresInput: false, +} + +func transformerTeeParseCLI( + pargi *int, + argc int, + args []string, + errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError + _ *clitypes.TReaderOptions, + mainRecordWriterOptions *clitypes.TWriterOptions, +) transforming.IRecordTransformer { + + // Get the verb name from the current spot in the mlr command line + argi := *pargi + verb := args[argi] + argi++ + + filename := "" + appending := false + // TODO: make sure this is a full nested-struct copy. + var recordWriterOptions *clitypes.TWriterOptions = nil + if mainRecordWriterOptions != nil { + copyThereof := *mainRecordWriterOptions + recordWriterOptions = ©Thereof + } + + // Parse local flags. + + for argi < argc /* variable increment: 1 or 2 depending on flag */ { + if !strings.HasPrefix(args[argi], "-") { + break // No more flag options to process + + } else if args[argi] == "-h" || args[argi] == "--help" { + transformerTeeUsage(os.Stdout, 0, errorHandling, args[0], verb) + return nil // help intentionally requested + + } else if args[argi] == "-a" { + appending = true + argi += 1 + + } else if clitypes.ParseWriterOptions(args, argc, &argi, recordWriterOptions) { + // This lets mlr main and mlr tee have different output formats. + // Nothing else to handle here. + + } else { + transformerTeeUsage(os.Stderr, 1, flag.ExitOnError, args[0], verb) + os.Exit(1) + } + } + + // Get the filename from the command line, after the flags + if argi >= argc { + transformerTeeUsage(os.Stderr, 1, flag.ExitOnError, args[0], verb) + os.Exit(1) + } + filename = args[argi] + argi += 1 + + transformer, err := NewTransformerTee( + appending, + filename, + recordWriterOptions, + ) + if err != nil { + // Error message already printed out + os.Exit(1) + } + + *pargi = argi + return transformer +} + +func transformerTeeUsage( + o *os.File, + exitCode int, + errorHandling flag.ErrorHandling, // ContinueOnError or ExitOnError + argv0 string, + verb string, +) { + fmt.Fprintf(o, "Usage: %s %s [options] {filename}\n", argv0, verb) + fmt.Fprintf(o, + `-a append to existing file, if any, rather than overwriting. +Any of the output-format command-line flags (see mlr -h). Example: using + mlr --icsv --opprint put '...' then tee --ojson ./mytap.dat then stats1 ... +the input is CSV, the output is pretty-print tabular, but the tee-file output +is written in JSON format. + +-h|--help Show this message. +`) +} + +// ---------------------------------------------------------------- +type TransformerTee struct { + filename string + fileOutputHandler *output.FileOutputHandler +} + +func NewTransformerTee( + appending bool, + filename string, + recordWriterOptions *clitypes.TWriterOptions, +) (*TransformerTee, error) { + var fileOutputHandler *output.FileOutputHandler = nil + var err error = nil + if appending { + fileOutputHandler, err = output.NewFileAppendOutputHandler(filename, recordWriterOptions) + } else { + fileOutputHandler, err = output.NewFileWriteOutputHandler(filename, recordWriterOptions) + } + if err != nil { + return nil, err + } + + return &TransformerTee{ + filename: filename, + fileOutputHandler: fileOutputHandler, + }, nil +} + +func (this *TransformerTee) Transform( + inrecAndContext *types.RecordAndContext, + outputChannel chan<- *types.RecordAndContext, +) { + if !inrecAndContext.EndOfStream { + err := this.fileOutputHandler.WriteRecordAndContext(inrecAndContext) + if err != nil { + fmt.Fprintf( + os.Stderr, + "%s: error writing to tee-file \"%s\":\n", + os.Args[0], this.filename, + ) + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + outputChannel <- inrecAndContext + } else { + err := this.fileOutputHandler.Close() + if err != nil { + fmt.Fprintf( + os.Stderr, + "%s: error closing tee-file \"%s\":\n", + os.Args[0], this.filename, + ) + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + outputChannel <- inrecAndContext + } +} diff --git a/go/todo.txt b/go/todo.txt index d75758d0f..3f696035f 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -10,7 +10,6 @@ TOP OF LIST: * in-place processing * prepipe - gzin; bz2in ... -* tee verb * c/go both: o https://brandur.org/logfmt is simply DKVP w/ IFS = space (need dquot though) @@ -22,9 +21,8 @@ TOP OF LIST: bugs: * case-c-dsl-redirects.sh - --no-fflush - tee verb missing - mlr put -q 'print | "tr \[a-z\] \[A-Z\]", "abi:".$a.$b.$i' s + k --no-fflush + o mlr put -q 'print | "tr \[a-z\] \[A-Z\]", "abi:".$a.$b.$i' s mlr: cannot parse DSL expression. * case-c-dsl-oosvars.sh @@ -204,7 +202,6 @@ histogram least-frequent most-frequent sec2gmtdate -tee top uniq @@ -470,3 +467,5 @@ i https://en.wikipedia.org/wiki/Delimiter#Delimiter_collision * doc-note: 'emit > stdout, @foo' and 'emit @foo' are not the same: o the former allows put --oxxx o the former is not necessarily in sync with the output record stream + +* tee -p {command} -- ?