mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-17 16:38:54 +00:00
570fcf0de4
Phase 3 of plans/exit.md: the streaming-interface change, the load-bearing piece for #341 (DSL exit statement) and #440 (strict mode). - RecordTransformer.Transform and RecordTransformerFunc now return error. All 69 Transform implementations and their dispatch helpers updated (mechanical rewrite, compiler- and errcheck-verified). - runSingleTransformerBatch, on a Transform error, forwards any output produced before the failure plus an end-of-stream marker downstream, so the rest of the chain and the record-writer drain and finish cleanly; runSingleTransformer then surfaces the error to stream.Stream's select loop (non-blocking send; first error wins) and signals upstream-done so the record-reader stops. This is exactly the flush-then-exit sequencing a future DSL 'exit N' needs. - dataProcessingErrorChannel and FileOutputHandler.recordErroredChannel are now chan error instead of chan bool; ChannelWriter still prints write- error details at the site and sends the 'exiting due to data error' sentinel, preserving the exact stderr shape pinned by regression cases. - Mid-stream os.Exit sites converted to returned errors: put/filter DSL begin/main/end-block errors and the non-boolean filter-expression case, tee write/close failures, split write/open/close failures, join left-file ingest failures (both half-streaming and sorted paths, with full error plumbing through JoinBucketKeeper), histogram/stats2 ingest errors, surv fit errors, and step stepper allocation (tStepperAllocator now returns (tStepper, error); bad EWMA coefficients propagate; negative slwin parameters are reported by the CLI parser via the existing bad-stepper-name pattern). - The two genuinely internal join-bucket-keeper states now use lib.InternalCodingErrorWithMessageIf instead of hand-rolled print+exit. - pkg/transformers is now os.Exit-free. Behavior notes: the non-boolean filter message gains the standard 'mlr: ' prefix and a newline (it previously printed with neither); tee errors now include the underlying cause. All 4779 regression cases pass unchanged; mlr head early-out latency is unaffected (0.02s over 50M records). Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
123 lines
3.3 KiB
Go
123 lines
3.3 KiB
Go
package transformers
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
|
|
"github.com/johnkerl/miller/v6/pkg/bifs"
|
|
"github.com/johnkerl/miller/v6/pkg/cli"
|
|
"github.com/johnkerl/miller/v6/pkg/lib"
|
|
"github.com/johnkerl/miller/v6/pkg/types"
|
|
)
|
|
|
|
const verbNameSec2GMTDate = "sec2gmtdate"
|
|
|
|
var sec2GMTDateOptions = []OptionSpec{}
|
|
|
|
var Sec2GMTDateSetup = TransformerSetup{
|
|
Verb: verbNameSec2GMTDate,
|
|
UsageFunc: transformerSec2GMTDateUsage,
|
|
ParseCLIFunc: transformerSec2GMTDateParseCLI,
|
|
IgnoresInput: false,
|
|
Options: sec2GMTDateOptions,
|
|
}
|
|
|
|
func transformerSec2GMTDateUsage(
|
|
o *os.File,
|
|
) {
|
|
fmt.Fprintf(o, "Usage: mlr sec2gmtdate {comma-separated list of field names}\n")
|
|
fmt.Fprintf(o, "Replaces a numeric field representing seconds since the epoch with the\n")
|
|
fmt.Fprintf(o, "corresponding GMT year-month-day timestamp; leaves non-numbers as-is.\n")
|
|
fmt.Fprintf(o, "This is nothing more than a keystroke-saver for the sec2gmtdate function:\n")
|
|
fmt.Fprintf(o, " mlr sec2gmtdate time1,time2\n")
|
|
fmt.Fprintf(o, "is the same as\n")
|
|
fmt.Fprintf(o, " mlr put '$time1=sec2gmtdate($time1);$time2=sec2gmtdate($time2)'\n")
|
|
WriteVerbOptions(o, sec2GMTDateOptions)
|
|
}
|
|
|
|
func transformerSec2GMTDateParseCLI(
|
|
pargi *int,
|
|
argc int,
|
|
args []string,
|
|
_ *cli.TOptions,
|
|
doConstruct bool, // false for first pass of CLI-parse, true for second pass
|
|
) (RecordTransformer, error) {
|
|
|
|
// Skip the verb name from the current spot in the mlr command line
|
|
argi := *pargi
|
|
argi++
|
|
|
|
for argi < argc /* variable increment: 1 or 2 depending on flag */ {
|
|
opt := args[argi]
|
|
if opt[0] != '-' {
|
|
break // No more flag options to process
|
|
}
|
|
if args[argi] == "--" {
|
|
break // All transformers must do this so main-flags can follow verb-flags
|
|
}
|
|
|
|
if opt == "-h" || opt == "--help" {
|
|
transformerSec2GMTDateUsage(os.Stdout)
|
|
return nil, cli.ErrHelpRequested
|
|
|
|
} else {
|
|
return nil, cli.VerbErrorf(verbNameSec2GMTDate, "option \"%s\" not recognized", opt)
|
|
}
|
|
}
|
|
|
|
if argi >= argc {
|
|
return nil, cli.VerbErrorf(verbNameSec2GMTDate, "field names required")
|
|
}
|
|
fieldNames := args[argi]
|
|
argi++
|
|
|
|
*pargi = argi
|
|
if !doConstruct { // All transformers must do this for main command-line parsing
|
|
return nil, nil
|
|
}
|
|
|
|
transformer, err := NewTransformerSec2GMTDate(
|
|
fieldNames,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return transformer, nil
|
|
}
|
|
|
|
type TransformerSec2GMTDate struct {
|
|
fieldNameList []string
|
|
}
|
|
|
|
func NewTransformerSec2GMTDate(
|
|
fieldNames string,
|
|
) (*TransformerSec2GMTDate, error) {
|
|
tr := &TransformerSec2GMTDate{
|
|
fieldNameList: lib.SplitString(fieldNames, ","),
|
|
}
|
|
return tr, nil
|
|
}
|
|
|
|
func (tr *TransformerSec2GMTDate) Transform(
|
|
inrecAndContext *types.RecordAndContext,
|
|
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
|
|
inputDownstreamDoneChannel <-chan bool,
|
|
outputDownstreamDoneChannel chan<- bool,
|
|
) error {
|
|
HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel)
|
|
if !inrecAndContext.EndOfStream {
|
|
inrec := inrecAndContext.Record
|
|
for _, fieldName := range tr.fieldNameList {
|
|
value := inrec.Get(fieldName)
|
|
if value != nil {
|
|
inrec.PutReference(fieldName, bifs.BIF_sec2gmtdate(value))
|
|
}
|
|
}
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
|
|
|
|
} else { // End of record stream
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
|
|
}
|
|
return nil
|
|
}
|