mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-17 16:38:54 +00:00
Phase 3 of plans/exit.md: the streaming-interface change, the load-bearing piece for #341 (DSL exit statement) and #440 (strict mode). - RecordTransformer.Transform and RecordTransformerFunc now return error. All 69 Transform implementations and their dispatch helpers updated (mechanical rewrite, compiler- and errcheck-verified). - runSingleTransformerBatch, on a Transform error, forwards any output produced before the failure plus an end-of-stream marker downstream, so the rest of the chain and the record-writer drain and finish cleanly; runSingleTransformer then surfaces the error to stream.Stream's select loop (non-blocking send; first error wins) and signals upstream-done so the record-reader stops. This is exactly the flush-then-exit sequencing a future DSL 'exit N' needs. - dataProcessingErrorChannel and FileOutputHandler.recordErroredChannel are now chan error instead of chan bool; ChannelWriter still prints write- error details at the site and sends the 'exiting due to data error' sentinel, preserving the exact stderr shape pinned by regression cases. - Mid-stream os.Exit sites converted to returned errors: put/filter DSL begin/main/end-block errors and the non-boolean filter-expression case, tee write/close failures, split write/open/close failures, join left-file ingest failures (both half-streaming and sorted paths, with full error plumbing through JoinBucketKeeper), histogram/stats2 ingest errors, surv fit errors, and step stepper allocation (tStepperAllocator now returns (tStepper, error); bad EWMA coefficients propagate; negative slwin parameters are reported by the CLI parser via the existing bad-stepper-name pattern). - The two genuinely internal join-bucket-keeper states now use lib.InternalCodingErrorWithMessageIf instead of hand-rolled print+exit. - pkg/transformers is now os.Exit-free. Behavior notes: the non-boolean filter message gains the standard 'mlr: ' prefix and a newline (it previously printed with neither); tee errors now include the underlying cause. All 4779 regression cases pass unchanged; mlr head early-out latency is unaffected (0.02s over 50M records). Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
214 lines
5.7 KiB
Go
214 lines
5.7 KiB
Go
package transformers
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/johnkerl/miller/v6/pkg/mlrval"
|
|
)
|
|
|
|
func TestParseStepperCount(t *testing.T) {
|
|
cases := []struct {
|
|
stepperName string
|
|
baseName string
|
|
wantCount int
|
|
wantOK bool
|
|
}{
|
|
{"shift_lag", "shift_lag", 1, true},
|
|
{"shift_lag_1", "shift_lag", 1, true},
|
|
{"shift_lag_12", "shift_lag", 12, true},
|
|
{"shift_lag_007", "shift_lag", 7, true},
|
|
{"shift_lag_0", "shift_lag", 0, false},
|
|
{"shift_lag_-3", "shift_lag", 0, false},
|
|
{"shift_lag_+3", "shift_lag", 0, false},
|
|
{"shift_lag_", "shift_lag", 0, false},
|
|
{"shift_lag_x", "shift_lag", 0, false},
|
|
{"shift_lag_3_4", "shift_lag", 0, false},
|
|
{"shift_lead", "shift_lag", 0, false},
|
|
{"shift_lag", "shift", 0, false}, // "lag" is not a count
|
|
{"shift_12", "shift", 12, true},
|
|
{"delta_4", "delta", 4, true},
|
|
{"ratio_4", "ratio", 4, true},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
count, ok := parseStepperCount(tc.stepperName, tc.baseName)
|
|
if ok != tc.wantOK || (ok && count != tc.wantCount) {
|
|
t.Errorf(
|
|
"parseStepperCount(%q, %q) = (%d, %v); want (%d, %v)",
|
|
tc.stepperName, tc.baseName, count, ok, tc.wantCount, tc.wantOK,
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestStepperNameHasBadCount(t *testing.T) {
|
|
cases := []struct {
|
|
stepperName string
|
|
want bool
|
|
}{
|
|
{"delta_0", true},
|
|
{"delta_-1", true},
|
|
{"delta_x", true},
|
|
{"delta_3_4", true},
|
|
{"shift_lag_0", true},
|
|
{"shift_lead_-2", true},
|
|
{"ratio_", true},
|
|
{"shift_", true},
|
|
{"delta", false}, // valid; never reaches the error path anyway
|
|
{"delta_7", false}, // valid; never reaches the error path anyway
|
|
{"foo", false},
|
|
{"foo_0", false},
|
|
{"slwin_x_y", false}, // slwin has its own handling
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
got := stepperNameHasBadCount(tc.stepperName)
|
|
if got != tc.want {
|
|
t.Errorf("stepperNameHasBadCount(%q) = %v; want %v", tc.stepperName, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestStepperInputFromNameWithCounts(t *testing.T) {
|
|
cases := []struct {
|
|
stepperName string
|
|
wantFound bool
|
|
wantBackward int
|
|
wantForward int
|
|
}{
|
|
{"shift_lag", true, 0, 0},
|
|
{"shift_lag_12", true, 0, 0},
|
|
{"shift_lead", true, 0, 1},
|
|
{"shift_lead_4", true, 0, 4},
|
|
{"shift", true, 0, 0},
|
|
{"shift_3", true, 0, 0},
|
|
{"delta", true, 0, 0},
|
|
{"delta_2", true, 0, 0},
|
|
{"ratio_2", true, 0, 0},
|
|
{"slwin_2_3", true, 2, 3},
|
|
{"shift_lag_0", false, 0, 0},
|
|
{"delta_x", false, 0, 0},
|
|
{"nonesuch", false, 0, 0},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
stepperInput := stepperInputFromName(tc.stepperName)
|
|
if tc.wantFound {
|
|
if stepperInput == nil {
|
|
t.Errorf("stepperInputFromName(%q) = nil; want non-nil", tc.stepperName)
|
|
continue
|
|
}
|
|
if stepperInput.numRecordsBackward != tc.wantBackward ||
|
|
stepperInput.numRecordsForward != tc.wantForward {
|
|
t.Errorf(
|
|
"stepperInputFromName(%q) = {backward:%d, forward:%d}; want {backward:%d, forward:%d}",
|
|
tc.stepperName,
|
|
stepperInput.numRecordsBackward, stepperInput.numRecordsForward,
|
|
tc.wantBackward, tc.wantForward,
|
|
)
|
|
}
|
|
} else {
|
|
if stepperInput != nil {
|
|
t.Errorf("stepperInputFromName(%q) = %v; want nil", tc.stepperName, stepperInput)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAllocateStepperOutputFieldNames(t *testing.T) {
|
|
cases := []struct {
|
|
stepperName string
|
|
wantOutputFieldName string
|
|
}{
|
|
{"shift", "x_shift"},
|
|
{"shift_lag", "x_shift_lag"},
|
|
{"shift_lag_12", "x_shift_lag_12"},
|
|
{"shift_lead", "x_shift_lead"},
|
|
{"shift_lead_4", "x_shift_lead_4"},
|
|
{"delta", "x_delta"},
|
|
{"delta_2", "x_delta_2"},
|
|
{"ratio", "x_ratio"},
|
|
{"ratio_2", "x_ratio_2"},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
stepperInput := stepperInputFromName(tc.stepperName)
|
|
if stepperInput == nil {
|
|
t.Errorf("stepperInputFromName(%q) = nil; want non-nil", tc.stepperName)
|
|
continue
|
|
}
|
|
stepper, err := allocateStepper(stepperInput, "x", nil, nil)
|
|
if err != nil {
|
|
t.Errorf("allocateStepper for %q: %v", tc.stepperName, err)
|
|
continue
|
|
}
|
|
if stepper == nil {
|
|
t.Errorf("allocateStepper for %q = nil; want non-nil", tc.stepperName)
|
|
continue
|
|
}
|
|
var got string
|
|
switch s := stepper.(type) {
|
|
case *tStepperShiftLag:
|
|
got = s.outputFieldName
|
|
case *tStepperShiftLead:
|
|
got = s.outputFieldName
|
|
case *tStepperDelta:
|
|
got = s.outputFieldName
|
|
case *tStepperRatio:
|
|
got = s.outputFieldName
|
|
default:
|
|
t.Errorf("allocateStepper for %q: unexpected stepper type %T", tc.stepperName, stepper)
|
|
continue
|
|
}
|
|
if got != tc.wantOutputFieldName {
|
|
t.Errorf(
|
|
"allocateStepper for %q: output field name %q; want %q",
|
|
tc.stepperName, got, tc.wantOutputFieldName,
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestValueRing(t *testing.T) {
|
|
ring := newValueRing(3)
|
|
|
|
// Fewer than 3 values seen: has must be false.
|
|
for i := 1; i <= 3; i++ {
|
|
nBack, has := ring.push(mlrval.FromInt(int64(i)))
|
|
if has {
|
|
t.Errorf("push %d: has = true; want false", i)
|
|
}
|
|
if nBack != nil {
|
|
t.Errorf("push %d: nBack = %v; want nil", i, nBack)
|
|
}
|
|
}
|
|
|
|
// From the 4th value on, the value from 3 records back is returned.
|
|
for i := 4; i <= 8; i++ {
|
|
nBack, has := ring.push(mlrval.FromInt(int64(i)))
|
|
if !has {
|
|
t.Errorf("push %d: has = false; want true", i)
|
|
}
|
|
if nBack == nil {
|
|
t.Errorf("push %d: nBack = nil; want non-nil", i)
|
|
continue
|
|
}
|
|
want := int64(i - 3)
|
|
got, ok := nBack.GetIntValue()
|
|
if !ok || got != want {
|
|
t.Errorf("push %d: nBack = %v; want %d", i, nBack, want)
|
|
}
|
|
}
|
|
|
|
// A nil push (absent field) occupies a slot.
|
|
ring.push(nil) // 9th slot: nil
|
|
ring.push(mlrval.FromInt(10)) // 10th
|
|
ring.push(mlrval.FromInt(11)) // 11th
|
|
nBack, has := ring.push(mlrval.FromInt(12)) // 12th: 3 back is the nil slot
|
|
if !has {
|
|
t.Errorf("push 12: has = false; want true")
|
|
}
|
|
if nBack != nil {
|
|
t.Errorf("push 12: nBack = %v; want nil", nBack)
|
|
}
|
|
}
|