From f8a8f295849c7a8d7db7000fed55e69bf9a96fd8 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Thu, 5 Mar 2026 20:14:32 -0500 Subject: [PATCH] Experiment with `mlr script` (#2009) --- docs/src/manpage.md | 7 +- docs/src/manpage.txt | 7 +- docs/src/reference-dsl-builtin-functions.md | 8 +- docs/src/reference-dsl-differences.md | 2 +- man/manpage.txt | 7 +- man/mlr.1 | 15 +- pkg/dsl/cst/builtin_function_manager.go | 11 +- pkg/dsl/cst/builtin_functions.go | 50 ++++++ pkg/dsl/cst/hofs.go | 3 + pkg/dsl/cst/leaves.go | 4 + pkg/dsl/cst/types.go | 1 + pkg/dsl/cst/validate.go | 20 +-- pkg/runtime/state.go | 7 + pkg/terminals/script/doc.go | 3 + pkg/terminals/script/entry.go | 153 ++++++++++++++++++ pkg/terminals/script/runner.go | 169 ++++++++++++++++++++ pkg/terminals/script/types.go | 37 +++++ pkg/terminals/terminals.go | 2 + test/cases/script/0001/cmd | 1 + test/cases/script/0001/experr | 0 test/cases/script/0001/expout | 1 + test/cases/script/0001/input | 4 + 22 files changed, 490 insertions(+), 22 deletions(-) create mode 100644 pkg/terminals/script/doc.go create mode 100644 pkg/terminals/script/entry.go create mode 100644 pkg/terminals/script/runner.go create mode 100644 pkg/terminals/script/types.go create mode 100644 test/cases/script/0001/cmd create mode 100644 test/cases/script/0001/experr create mode 100644 test/cases/script/0001/expout create mode 100644 test/cases/script/0001/input diff --git a/docs/src/manpage.md b/docs/src/manpage.md index df67c8b8b..6d07bfc1a 100644 --- a/docs/src/manpage.md +++ b/docs/src/manpage.md @@ -246,7 +246,7 @@ This is simply a copy of what you should see on running `man mlr` at a command p is_numeric is_present is_string joink joinkv joinv json_parse json_stringify kurtosis latin1_to_utf8 leafcount leftpad length localtime2gmt localtime2nsec localtime2sec log log10 log1p logifit lstrip madd mapdiff mapexcept mapselect - mapsum max maxlen md5 mean meaneb median mexp min minlen mmul mode msub + mapsum max maxlen md5 mean meaneb median mexp min minlen mmul mode msub next nsec2gmt nsec2gmtdate nsec2localdate nsec2localtime null_count os percentile percentiles pow qnorm reduce regextract regextract_or_else rightpad round roundm rstrip sec2dhms sec2gmt sec2gmtdate sec2hms sec2localdate sec2localtime @@ -2825,6 +2825,9 @@ This is simply a copy of what you should see on running `man mlr` at a command p 1mmsub0m (class=arithmetic #args=3) a - b mod m (integers) + 1mnext0m + (class=system #args=0) In mlr script mode only: reads the next record from input. Returns true if a record was read (and sets $* to it), false at end of stream (and $* becomes false). + 1mnsec2gmt0m (class=time #args=1,2) Formats integer nanoseconds since epoch as GMT timestamp. Leaves non-numbers as-is. With second integer argument n, includes n decimal places for the seconds part. Examples: @@ -3817,5 +3820,5 @@ This is simply a copy of what you should see on running `man mlr` at a command p MIME Type for Comma-Separated Values (CSV) Files, the Miller docsite https://miller.readthedocs.io - 2026-03-03 4mMILLER24m(1) + 2026-03-06 4mMILLER24m(1) diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt index 0fa0c0ce3..1ac8d7917 100644 --- a/docs/src/manpage.txt +++ b/docs/src/manpage.txt @@ -225,7 +225,7 @@ is_numeric is_present is_string joink joinkv joinv json_parse json_stringify kurtosis latin1_to_utf8 leafcount leftpad length localtime2gmt localtime2nsec localtime2sec log log10 log1p logifit lstrip madd mapdiff mapexcept mapselect - mapsum max maxlen md5 mean meaneb median mexp min minlen mmul mode msub + mapsum max maxlen md5 mean meaneb median mexp min minlen mmul mode msub next nsec2gmt nsec2gmtdate nsec2localdate nsec2localtime null_count os percentile percentiles pow qnorm reduce regextract regextract_or_else rightpad round roundm rstrip sec2dhms sec2gmt sec2gmtdate sec2hms sec2localdate sec2localtime @@ -2804,6 +2804,9 @@ 1mmsub0m (class=arithmetic #args=3) a - b mod m (integers) + 1mnext0m + (class=system #args=0) In mlr script mode only: reads the next record from input. Returns true if a record was read (and sets $* to it), false at end of stream (and $* becomes false). + 1mnsec2gmt0m (class=time #args=1,2) Formats integer nanoseconds since epoch as GMT timestamp. Leaves non-numbers as-is. With second integer argument n, includes n decimal places for the seconds part. Examples: @@ -3796,4 +3799,4 @@ MIME Type for Comma-Separated Values (CSV) Files, the Miller docsite https://miller.readthedocs.io - 2026-03-03 4mMILLER24m(1) + 2026-03-06 4mMILLER24m(1) diff --git a/docs/src/reference-dsl-builtin-functions.md b/docs/src/reference-dsl-builtin-functions.md index 880ffb19f..97fc182f6 100644 --- a/docs/src/reference-dsl-builtin-functions.md +++ b/docs/src/reference-dsl-builtin-functions.md @@ -61,7 +61,7 @@ Operators are listed here along with functions. In this case, the argument count * [**Math functions**](#math-functions): [abs](#abs), [acos](#acos), [acosh](#acosh), [asin](#asin), [asinh](#asinh), [atan](#atan), [atan2](#atan2), [atanh](#atanh), [cbrt](#cbrt), [ceil](#ceil), [cos](#cos), [cosh](#cosh), [erf](#erf), [erfc](#erfc), [exp](#exp), [expm1](#expm1), [floor](#floor), [invqnorm](#invqnorm), [log](#log), [log10](#log10), [log1p](#log1p), [logifit](#logifit), [max](#max), [min](#min), [qnorm](#qnorm), [round](#round), [roundm](#roundm), [sgn](#sgn), [sin](#sin), [sinh](#sinh), [sqrt](#sqrt), [tan](#tan), [tanh](#tanh), [urand](#urand), [urand32](#urand32), [urandelement](#urandelement), [urandint](#urandint), [urandrange](#urandrange). * [**Stats functions**](#stats-functions): [antimode](#antimode), [count](#count), [distinct_count](#distinct_count), [kurtosis](#kurtosis), [maxlen](#maxlen), [mean](#mean), [meaneb](#meaneb), [median](#median), [minlen](#minlen), [mode](#mode), [null_count](#null_count), [percentile](#percentile), [percentiles](#percentiles), [skewness](#skewness), [sort_collection](#sort_collection), [stddev](#stddev), [sum](#sum), [sum2](#sum2), [sum3](#sum3), [sum4](#sum4), [variance](#variance). * [**String functions**](#string-functions): [capitalize](#capitalize), [clean_whitespace](#clean_whitespace), [collapse_whitespace](#collapse_whitespace), [contains](#contains), [format](#format), [gssub](#gssub), [gsub](#gsub), [index](#index), [latin1_to_utf8](#latin1_to_utf8), [leftpad](#leftpad), [lstrip](#lstrip), [regextract](#regextract), [regextract_or_else](#regextract_or_else), [rightpad](#rightpad), [rstrip](#rstrip), [ssub](#ssub), [strip](#strip), [strlen](#strlen), [strmatch](#strmatch), [strmatchx](#strmatchx), [sub](#sub), [substr](#substr), [substr0](#substr0), [substr1](#substr1), [tolower](#tolower), [toupper](#toupper), [truncate](#truncate), [unformat](#unformat), [unformatx](#unformatx), [utf8_to_latin1](#utf8_to_latin1), [\.](#dot). -* [**System functions**](#system-functions): [exec](#exec), [hostname](#hostname), [os](#os), [stat](#stat), [system](#system), [version](#version). +* [**System functions**](#system-functions): [exec](#exec), [hostname](#hostname), [next](#next), [os](#os), [stat](#stat), [system](#system), [version](#version). * [**Time functions**](#time-functions): [dhms2fsec](#dhms2fsec), [dhms2sec](#dhms2sec), [fsec2dhms](#fsec2dhms), [fsec2hms](#fsec2hms), [gmt2localtime](#gmt2localtime), [gmt2nsec](#gmt2nsec), [gmt2sec](#gmt2sec), [hms2fsec](#hms2fsec), [hms2sec](#hms2sec), [localtime2gmt](#localtime2gmt), [localtime2nsec](#localtime2nsec), [localtime2sec](#localtime2sec), [nsec2gmt](#nsec2gmt), [nsec2gmtdate](#nsec2gmtdate), [nsec2localdate](#nsec2localdate), [nsec2localtime](#nsec2localtime), [sec2dhms](#sec2dhms), [sec2gmt](#sec2gmt), [sec2gmtdate](#sec2gmtdate), [sec2hms](#sec2hms), [sec2localdate](#sec2localdate), [sec2localtime](#sec2localtime), [strfntime](#strfntime), [strfntime_local](#strfntime_local), [strftime](#strftime), [strftime_local](#strftime_local), [strpntime](#strpntime), [strpntime_local](#strpntime_local), [strptime](#strptime), [strptime_local](#strptime_local), [sysntime](#sysntime), [systime](#systime), [systimeint](#systimeint), [upntime](#upntime), [uptime](#uptime). * [**Typing functions**](#typing-functions): [asserting_absent](#asserting_absent), [asserting_array](#asserting_array), [asserting_bool](#asserting_bool), [asserting_boolean](#asserting_boolean), [asserting_empty](#asserting_empty), [asserting_empty_map](#asserting_empty_map), [asserting_error](#asserting_error), [asserting_float](#asserting_float), [asserting_int](#asserting_int), [asserting_map](#asserting_map), [asserting_nonempty_map](#asserting_nonempty_map), [asserting_not_array](#asserting_not_array), [asserting_not_empty](#asserting_not_empty), [asserting_not_map](#asserting_not_map), [asserting_not_null](#asserting_not_null), [asserting_null](#asserting_null), [asserting_numeric](#asserting_numeric), [asserting_present](#asserting_present), [asserting_string](#asserting_string), [is_absent](#is_absent), [is_array](#is_array), [is_bool](#is_bool), [is_boolean](#is_boolean), [is_empty](#is_empty), [is_empty_map](#is_empty_map), [is_error](#is_error), [is_float](#is_float), [is_int](#is_int), [is_map](#is_map), [is_nan](#is_nan), [is_nonempty_map](#is_nonempty_map), [is_not_array](#is_not_array), [is_not_empty](#is_not_empty), [is_not_map](#is_not_map), [is_not_null](#is_not_null), [is_null](#is_null), [is_numeric](#is_numeric), [is_present](#is_present), [is_string](#is_string), [typeof](#typeof). @@ -1481,6 +1481,12 @@ hostname (class=system #args=0) Returns the hostname as a string. +### next +
+next  (class=system #args=0) In mlr script mode only: reads the next record from input. Returns true if a record was read (and sets $* to it), false at end of stream (and $* becomes false).
+
+ + ### os 
 os  (class=system #args=0) Returns the operating-system name as a string.
diff --git a/docs/src/reference-dsl-differences.md b/docs/src/reference-dsl-differences.md
index 23e0ee17d..b81860836 100644
--- a/docs/src/reference-dsl-differences.md
+++ b/docs/src/reference-dsl-differences.md
@@ -68,7 +68,7 @@ mlr: Parse error on token "$y" at line 6 column 3.
 Please check for missing semicolon.
 Expected one of:
   ␚ ; > >> | ? || ^^ && =~ !=~ == != <=> >= < <= ^ & << >>> + - .+ .- *
-  / // % .* ./ .// . ?? ??? ** [ [[ [[[
+  / // % .* ./ .// . ?? ??? ** [
diff --git a/man/manpage.txt b/man/manpage.txt index 0fa0c0ce3..1ac8d7917 100644 --- a/man/manpage.txt +++ b/man/manpage.txt @@ -225,7 +225,7 @@ is_numeric is_present is_string joink joinkv joinv json_parse json_stringify kurtosis latin1_to_utf8 leafcount leftpad length localtime2gmt localtime2nsec localtime2sec log log10 log1p logifit lstrip madd mapdiff mapexcept mapselect - mapsum max maxlen md5 mean meaneb median mexp min minlen mmul mode msub + mapsum max maxlen md5 mean meaneb median mexp min minlen mmul mode msub next nsec2gmt nsec2gmtdate nsec2localdate nsec2localtime null_count os percentile percentiles pow qnorm reduce regextract regextract_or_else rightpad round roundm rstrip sec2dhms sec2gmt sec2gmtdate sec2hms sec2localdate sec2localtime @@ -2804,6 +2804,9 @@ 1mmsub0m (class=arithmetic #args=3) a - b mod m (integers) + 1mnext0m + (class=system #args=0) In mlr script mode only: reads the next record from input. Returns true if a record was read (and sets $* to it), false at end of stream (and $* becomes false). + 1mnsec2gmt0m (class=time #args=1,2) Formats integer nanoseconds since epoch as GMT timestamp. Leaves non-numbers as-is. With second integer argument n, includes n decimal places for the seconds part. Examples: @@ -3796,4 +3799,4 @@ MIME Type for Comma-Separated Values (CSV) Files, the Miller docsite https://miller.readthedocs.io - 2026-03-03 4mMILLER24m(1) + 2026-03-06 4mMILLER24m(1) diff --git a/man/mlr.1 b/man/mlr.1 index 23bb35be7..cd6fd0c9a 100644 --- a/man/mlr.1 +++ b/man/mlr.1 @@ -2,12 +2,12 @@ .\" Title: mlr .\" Author: [see the "AUTHOR" section] .\" Generator: ./mkman.rb -.\" Date: 2026-03-03 +.\" Date: 2026-03-06 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "MILLER" "1" "2026-03-03" "\ \&" "\ \&" +.TH "MILLER" "1" "2026-03-06" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Portability definitions .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -274,7 +274,7 @@ is_nonempty_map is_not_array is_not_empty is_not_map is_not_null is_null is_numeric is_present is_string joink joinkv joinv json_parse json_stringify kurtosis latin1_to_utf8 leafcount leftpad length localtime2gmt localtime2nsec localtime2sec log log10 log1p logifit lstrip madd mapdiff mapexcept mapselect -mapsum max maxlen md5 mean meaneb median mexp min minlen mmul mode msub +mapsum max maxlen md5 mean meaneb median mexp min minlen mmul mode msub next nsec2gmt nsec2gmtdate nsec2localdate nsec2localtime null_count os percentile percentiles pow qnorm reduce regextract regextract_or_else rightpad round roundm rstrip sec2dhms sec2gmt sec2gmtdate sec2hms sec2localdate sec2localtime @@ -4207,6 +4207,15 @@ mode([3,3,4,4]) is 3 .fi .if n \{\ .RE +.SS "next" +.if n \{\ +.RS 0 +.\} +.nf + (class=system #args=0) In mlr script mode only: reads the next record from input. Returns true if a record was read (and sets $* to it), false at end of stream (and $* becomes false). +.fi +.if n \{\ +.RE .SS "nsec2gmt" .if n \{\ .RS 0 diff --git a/pkg/dsl/cst/builtin_function_manager.go b/pkg/dsl/cst/builtin_function_manager.go index a75fca274..374e87b68 100644 --- a/pkg/dsl/cst/builtin_function_manager.go +++ b/pkg/dsl/cst/builtin_function_manager.go @@ -58,6 +58,7 @@ type BuiltinFunctionInfo struct { variadicFunc bifs.VariadicFunc unaryFuncWithContext bifs.UnaryFuncWithContext // asserting_{typename} regexCaptureBinaryFunc bifs.RegexCaptureBinaryFunc // =~ and !=~ + zaryFuncWithState ZaryFuncWithState // next binaryFuncWithState BinaryFuncWithState // select, apply, reduce ternaryFuncWithState TernaryFuncWithState // fold variadicFuncWithState VariadicFuncWithState // sort @@ -2504,6 +2505,14 @@ Run a command via executable, path, args and environment, yielding its stdout mi help: `Returns the Miller version as a string.`, zaryFunc: bifs.BIF_version, }, + + // next() is for mlr script: reads next record from input, returns true/false. + { + name: "next", + class: FUNC_CLASS_SYSTEM, + help: `In mlr script mode only: reads the next record from input. Returns true if a record was read (and sets $* to it), false at end of stream (and $* becomes false).`, + zaryFuncWithState: BIF_next, + }, } // Sort the function table. Useful for online help and autogenned docs / manpage. @@ -2734,7 +2743,7 @@ func describeNargs(info *BuiltinFunctionInfo) string { return strings.Join(pieces, ",") } - if info.zaryFunc != nil { + if info.zaryFunc != nil || info.zaryFuncWithState != nil { return "0" } if info.unaryFunc != nil { diff --git a/pkg/dsl/cst/builtin_functions.go b/pkg/dsl/cst/builtin_functions.go index fa680a687..064f68de8 100644 --- a/pkg/dsl/cst/builtin_functions.go +++ b/pkg/dsl/cst/builtin_functions.go @@ -13,6 +13,23 @@ import ( "github.com/johnkerl/miller/v6/pkg/runtime" ) +// BIF_next is for mlr script: reads next record from input stream. +func BIF_next(state *runtime.State) *mlrval.Mlrval { + if state.NextRecordFunc == nil { + return mlrval.FromErrorString("next() is only available in mlr script mode") + } + record, context, hasMore := state.NextRecordFunc() + if hasMore { + state.Update(record, context) + state.AtEndOfStream = false + return mlrval.FromBool(true) + } + state.Inrec = nil + state.Context = context + state.AtEndOfStream = true + return mlrval.FromBool(false) +} + func (root *RootNode) BuildBuiltinFunctionCallsiteNode( astNode *dsl.ASTNode, ) (IEvaluable, error) { @@ -29,6 +46,8 @@ func (root *RootNode) BuildBuiltinFunctionCallsiteNode( if builtinFunctionInfo != nil { if builtinFunctionInfo.hasMultipleArities { // E.g. "+" and "-" return root.BuildMultipleArityFunctionCallsiteNode(astNode, builtinFunctionInfo) + } else if builtinFunctionInfo.zaryFuncWithState != nil { + return BuildZaryFunctionWithStateCallsiteNode(astNode, builtinFunctionInfo) } else if builtinFunctionInfo.zaryFunc != nil { return BuildZaryFunctionCallsiteNode(astNode, builtinFunctionInfo) } else if builtinFunctionInfo.unaryFunc != nil { @@ -105,6 +124,37 @@ func BuildZaryFunctionCallsiteNode( }, nil } +func BuildZaryFunctionWithStateCallsiteNode( + astNode *dsl.ASTNode, + builtinFunctionInfo *BuiltinFunctionInfo, +) (IEvaluable, error) { + callsiteArity := len(astNode.Children) + expectedArity := 0 + if callsiteArity != expectedArity { + return nil, fmt.Errorf( + "function %s invoked with %d argument%s; expected %d", + builtinFunctionInfo.name, + callsiteArity, + lib.Plural(callsiteArity), + expectedArity, + ) + } + + return &ZaryFunctionWithStateCallsiteNode{ + zaryFuncWithState: builtinFunctionInfo.zaryFuncWithState, + }, nil +} + +type ZaryFunctionWithStateCallsiteNode struct { + zaryFuncWithState ZaryFuncWithState +} + +func (node *ZaryFunctionWithStateCallsiteNode) Evaluate( + state *runtime.State, +) *mlrval.Mlrval { + return node.zaryFuncWithState(state) +} + func (node *ZaryFunctionCallsiteNode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { diff --git a/pkg/dsl/cst/hofs.go b/pkg/dsl/cst/hofs.go index 2ef089996..22f2ddbf9 100644 --- a/pkg/dsl/cst/hofs.go +++ b/pkg/dsl/cst/hofs.go @@ -29,6 +29,9 @@ type BinaryFuncWithState func( state *runtime.State, ) *mlrval.Mlrval +// ZaryFuncWithState is for next() in mlr script. +type ZaryFuncWithState func(state *runtime.State) *mlrval.Mlrval + // TernaryFuncWithState is for fold. type TernaryFuncWithState func( input1 *mlrval.Mlrval, diff --git a/pkg/dsl/cst/leaves.go b/pkg/dsl/cst/leaves.go index da32583ed..692afcd08 100644 --- a/pkg/dsl/cst/leaves.go +++ b/pkg/dsl/cst/leaves.go @@ -122,6 +122,10 @@ func (node *FullSrecRvalueNode) Evaluate( // print inrec attributes. Also, a UDF/UDS invoked from begin/end could try // to access the inrec, and that would get past the validator. if state.Inrec == nil { + // In mlr script, after next() returns false, $* is boolean false. + if state.AtEndOfStream { + return mlrval.FromBool(false) + } return mlrval.ABSENT.StrictModeCheck(state.StrictMode, "$*") } return mlrval.FromMap(state.Inrec) diff --git a/pkg/dsl/cst/types.go b/pkg/dsl/cst/types.go index 251d8610e..209b15fd6 100644 --- a/pkg/dsl/cst/types.go +++ b/pkg/dsl/cst/types.go @@ -28,6 +28,7 @@ const ( DSLInstanceTypePut = iota DSLInstanceTypeFilter DSLInstanceTypeREPL + DSLInstanceTypeScript ) // Please see root.go for context and comments. diff --git a/pkg/dsl/cst/validate.go b/pkg/dsl/cst/validate.go index bc5c5442d..185f30331 100644 --- a/pkg/dsl/cst/validate.go +++ b/pkg/dsl/cst/validate.go @@ -298,15 +298,15 @@ func validateForLoopMultivariableUniqueNames(astNode *dsl.ASTNode) error { } var VALID_LHS_NODE_TYPES = map[dsl.TNodeType]bool{ - dsl.NodeTypeArrayOrMapIndexAccess: true, - dsl.NodeTypeDotOperator: true, + dsl.NodeTypeArrayOrMapIndexAccess: true, + dsl.NodeTypeDotOperator: true, dsl.NodeTypeArraySliceAccess: true, - dsl.NodeTypeDirectFieldValue: true, - dsl.NodeTypeDirectOosvarValue: true, - dsl.NodeTypeEnvironmentVariable: true, - dsl.NodeTypeFullOosvar: true, - dsl.NodeTypeFullSrec: true, - dsl.NodeTypeIndirectFieldValue: true, // includes $[[n]] and $[[[n]]] - dsl.NodeTypeIndirectOosvarValue: true, - dsl.NodeTypeLocalVariable: true, + dsl.NodeTypeDirectFieldValue: true, + dsl.NodeTypeDirectOosvarValue: true, + dsl.NodeTypeEnvironmentVariable: true, + dsl.NodeTypeFullOosvar: true, + dsl.NodeTypeFullSrec: true, + dsl.NodeTypeIndirectFieldValue: true, // includes $[[n]] and $[[[n]]] + dsl.NodeTypeIndirectOosvarValue: true, + dsl.NodeTypeLocalVariable: true, } diff --git a/pkg/runtime/state.go b/pkg/runtime/state.go index 82fbf5339..1681f2595 100644 --- a/pkg/runtime/state.go +++ b/pkg/runtime/state.go @@ -43,6 +43,13 @@ type State struct { // NoExitOnFunctionNotFound is used by the REPL: when true, a call to an // undefined function returns an error mlrval instead of exiting the process. NoExitOnFunctionNotFound bool + + // NextRecordFunc is set by mlr script. When called, reads next record from input. + // Returns (record, context, hasMore). When hasMore is false, record and context are for end-of-stream. + NextRecordFunc func() (*mlrval.Mlrmap, *types.Context, bool) + + // AtEndOfStream is set when next() returns false. Makes $* evaluate to boolean false. + AtEndOfStream bool } func NewEmptyState(options *cli.TOptions, strictMode bool) *State { diff --git a/pkg/terminals/script/doc.go b/pkg/terminals/script/doc.go new file mode 100644 index 000000000..fbe02fb04 --- /dev/null +++ b/pkg/terminals/script/doc.go @@ -0,0 +1,3 @@ +// Package script provides the mlr script terminal: non-interactive execution +// of Miller DSL scripts with next()-driven record iteration. +package script diff --git a/pkg/terminals/script/entry.go b/pkg/terminals/script/entry.go new file mode 100644 index 000000000..aa2fe39c1 --- /dev/null +++ b/pkg/terminals/script/entry.go @@ -0,0 +1,153 @@ +// Entry point for mlr script: executes a Miller DSL script with next()-driven +// record iteration. No prompt, no REPL. Script from -f or -e, data from +// filenames or stdin. +// +// Example: +// +// mlr script -f sum.mlr data.csv +// mlr script -e '@sum=0; while(next()){@sum+=$x}; print @sum' --csv data.csv + +package script + +import ( + "fmt" + "os" + "path" + "strings" + + "github.com/johnkerl/miller/v6/pkg/cli" + "github.com/johnkerl/miller/v6/pkg/lib" +) + +func scriptUsage(verbName string, o *os.File, exitCode int) { + exeName := path.Base(os.Args[0]) + fmt.Fprintf(o, "Usage: %s %s [options] -f {script file} | -e {expression} [zero or more data-file names]\n", exeName, verbName) + fmt.Fprint(o, + `-f {file} Load DSL script from file. If file is a directory, load all *.mlr files. +-e {expression} DSL script from command line. May be combined with -f. + +--load {file} Preload DSL before script (e.g. function library). +--mload {files} Like --load but multiple files. Use -- to terminate list. + +-w Show warnings about uninitialized variables. +-z Strict mode. + +-h|--help Show this message. + +Or any --icsv, --ojson, etc. reader/writer options. + +Data-file names (or stdin if none) are the record input stream. +`) + os.Exit(exitCode) +} + +func ScriptMain(args []string) int { + scriptName := args[0] + argc := len(args) + argi := 1 + + doWarnings := false + strictMode := false + options := cli.DefaultOptions() + var dslStrings []string + haveDSLStrings := false + + for argi < argc { + if !strings.HasPrefix(args[argi], "-") { + break + } + + if args[argi] == "-h" || args[argi] == "--help" { + scriptUsage(scriptName, os.Stdout, 0) + } else if args[argi] == "-w" { + doWarnings = true + argi++ + } else if args[argi] == "-z" { + strictMode = true + argi++ + } else if args[argi] == "-f" { + if argc-argi < 2 { + scriptUsage(scriptName, os.Stderr, 1) + } + argi++ + filename, err := cli.VerbGetStringArg("script", "-f", args, &argi, argc) + if err != nil { + fmt.Fprintf(os.Stderr, "mlr: %v\n", err) + os.Exit(1) + } + theseStrings, err := lib.LoadStringsFromFileOrDir(filename, ".mlr") + if err != nil { + fmt.Fprintf(os.Stderr, "mlr script: cannot load script from \"%s\": %v\n", filename, err) + os.Exit(1) + } + dslStrings = append(dslStrings, theseStrings...) + haveDSLStrings = true + } else if args[argi] == "-e" { + if argc-argi < 2 { + scriptUsage(scriptName, os.Stderr, 1) + } + argi++ + expr, err := cli.VerbGetStringArg("script", "-e", args, &argi, argc) + if err != nil { + fmt.Fprintf(os.Stderr, "mlr: %v\n", err) + os.Exit(1) + } + dslStrings = append(dslStrings, expr) + haveDSLStrings = true + } else if args[argi] == "--load" { + if argc-argi < 2 { + scriptUsage(scriptName, os.Stderr, 1) + } + options.DSLPreloadFileNames = append(options.DSLPreloadFileNames, args[argi+1]) + argi += 2 + } else if args[argi] == "--mload" { + if argc-argi < 2 { + scriptUsage(scriptName, os.Stderr, 1) + } + argi++ + for argi < argc && args[argi] != "--" { + options.DSLPreloadFileNames = append(options.DSLPreloadFileNames, args[argi]) + argi++ + } + if argi < argc && args[argi] == "--" { + argi++ + } + } else if cli.FLAG_TABLE.Parse(args, argc, &argi, options) { + } else { + scriptUsage(scriptName, os.Stderr, 1) + } + } + + if !haveDSLStrings { + fmt.Fprintf(os.Stderr, "mlr script: -f or -e is required\n") + scriptUsage(scriptName, os.Stderr, 1) + } + + cli.FinalizeReaderOptions(&options.ReaderOptions) + cli.FinalizeWriterOptions(&options.WriterOptions) + options.WriterOptions.AutoFlatten = cli.DecideFinalFlatten(&options.WriterOptions) + options.WriterOptions.AutoUnflatten = cli.DecideFinalUnflatten(options, [][]string{}) + + filenames := args[argi:] + + scr, err := NewScript(options, doWarnings, strictMode, dslStrings) + if err != nil { + fmt.Fprintf(os.Stderr, "mlr script: %v\n", err) + os.Exit(1) + } + + scr.openFiles(filenames) + err = scr.run() + if err != nil { + fmt.Fprintf(os.Stderr, "mlr script: %v\n", err) + os.Exit(1) + } + + scr.bufferedRecordOutputStream.Flush() + err = scr.closeBufferedOutputStream() + if err != nil { + fmt.Fprintf(os.Stderr, "mlr script: %v\n", err) + os.Exit(1) + } + return 0 +} diff --git a/pkg/terminals/script/runner.go b/pkg/terminals/script/runner.go new file mode 100644 index 000000000..6c8d2e919 --- /dev/null +++ b/pkg/terminals/script/runner.go @@ -0,0 +1,169 @@ +package script + +import ( + "bufio" + "fmt" + "os" + + "github.com/johnkerl/miller/v6/pkg/cli" + "github.com/johnkerl/miller/v6/pkg/dsl/cst" + "github.com/johnkerl/miller/v6/pkg/input" + "github.com/johnkerl/miller/v6/pkg/lib" + "github.com/johnkerl/miller/v6/pkg/mlrval" + "github.com/johnkerl/miller/v6/pkg/output" + "github.com/johnkerl/miller/v6/pkg/runtime" + "github.com/johnkerl/miller/v6/pkg/types" +) + +func NewScript( + options *cli.TOptions, + doWarnings bool, + strictMode bool, + dslStrings []string, +) (*Script, error) { + recordReader, err := input.Create(&options.ReaderOptions, 1) + if err != nil { + return nil, err + } + + recordWriter, err := output.Create(&options.WriterOptions) + if err != nil { + return nil, err + } + + context := types.NewContext() + runtimeState := runtime.NewEmptyState(options, strictMode) + runtimeState.NoExitOnFunctionNotFound = true + runtimeState.Update(nil, context) + runtimeState.FilterExpression = mlrval.NULL + + cstRootNode := cst.NewEmptyRoot( + &options.WriterOptions, cst.DSLInstanceTypeScript, + ).WithRedefinableUDFUDS().WithStrictMode(strictMode) + + // Collect all DSL: preloads first, then main script + allDSLStrings := []string{} + for _, filename := range options.DSLPreloadFileNames { + theseStrings, err := lib.LoadStringsFromFileOrDir(filename, ".mlr") + if err != nil { + return nil, fmt.Errorf("cannot load from \"%s\": %w", filename, err) + } + allDSLStrings = append(allDSLStrings, theseStrings...) + } + allDSLStrings = append(allDSLStrings, dslStrings...) + + scr := &Script{ + name: "script", + doWarnings: doWarnings, + cstRootNode: cstRootNode, + options: options, + recordReader: recordReader, + recordWriter: recordWriter, + runtimeState: runtimeState, + } + + scr.recordOutputFileName = "(stdout)" + scr.recordOutputStream = os.Stdout + scr.bufferedRecordOutputStream = bufio.NewWriter(os.Stdout) + + _, err = cstRootNode.Build( + allDSLStrings, + cst.DSLInstanceTypeScript, + false, + doWarnings, + nil, + ) + if err != nil { + return nil, err + } + + return scr, nil +} + +func (scr *Script) openFiles(filenames []string) { + scr.options.FileNames = filenames + scr.readerChannel = make(chan []*types.RecordAndContext, 2) + scr.errorChannel = make(chan error, 1) + scr.downstreamDoneChannel = make(chan bool, 1) + + go scr.recordReader.Read( + filenames, + *scr.runtimeState.Context, + scr.readerChannel, + scr.errorChannel, + scr.downstreamDoneChannel, + ) +} + +func (scr *Script) run() error { + // Wire NextRecordFunc before running + readerChannel := scr.readerChannel + errorChannel := scr.errorChannel + bufferedOutput := scr.bufferedRecordOutputStream + + scr.runtimeState.NextRecordFunc = func() (*mlrval.Mlrmap, *types.Context, bool) { + for { + var recordsAndContexts []*types.RecordAndContext + var err error + select { + case recordsAndContexts = <-readerChannel: + case err = <-errorChannel: + } + + if err != nil { + return nil, scr.runtimeState.Context, false + } + + if recordsAndContexts == nil { + return nil, scr.runtimeState.Context, false + } + + lib.InternalCodingErrorIf(len(recordsAndContexts) != 1) + rac := recordsAndContexts[0] + + if rac.EndOfStream { + return nil, &rac.Context, false + } + + if rac.Record == nil { + // Output string from print/dump etc + bufferedOutput.WriteString(rac.OutputString) + bufferedOutput.Flush() + continue + } + + // Actual record + return rac.Record, &rac.Context, true + } + } + + // Execute begin blocks + err := scr.cstRootNode.ExecuteBeginBlocks(scr.runtimeState) + if err != nil { + return err + } + + // Execute main block (script drives itself via next()) + _, err = scr.cstRootNode.ExecuteMainBlock(scr.runtimeState) + if err != nil { + return err + } + + // Execute end blocks + err = scr.cstRootNode.ExecuteEndBlocks(scr.runtimeState) + if err != nil { + return err + } + + return nil +} + +func (scr *Script) closeBufferedOutputStream() error { + if scr.recordOutputStream != os.Stdout { + err := scr.recordOutputStream.Close() + if err != nil { + return fmt.Errorf("error on close: %w", err) + } + } + return nil +} diff --git a/pkg/terminals/script/types.go b/pkg/terminals/script/types.go new file mode 100644 index 000000000..89a255f98 --- /dev/null +++ b/pkg/terminals/script/types.go @@ -0,0 +1,37 @@ +// Data types for the Miller script terminal. + +package script + +import ( + "bufio" + "os" + + "github.com/johnkerl/miller/v6/pkg/cli" + "github.com/johnkerl/miller/v6/pkg/dsl/cst" + "github.com/johnkerl/miller/v6/pkg/input" + "github.com/johnkerl/miller/v6/pkg/output" + "github.com/johnkerl/miller/v6/pkg/runtime" + "github.com/johnkerl/miller/v6/pkg/types" +) + +type Script struct { + exeName string + name string + + doWarnings bool + cstRootNode *cst.RootNode + + options *cli.TOptions + + readerChannel chan []*types.RecordAndContext + errorChannel chan error + downstreamDoneChannel chan bool + recordReader input.IRecordReader + recordWriter output.IRecordWriter + + recordOutputFileName string + recordOutputStream *os.File + bufferedRecordOutputStream *bufio.Writer + + runtimeState *runtime.State +} diff --git a/pkg/terminals/terminals.go b/pkg/terminals/terminals.go index d88dcf709..596bc81c2 100644 --- a/pkg/terminals/terminals.go +++ b/pkg/terminals/terminals.go @@ -11,6 +11,7 @@ import ( "github.com/johnkerl/miller/v6/pkg/terminals/help" "github.com/johnkerl/miller/v6/pkg/terminals/regtest" "github.com/johnkerl/miller/v6/pkg/terminals/repl" + "github.com/johnkerl/miller/v6/pkg/terminals/script" "github.com/johnkerl/miller/v6/pkg/version" ) @@ -34,6 +35,7 @@ func init() { {"help", help.HelpMain}, {"regtest", regtest.RegTestMain}, {"repl", repl.ReplMain}, + {"script", script.ScriptMain}, {"version", showVersion}, } } diff --git a/test/cases/script/0001/cmd b/test/cases/script/0001/cmd new file mode 100644 index 000000000..8902f1593 --- /dev/null +++ b/test/cases/script/0001/cmd @@ -0,0 +1 @@ +mlr script --csv -e '@sum=0; while(next()){@sum+=$x}; print @sum' ./${CASEDIR}/input diff --git a/test/cases/script/0001/experr b/test/cases/script/0001/experr new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/script/0001/expout b/test/cases/script/0001/expout new file mode 100644 index 000000000..abdfb053e --- /dev/null +++ b/test/cases/script/0001/expout @@ -0,0 +1 @@ +60 diff --git a/test/cases/script/0001/input b/test/cases/script/0001/input new file mode 100644 index 000000000..62dc06f75 --- /dev/null +++ b/test/cases/script/0001/input @@ -0,0 +1,4 @@ +a,b,x +1,2,10 +3,4,20 +5,6,30