From 844fb23e5eff3161c903981f9c19ebb0a0866792 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 1 Feb 2021 22:48:10 -0500 Subject: [PATCH] AST-printing and function redefines in the REPL --- go/src/miller/auxents/repl.go | 157 ++++++++++++++++++++++++++++----- go/src/miller/dsl/cst/print.go | 2 +- go/src/miller/dsl/cst/root.go | 9 ++ go/src/miller/dsl/cst/types.go | 1 + go/src/miller/dsl/cst/udf.go | 16 ++-- go/src/miller/dsl/cst/uds.go | 16 ++-- go/todo.txt | 20 ++--- go/u/mand.mlr | 1 - 8 files changed, 172 insertions(+), 50 deletions(-) diff --git a/go/src/miller/auxents/repl.go b/go/src/miller/auxents/repl.go index 57badaf0c..ee897bae2 100644 --- a/go/src/miller/auxents/repl.go +++ b/go/src/miller/auxents/repl.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "io/ioutil" "os" "strings" @@ -25,6 +26,7 @@ import ( "miller/types" ) +// ================================================================ func replUsage(verbName string, o *os.File, exitCode int) { fmt.Fprintf(o, "Usage: %s %s with no arguments\n", mlrExeName(), verbName) os.Exit(exitCode) @@ -40,6 +42,16 @@ func replMain(args []string) int { return 0 } +// ================================================================ +type ASTPrintMode int + +const ( + ASTPrintNone ASTPrintMode = iota + ASTPrintParex + ASTPrintParexOneLine + ASTPrintIndent +) + // ================================================================ type Repl struct { inputIsTerminal bool @@ -47,9 +59,9 @@ type Repl struct { prompt2 string doingMultilineInput bool - verboseASTParse bool - isFilter bool - cstRootNode *cst.RootNode + astPrintMode ASTPrintMode + isFilter bool + cstRootNode *cst.RootNode options cliutil.TOptions context *types.Context @@ -89,9 +101,9 @@ func NewRepl() (*Repl, error) { prompt2: prompt2, doingMultilineInput: doingMultilineInput, - verboseASTParse: false, - isFilter: false, - cstRootNode: cst.NewEmptyRoot(&options.WriterOptions), + astPrintMode: ASTPrintNone, + isFilter: false, + cstRootNode: cst.NewEmptyRoot(&options.WriterOptions).WithRedefinableUDFS(), options: options, context: context, @@ -172,21 +184,122 @@ func (this *Repl) handleNonDSLLine(trimmedLine string) bool { // Make a lookup-table maybe if verb == ":help" || verb == "?" || verb == "help" { this.handleHelp(args) - return true + } else if verb == ":astprint" { + this.handleASTPrint(args) + } else if verb == ":load" { + this.handleLoad(args) + } else if verb == ":begin" { + err := this.cstRootNode.ExecuteBeginBlocks(this.runtimeState) + if err != nil { + fmt.Fprintln(os.Stderr, err) + } + } else if verb == ":end" { + err := this.cstRootNode.ExecuteEndBlocks(this.runtimeState) + if err != nil { + fmt.Fprintln(os.Stderr, err) + } + } else { + fmt.Printf("Unrecognized command:%s\n", verb) } - - fmt.Printf("Unrecognized command:%s\n", verb) return true } -// ---------------------------------------------------------------- func (this *Repl) handleHelp(args []string) { args = args[1:] // strip off verb if len(args) == 0 { - cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionUsages(os.Stdout) + fmt.Println("Options:") + fmt.Println(":help repl") + fmt.Println(":help functions") + fmt.Println(":help {function name}, e.g. :help sec2gmt") } else { for _, arg := range args { - cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionUsage(arg, os.Stdout) + if arg == "repl" { + this.showREPLHelp() + } else if arg == "functions" { + cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionUsages(os.Stdout) + } else { + cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionUsage(arg, os.Stdout) + } + } + } +} + +// TODO: make this more like ':help repl explain' or somesuch +func (this *Repl) showREPLHelp() { + fmt.Println( + `Enter any Miller DSL expression. +Non-DSL commands (REPL-only statements) start with ':', such as ':help' or ':quit'. +Type ':help functions' for help with DSL functions; type ':help repl' for help with non-DSL expressions. + +The input "record" by default is the empty map but you can do things like '$x=3', +or 'unset $y', or '$* = {"x": 3, "y": 4}' to populate it. + +Enter '<' on a line by itself to enter multi-line mode, e.g. to enter a function definition; +enter '>' on a line by itself to exit multi-line mode. + +In multi-line mode, semicolons are required between statements; otherwise they are not needed. +Non-assignment expressions, such as '7' or 'true', in mlr put are filter statements; here, they +are simply printed to the terminal, e.g. if you type '1+2', you will see '3'. + +Examples, assuming the prompt is 'mlr: ' + +mlr: 1+2 +3 +mlr: x=3 +mlr: y=4 +mlr: x+y +7 +mlr: < +func f(a,b) { + return a**b +} +> +mlr: f(7,5) +16807 +`) +} + +func (this *Repl) handleASTPrint(args []string) { + args = args[1:] // strip off verb + if len(args) != 1 { + fmt.Println("Need argument: see ':help :astprint'.") + return + } + style := args[0] + if style == "parex" { + this.astPrintMode = ASTPrintParex + } else if style == "parex1" { + this.astPrintMode = ASTPrintParexOneLine + } else if style == "indent" { + this.astPrintMode = ASTPrintIndent + } else if style == "none" { + this.astPrintMode = ASTPrintNone + } else { + fmt.Printf("Unrecognized style %s: see ':help :astprint'.\n", style) + return + } +} + +func (this *Repl) handleLoad(args []string) { + args = args[1:] // strip off verb + if len(args) < 1 { + fmt.Println("Need filenames: see ':help :load'.") + return + } + for _, filename := range args { + + dslBytes, err := ioutil.ReadFile(filename) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot load DSL expression from file \"%s\": ", + filename) + fmt.Println(err) + return + } + dslString := string(dslBytes) + + err = this.HandleDSLString(dslString) + if err != nil { + fmt.Fprintln(os.Stderr, err) } } } @@ -194,7 +307,7 @@ func (this *Repl) handleHelp(args []string) { // ---------------------------------------------------------------- func (this *Repl) HandleDSLString(dslString string) error { - astRootNode, err := this.BuildASTFromStringWithMessage(dslString, this.verboseASTParse) + astRootNode, err := this.BuildASTFromStringWithMessage(dslString) if err != nil { // Error message already printed out return err @@ -230,7 +343,6 @@ func (this *Repl) HandleDSLString(dslString string) error { // ---------------------------------------------------------------- func (this *Repl) BuildASTFromStringWithMessage( dslString string, - verbose bool, ) (*dsl.AST, error) { astRootNode, err := this.BuildASTFromString(dslString) if err != nil { @@ -239,19 +351,18 @@ func (this *Repl) BuildASTFromStringWithMessage( // fmt.Fprintln(os.Stderr, err) fmt.Fprintf(os.Stderr, "%s: cannot parse DSL expression.\n", lib.MlrExeName()) - if verbose { + if this.astPrintMode != ASTPrintNone { fmt.Fprintln(os.Stderr, dslString) } return nil, err } else { - - // if printASTSingleLine { - // astRootNode.PrintParexOneLine() - // } else if xxx { - // astRootNode.PrintParex() - // } else { - // astRootNode.Print() - // } + if this.astPrintMode == ASTPrintParex { + astRootNode.PrintParex() + } else if this.astPrintMode == ASTPrintParexOneLine { + astRootNode.PrintParexOneLine() + } else if this.astPrintMode == ASTPrintIndent { + astRootNode.Print() + } return astRootNode, nil } diff --git a/go/src/miller/dsl/cst/print.go b/go/src/miller/dsl/cst/print.go index 09823e01e..913a0da1f 100644 --- a/go/src/miller/dsl/cst/print.go +++ b/go/src/miller/dsl/cst/print.go @@ -341,7 +341,7 @@ func (this *PrintStatementNode) printToStdout( if state.OutputChannel != nil { state.OutputChannel <- types.NewOutputString(outputString, state.Context) } else { - fmt.Println(outputString) + fmt.Print(outputString) } return nil diff --git a/go/src/miller/dsl/cst/root.go b/go/src/miller/dsl/cst/root.go index 1f526a6b3..5ac816ece 100644 --- a/go/src/miller/dsl/cst/root.go +++ b/go/src/miller/dsl/cst/root.go @@ -29,6 +29,7 @@ func NewEmptyRoot( endBlocks: make([]*StatementBlockNode, 0), udfManager: NewUDFManager(), udsManager: NewUDSManager(), + allowUDFSRedefinitions: false, unresolvedFunctionCallsites: list.New(), unresolvedSubroutineCallsites: list.New(), outputHandlerManagers: list.New(), @@ -36,6 +37,14 @@ func NewEmptyRoot( } } +// Nominally for mlr put/filter we want to flag overwritten UDFs/UDSs as an +// error. But in the REPL, which is interactive, people should be able to +// redefine. This method allows the latter use-case. +func (this *RootNode) WithRedefinableUDFS() *RootNode { + this.allowUDFSRedefinitions = true + return this +} + // ---------------------------------------------------------------- func (this *RootNode) Build( ast *dsl.AST, diff --git a/go/src/miller/dsl/cst/types.go b/go/src/miller/dsl/cst/types.go index e891b19e5..31698e0c0 100644 --- a/go/src/miller/dsl/cst/types.go +++ b/go/src/miller/dsl/cst/types.go @@ -21,6 +21,7 @@ type RootNode struct { endBlocks []*StatementBlockNode udfManager *UDFManager udsManager *UDSManager + allowUDFSRedefinitions bool unresolvedFunctionCallsites *list.List unresolvedSubroutineCallsites *list.List outputHandlerManagers *list.List diff --git a/go/src/miller/dsl/cst/udf.go b/go/src/miller/dsl/cst/udf.go index 737e73926..9e43f7f1d 100644 --- a/go/src/miller/dsl/cst/udf.go +++ b/go/src/miller/dsl/cst/udf.go @@ -267,13 +267,15 @@ func (this *RootNode) BuildAndInstallUDF(astNode *dsl.ASTNode) error { ) } - if this.udfManager.ExistsByName(functionName) { - return errors.New( - fmt.Sprintf( - "Miller: function named \"%s\" has already been defined.", - functionName, - ), - ) + if !this.allowUDFSRedefinitions { + if this.udfManager.ExistsByName(functionName) { + return errors.New( + fmt.Sprintf( + "Miller: function named \"%s\" has already been defined.", + functionName, + ), + ) + } } parameterListASTNode := astNode.Children[0] diff --git a/go/src/miller/dsl/cst/uds.go b/go/src/miller/dsl/cst/uds.go index fbb618817..47a1e9c91 100644 --- a/go/src/miller/dsl/cst/uds.go +++ b/go/src/miller/dsl/cst/uds.go @@ -237,13 +237,15 @@ func (this *RootNode) BuildAndInstallUDS(astNode *dsl.ASTNode) error { subroutineName := string(astNode.Token.Lit) - if this.udsManager.ExistsByName(subroutineName) { - return errors.New( - fmt.Sprintf( - "Miller: subroutine named \"%s\" has already been defined.", - subroutineName, - ), - ) + if !this.allowUDFSRedefinitions { + if this.udsManager.ExistsByName(subroutineName) { + return errors.New( + fmt.Sprintf( + "Miller: subroutine named \"%s\" has already been defined.", + subroutineName, + ), + ) + } } parameterListASTNode := astNode.Children[0] diff --git a/go/todo.txt b/go/todo.txt index 9656100d6..d587e624c 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -19,27 +19,25 @@ TOP OF LIST: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - * repl foo: - o MLR_REPL_PS1/MLR_REPL_PS2 env vars + d doc MLR_REPL_PS1/MLR_REPL_PS2 env vars + d :load {source files} + d :begin and :end ... + + o func-handle lookup tables as usual ... o UTs - o take filenames & reader/writer options from CLI -- ? - o :some flag settings ... ? :--o json -- ? - - especially toggle parex/etc - - o :help -> :help shortlist - - :help functions - - :help repl - - o :load source file o :read data file o :open file (or stdin?) - o :next .... + o :next o :close o :continue o :break conditions -- ? o ^C -- ?!? + o take filenames & reader/writer options from CLI -- ? + o :some flag settings ... ? :--o json -- ? + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/go/u/mand.mlr b/go/u/mand.mlr index 36947080a..d2406da57 100644 --- a/go/u/mand.mlr +++ b/go/u/mand.mlr @@ -9,7 +9,6 @@ begin { @maxits = 100; @levelstep = 5; @chars = "@X*o-."; - } # Override defaults