From 3dfaf9bb92fe79cc598768560fd686e564a5bd72 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Fri, 28 Aug 2020 10:53:01 -0400 Subject: [PATCH] AST iterate --- go/src/miller/dsl/ast.go | 90 ++++++++----------- go/src/miller/dsl/ast_test.go | 19 ++-- go/src/miller/parsing/mlr.bnf | 33 +++---- go/src/miller/parsing/parser/actiontable.go | 64 +++++++++---- go/src/miller/parsing/parser/gototable.go | 75 +++++++++++++--- go/src/miller/parsing/parser/parser.go | 6 +- .../miller/parsing/parser/productionstable.go | 64 +++++++++---- 7 files changed, 225 insertions(+), 126 deletions(-) diff --git a/go/src/miller/dsl/ast.go b/go/src/miller/dsl/ast.go index 5bba2601d..dd970cfb0 100644 --- a/go/src/miller/dsl/ast.go +++ b/go/src/miller/dsl/ast.go @@ -5,85 +5,65 @@ import ( ) // ---------------------------------------------------------------- -// AST type: -// root *Node - -// Node type: -// * text string -// * type enum -// * children []Node +// xxx comment interface{} everywhere vs. true types due to gocc polymorphism API. +// and, line-count for casts here vs in the BNF: +// +// Statement : +// md_token_field_name md_token_assign md_token_number +// +// Statement : +// md_token_field_name md_token_assign md_token_number +// << dsl.NewASTNodeTernary("foo", $0, $1, $2) >> ; // ---------------------------------------------------------------- type AST struct { Root *ASTNode } -func NewAST(root *ASTNode) *AST { +func NewAST(root interface{}) (*AST, error) { return &AST { - root, - } + root.(*ASTNode), + }, nil } // ---------------------------------------------------------------- type ASTNode struct { - Text string + Token token.Token // Type enum + // There need to be tokenless nodes + // text string Children []*ASTNode } -func NewASTNode(text string) *ASTNode { +func NewASTNode(itok interface{}) (*ASTNode, error) { + tok := itok.(*token.Token) return &ASTNode { - text, + *tok, // type nil, - } + }, nil } -func NewASTNodeZary(text string) *ASTNode { +func MakeZary(iparent interface{}) (*ASTNode, error) { + parent := iparent.(*ASTNode) children := make([]*ASTNode, 0) - return &ASTNode { - text, - // type - children, - } + parent.Children = children + return parent, nil } -func NewASTNodeUnary(text string, childA *ASTNode) *ASTNode { +func MakeUnary(iparent interface{}, childA interface{}) (*ASTNode, error) { + parent := iparent.(*ASTNode) children := make([]*ASTNode, 1) - children[0] = childA - return &ASTNode { - text, - // type - children, - } + children[0] = childA.(*ASTNode) + parent.Children = children + return parent, nil } -func NewASTNodeBinary(text string, childA *ASTNode, childB *ASTNode) *ASTNode { - children := make([]*ASTNode, 1) - children[0] = childA - children[1] = childB - return &ASTNode { - text, - // type - children, - } -} - -// ---------------------------------------------------------------- -// prototype stuff from gocc example -type ( - StatementList []Statement - Statement string -) - -func NewStatementList(statement interface{}) (StatementList, error) { - return StatementList{statement.(Statement)}, nil -} - -func AppendStatement(statementList, statement interface{}) (StatementList, error) { - return append(statementList.(StatementList), statement.(Statement)), nil -} - -func NewStatement(statementList interface{}) (Statement, error) { - return Statement(statementList.(*token.Token).Lit), nil +func MakeBinary(iparent interface{}, childA interface{}, childB interface{}) (*ASTNode, error) { + parent := iparent.(*ASTNode) + children := make([]*ASTNode, 2) + children[0] = childA.(*ASTNode) + children[1] = childB.(*ASTNode) + parent.Children = children + return parent, nil } diff --git a/go/src/miller/dsl/ast_test.go b/go/src/miller/dsl/ast_test.go index 54df1cf79..c529e6949 100644 --- a/go/src/miller/dsl/ast_test.go +++ b/go/src/miller/dsl/ast_test.go @@ -9,23 +9,24 @@ import ( "miller/parsing/parser" ) -func testOne(src []byte) (astree dsl.StatementList, err error) { - fmt.Printf("Input: %s\n", src) - s := lexer.NewLexer(src) - p := parser.NewParser() - a, err := p.Parse(s) +func testOne(sourceString []byte) (*dsl.AST, error) { + fmt.Printf("Input: %s\n", sourceString) + theLexer := lexer.NewLexer(sourceString) + theParser := parser.NewParser() + interfaceAST, err := theParser.Parse(theLexer) if err == nil { - astree = a.(dsl.StatementList) + return interfaceAST.(*dsl.AST), nil + } else { + return nil, err } - return } func TestPass(t *testing.T) { - sml, err := testOne([]byte("$x = 3")) + ast, err := testOne([]byte("$x = 3")) if err != nil { t.Fatal(err.Error()) } - fmt.Printf("Output: %s\n", sml) + fmt.Printf("Output: %v\n", ast) } func TestFail(t *testing.T) { diff --git a/go/src/miller/parsing/mlr.bnf b/go/src/miller/parsing/mlr.bnf index 10a801380..7623e40a8 100644 --- a/go/src/miller/parsing/mlr.bnf +++ b/go/src/miller/parsing/mlr.bnf @@ -628,19 +628,10 @@ md_token_number : _digit { _digit } | '-' _digit { _digit } ; << import "miller/dsl" >> -StatementList : - Statement << dsl.NewStatementList($0) >> - | StatementList Statement << dsl.AppendStatement($0, $1) >> -; - -Statement : - md_token_field_name md_token_assign md_token_number << dsl.NewStatement($0) >> -; - //// The caller is expected to provide more context. //%syntax_error { // fprintf(stderr, "mlr DSL: syntax error at \"%s\"\n", TOKEN->text); - +// //// This is confusing and (as is) worse than nothing. //// Ideally we want to show the position within the input of the syntax error. //// @@ -651,13 +642,25 @@ Statement : //// fprintf(stderr, "Possible token \"%s\"\n", yyTokenName[i]); //// } //// } - +// //} -//// ================================================================ -//md_body ::= md_statement_block(B). { -// past->proot = B; -//} +// ================================================================ +Body : + StatementBlock + <> ; + +StatementBlock : + Statement + << dsl.MakeZary($0) >> ; // xxx temp + +Statement : + FieldName Assign Number + << dsl.MakeBinary($1, $0, $2) >> ; + +FieldName: md_token_field_name << dsl.NewASTNode($0) >> ; +Assign: md_token_assign << dsl.NewASTNode($0) >> ; +Number: md_token_number << dsl.NewASTNode($0) >> ; // ---------------------------------------------------------------- // Given "$a=1;$b=2;$c=3": since this is a bottom-up parser, we get first the "$a=1", then diff --git a/go/src/miller/parsing/parser/actiontable.go b/go/src/miller/parsing/parser/actiontable.go index ba2ab7e04..fcb2aaaae 100644 --- a/go/src/miller/parsing/parser/actiontable.go +++ b/go/src/miller/parsing/parser/actiontable.go @@ -16,7 +16,7 @@ var actionTab = actionTable{ actions: [numSymbols]action{ nil, // INVALID nil, // $ - shift(3), // md_token_field_name + shift(5), // md_token_field_name nil, // md_token_assign nil, // md_token_number }, @@ -26,7 +26,7 @@ var actionTab = actionTable{ actions: [numSymbols]action{ nil, // INVALID accept(true), // $ - shift(3), // md_token_field_name + nil, // md_token_field_name nil, // md_token_assign nil, // md_token_number }, @@ -35,8 +35,8 @@ var actionTab = actionTable{ canRecover: false, actions: [numSymbols]action{ nil, // INVALID - reduce(1), // $, reduce: StatementList - reduce(1), // md_token_field_name, reduce: StatementList + reduce(1), // $, reduce: Body + nil, // md_token_field_name nil, // md_token_assign nil, // md_token_number }, @@ -44,39 +44,69 @@ var actionTab = actionTable{ actionRow{ // S3 canRecover: false, actions: [numSymbols]action{ - nil, // INVALID - nil, // $ - nil, // md_token_field_name - shift(5), // md_token_assign - nil, // md_token_number + nil, // INVALID + reduce(2), // $, reduce: StatementBlock + nil, // md_token_field_name + nil, // md_token_assign + nil, // md_token_number }, }, actionRow{ // S4 canRecover: false, actions: [numSymbols]action{ - nil, // INVALID - reduce(2), // $, reduce: StatementList - reduce(2), // md_token_field_name, reduce: StatementList - nil, // md_token_assign - nil, // md_token_number + nil, // INVALID + nil, // $ + nil, // md_token_field_name + shift(7), // md_token_assign + nil, // md_token_number }, }, actionRow{ // S5 + canRecover: false, + actions: [numSymbols]action{ + nil, // INVALID + nil, // $ + nil, // md_token_field_name + reduce(4), // md_token_assign, reduce: FieldName + nil, // md_token_number + }, + }, + actionRow{ // S6 canRecover: false, actions: [numSymbols]action{ nil, // INVALID nil, // $ nil, // md_token_field_name nil, // md_token_assign - shift(6), // md_token_number + shift(9), // md_token_number }, }, - actionRow{ // S6 + actionRow{ // S7 + canRecover: false, + actions: [numSymbols]action{ + nil, // INVALID + nil, // $ + nil, // md_token_field_name + nil, // md_token_assign + reduce(5), // md_token_number, reduce: Assign + }, + }, + actionRow{ // S8 canRecover: false, actions: [numSymbols]action{ nil, // INVALID reduce(3), // $, reduce: Statement - reduce(3), // md_token_field_name, reduce: Statement + nil, // md_token_field_name + nil, // md_token_assign + nil, // md_token_number + }, + }, + actionRow{ // S9 + canRecover: false, + actions: [numSymbols]action{ + nil, // INVALID + reduce(6), // $, reduce: Number + nil, // md_token_field_name nil, // md_token_assign nil, // md_token_number }, diff --git a/go/src/miller/parsing/parser/gototable.go b/go/src/miller/parsing/parser/gototable.go index 4d51839f0..6c39f85e1 100644 --- a/go/src/miller/parsing/parser/gototable.go +++ b/go/src/miller/parsing/parser/gototable.go @@ -2,7 +2,7 @@ package parser -const numNTSymbols = 3 +const numNTSymbols = 7 type ( gotoTable [numStates]gotoRow @@ -12,37 +12,92 @@ type ( var gotoTab = gotoTable{ gotoRow{ // S0 -1, // S' - 1, // StatementList - 2, // Statement + 1, // Body + 2, // StatementBlock + 3, // Statement + 4, // FieldName + -1, // Assign + -1, // Number }, gotoRow{ // S1 -1, // S' - -1, // StatementList - 4, // Statement + -1, // Body + -1, // StatementBlock + -1, // Statement + -1, // FieldName + -1, // Assign + -1, // Number }, gotoRow{ // S2 -1, // S' - -1, // StatementList + -1, // Body + -1, // StatementBlock -1, // Statement + -1, // FieldName + -1, // Assign + -1, // Number }, gotoRow{ // S3 -1, // S' - -1, // StatementList + -1, // Body + -1, // StatementBlock -1, // Statement + -1, // FieldName + -1, // Assign + -1, // Number }, gotoRow{ // S4 -1, // S' - -1, // StatementList + -1, // Body + -1, // StatementBlock -1, // Statement + -1, // FieldName + 6, // Assign + -1, // Number }, gotoRow{ // S5 -1, // S' - -1, // StatementList + -1, // Body + -1, // StatementBlock -1, // Statement + -1, // FieldName + -1, // Assign + -1, // Number }, gotoRow{ // S6 -1, // S' - -1, // StatementList + -1, // Body + -1, // StatementBlock -1, // Statement + -1, // FieldName + -1, // Assign + 8, // Number + }, + gotoRow{ // S7 + -1, // S' + -1, // Body + -1, // StatementBlock + -1, // Statement + -1, // FieldName + -1, // Assign + -1, // Number + }, + gotoRow{ // S8 + -1, // S' + -1, // Body + -1, // StatementBlock + -1, // Statement + -1, // FieldName + -1, // Assign + -1, // Number + }, + gotoRow{ // S9 + -1, // S' + -1, // Body + -1, // StatementBlock + -1, // Statement + -1, // FieldName + -1, // Assign + -1, // Number }, } diff --git a/go/src/miller/parsing/parser/parser.go b/go/src/miller/parsing/parser/parser.go index b3cb554ac..0ea9e08fb 100644 --- a/go/src/miller/parsing/parser/parser.go +++ b/go/src/miller/parsing/parser/parser.go @@ -11,9 +11,9 @@ import ( ) const ( - numProductions = 4 - numStates = 7 - numSymbols = 8 + numProductions = 7 + numStates = 10 + numSymbols = 12 ) // Stack diff --git a/go/src/miller/parsing/parser/productionstable.go b/go/src/miller/parsing/parser/productionstable.go index 06e59258a..794951810 100644 --- a/go/src/miller/parsing/parser/productionstable.go +++ b/go/src/miller/parsing/parser/productionstable.go @@ -21,7 +21,7 @@ type ( var productionsTable = ProdTab{ ProdTabEntry{ - String: `S' : StatementList << >>`, + String: `S' : Body << >>`, Id: "S'", NTType: 0, Index: 0, @@ -31,33 +31,63 @@ var productionsTable = ProdTab{ }, }, ProdTabEntry{ - String: `StatementList : Statement << dsl.NewStatementList(X[0]) >>`, - Id: "StatementList", + String: `Body : StatementBlock << dsl.NewAST(X[0]) >>`, + Id: "Body", NTType: 1, Index: 1, NumSymbols: 1, ReduceFunc: func(X []Attrib) (Attrib, error) { - return dsl.NewStatementList(X[0]) + return dsl.NewAST(X[0]) }, }, ProdTabEntry{ - String: `StatementList : StatementList Statement << dsl.AppendStatement(X[0], X[1]) >>`, - Id: "StatementList", - NTType: 1, - Index: 2, - NumSymbols: 2, - ReduceFunc: func(X []Attrib) (Attrib, error) { - return dsl.AppendStatement(X[0], X[1]) - }, - }, - ProdTabEntry{ - String: `Statement : md_token_field_name md_token_assign md_token_number << dsl.NewStatement(X[0]) >>`, - Id: "Statement", + String: `StatementBlock : Statement << dsl.MakeZary(X[0]) >>`, + Id: "StatementBlock", NTType: 2, + Index: 2, + NumSymbols: 1, + ReduceFunc: func(X []Attrib) (Attrib, error) { + return dsl.MakeZary(X[0]) + }, + }, + ProdTabEntry{ + String: `Statement : FieldName Assign Number << dsl.MakeBinary(X[1], X[0], X[2]) >>`, + Id: "Statement", + NTType: 3, Index: 3, NumSymbols: 3, ReduceFunc: func(X []Attrib) (Attrib, error) { - return dsl.NewStatement(X[0]) + return dsl.MakeBinary(X[1], X[0], X[2]) + }, + }, + ProdTabEntry{ + String: `FieldName : md_token_field_name << dsl.NewASTNode(X[0]) >>`, + Id: "FieldName", + NTType: 4, + Index: 4, + NumSymbols: 1, + ReduceFunc: func(X []Attrib) (Attrib, error) { + return dsl.NewASTNode(X[0]) + }, + }, + ProdTabEntry{ + String: `Assign : md_token_assign << dsl.NewASTNode(X[0]) >>`, + Id: "Assign", + NTType: 5, + Index: 5, + NumSymbols: 1, + ReduceFunc: func(X []Attrib) (Attrib, error) { + return dsl.NewASTNode(X[0]) + }, + }, + ProdTabEntry{ + String: `Number : md_token_number << dsl.NewASTNode(X[0]) >>`, + Id: "Number", + NTType: 6, + Index: 6, + NumSymbols: 1, + ReduceFunc: func(X []Attrib) (Attrib, error) { + return dsl.NewASTNode(X[0]) }, }, }