mirror of
https://github.com/johnkerl/miller.git
synced 2026-08-02 20:42:52 +00:00
AST iterate
This commit is contained in:
John Kerl
parent
491a902533
commit
3dfaf9bb92
7 changed files with 225 additions and 126 deletions
|
|
@ -5,85 +5,65 @@ import (
|
|||
)
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// AST type:
|
||||
// root *Node
|
||||
|
||||
// Node type:
|
||||
// * text string
|
||||
// * type enum
|
||||
// * children []Node
|
||||
// xxx comment interface{} everywhere vs. true types due to gocc polymorphism API.
|
||||
// and, line-count for casts here vs in the BNF:
|
||||
//
|
||||
// Statement :
|
||||
// md_token_field_name md_token_assign md_token_number
|
||||
//
|
||||
// Statement :
|
||||
// md_token_field_name md_token_assign md_token_number
|
||||
// << dsl.NewASTNodeTernary("foo", $0, $1, $2) >> ;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
type AST struct {
|
||||
Root *ASTNode
|
||||
}
|
||||
|
||||
func NewAST(root *ASTNode) *AST {
|
||||
func NewAST(root interface{}) (*AST, error) {
|
||||
return &AST {
|
||||
root,
|
||||
}
|
||||
root.(*ASTNode),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
type ASTNode struct {
|
||||
Text string
|
||||
Token token.Token
|
||||
// Type enum
|
||||
// There need to be tokenless nodes
|
||||
// text string
|
||||
Children []*ASTNode
|
||||
}
|
||||
|
||||
func NewASTNode(text string) *ASTNode {
|
||||
func NewASTNode(itok interface{}) (*ASTNode, error) {
|
||||
tok := itok.(*token.Token)
|
||||
return &ASTNode {
|
||||
text,
|
||||
*tok,
|
||||
// type
|
||||
nil,
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
func NewASTNodeZary(text string) *ASTNode {
|
||||
func MakeZary(iparent interface{}) (*ASTNode, error) {
|
||||
parent := iparent.(*ASTNode)
|
||||
children := make([]*ASTNode, 0)
|
||||
return &ASTNode {
|
||||
text,
|
||||
// type
|
||||
children,
|
||||
}
|
||||
parent.Children = children
|
||||
return parent, nil
|
||||
}
|
||||
|
||||
func NewASTNodeUnary(text string, childA *ASTNode) *ASTNode {
|
||||
func MakeUnary(iparent interface{}, childA interface{}) (*ASTNode, error) {
|
||||
parent := iparent.(*ASTNode)
|
||||
children := make([]*ASTNode, 1)
|
||||
children[0] = childA
|
||||
return &ASTNode {
|
||||
text,
|
||||
// type
|
||||
children,
|
||||
}
|
||||
children[0] = childA.(*ASTNode)
|
||||
parent.Children = children
|
||||
return parent, nil
|
||||
}
|
||||
|
||||
func NewASTNodeBinary(text string, childA *ASTNode, childB *ASTNode) *ASTNode {
|
||||
children := make([]*ASTNode, 1)
|
||||
children[0] = childA
|
||||
children[1] = childB
|
||||
return &ASTNode {
|
||||
text,
|
||||
// type
|
||||
children,
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// prototype stuff from gocc example
|
||||
type (
|
||||
StatementList []Statement
|
||||
Statement string
|
||||
)
|
||||
|
||||
func NewStatementList(statement interface{}) (StatementList, error) {
|
||||
return StatementList{statement.(Statement)}, nil
|
||||
}
|
||||
|
||||
func AppendStatement(statementList, statement interface{}) (StatementList, error) {
|
||||
return append(statementList.(StatementList), statement.(Statement)), nil
|
||||
}
|
||||
|
||||
func NewStatement(statementList interface{}) (Statement, error) {
|
||||
return Statement(statementList.(*token.Token).Lit), nil
|
||||
func MakeBinary(iparent interface{}, childA interface{}, childB interface{}) (*ASTNode, error) {
|
||||
parent := iparent.(*ASTNode)
|
||||
children := make([]*ASTNode, 2)
|
||||
children[0] = childA.(*ASTNode)
|
||||
children[1] = childB.(*ASTNode)
|
||||
parent.Children = children
|
||||
return parent, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,23 +9,24 @@ import (
|
|||
"miller/parsing/parser"
|
||||
)
|
||||
|
||||
func testOne(src []byte) (astree dsl.StatementList, err error) {
|
||||
fmt.Printf("Input: %s\n", src)
|
||||
s := lexer.NewLexer(src)
|
||||
p := parser.NewParser()
|
||||
a, err := p.Parse(s)
|
||||
func testOne(sourceString []byte) (*dsl.AST, error) {
|
||||
fmt.Printf("Input: %s\n", sourceString)
|
||||
theLexer := lexer.NewLexer(sourceString)
|
||||
theParser := parser.NewParser()
|
||||
interfaceAST, err := theParser.Parse(theLexer)
|
||||
if err == nil {
|
||||
astree = a.(dsl.StatementList)
|
||||
return interfaceAST.(*dsl.AST), nil
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func TestPass(t *testing.T) {
|
||||
sml, err := testOne([]byte("$x = 3"))
|
||||
ast, err := testOne([]byte("$x = 3"))
|
||||
if err != nil {
|
||||
t.Fatal(err.Error())
|
||||
}
|
||||
fmt.Printf("Output: %s\n", sml)
|
||||
fmt.Printf("Output: %v\n", ast)
|
||||
}
|
||||
|
||||
func TestFail(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -628,19 +628,10 @@ md_token_number : _digit { _digit } | '-' _digit { _digit } ;
|
|||
|
||||
<< import "miller/dsl" >>
|
||||
|
||||
StatementList :
|
||||
Statement << dsl.NewStatementList($0) >>
|
||||
| StatementList Statement << dsl.AppendStatement($0, $1) >>
|
||||
;
|
||||
|
||||
Statement :
|
||||
md_token_field_name md_token_assign md_token_number << dsl.NewStatement($0) >>
|
||||
;
|
||||
|
||||
//// The caller is expected to provide more context.
|
||||
//%syntax_error {
|
||||
// fprintf(stderr, "mlr DSL: syntax error at \"%s\"\n", TOKEN->text);
|
||||
|
||||
//
|
||||
//// This is confusing and (as is) worse than nothing.
|
||||
//// Ideally we want to show the position within the input of the syntax error.
|
||||
////
|
||||
|
|
@ -651,13 +642,25 @@ Statement :
|
|||
//// fprintf(stderr, "Possible token \"%s\"\n", yyTokenName[i]);
|
||||
//// }
|
||||
//// }
|
||||
|
||||
//
|
||||
//}
|
||||
|
||||
//// ================================================================
|
||||
//md_body ::= md_statement_block(B). {
|
||||
// past->proot = B;
|
||||
//}
|
||||
// ================================================================
|
||||
Body :
|
||||
StatementBlock
|
||||
<<dsl.NewAST($0) >> ;
|
||||
|
||||
StatementBlock :
|
||||
Statement
|
||||
<< dsl.MakeZary($0) >> ; // xxx temp
|
||||
|
||||
Statement :
|
||||
FieldName Assign Number
|
||||
<< dsl.MakeBinary($1, $0, $2) >> ;
|
||||
|
||||
FieldName: md_token_field_name << dsl.NewASTNode($0) >> ;
|
||||
Assign: md_token_assign << dsl.NewASTNode($0) >> ;
|
||||
Number: md_token_number << dsl.NewASTNode($0) >> ;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Given "$a=1;$b=2;$c=3": since this is a bottom-up parser, we get first the "$a=1", then
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ var actionTab = actionTable{
|
|||
actions: [numSymbols]action{
|
||||
nil, // INVALID
|
||||
nil, // $
|
||||
shift(3), // md_token_field_name
|
||||
shift(5), // md_token_field_name
|
||||
nil, // md_token_assign
|
||||
nil, // md_token_number
|
||||
},
|
||||
|
|
@ -26,7 +26,7 @@ var actionTab = actionTable{
|
|||
actions: [numSymbols]action{
|
||||
nil, // INVALID
|
||||
accept(true), // $
|
||||
shift(3), // md_token_field_name
|
||||
nil, // md_token_field_name
|
||||
nil, // md_token_assign
|
||||
nil, // md_token_number
|
||||
},
|
||||
|
|
@ -35,8 +35,8 @@ var actionTab = actionTable{
|
|||
canRecover: false,
|
||||
actions: [numSymbols]action{
|
||||
nil, // INVALID
|
||||
reduce(1), // $, reduce: StatementList
|
||||
reduce(1), // md_token_field_name, reduce: StatementList
|
||||
reduce(1), // $, reduce: Body
|
||||
nil, // md_token_field_name
|
||||
nil, // md_token_assign
|
||||
nil, // md_token_number
|
||||
},
|
||||
|
|
@ -44,39 +44,69 @@ var actionTab = actionTable{
|
|||
actionRow{ // S3
|
||||
canRecover: false,
|
||||
actions: [numSymbols]action{
|
||||
nil, // INVALID
|
||||
nil, // $
|
||||
nil, // md_token_field_name
|
||||
shift(5), // md_token_assign
|
||||
nil, // md_token_number
|
||||
nil, // INVALID
|
||||
reduce(2), // $, reduce: StatementBlock
|
||||
nil, // md_token_field_name
|
||||
nil, // md_token_assign
|
||||
nil, // md_token_number
|
||||
},
|
||||
},
|
||||
actionRow{ // S4
|
||||
canRecover: false,
|
||||
actions: [numSymbols]action{
|
||||
nil, // INVALID
|
||||
reduce(2), // $, reduce: StatementList
|
||||
reduce(2), // md_token_field_name, reduce: StatementList
|
||||
nil, // md_token_assign
|
||||
nil, // md_token_number
|
||||
nil, // INVALID
|
||||
nil, // $
|
||||
nil, // md_token_field_name
|
||||
shift(7), // md_token_assign
|
||||
nil, // md_token_number
|
||||
},
|
||||
},
|
||||
actionRow{ // S5
|
||||
canRecover: false,
|
||||
actions: [numSymbols]action{
|
||||
nil, // INVALID
|
||||
nil, // $
|
||||
nil, // md_token_field_name
|
||||
reduce(4), // md_token_assign, reduce: FieldName
|
||||
nil, // md_token_number
|
||||
},
|
||||
},
|
||||
actionRow{ // S6
|
||||
canRecover: false,
|
||||
actions: [numSymbols]action{
|
||||
nil, // INVALID
|
||||
nil, // $
|
||||
nil, // md_token_field_name
|
||||
nil, // md_token_assign
|
||||
shift(6), // md_token_number
|
||||
shift(9), // md_token_number
|
||||
},
|
||||
},
|
||||
actionRow{ // S6
|
||||
actionRow{ // S7
|
||||
canRecover: false,
|
||||
actions: [numSymbols]action{
|
||||
nil, // INVALID
|
||||
nil, // $
|
||||
nil, // md_token_field_name
|
||||
nil, // md_token_assign
|
||||
reduce(5), // md_token_number, reduce: Assign
|
||||
},
|
||||
},
|
||||
actionRow{ // S8
|
||||
canRecover: false,
|
||||
actions: [numSymbols]action{
|
||||
nil, // INVALID
|
||||
reduce(3), // $, reduce: Statement
|
||||
reduce(3), // md_token_field_name, reduce: Statement
|
||||
nil, // md_token_field_name
|
||||
nil, // md_token_assign
|
||||
nil, // md_token_number
|
||||
},
|
||||
},
|
||||
actionRow{ // S9
|
||||
canRecover: false,
|
||||
actions: [numSymbols]action{
|
||||
nil, // INVALID
|
||||
reduce(6), // $, reduce: Number
|
||||
nil, // md_token_field_name
|
||||
nil, // md_token_assign
|
||||
nil, // md_token_number
|
||||
},
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
package parser
|
||||
|
||||
const numNTSymbols = 3
|
||||
const numNTSymbols = 7
|
||||
|
||||
type (
|
||||
gotoTable [numStates]gotoRow
|
||||
|
|
@ -12,37 +12,92 @@ type (
|
|||
var gotoTab = gotoTable{
|
||||
gotoRow{ // S0
|
||||
-1, // S'
|
||||
1, // StatementList
|
||||
2, // Statement
|
||||
1, // Body
|
||||
2, // StatementBlock
|
||||
3, // Statement
|
||||
4, // FieldName
|
||||
-1, // Assign
|
||||
-1, // Number
|
||||
},
|
||||
gotoRow{ // S1
|
||||
-1, // S'
|
||||
-1, // StatementList
|
||||
4, // Statement
|
||||
-1, // Body
|
||||
-1, // StatementBlock
|
||||
-1, // Statement
|
||||
-1, // FieldName
|
||||
-1, // Assign
|
||||
-1, // Number
|
||||
},
|
||||
gotoRow{ // S2
|
||||
-1, // S'
|
||||
-1, // StatementList
|
||||
-1, // Body
|
||||
-1, // StatementBlock
|
||||
-1, // Statement
|
||||
-1, // FieldName
|
||||
-1, // Assign
|
||||
-1, // Number
|
||||
},
|
||||
gotoRow{ // S3
|
||||
-1, // S'
|
||||
-1, // StatementList
|
||||
-1, // Body
|
||||
-1, // StatementBlock
|
||||
-1, // Statement
|
||||
-1, // FieldName
|
||||
-1, // Assign
|
||||
-1, // Number
|
||||
},
|
||||
gotoRow{ // S4
|
||||
-1, // S'
|
||||
-1, // StatementList
|
||||
-1, // Body
|
||||
-1, // StatementBlock
|
||||
-1, // Statement
|
||||
-1, // FieldName
|
||||
6, // Assign
|
||||
-1, // Number
|
||||
},
|
||||
gotoRow{ // S5
|
||||
-1, // S'
|
||||
-1, // StatementList
|
||||
-1, // Body
|
||||
-1, // StatementBlock
|
||||
-1, // Statement
|
||||
-1, // FieldName
|
||||
-1, // Assign
|
||||
-1, // Number
|
||||
},
|
||||
gotoRow{ // S6
|
||||
-1, // S'
|
||||
-1, // StatementList
|
||||
-1, // Body
|
||||
-1, // StatementBlock
|
||||
-1, // Statement
|
||||
-1, // FieldName
|
||||
-1, // Assign
|
||||
8, // Number
|
||||
},
|
||||
gotoRow{ // S7
|
||||
-1, // S'
|
||||
-1, // Body
|
||||
-1, // StatementBlock
|
||||
-1, // Statement
|
||||
-1, // FieldName
|
||||
-1, // Assign
|
||||
-1, // Number
|
||||
},
|
||||
gotoRow{ // S8
|
||||
-1, // S'
|
||||
-1, // Body
|
||||
-1, // StatementBlock
|
||||
-1, // Statement
|
||||
-1, // FieldName
|
||||
-1, // Assign
|
||||
-1, // Number
|
||||
},
|
||||
gotoRow{ // S9
|
||||
-1, // S'
|
||||
-1, // Body
|
||||
-1, // StatementBlock
|
||||
-1, // Statement
|
||||
-1, // FieldName
|
||||
-1, // Assign
|
||||
-1, // Number
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,9 +11,9 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
numProductions = 4
|
||||
numStates = 7
|
||||
numSymbols = 8
|
||||
numProductions = 7
|
||||
numStates = 10
|
||||
numSymbols = 12
|
||||
)
|
||||
|
||||
// Stack
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ type (
|
|||
|
||||
var productionsTable = ProdTab{
|
||||
ProdTabEntry{
|
||||
String: `S' : StatementList << >>`,
|
||||
String: `S' : Body << >>`,
|
||||
Id: "S'",
|
||||
NTType: 0,
|
||||
Index: 0,
|
||||
|
|
@ -31,33 +31,63 @@ var productionsTable = ProdTab{
|
|||
},
|
||||
},
|
||||
ProdTabEntry{
|
||||
String: `StatementList : Statement << dsl.NewStatementList(X[0]) >>`,
|
||||
Id: "StatementList",
|
||||
String: `Body : StatementBlock << dsl.NewAST(X[0]) >>`,
|
||||
Id: "Body",
|
||||
NTType: 1,
|
||||
Index: 1,
|
||||
NumSymbols: 1,
|
||||
ReduceFunc: func(X []Attrib) (Attrib, error) {
|
||||
return dsl.NewStatementList(X[0])
|
||||
return dsl.NewAST(X[0])
|
||||
},
|
||||
},
|
||||
ProdTabEntry{
|
||||
String: `StatementList : StatementList Statement << dsl.AppendStatement(X[0], X[1]) >>`,
|
||||
Id: "StatementList",
|
||||
NTType: 1,
|
||||
Index: 2,
|
||||
NumSymbols: 2,
|
||||
ReduceFunc: func(X []Attrib) (Attrib, error) {
|
||||
return dsl.AppendStatement(X[0], X[1])
|
||||
},
|
||||
},
|
||||
ProdTabEntry{
|
||||
String: `Statement : md_token_field_name md_token_assign md_token_number << dsl.NewStatement(X[0]) >>`,
|
||||
Id: "Statement",
|
||||
String: `StatementBlock : Statement << dsl.MakeZary(X[0]) >>`,
|
||||
Id: "StatementBlock",
|
||||
NTType: 2,
|
||||
Index: 2,
|
||||
NumSymbols: 1,
|
||||
ReduceFunc: func(X []Attrib) (Attrib, error) {
|
||||
return dsl.MakeZary(X[0])
|
||||
},
|
||||
},
|
||||
ProdTabEntry{
|
||||
String: `Statement : FieldName Assign Number << dsl.MakeBinary(X[1], X[0], X[2]) >>`,
|
||||
Id: "Statement",
|
||||
NTType: 3,
|
||||
Index: 3,
|
||||
NumSymbols: 3,
|
||||
ReduceFunc: func(X []Attrib) (Attrib, error) {
|
||||
return dsl.NewStatement(X[0])
|
||||
return dsl.MakeBinary(X[1], X[0], X[2])
|
||||
},
|
||||
},
|
||||
ProdTabEntry{
|
||||
String: `FieldName : md_token_field_name << dsl.NewASTNode(X[0]) >>`,
|
||||
Id: "FieldName",
|
||||
NTType: 4,
|
||||
Index: 4,
|
||||
NumSymbols: 1,
|
||||
ReduceFunc: func(X []Attrib) (Attrib, error) {
|
||||
return dsl.NewASTNode(X[0])
|
||||
},
|
||||
},
|
||||
ProdTabEntry{
|
||||
String: `Assign : md_token_assign << dsl.NewASTNode(X[0]) >>`,
|
||||
Id: "Assign",
|
||||
NTType: 5,
|
||||
Index: 5,
|
||||
NumSymbols: 1,
|
||||
ReduceFunc: func(X []Attrib) (Attrib, error) {
|
||||
return dsl.NewASTNode(X[0])
|
||||
},
|
||||
},
|
||||
ProdTabEntry{
|
||||
String: `Number : md_token_number << dsl.NewASTNode(X[0]) >>`,
|
||||
Id: "Number",
|
||||
NTType: 6,
|
||||
Index: 6,
|
||||
NumSymbols: 1,
|
||||
ReduceFunc: func(X []Attrib) (Attrib, error) {
|
||||
return dsl.NewASTNode(X[0])
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue