diff --git a/go/.gitignore b/go/.gitignore index 3f43b1385..b9c125cb8 100644 --- a/go/.gitignore +++ b/go/.gitignore @@ -7,5 +7,10 @@ s mlrgo gg -parser-experiments/src -parser-experiments/main +parser-experiments/one/src/github.com +parser-experiments/one/src/experimental +parser-experiments/one/main + +parser-experiments/two/src/github.com +parser-experiments/two/src/experimental +parser-experiments/two/main diff --git a/go/parser-experiments/README.md b/go/parser-experiments/README.md index 943b73e05..8d333720a 100644 --- a/go/parser-experiments/README.md +++ b/go/parser-experiments/README.md @@ -1,6 +1,2 @@ Running the GOCC parser-generator for the full Miller grammar takes a few minutes. That's a bit painful for experimentation; hence this. - -Building: `build` - -Running: `main` for canned tests; `main 'exprssion goes here'` for ad-hoc tests. diff --git a/go/parser-experiments/build b/go/parser-experiments/one/build similarity index 100% rename from go/parser-experiments/build rename to go/parser-experiments/one/build diff --git a/go/parser-experiments/main.go b/go/parser-experiments/one/main.go similarity index 100% rename from go/parser-experiments/main.go rename to go/parser-experiments/one/main.go diff --git a/go/parser-experiments/run b/go/parser-experiments/one/run similarity index 100% rename from go/parser-experiments/run rename to go/parser-experiments/one/run diff --git a/go/parser-experiments/semi1.bnf b/go/parser-experiments/one/semi1.bnf similarity index 100% rename from go/parser-experiments/semi1.bnf rename to go/parser-experiments/one/semi1.bnf diff --git a/go/parser-experiments/semi2.bnf b/go/parser-experiments/one/semi2.bnf similarity index 74% rename from go/parser-experiments/semi2.bnf rename to go/parser-experiments/one/semi2.bnf index 00d4e85f3..de5a381c7 100644 --- a/go/parser-experiments/semi2.bnf +++ b/go/parser-experiments/one/semi2.bnf @@ -4,13 +4,6 @@ Root : StatementBlock ; -// OK: 'x ; {}' -// OK: 'x ; x' -// Not OK: 'x x' -// Not OK: 'x {}' -// OK: '{} ; x' -// OK: '{} x' - StatementBlock : empty | ";" StatementBlock diff --git a/go/parser-experiments/two/build b/go/parser-experiments/two/build new file mode 100755 index 000000000..ebe69e3f3 --- /dev/null +++ b/go/parser-experiments/two/build @@ -0,0 +1,49 @@ +#!/bin/bash + +# ---------------------------------------------------------------- +# Setup +us=$(basename $0) +set -euo pipefail + +if [ $# -ne 1 ]; then + echo "Usage: $0 {.bnf file}" 1>&2 + exit 1 +fi +bnf="$1" + +dir=src/experimental +#verbose="true" +verbose="false" + +mkdir -p $dir + +# ---------------------------------------------------------------- +# Run the parser-generator +export GOPATH=$(pwd) + +# Build the bin/gocc executable: +go get github.com/goccmack/gocc + +rm -f $dir/*.txt +if [ "$verbose" = "true" ]; then + lr1="$dir/LR1_conflicts.txt" + # The -o specifies the package name within the autogen + bin/gocc -v -o $dir $bnf || expand -2 $lr1 +else + bin/gocc -o $dir $bnf +fi + +echo "Parser-autogen OK" + +# Code-gen directories: +# $dir/errors/ +# $dir/lexer/ +# $dir/parser/ +# $dir/token/ +# $dir/util/ + +# ---------------------------------------------------------------- +# Compile the main and the parser-autogen + +go build main.go +echo "Compile OK" diff --git a/go/parser-experiments/two/main.go b/go/parser-experiments/two/main.go new file mode 100644 index 000000000..20179d93a --- /dev/null +++ b/go/parser-experiments/two/main.go @@ -0,0 +1,62 @@ +package main + +import ( + "fmt" + "os" + + "experimental/lexer" + "experimental/parser" +) + +func parseOne(input string) { + theLexer := lexer.NewLexer([]byte(input)) + theParser := parser.NewParser() + _, err := theParser.Parse(theLexer) + + green := "\033[32;01m" + red := "\033[31;01m" + textdefault := "\033[0m" + + if err != nil { + //fmt.Println(err) + fmt.Printf("%sFail%s %s\n", red, textdefault, input) + } else { + fmt.Printf("%sOK%s %s\n", green, textdefault, input) + } +} + +func main() { + if len(os.Args) == 1 { + inputs := []string{ + // Expect pass + "", + ";", + ";;", + "x", + "x;x", + "x;", + "x;;", + ";x", + ";;x", + "x ; {}", + "{} ; x", + "{} x", + "{ x }", + "{ x; x }", + "x; { x; x }", + "{ x; x } x", + "{ x; x } ; x", + + // Expect fail + "x x", + "x {}", + } + for _, input := range inputs { + parseOne(input) + } + } else { + for _, arg := range os.Args[1:] { + parseOne(arg) + } + } +} diff --git a/go/parser-experiments/two/run b/go/parser-experiments/two/run new file mode 100755 index 000000000..95a327265 --- /dev/null +++ b/go/parser-experiments/two/run @@ -0,0 +1,9 @@ +#!/bin/bash + +if [ $# -lt 1 ]; then + echo "Usage: $0 {.bnf file} [expressions]" 1>&2 + exit 1 +fi +bnf="$1" +shift +./build $bnf && echo && main "$@" diff --git a/go/parser-experiments/two/semi3.bnf b/go/parser-experiments/two/semi3.bnf new file mode 100644 index 000000000..da8915275 --- /dev/null +++ b/go/parser-experiments/two/semi3.bnf @@ -0,0 +1,35 @@ +// ---------------------------------------------------------------- +// LEXER + +!whitespace : ' ' | '\t' | '\n' | '\r' ; + +// ---------------------------------------------------------------- +// IMPORT + +<< import "dsl" >> + +// ---------------------------------------------------------------- +// PARSER + +Root + : StatementBlock + << dsl.NewAST($0) >> +; + +StatementBlock + : empty + | ";" StatementBlock + | BracelessStatement + | BracefulStatement + | BracelessStatement ";" StatementBlock + | BracefulStatement ";" BracelessStatement + | BracefulStatement BracelessStatement +; + +BracelessStatement + : "x" +; + +BracefulStatement + : "{" StatementBlock "}" +; diff --git a/go/parser-experiments/two/src/dsl/ast.go b/go/parser-experiments/two/src/dsl/ast.go new file mode 100644 index 000000000..b3883c0b8 --- /dev/null +++ b/go/parser-experiments/two/src/dsl/ast.go @@ -0,0 +1,296 @@ +package dsl + +import ( + "fmt" + + "experimental/token" +) + +type TNodeType string + +const ( + NodeTypeEmptyStatement TNodeType = "empty statement" + + NodeTypeStatementBlock = "statement block" +) + +// ---------------------------------------------------------------- +type AST struct { + RootNode *ASTNode +} + +// This is for the GOCC/BNF parser, which produces an AST +func NewAST(iroot interface{}) (*AST, error) { + return &AST{ + RootNode: iroot.(*ASTNode), + }, nil +} + +func (this *AST) Print() { + this.RootNode.Print() +} + +// ---------------------------------------------------------------- +type ASTNode struct { + Token *token.Token // Nil for tokenless/structural nodes + Type TNodeType + Children []*ASTNode +} + +func (this *ASTNode) Print() { + this.PrintAux(0) +} +func (this *ASTNode) PrintAux(depth int) { + for i := 0; i < depth; i++ { + fmt.Print(" ") + } + tok := this.Token + fmt.Print("* " + this.Type) + + if tok != nil { + //fmt.Printf(" \"%s\" \"%s\"", + // token.TokMap.Id(tok.Type), string(tok.Lit)) + fmt.Printf(" \"%s\"", string(tok.Lit)) + } + fmt.Println() + if this.Children != nil { + for _, child := range this.Children { + child.PrintAux(depth + 1) + } + } +} + +// func NewASTNode(itok interface{}, nodeType TNodeType) (*ASTNode, error) { +// return NewASTNodeNestable(itok, nodeType), nil +// } +// +// // For handling empty expressions. +// func NewASTNodeEmptyNestable(nodeType TNodeType) *ASTNode { +// return &ASTNode{ +// Token: nil, +// Type: nodeType, +// Children: nil, +// } +// } +// +// // For handling empty expressions. +// func NewASTNodeEmpty(nodeType TNodeType) (*ASTNode, error) { +// return NewASTNodeEmptyNestable(nodeType), nil +// } +// +// // Strips the leading '$' from field names, or '@' from oosvar names. Not done +// // in the parser itself due to LR-1 conflicts. +// func NewASTNodeStripDollarOrAtSign(itok interface{}, nodeType TNodeType) (*ASTNode, error) { +// oldToken := itok.(*token.Token) +// lib.InternalCodingErrorIf(len(oldToken.Lit) < 2) +// lib.InternalCodingErrorIf(oldToken.Lit[0] != '$' && oldToken.Lit[0] != '@') +// newToken := &token.Token{ +// Type: oldToken.Type, +// Lit: oldToken.Lit[1:], +// Pos: oldToken.Pos, +// } +// return NewASTNodeNestable(newToken, nodeType), nil +// } +// +// // Strips the leading '${' and trailing '}' from braced field names, or '@{' +// // and '}' from oosvar names. Not done in the parser itself due to LR-1 +// // conflicts. +// func NewASTNodeStripDollarOrAtSignAndCurlyBraces( +// itok interface{}, +// nodeType TNodeType, +// ) (*ASTNode, error) { +// oldToken := itok.(*token.Token) +// n := len(oldToken.Lit) +// lib.InternalCodingErrorIf(n < 4) +// lib.InternalCodingErrorIf(oldToken.Lit[0] != '$' && oldToken.Lit[0] != '@') +// lib.InternalCodingErrorIf(oldToken.Lit[1] != '{') +// lib.InternalCodingErrorIf(oldToken.Lit[n-1] != '}') +// newToken := &token.Token{ +// Type: oldToken.Type, +// Lit: oldToken.Lit[2 : n-1], +// Pos: oldToken.Pos, +// } +// return NewASTNodeNestable(newToken, nodeType), nil +// } +// +// // Likewise for the leading/trailing double quotes on string literals. Also, +// // since string literals can have backslash-escaped double-quotes like +// // "...\"...\"...", we also unbackslash here. +// func NewASTNodeStripDoubleQuotePair( +// itok interface{}, +// nodeType TNodeType, +// ) (*ASTNode, error) { +// oldToken := itok.(*token.Token) +// n := len(oldToken.Lit) +// contents := string(oldToken.Lit[1 : n-1]) +// contents = strings.Replace(contents, "\\\"", "\"", -1) +// newToken := &token.Token{ +// Type: oldToken.Type, +// Lit: []byte(contents), +// Pos: oldToken.Pos, +// } +// return NewASTNodeNestable(newToken, nodeType), nil +// } +// +// // xxx comment why grammar use +// func NewASTNodeNestable(itok interface{}, nodeType TNodeType) *ASTNode { +// var tok *token.Token = nil +// if itok != nil { +// tok = itok.(*token.Token) +// } +// return &ASTNode{ +// Token: tok, +// Type: nodeType, +// Children: nil, +// } +// } +// +// func NewASTNodeZary(itok interface{}, nodeType TNodeType) (*ASTNode, error) { +// parent := NewASTNodeNestable(itok, nodeType) +// convertToZary(parent) +// return parent, nil +// } +// +// func NewASTNodeUnary(itok, childA interface{}, nodeType TNodeType) (*ASTNode, error) { +// parent := NewASTNodeNestable(itok, nodeType) +// convertToUnary(parent, childA) +// return parent, nil +// } +// +// // Signature: Token Node Node Type +// func NewASTNodeBinaryNestable(itok, childA, childB interface{}, nodeType TNodeType) *ASTNode { +// parent := NewASTNodeNestable(itok, nodeType) +// convertToBinary(parent, childA, childB) +// return parent +// } +// +// // Signature: Token Node Node Type +// func NewASTNodeBinary( +// itok, childA, childB interface{}, nodeType TNodeType, +// ) (*ASTNode, error) { +// return NewASTNodeBinaryNestable(itok, childA, childB, nodeType), nil +// } +// +// func NewASTNodeTernary(itok, childA, childB, childC interface{}, nodeType TNodeType) (*ASTNode, error) { +// parent := NewASTNodeNestable(itok, nodeType) +// convertToTernary(parent, childA, childB, childC) +// return parent, nil +// } +// +// func NewASTNodeQuaternary( +// itok, childA, childB, childC, childD interface{}, nodeType TNodeType, +// ) (*ASTNode, error) { +// parent := NewASTNodeNestable(itok, nodeType) +// convertToQuaternary(parent, childA, childB, childC, childD) +// return parent, nil +// } +// +// // Pass-through expressions in the grammar sometimes need to be turned from +// // (ASTNode) to (ASTNode, error) +// func Nestable(iparent interface{}) (*ASTNode, error) { +// return iparent.(*ASTNode), nil +// } +// +// func convertToZary(iparent interface{}) { +// parent := iparent.(*ASTNode) +// children := make([]*ASTNode, 0) +// parent.Children = children +// } +// +// // xxx inline this. can be a one-liner. +// func convertToUnary(iparent interface{}, childA interface{}) { +// parent := iparent.(*ASTNode) +// children := make([]*ASTNode, 1) +// children[0] = childA.(*ASTNode) +// parent.Children = children +// } +// +// func convertToBinary(iparent interface{}, childA, childB interface{}) { +// parent := iparent.(*ASTNode) +// children := make([]*ASTNode, 2) +// children[0] = childA.(*ASTNode) +// children[1] = childB.(*ASTNode) +// parent.Children = children +// } +// +// func convertToTernary(iparent interface{}, childA, childB, childC interface{}) { +// parent := iparent.(*ASTNode) +// children := make([]*ASTNode, 3) +// children[0] = childA.(*ASTNode) +// children[1] = childB.(*ASTNode) +// children[2] = childC.(*ASTNode) +// parent.Children = children +// } +// +// func convertToQuaternary(iparent interface{}, childA, childB, childC, childD interface{}) { +// parent := iparent.(*ASTNode) +// children := make([]*ASTNode, 4) +// children[0] = childA.(*ASTNode) +// children[1] = childB.(*ASTNode) +// children[2] = childC.(*ASTNode) +// children[3] = childD.(*ASTNode) +// parent.Children = children +// } +// +// func PrependChild(iparent interface{}, ichild interface{}) (*ASTNode, error) { +// parent := iparent.(*ASTNode) +// child := ichild.(*ASTNode) +// if parent.Children == nil { +// convertToUnary(iparent, ichild) +// } else { +// parent.Children = append([]*ASTNode{child}, parent.Children...) +// } +// return parent, nil +// } +// +// func AppendChild(iparent interface{}, child interface{}) (*ASTNode, error) { +// parent := iparent.(*ASTNode) +// if parent.Children == nil { +// convertToUnary(iparent, child) +// } else { +// parent.Children = append(parent.Children, child.(*ASTNode)) +// } +// return parent, nil +// } +// +// func AdoptChildren(iparent interface{}, ichild interface{}) (*ASTNode, error) { +// parent := iparent.(*ASTNode) +// child := ichild.(*ASTNode) +// parent.Children = child.Children +// child.Children = nil +// return parent, nil +// } +// +// // TODO: comment +// func Wrap(inode interface{}) (*ASTNode, error) { +// node := inode.(*ASTNode) +// return node, nil +// } +// +// func (this *ASTNode) CheckArity( +// arity int, +// ) error { +// if len(this.Children) != arity { +// return errors.New( +// fmt.Sprintf( +// "AST node arity %d, expected %d", +// len(this.Children), arity, +// ), +// ) +// } else { +// return nil +// } +// } +// +// // Tokens are produced by GOCC. However there is an exception: for the ternary +// // operator I want the AST to have a "?:" token, which GOCC doesn't produce +// // since nothing is actually spelled like that in the DSL. +// func NewASTToken(iliteral interface{}, iclonee interface{}) *token.Token { +// literal := iliteral.(string) +// clonee := iclonee.(*token.Token) +// return &token.Token{ +// Type: clonee.Type, +// Lit: []byte(literal), +// Pos: clonee.Pos, +// } +// }