parser experiments iterate

go/.gitignore

@@ -7,5 +7,10 @@ s
 mlrgo
 gg
 
-parser-experiments/src
-parser-experiments/main
+parser-experiments/one/src/github.com
+parser-experiments/one/src/experimental
+parser-experiments/one/main
+
+parser-experiments/two/src/github.com
+parser-experiments/two/src/experimental
+parser-experiments/two/main

go/parser-experiments/README.md

@@ -1,6 +1,2 @@
 Running the GOCC parser-generator for the full Miller grammar takes a few minutes. That's
 a bit painful for experimentation; hence this.
-
-Building: `build`
-
-Running: `main` for canned tests; `main 'exprssion goes here'` for ad-hoc tests.

go/parser-experiments/one/semi2.bnf

@@ -4,13 +4,6 @@ Root
   : StatementBlock
 ;
 
-// OK:     'x  ;  {}'
-// OK:     'x  ;  x'
-// Not OK: 'x     x'
-// Not OK: 'x     {}'
-// OK:     '{} ;  x'
-// OK:     '{}    x'
-
 StatementBlock
   : empty
   | ";" StatementBlock

go/parser-experiments/two/build

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# ----------------------------------------------------------------
+# Setup
+us=$(basename $0)
+set -euo pipefail
+
+if [ $# -ne 1 ]; then
+  echo "Usage: $0 {.bnf file}" 1>&2
+  exit 1
+fi
+bnf="$1"
+
+dir=src/experimental
+#verbose="true"
+verbose="false"
+
+mkdir -p $dir
+
+# ----------------------------------------------------------------
+# Run the parser-generator
+export GOPATH=$(pwd)
+
+# Build the bin/gocc executable:
+go get github.com/goccmack/gocc
+
+rm -f $dir/*.txt
+if [ "$verbose" = "true" ]; then
+  lr1="$dir/LR1_conflicts.txt"
+  # The -o specifies the package name within the autogen
+  bin/gocc -v -o $dir $bnf || expand -2 $lr1
+else
+  bin/gocc -o $dir $bnf
+fi
+
+echo "Parser-autogen OK"
+
+# Code-gen directories:
+#   $dir/errors/
+#   $dir/lexer/
+#   $dir/parser/
+#   $dir/token/
+#   $dir/util/
+
+# ----------------------------------------------------------------
+# Compile the main and the parser-autogen
+
+go build main.go
+echo "Compile OK"

go/parser-experiments/two/main.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"experimental/lexer"
+	"experimental/parser"
+)
+
+func parseOne(input string) {
+	theLexer := lexer.NewLexer([]byte(input))
+	theParser := parser.NewParser()
+	_, err := theParser.Parse(theLexer)
+
+	green := "\033[32;01m"
+	red := "\033[31;01m"
+	textdefault := "\033[0m"
+
+	if err != nil {
+		//fmt.Println(err)
+		fmt.Printf("%sFail%s %s\n", red, textdefault, input)
+	} else {
+		fmt.Printf("%sOK%s   %s\n", green, textdefault, input)
+	}
+}
+
+func main() {
+	if len(os.Args) == 1 {
+		inputs := []string{
+			// Expect pass
+			"",
+			";",
+			";;",
+			"x",
+			"x;x",
+			"x;",
+			"x;;",
+			";x",
+			";;x",
+			"x ; {}",
+			"{} ; x",
+			"{} x",
+			"{ x }",
+			"{ x; x }",
+			"x; { x; x }",
+			"{ x; x } x",
+			"{ x; x } ; x",
+
+			// Expect fail
+			"x x",
+			"x {}",
+		}
+		for _, input := range inputs {
+			parseOne(input)
+		}
+	} else {
+		for _, arg := range os.Args[1:] {
+			parseOne(arg)
+		}
+	}
+}

go/parser-experiments/two/run

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [ $# -lt 1 ]; then
+  echo "Usage: $0 {.bnf file} [expressions]" 1>&2
+  exit 1
+fi
+bnf="$1"
+shift
+./build $bnf && echo && main "$@"

go/parser-experiments/two/semi3.bnf

@@ -0,0 +1,35 @@
+// ----------------------------------------------------------------
+// LEXER
+
+!whitespace : ' ' | '\t' | '\n' | '\r' ;
+
+// ----------------------------------------------------------------
+// IMPORT
+
+<< import "dsl" >>
+
+// ----------------------------------------------------------------
+// PARSER
+
+Root
+  : StatementBlock
+   << dsl.NewAST($0) >>
+;
+
+StatementBlock
+  : empty
+  | ";" StatementBlock
+  | BracelessStatement
+  | BracefulStatement
+  | BracelessStatement ";" StatementBlock
+  | BracefulStatement  ";" BracelessStatement
+  | BracefulStatement      BracelessStatement
+;
+
+BracelessStatement
+  : "x"
+;
+
+BracefulStatement
+  : "{" StatementBlock "}"
+;

go/parser-experiments/two/src/dsl/ast.go

@@ -0,0 +1,296 @@
+package dsl
+
+import (
+	"fmt"
+
+	"experimental/token"
+)
+
+type TNodeType string
+
+const (
+	NodeTypeEmptyStatement TNodeType = "empty statement"
+
+	NodeTypeStatementBlock = "statement block"
+)
+
+// ----------------------------------------------------------------
+type AST struct {
+	RootNode *ASTNode
+}
+
+// This is for the GOCC/BNF parser, which produces an AST
+func NewAST(iroot interface{}) (*AST, error) {
+	return &AST{
+		RootNode: iroot.(*ASTNode),
+	}, nil
+}
+
+func (this *AST) Print() {
+	this.RootNode.Print()
+}
+
+// ----------------------------------------------------------------
+type ASTNode struct {
+	Token    *token.Token // Nil for tokenless/structural nodes
+	Type     TNodeType
+	Children []*ASTNode
+}
+
+func (this *ASTNode) Print() {
+	this.PrintAux(0)
+}
+func (this *ASTNode) PrintAux(depth int) {
+	for i := 0; i < depth; i++ {
+		fmt.Print("    ")
+	}
+	tok := this.Token
+	fmt.Print("* " + this.Type)
+
+	if tok != nil {
+		//fmt.Printf(" \"%s\" \"%s\"",
+		//	token.TokMap.Id(tok.Type), string(tok.Lit))
+		fmt.Printf(" \"%s\"", string(tok.Lit))
+	}
+	fmt.Println()
+	if this.Children != nil {
+		for _, child := range this.Children {
+			child.PrintAux(depth + 1)
+		}
+	}
+}
+
+// func NewASTNode(itok interface{}, nodeType TNodeType) (*ASTNode, error) {
+// 	return NewASTNodeNestable(itok, nodeType), nil
+// }
+// 
+// // For handling empty expressions.
+// func NewASTNodeEmptyNestable(nodeType TNodeType) *ASTNode {
+// 	return &ASTNode{
+// 		Token:    nil,
+// 		Type:     nodeType,
+// 		Children: nil,
+// 	}
+// }
+// 
+// // For handling empty expressions.
+// func NewASTNodeEmpty(nodeType TNodeType) (*ASTNode, error) {
+// 	return NewASTNodeEmptyNestable(nodeType), nil
+// }
+// 
+// // Strips the leading '$' from field names, or '@' from oosvar names. Not done
+// // in the parser itself due to LR-1 conflicts.
+// func NewASTNodeStripDollarOrAtSign(itok interface{}, nodeType TNodeType) (*ASTNode, error) {
+// 	oldToken := itok.(*token.Token)
+// 	lib.InternalCodingErrorIf(len(oldToken.Lit) < 2)
+// 	lib.InternalCodingErrorIf(oldToken.Lit[0] != '$' && oldToken.Lit[0] != '@')
+// 	newToken := &token.Token{
+// 		Type: oldToken.Type,
+// 		Lit:  oldToken.Lit[1:],
+// 		Pos:  oldToken.Pos,
+// 	}
+// 	return NewASTNodeNestable(newToken, nodeType), nil
+// }
+// 
+// // Strips the leading '${' and trailing '}' from braced field names, or '@{'
+// // and '}' from oosvar names. Not done in the parser itself due to LR-1
+// // conflicts.
+// func NewASTNodeStripDollarOrAtSignAndCurlyBraces(
+// 	itok interface{},
+// 	nodeType TNodeType,
+// ) (*ASTNode, error) {
+// 	oldToken := itok.(*token.Token)
+// 	n := len(oldToken.Lit)
+// 	lib.InternalCodingErrorIf(n < 4)
+// 	lib.InternalCodingErrorIf(oldToken.Lit[0] != '$' && oldToken.Lit[0] != '@')
+// 	lib.InternalCodingErrorIf(oldToken.Lit[1] != '{')
+// 	lib.InternalCodingErrorIf(oldToken.Lit[n-1] != '}')
+// 	newToken := &token.Token{
+// 		Type: oldToken.Type,
+// 		Lit:  oldToken.Lit[2 : n-1],
+// 		Pos:  oldToken.Pos,
+// 	}
+// 	return NewASTNodeNestable(newToken, nodeType), nil
+// }
+// 
+// // Likewise for the leading/trailing double quotes on string literals.  Also,
+// // since string literals can have backslash-escaped double-quotes like
+// // "...\"...\"...", we also unbackslash here.
+// func NewASTNodeStripDoubleQuotePair(
+// 	itok interface{},
+// 	nodeType TNodeType,
+// ) (*ASTNode, error) {
+// 	oldToken := itok.(*token.Token)
+// 	n := len(oldToken.Lit)
+// 	contents := string(oldToken.Lit[1 : n-1])
+// 	contents = strings.Replace(contents, "\\\"", "\"", -1)
+// 	newToken := &token.Token{
+// 		Type: oldToken.Type,
+// 		Lit:  []byte(contents),
+// 		Pos:  oldToken.Pos,
+// 	}
+// 	return NewASTNodeNestable(newToken, nodeType), nil
+// }
+// 
+// // xxx comment why grammar use
+// func NewASTNodeNestable(itok interface{}, nodeType TNodeType) *ASTNode {
+// 	var tok *token.Token = nil
+// 	if itok != nil {
+// 		tok = itok.(*token.Token)
+// 	}
+// 	return &ASTNode{
+// 		Token:    tok,
+// 		Type:     nodeType,
+// 		Children: nil,
+// 	}
+// }
+// 
+// func NewASTNodeZary(itok interface{}, nodeType TNodeType) (*ASTNode, error) {
+// 	parent := NewASTNodeNestable(itok, nodeType)
+// 	convertToZary(parent)
+// 	return parent, nil
+// }
+// 
+// func NewASTNodeUnary(itok, childA interface{}, nodeType TNodeType) (*ASTNode, error) {
+// 	parent := NewASTNodeNestable(itok, nodeType)
+// 	convertToUnary(parent, childA)
+// 	return parent, nil
+// }
+// 
+// // Signature: Token Node Node Type
+// func NewASTNodeBinaryNestable(itok, childA, childB interface{}, nodeType TNodeType) *ASTNode {
+// 	parent := NewASTNodeNestable(itok, nodeType)
+// 	convertToBinary(parent, childA, childB)
+// 	return parent
+// }
+// 
+// // Signature: Token Node Node Type
+// func NewASTNodeBinary(
+// 	itok, childA, childB interface{}, nodeType TNodeType,
+// ) (*ASTNode, error) {
+// 	return NewASTNodeBinaryNestable(itok, childA, childB, nodeType), nil
+// }
+// 
+// func NewASTNodeTernary(itok, childA, childB, childC interface{}, nodeType TNodeType) (*ASTNode, error) {
+// 	parent := NewASTNodeNestable(itok, nodeType)
+// 	convertToTernary(parent, childA, childB, childC)
+// 	return parent, nil
+// }
+// 
+// func NewASTNodeQuaternary(
+// 	itok, childA, childB, childC, childD interface{}, nodeType TNodeType,
+// ) (*ASTNode, error) {
+// 	parent := NewASTNodeNestable(itok, nodeType)
+// 	convertToQuaternary(parent, childA, childB, childC, childD)
+// 	return parent, nil
+// }
+// 
+// // Pass-through expressions in the grammar sometimes need to be turned from
+// // (ASTNode) to (ASTNode, error)
+// func Nestable(iparent interface{}) (*ASTNode, error) {
+// 	return iparent.(*ASTNode), nil
+// }
+// 
+// func convertToZary(iparent interface{}) {
+// 	parent := iparent.(*ASTNode)
+// 	children := make([]*ASTNode, 0)
+// 	parent.Children = children
+// }
+// 
+// // xxx inline this. can be a one-liner.
+// func convertToUnary(iparent interface{}, childA interface{}) {
+// 	parent := iparent.(*ASTNode)
+// 	children := make([]*ASTNode, 1)
+// 	children[0] = childA.(*ASTNode)
+// 	parent.Children = children
+// }
+// 
+// func convertToBinary(iparent interface{}, childA, childB interface{}) {
+// 	parent := iparent.(*ASTNode)
+// 	children := make([]*ASTNode, 2)
+// 	children[0] = childA.(*ASTNode)
+// 	children[1] = childB.(*ASTNode)
+// 	parent.Children = children
+// }
+// 
+// func convertToTernary(iparent interface{}, childA, childB, childC interface{}) {
+// 	parent := iparent.(*ASTNode)
+// 	children := make([]*ASTNode, 3)
+// 	children[0] = childA.(*ASTNode)
+// 	children[1] = childB.(*ASTNode)
+// 	children[2] = childC.(*ASTNode)
+// 	parent.Children = children
+// }
+// 
+// func convertToQuaternary(iparent interface{}, childA, childB, childC, childD interface{}) {
+// 	parent := iparent.(*ASTNode)
+// 	children := make([]*ASTNode, 4)
+// 	children[0] = childA.(*ASTNode)
+// 	children[1] = childB.(*ASTNode)
+// 	children[2] = childC.(*ASTNode)
+// 	children[3] = childD.(*ASTNode)
+// 	parent.Children = children
+// }
+// 
+// func PrependChild(iparent interface{}, ichild interface{}) (*ASTNode, error) {
+// 	parent := iparent.(*ASTNode)
+// 	child := ichild.(*ASTNode)
+// 	if parent.Children == nil {
+// 		convertToUnary(iparent, ichild)
+// 	} else {
+// 		parent.Children = append([]*ASTNode{child}, parent.Children...)
+// 	}
+// 	return parent, nil
+// }
+// 
+// func AppendChild(iparent interface{}, child interface{}) (*ASTNode, error) {
+// 	parent := iparent.(*ASTNode)
+// 	if parent.Children == nil {
+// 		convertToUnary(iparent, child)
+// 	} else {
+// 		parent.Children = append(parent.Children, child.(*ASTNode))
+// 	}
+// 	return parent, nil
+// }
+// 
+// func AdoptChildren(iparent interface{}, ichild interface{}) (*ASTNode, error) {
+// 	parent := iparent.(*ASTNode)
+// 	child := ichild.(*ASTNode)
+// 	parent.Children = child.Children
+// 	child.Children = nil
+// 	return parent, nil
+// }
+// 
+// // TODO: comment
+// func Wrap(inode interface{}) (*ASTNode, error) {
+// 	node := inode.(*ASTNode)
+// 	return node, nil
+// }
+// 
+// func (this *ASTNode) CheckArity(
+// 	arity int,
+// ) error {
+// 	if len(this.Children) != arity {
+// 		return errors.New(
+// 			fmt.Sprintf(
+// 				"AST node arity %d, expected %d",
+// 				len(this.Children), arity,
+// 			),
+// 		)
+// 	} else {
+// 		return nil
+// 	}
+// }
+// 
+// // Tokens are produced by GOCC. However there is an exception: for the ternary
+// // operator I want the AST to have a "?:" token, which GOCC doesn't produce
+// // since nothing is actually spelled like that in the DSL.
+// func NewASTToken(iliteral interface{}, iclonee interface{}) *token.Token {
+// 	literal := iliteral.(string)
+// 	clonee := iclonee.(*token.Token)
+// 	return &token.Token{
+// 		Type: clonee.Type,
+// 		Lit:  []byte(literal),
+// 		Pos:  clonee.Pos,
+// 	}
+// }