grammar iterate

go/build

@@ -1,4 +1,5 @@
 #!/bin/bash
 
 export GOPATH=$(pwd)
+./build-dsl
 go build mlr.go

go/build-dsl

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -euo pipefail
+
 export GOPATH=$(pwd)
 
 # Build the bin/gocc executable:
@@ -13,6 +15,3 @@ bin/gocc -o src/miller/parsing src/miller/parsing/mlr.bnf
 #   src/miller/parsing/parser/
 #   src/miller/parsing/token/
 #   src/miller/parsing/util/
-
-# Run tests:
-go test -v src/miller/dsl/ast_test.go

go/mlr.go

@@ -10,6 +10,10 @@ import (
 	"runtime/pprof"
 	// Miller:
 	"miller/stream"
+	// Temp:
+	"miller/dsl"
+	"miller/parsing/lexer"
+	"miller/parsing/parser"
 )
 
 // ----------------------------------------------------------------
@@ -35,6 +39,11 @@ func main() {
 
 	args := flag.Args()
 
+	// xxx temp
+	if len(args) == 2 && args[0] == "parse" {
+		parse(args[1])
+		return
+	}
 
 	if len(args) < 3 {
 		usage()
@@ -50,6 +59,21 @@ func main() {
 	}
 }
 
+// ----------------------------------------------------------------
+// xxx temp
+func parse(sourceString string) {
+	fmt.Printf("Input: %s\n", sourceString)
+	theLexer := lexer.NewLexer([]byte(sourceString))
+	theParser := parser.NewParser()
+	interfaceAST, err := theParser.Parse(theLexer)
+	if err == nil {
+		interfaceAST.(*dsl.AST).Print()
+	} else {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
 // ----------------------------------------------------------------
 func maybeProfile(cpuprofile *string) {
 	// to do: move to method

go/src/miller/dsl/ast.go

@@ -10,7 +10,7 @@ type TNodeType string
 const (
 	NodeTypeStatementBlock = "StatementBlock"
 	NodeTypeStatement      = "Statement"
-	NodeTypeLeaf           = "Leaf" // xxx temp
+	NodeTypeToken          = "Token"
 )
 
 // ----------------------------------------------------------------
@@ -89,11 +89,11 @@ func NewASTNodeUnary(childA interface{}, nodeType TNodeType) (*ASTNode, error) {
 	return parent, nil
 }
 
-//// xxx temp
-//func Wrap(iparent interface{}) (*ASTNode, error) {
-//	parent := iparent.(*ASTNode)
-//	return parent, nil
-//}
+// Pass-through expressions in the grammar sometimes need to be turned from
+// (ASTNode) to (ASTNode, error)
+func PairNil(iparent interface{}) (*ASTNode, error) {
+	return iparent.(*ASTNode), nil
+}
 
 func MakeZary(iparent interface{}) (*ASTNode, error) {
 	parent := iparent.(*ASTNode)

go/src/miller/dsl/ast_test.go

@@ -40,7 +40,16 @@ func TestPassOne(t *testing.T) {
 }
 
 func TestPassTwo(t *testing.T) {
-	ast, err := testSingle([]byte("$x = 3; $y = 4"))
+	ast, err := testSingle([]byte("$x = 3; $y = 0xef"))
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	fmt.Println("AST:")
+	ast.Print()
+}
+
+func TestPassThree(t *testing.T) {
+	ast, err := testSingle([]byte("$x = 3; $y = 0xef; $z = true"))
 	if err != nil {
 		t.Fatal(err.Error())
 	}

go/src/miller/parsing/lexer/acttab.go

@@ -41,27 +41,75 @@ var ActTab = ActionTable{
 		Ignore: "",
 	},
 	ActionRow{ // S5
-		Accept: 2,
-		Ignore: "",
-	},
-	ActionRow{ // S6
-		Accept: 4,
-		Ignore: "",
-	},
-	ActionRow{ // S7
-		Accept: 3,
-		Ignore: "",
-	},
-	ActionRow{ // S8
-		Accept: 3,
-		Ignore: "",
-	},
-	ActionRow{ // S9
-		Accept: 3,
-		Ignore: "",
-	},
-	ActionRow{ // S10
 		Accept: 5,
 		Ignore: "",
 	},
+	ActionRow{ // S6
+		Accept: 2,
+		Ignore: "",
+	},
+	ActionRow{ // S7
+		Accept: 4,
+		Ignore: "",
+	},
+	ActionRow{ // S8
+		Accept: 0,
+		Ignore: "",
+	},
+	ActionRow{ // S9
+		Accept: 0,
+		Ignore: "",
+	},
+	ActionRow{ // S10
+		Accept: 3,
+		Ignore: "",
+	},
+	ActionRow{ // S11
+		Accept: 3,
+		Ignore: "",
+	},
+	ActionRow{ // S12
+		Accept: 3,
+		Ignore: "",
+	},
+	ActionRow{ // S13
+		Accept: 5,
+		Ignore: "",
+	},
+	ActionRow{ // S14
+		Accept: 0,
+		Ignore: "",
+	},
+	ActionRow{ // S15
+		Accept: 0,
+		Ignore: "",
+	},
+	ActionRow{ // S16
+		Accept: 0,
+		Ignore: "",
+	},
+	ActionRow{ // S17
+		Accept: 5,
+		Ignore: "",
+	},
+	ActionRow{ // S18
+		Accept: 0,
+		Ignore: "",
+	},
+	ActionRow{ // S19
+		Accept: 0,
+		Ignore: "",
+	},
+	ActionRow{ // S20
+		Accept: 0,
+		Ignore: "",
+	},
+	ActionRow{ // S21
+		Accept: 6,
+		Ignore: "",
+	},
+	ActionRow{ // S22
+		Accept: 7,
+		Ignore: "",
+	},
 }

go/src/miller/parsing/lexer/lexer.go

@@ -11,8 +11,8 @@ import (
 
 const (
 	NoState    = -1
-	NumStates  = 11
-	NumSymbols = 13
+	NumStates  = 23
+	NumSymbols = 27
 )
 
 type Lexer struct {
@@ -116,14 +116,28 @@ Lexer symbols:
 0: '$'
 1: '='
 2: '-'
-3: ';'
-4: '_'
-5: ' '
-6: '\t'
-7: '\n'
-8: '\r'
-9: 'a'-'z'
-10: 'A'-'Z'
-11: '0'-'9'
-12: .
+3: '0'
+4: 'x'
+5: ';'
+6: 't'
+7: 'r'
+8: 'u'
+9: 'e'
+10: 'f'
+11: 'a'
+12: 'l'
+13: 's'
+14: 'e'
+15: '_'
+16: ' '
+17: '\t'
+18: '\n'
+19: '\r'
+20: 'a'-'z'
+21: 'A'-'Z'
+22: '0'-'9'
+23: '0'-'9'
+24: 'a'-'f'
+25: 'A'-'F'
+26: .
 */

go/src/miller/parsing/lexer/transitiontable.go

@@ -25,12 +25,18 @@ var TransTab = TransitionTable{
 			return 2
 		case r == 45: // ['-','-']
 			return 3
-		case 48 <= r && r <= 57: // ['0','9']
+		case r == 48: // ['0','0']
 			return 4
-		case r == 59: // [';',';']
+		case 49 <= r && r <= 57: // ['1','9']
 			return 5
-		case r == 61: // ['=','=']
+		case r == 59: // [';',';']
 			return 6
+		case r == 61: // ['=','=']
+			return 7
+		case r == 102: // ['f','f']
+			return 8
+		case r == 116: // ['t','t']
+			return 9
 		}
 		return NoState
 	},
@@ -44,13 +50,13 @@ var TransTab = TransitionTable{
 	func(r rune) int {
 		switch {
 		case 48 <= r && r <= 57: // ['0','9']
-			return 7
+			return 10
 		case 65 <= r && r <= 90: // ['A','Z']
-			return 8
+			return 11
 		case r == 95: // ['_','_']
-			return 9
+			return 12
 		case 97 <= r && r <= 122: // ['a','z']
-			return 8
+			return 11
 		}
 		return NoState
 	},
@@ -58,7 +64,7 @@ var TransTab = TransitionTable{
 	func(r rune) int {
 		switch {
 		case 48 <= r && r <= 57: // ['0','9']
-			return 10
+			return 13
 		}
 		return NoState
 	},
@@ -66,13 +72,17 @@ var TransTab = TransitionTable{
 	func(r rune) int {
 		switch {
 		case 48 <= r && r <= 57: // ['0','9']
-			return 4
+			return 5
+		case r == 120: // ['x','x']
+			return 14
 		}
 		return NoState
 	},
 	// S5
 	func(r rune) int {
 		switch {
+		case 48 <= r && r <= 57: // ['0','9']
+			return 5
 		}
 		return NoState
 	},
@@ -85,42 +95,22 @@ var TransTab = TransitionTable{
 	// S7
 	func(r rune) int {
 		switch {
-		case 48 <= r && r <= 57: // ['0','9']
-			return 7
-		case 65 <= r && r <= 90: // ['A','Z']
-			return 8
-		case r == 95: // ['_','_']
-			return 9
-		case 97 <= r && r <= 122: // ['a','z']
-			return 8
 		}
 		return NoState
 	},
 	// S8
 	func(r rune) int {
 		switch {
-		case 48 <= r && r <= 57: // ['0','9']
-			return 7
-		case 65 <= r && r <= 90: // ['A','Z']
-			return 8
-		case r == 95: // ['_','_']
-			return 9
-		case 97 <= r && r <= 122: // ['a','z']
-			return 8
+		case r == 97: // ['a','a']
+			return 15
 		}
 		return NoState
 	},
 	// S9
 	func(r rune) int {
 		switch {
-		case 48 <= r && r <= 57: // ['0','9']
-			return 7
-		case 65 <= r && r <= 90: // ['A','Z']
-			return 8
-		case r == 95: // ['_','_']
-			return 9
-		case 97 <= r && r <= 122: // ['a','z']
-			return 8
+		case r == 114: // ['r','r']
+			return 16
 		}
 		return NoState
 	},
@@ -129,6 +119,124 @@ var TransTab = TransitionTable{
 		switch {
 		case 48 <= r && r <= 57: // ['0','9']
 			return 10
+		case 65 <= r && r <= 90: // ['A','Z']
+			return 11
+		case r == 95: // ['_','_']
+			return 12
+		case 97 <= r && r <= 122: // ['a','z']
+			return 11
+		}
+		return NoState
+	},
+	// S11
+	func(r rune) int {
+		switch {
+		case 48 <= r && r <= 57: // ['0','9']
+			return 10
+		case 65 <= r && r <= 90: // ['A','Z']
+			return 11
+		case r == 95: // ['_','_']
+			return 12
+		case 97 <= r && r <= 122: // ['a','z']
+			return 11
+		}
+		return NoState
+	},
+	// S12
+	func(r rune) int {
+		switch {
+		case 48 <= r && r <= 57: // ['0','9']
+			return 10
+		case 65 <= r && r <= 90: // ['A','Z']
+			return 11
+		case r == 95: // ['_','_']
+			return 12
+		case 97 <= r && r <= 122: // ['a','z']
+			return 11
+		}
+		return NoState
+	},
+	// S13
+	func(r rune) int {
+		switch {
+		case 48 <= r && r <= 57: // ['0','9']
+			return 13
+		}
+		return NoState
+	},
+	// S14
+	func(r rune) int {
+		switch {
+		case 48 <= r && r <= 57: // ['0','9']
+			return 17
+		case 65 <= r && r <= 70: // ['A','F']
+			return 17
+		case 97 <= r && r <= 102: // ['a','f']
+			return 17
+		}
+		return NoState
+	},
+	// S15
+	func(r rune) int {
+		switch {
+		case r == 108: // ['l','l']
+			return 18
+		}
+		return NoState
+	},
+	// S16
+	func(r rune) int {
+		switch {
+		case r == 117: // ['u','u']
+			return 19
+		}
+		return NoState
+	},
+	// S17
+	func(r rune) int {
+		switch {
+		case 48 <= r && r <= 57: // ['0','9']
+			return 17
+		case 65 <= r && r <= 70: // ['A','F']
+			return 17
+		case 97 <= r && r <= 102: // ['a','f']
+			return 17
+		}
+		return NoState
+	},
+	// S18
+	func(r rune) int {
+		switch {
+		case r == 115: // ['s','s']
+			return 20
+		}
+		return NoState
+	},
+	// S19
+	func(r rune) int {
+		switch {
+		case r == 101: // ['e','e']
+			return 21
+		}
+		return NoState
+	},
+	// S20
+	func(r rune) int {
+		switch {
+		case r == 101: // ['e','e']
+			return 22
+		}
+		return NoState
+	},
+	// S21
+	func(r rune) int {
+		switch {
+		}
+		return NoState
+	},
+	// S22
+	func(r rune) int {
+		switch {
 		}
 		return NoState
 	},

go/src/miller/parsing/parser/actiontable.go

@@ -16,10 +16,12 @@ var actionTab = actionTable{
 		actions: [numSymbols]action{
 			nil,      // INVALID
 			nil,      // $
-			nil,      // md_semicolon
+			nil,      // ;
 			shift(5), // md_token_field_name
 			nil,      // md_token_assign
 			nil,      // md_token_number
+			nil,      // true
+			nil,      // false
 		},
 	},
 	actionRow{ // S1
@@ -27,21 +29,25 @@ var actionTab = actionTable{
 		actions: [numSymbols]action{
 			nil,          // INVALID
 			accept(true), // $
-			nil,          // md_semicolon
+			nil,          // ;
 			nil,          // md_token_field_name
 			nil,          // md_token_assign
 			nil,          // md_token_number
+			nil,          // true
+			nil,          // false
 		},
 	},
 	actionRow{ // S2
 		canRecover: false,
 		actions: [numSymbols]action{
 			nil,       // INVALID
-			reduce(1), // $, reduce: Body
-			shift(6),  // md_semicolon
+			reduce(1), // $, reduce: Root
+			shift(6),  // ;
 			nil,       // md_token_field_name
 			nil,       // md_token_assign
 			nil,       // md_token_number
+			nil,       // true
+			nil,       // false
 		},
 	},
 	actionRow{ // S3
@@ -49,10 +55,12 @@ var actionTab = actionTable{
 		actions: [numSymbols]action{
 			nil,       // INVALID
 			reduce(2), // $, reduce: StatementBlock
-			reduce(2), // md_semicolon, reduce: StatementBlock
+			reduce(2), // ;, reduce: StatementBlock
 			nil,       // md_token_field_name
 			nil,       // md_token_assign
 			nil,       // md_token_number
+			nil,       // true
+			nil,       // false
 		},
 	},
 	actionRow{ // S4
@@ -60,10 +68,12 @@ var actionTab = actionTable{
 		actions: [numSymbols]action{
 			nil,      // INVALID
 			nil,      // $
-			nil,      // md_semicolon
+			nil,      // ;
 			nil,      // md_token_field_name
 			shift(8), // md_token_assign
 			nil,      // md_token_number
+			nil,      // true
+			nil,      // false
 		},
 	},
 	actionRow{ // S5
@@ -71,21 +81,25 @@ var actionTab = actionTable{
 		actions: [numSymbols]action{
 			nil,       // INVALID
 			nil,       // $
-			nil,       // md_semicolon
+			nil,       // ;
 			nil,       // md_token_field_name
-			reduce(5), // md_token_assign, reduce: FieldName
+			reduce(6), // md_token_assign, reduce: FieldName
 			nil,       // md_token_number
+			nil,       // true
+			nil,       // false
 		},
 	},
 	actionRow{ // S6
 		canRecover: false,
 		actions: [numSymbols]action{
-			nil,      // INVALID
-			nil,      // $
-			nil,      // md_semicolon
-			shift(5), // md_token_field_name
-			nil,      // md_token_assign
-			nil,      // md_token_number
+			nil,       // INVALID
+			reduce(3), // $, reduce: StatementBlock
+			reduce(3), // ;, reduce: StatementBlock
+			shift(5),  // md_token_field_name
+			nil,       // md_token_assign
+			nil,       // md_token_number
+			nil,       // true
+			nil,       // false
 		},
 	},
 	actionRow{ // S7
@@ -93,10 +107,12 @@ var actionTab = actionTable{
 		actions: [numSymbols]action{
 			nil,       // INVALID
 			nil,       // $
-			nil,       // md_semicolon
-			nil,       // md_token_field_name
+			nil,       // ;
+			shift(11), // md_token_field_name
 			nil,       // md_token_assign
-			shift(11), // md_token_number
+			shift(12), // md_token_number
+			shift(13), // true
+			shift(14), // false
 		},
 	},
 	actionRow{ // S8
@@ -104,43 +120,90 @@ var actionTab = actionTable{
 		actions: [numSymbols]action{
 			nil,       // INVALID
 			nil,       // $
-			nil,       // md_semicolon
-			nil,       // md_token_field_name
+			nil,       // ;
+			reduce(7), // md_token_field_name, reduce: Assign
 			nil,       // md_token_assign
-			reduce(6), // md_token_number, reduce: Assign
+			reduce(7), // md_token_number, reduce: Assign
+			reduce(7), // true, reduce: Assign
+			reduce(7), // false, reduce: Assign
 		},
 	},
 	actionRow{ // S9
 		canRecover: false,
 		actions: [numSymbols]action{
 			nil,       // INVALID
-			reduce(3), // $, reduce: StatementBlock
-			reduce(3), // md_semicolon, reduce: StatementBlock
+			reduce(4), // $, reduce: StatementBlock
+			reduce(4), // ;, reduce: StatementBlock
 			nil,       // md_token_field_name
 			nil,       // md_token_assign
 			nil,       // md_token_number
+			nil,       // true
+			nil,       // false
 		},
 	},
 	actionRow{ // S10
 		canRecover: false,
 		actions: [numSymbols]action{
 			nil,       // INVALID
-			reduce(4), // $, reduce: Statement
-			reduce(4), // md_semicolon, reduce: Statement
+			reduce(5), // $, reduce: Statement
+			reduce(5), // ;, reduce: Statement
 			nil,       // md_token_field_name
 			nil,       // md_token_assign
 			nil,       // md_token_number
+			nil,       // true
+			nil,       // false
 		},
 	},
 	actionRow{ // S11
 		canRecover: false,
 		actions: [numSymbols]action{
 			nil,       // INVALID
-			reduce(7), // $, reduce: Number
-			reduce(7), // md_semicolon, reduce: Number
+			reduce(8), // $, reduce: AtomOrFunction
+			reduce(8), // ;, reduce: AtomOrFunction
 			nil,       // md_token_field_name
 			nil,       // md_token_assign
 			nil,       // md_token_number
+			nil,       // true
+			nil,       // false
+		},
+	},
+	actionRow{ // S12
+		canRecover: false,
+		actions: [numSymbols]action{
+			nil,       // INVALID
+			reduce(9), // $, reduce: AtomOrFunction
+			reduce(9), // ;, reduce: AtomOrFunction
+			nil,       // md_token_field_name
+			nil,       // md_token_assign
+			nil,       // md_token_number
+			nil,       // true
+			nil,       // false
+		},
+	},
+	actionRow{ // S13
+		canRecover: false,
+		actions: [numSymbols]action{
+			nil,        // INVALID
+			reduce(10), // $, reduce: AtomOrFunction
+			reduce(10), // ;, reduce: AtomOrFunction
+			nil,        // md_token_field_name
+			nil,        // md_token_assign
+			nil,        // md_token_number
+			nil,        // true
+			nil,        // false
+		},
+	},
+	actionRow{ // S14
+		canRecover: false,
+		actions: [numSymbols]action{
+			nil,        // INVALID
+			reduce(11), // $, reduce: AtomOrFunction
+			reduce(11), // ;, reduce: AtomOrFunction
+			nil,        // md_token_field_name
+			nil,        // md_token_assign
+			nil,        // md_token_number
+			nil,        // true
+			nil,        // false
 		},
 	},
 }

go/src/miller/parsing/parser/gototable.go

@@ -12,110 +12,137 @@ type (
 var gotoTab = gotoTable{
 	gotoRow{ // S0
 		-1, // S'
-		1,  // Body
+		1,  // Root
 		2,  // StatementBlock
 		3,  // Statement
 		4,  // FieldName
 		-1, // Assign
-		-1, // Number
+		-1, // AtomOrFunction
 	},
 	gotoRow{ // S1
 		-1, // S'
-		-1, // Body
+		-1, // Root
 		-1, // StatementBlock
 		-1, // Statement
 		-1, // FieldName
 		-1, // Assign
-		-1, // Number
+		-1, // AtomOrFunction
 	},
 	gotoRow{ // S2
 		-1, // S'
-		-1, // Body
+		-1, // Root
 		-1, // StatementBlock
 		-1, // Statement
 		-1, // FieldName
 		-1, // Assign
-		-1, // Number
+		-1, // AtomOrFunction
 	},
 	gotoRow{ // S3
 		-1, // S'
-		-1, // Body
+		-1, // Root
 		-1, // StatementBlock
 		-1, // Statement
 		-1, // FieldName
 		-1, // Assign
-		-1, // Number
+		-1, // AtomOrFunction
 	},
 	gotoRow{ // S4
 		-1, // S'
-		-1, // Body
+		-1, // Root
 		-1, // StatementBlock
 		-1, // Statement
 		-1, // FieldName
 		7,  // Assign
-		-1, // Number
+		-1, // AtomOrFunction
 	},
 	gotoRow{ // S5
 		-1, // S'
-		-1, // Body
+		-1, // Root
 		-1, // StatementBlock
 		-1, // Statement
 		-1, // FieldName
 		-1, // Assign
-		-1, // Number
+		-1, // AtomOrFunction
 	},
 	gotoRow{ // S6
 		-1, // S'
-		-1, // Body
+		-1, // Root
 		-1, // StatementBlock
 		9,  // Statement
 		4,  // FieldName
 		-1, // Assign
-		-1, // Number
+		-1, // AtomOrFunction
 	},
 	gotoRow{ // S7
 		-1, // S'
-		-1, // Body
+		-1, // Root
 		-1, // StatementBlock
 		-1, // Statement
 		-1, // FieldName
 		-1, // Assign
-		10, // Number
+		10, // AtomOrFunction
 	},
 	gotoRow{ // S8
 		-1, // S'
-		-1, // Body
+		-1, // Root
 		-1, // StatementBlock
 		-1, // Statement
 		-1, // FieldName
 		-1, // Assign
-		-1, // Number
+		-1, // AtomOrFunction
 	},
 	gotoRow{ // S9
 		-1, // S'
-		-1, // Body
+		-1, // Root
 		-1, // StatementBlock
 		-1, // Statement
 		-1, // FieldName
 		-1, // Assign
-		-1, // Number
+		-1, // AtomOrFunction
 	},
 	gotoRow{ // S10
 		-1, // S'
-		-1, // Body
+		-1, // Root
 		-1, // StatementBlock
 		-1, // Statement
 		-1, // FieldName
 		-1, // Assign
-		-1, // Number
+		-1, // AtomOrFunction
 	},
 	gotoRow{ // S11
 		-1, // S'
-		-1, // Body
+		-1, // Root
 		-1, // StatementBlock
 		-1, // Statement
 		-1, // FieldName
 		-1, // Assign
-		-1, // Number
+		-1, // AtomOrFunction
+	},
+	gotoRow{ // S12
+		-1, // S'
+		-1, // Root
+		-1, // StatementBlock
+		-1, // Statement
+		-1, // FieldName
+		-1, // Assign
+		-1, // AtomOrFunction
+	},
+	gotoRow{ // S13
+		-1, // S'
+		-1, // Root
+		-1, // StatementBlock
+		-1, // Statement
+		-1, // FieldName
+		-1, // Assign
+		-1, // AtomOrFunction
+	},
+	gotoRow{ // S14
+		-1, // S'
+		-1, // Root
+		-1, // StatementBlock
+		-1, // Statement
+		-1, // FieldName
+		-1, // Assign
+		-1, // AtomOrFunction
 	},
 }

go/src/miller/parsing/parser/parser.go

@@ -11,9 +11,9 @@ import (
 )
 
 const (
-	numProductions = 8
-	numStates      = 12
-	numSymbols     = 13
+	numProductions = 12
+	numStates      = 15
+	numSymbols     = 15
 )
 
 // Stack

go/src/miller/parsing/parser/productionstable.go

@@ -21,7 +21,7 @@ type (
 
 var productionsTable = ProdTab{
 	ProdTabEntry{
-		String: `S' : Body	<<  >>`,
+		String: `S' : Root	<<  >>`,
 		Id:         "S'",
 		NTType:     0,
 		Index:      0,
@@ -31,8 +31,8 @@ var productionsTable = ProdTab{
 		},
 	},
 	ProdTabEntry{
-		String: `Body : StatementBlock	<< dsl.NewAST(X[0]) >>`,
-		Id:         "Body",
+		String: `Root : StatementBlock	<< dsl.NewAST(X[0]) >>`,
+		Id:         "Root",
 		NTType:     1,
 		Index:      1,
 		NumSymbols: 1,
@@ -51,53 +51,93 @@ var productionsTable = ProdTab{
 		},
 	},
 	ProdTabEntry{
-		String: `StatementBlock : StatementBlock md_semicolon Statement	<< dsl.AppendChild(X[0], X[2]) >>`,
+		String: `StatementBlock : StatementBlock ";"	<< dsl.PairNil(X[0]) >>`,
 		Id:         "StatementBlock",
 		NTType:     2,
 		Index:      3,
+		NumSymbols: 2,
+		ReduceFunc: func(X []Attrib) (Attrib, error) {
+			return dsl.PairNil(X[0])
+		},
+	},
+	ProdTabEntry{
+		String: `StatementBlock : StatementBlock ";" Statement	<< dsl.AppendChild(X[0], X[2]) >>`,
+		Id:         "StatementBlock",
+		NTType:     2,
+		Index:      4,
 		NumSymbols: 3,
 		ReduceFunc: func(X []Attrib) (Attrib, error) {
 			return dsl.AppendChild(X[0], X[2])
 		},
 	},
 	ProdTabEntry{
-		String: `Statement : FieldName Assign Number	<< dsl.MakeBinary(X[1], X[0], X[2]) >>`,
+		String: `Statement : FieldName Assign AtomOrFunction	<< dsl.MakeBinary(X[1], X[0], X[2]) >>`,
 		Id:         "Statement",
 		NTType:     3,
-		Index:      4,
+		Index:      5,
 		NumSymbols: 3,
 		ReduceFunc: func(X []Attrib) (Attrib, error) {
 			return dsl.MakeBinary(X[1], X[0], X[2])
 		},
 	},
 	ProdTabEntry{
-		String: `FieldName : md_token_field_name	<< dsl.NewASTNode(X[0], dsl.NodeTypeLeaf) >>`,
+		String: `FieldName : md_token_field_name	<< dsl.NewASTNode(X[0], dsl.NodeTypeToken) >>`,
 		Id:         "FieldName",
 		NTType:     4,
-		Index:      5,
-		NumSymbols: 1,
-		ReduceFunc: func(X []Attrib) (Attrib, error) {
-			return dsl.NewASTNode(X[0], dsl.NodeTypeLeaf)
-		},
-	},
-	ProdTabEntry{
-		String: `Assign : md_token_assign	<< dsl.NewASTNode(X[0], dsl.NodeTypeLeaf) >>`,
-		Id:         "Assign",
-		NTType:     5,
 		Index:      6,
 		NumSymbols: 1,
 		ReduceFunc: func(X []Attrib) (Attrib, error) {
-			return dsl.NewASTNode(X[0], dsl.NodeTypeLeaf)
+			return dsl.NewASTNode(X[0], dsl.NodeTypeToken)
 		},
 	},
 	ProdTabEntry{
-		String: `Number : md_token_number	<< dsl.NewASTNode(X[0], dsl.NodeTypeLeaf) >>`,
-		Id:         "Number",
-		NTType:     6,
+		String: `Assign : md_token_assign	<< dsl.NewASTNode(X[0], dsl.NodeTypeToken) >>`,
+		Id:         "Assign",
+		NTType:     5,
 		Index:      7,
 		NumSymbols: 1,
 		ReduceFunc: func(X []Attrib) (Attrib, error) {
-			return dsl.NewASTNode(X[0], dsl.NodeTypeLeaf)
+			return dsl.NewASTNode(X[0], dsl.NodeTypeToken)
+		},
+	},
+	ProdTabEntry{
+		String: `AtomOrFunction : md_token_field_name	<< dsl.NewASTNode(X[0], dsl.NodeTypeToken) >>`,
+		Id:         "AtomOrFunction",
+		NTType:     6,
+		Index:      8,
+		NumSymbols: 1,
+		ReduceFunc: func(X []Attrib) (Attrib, error) {
+			return dsl.NewASTNode(X[0], dsl.NodeTypeToken)
+		},
+	},
+	ProdTabEntry{
+		String: `AtomOrFunction : md_token_number	<< dsl.NewASTNode(X[0], dsl.NodeTypeToken) >>`,
+		Id:         "AtomOrFunction",
+		NTType:     6,
+		Index:      9,
+		NumSymbols: 1,
+		ReduceFunc: func(X []Attrib) (Attrib, error) {
+			return dsl.NewASTNode(X[0], dsl.NodeTypeToken)
+		},
+	},
+	ProdTabEntry{
+		String: `AtomOrFunction : "true"	<< dsl.NewASTNode(X[0], dsl.NodeTypeToken) >>`,
+		Id:         "AtomOrFunction",
+		NTType:     6,
+		Index:      10,
+		NumSymbols: 1,
+		ReduceFunc: func(X []Attrib) (Attrib, error) {
+			return dsl.NewASTNode(X[0], dsl.NodeTypeToken)
+		},
+	},
+	ProdTabEntry{
+		String: `AtomOrFunction : "false"	<< dsl.NewASTNode(X[0], dsl.NodeTypeToken) >>`,
+		Id:         "AtomOrFunction",
+		NTType:     6,
+		Index:      11,
+		NumSymbols: 1,
+		ReduceFunc: func(X []Attrib) (Attrib, error) {
+			return dsl.NewASTNode(X[0], dsl.NodeTypeToken)
 		},
 	},
 }

go/src/miller/parsing/token/token.go

@@ -121,18 +121,22 @@ var TokMap = TokenMap{
 	typeMap: []string{
 		"INVALID",
 		"$",
-		"md_semicolon",
+		";",
 		"md_token_field_name",
 		"md_token_assign",
 		"md_token_number",
+		"true",
+		"false",
 	},
 
 	idMap: map[string]Type{
 		"INVALID":             0,
 		"$":                   1,
-		"md_semicolon":        2,
+		";":                   2,
 		"md_token_field_name": 3,
 		"md_token_assign":     4,
 		"md_token_number":     5,
+		"true":                6,
+		"false":               7,
 	},
 }