Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-08-04 21:43:38 +00:00
Code Issues Projects Releases Wiki Activity

allow zero statements in the DSL

Browse source
This commit is contained in:
John Kerl 2020-09-08 09:21:14 -04:00
parent 83e6dfd708
commit 670a6ca762
10 changed files with 12650 additions and 11568 deletions
Download patch file Download diff file Expand all files Collapse all files

33
go/src/miller/dsl/ast.go
View file

@ -11,17 +11,19 @@ import (
type TNodeType string
const (
NodeTypeStringLiteral TNodeType = "StringLiteral"
NodeTypeIntLiteral = "IntLiteral"
NodeTypeFloatLiteral = "FloatLiteral"
NodeTypeBoolLiteral = "BoolLiteral"
NodeTypeArrayLiteral = "ArrayLiteral"
NodeTypeMapLiteral = "MapLiteral"
NodeTypeMapLiteralKeyValuePair = "MapLiteralKeyValuePair"
NodeTypeArrayOrMapIndexAccess = "ArrayOrMapIndexAccess"
NodeTypeArraySliceAccess = "ArraySliceAccess"
NodeTypeArraySliceEmptyLowerIndex = "ArraySliceEmptyLowerIndex"
NodeTypeArraySliceEmptyUpperIndex = "ArraySliceEmptyUpperIndex"
NodeTypeEmptyStatement TNodeType = "EmptyStatement"
NodeTypeStringLiteral = "StringLiteral"
NodeTypeIntLiteral = "IntLiteral"
NodeTypeFloatLiteral = "FloatLiteral"
NodeTypeBoolLiteral = "BoolLiteral"
NodeTypeArrayLiteral = "ArrayLiteral"
NodeTypeMapLiteral = "MapLiteral"
NodeTypeMapLiteralKeyValuePair = "MapLiteralKeyValuePair"
NodeTypeArrayOrMapIndexAccess = "ArrayOrMapIndexAccess"
NodeTypeArraySliceAccess = "ArraySliceAccess"
NodeTypeArraySliceEmptyLowerIndex = "ArraySliceEmptyLowerIndex"
NodeTypeArraySliceEmptyUpperIndex = "ArraySliceEmptyUpperIndex"
NodeTypeDirectFieldName = "DirectFieldName"
NodeTypeIndirectFieldName = "IndirectFieldName"
@ -116,6 +118,15 @@ func NewASTNode(itok interface{}, nodeType TNodeType) (*ASTNode, error) {
return NewASTNodeNestable(itok, nodeType), nil
}
// For handling empty expressions.
func NewASTNodeEmpty(nodeType TNodeType) (*ASTNode, error) {
return &ASTNode{
Token: nil,
Type: nodeType,
Children: nil,
}, nil
}
// Strips the leading '$' from field names. Not done in the parser itself due
// to LR-1 conflicts.
func NewASTNodeStripDollarPlease(itok interface{}, nodeType TNodeType) (*ASTNode, error) {

6
go/src/miller/dsl/cst/lvalues.go
View file

@ -25,9 +25,9 @@ func BuildAssignableNode(
case dsl.NodeTypeFullSrec:
return BuildFullSrecLvalueNode(astNode)
break
//case dsl.NodeTypeIndexedLvalue:
// return xxx(astNode)
// break
//case dsl.NodeTypeIndexedLvalue:
// return xxx(astNode)
// break
}
// xxx temp

11
go/src/miller/dsl/cst/root.go
View file

@ -29,14 +29,21 @@ func Build(ast *dsl.AST) (*Root, error) {
if ast.Root == nil {
return nil, errors.New("Cannot build CST from nil AST root")
}
cstRoot := BuildRoot()
// They can do mlr put '': there are simply zero statements.
if ast.Root.Type == dsl.NodeTypeEmptyStatement {
return cstRoot, nil
}
if ast.Root.Type != dsl.NodeTypeStatementBlock {
return nil, errors.New(
"CST root build: on-statement-block AST root node unhandled",
"CST root build: non-statement-block AST root node unhandled",
)
}
astChildren := ast.Root.Children
cstRoot := BuildRoot()
for _, astChild := range astChildren {
statement, err := BuildStatementNode(astChild)
if err != nil {

264
go/src/miller/parsing/lexer/acttab.go
View file

@ -29,7 +29,7 @@ var ActTab = ActionTable{
Ignore: "!whitespace",
},
ActionRow{ // S2
Accept: 54,
Accept: 55,
Ignore: "",
},
ActionRow{ // S3
@ -41,75 +41,75 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S5
Accept: 50,
Accept: 51,
Ignore: "",
},
ActionRow{ // S6
Accept: 39,
Accept: 40,
Ignore: "",
},
ActionRow{ // S7
Accept: 57,
Ignore: "",
},
ActionRow{ // S8
Accept: 58,
Ignore: "",
},
ActionRow{ // S8
Accept: 59,
Ignore: "",
},
ActionRow{ // S9
Accept: 47,
Ignore: "",
},
ActionRow{ // S10
Accept: 42,
Ignore: "",
},
ActionRow{ // S11
Accept: 75,
Ignore: "",
},
ActionRow{ // S12
Accept: 43,
Ignore: "",
},
ActionRow{ // S13
Accept: 46,
Ignore: "",
},
ActionRow{ // S14
Accept: 48,
Ignore: "",
},
ActionRow{ // S10
Accept: 43,
Ignore: "",
},
ActionRow{ // S11
Accept: 76,
Ignore: "",
},
ActionRow{ // S12
Accept: 44,
Ignore: "",
},
ActionRow{ // S13
Accept: 47,
Ignore: "",
},
ActionRow{ // S14
Accept: 49,
Ignore: "",
},
ActionRow{ // S15
Accept: 60,
Accept: 61,
Ignore: "",
},
ActionRow{ // S16
Accept: 60,
Accept: 61,
Ignore: "",
},
ActionRow{ // S17
Accept: 25,
Accept: 26,
Ignore: "",
},
ActionRow{ // S18
Accept: 2,
Ignore: "",
},
ActionRow{ // S19
Accept: 35,
Ignore: "",
},
ActionRow{ // S20
Accept: 3,
Ignore: "",
},
ActionRow{ // S19
Accept: 36,
Ignore: "",
},
ActionRow{ // S20
Accept: 4,
Ignore: "",
},
ActionRow{ // S21
Accept: 33,
Accept: 34,
Ignore: "",
},
ActionRow{ // S22
Accept: 24,
Accept: 25,
Ignore: "",
},
ActionRow{ // S23
@ -129,15 +129,15 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S27
Accept: 4,
Ignore: "",
},
ActionRow{ // S28
Accept: 5,
Ignore: "",
},
ActionRow{ // S28
Accept: 6,
Ignore: "",
},
ActionRow{ // S29
Accept: 38,
Accept: 39,
Ignore: "",
},
ActionRow{ // S30
@ -149,23 +149,23 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S32
Accept: 76,
Ignore: "",
},
ActionRow{ // S33
Accept: 37,
Ignore: "",
},
ActionRow{ // S34
Accept: 77,
Ignore: "",
},
ActionRow{ // S33
Accept: 38,
Ignore: "",
},
ActionRow{ // S34
Accept: 78,
Ignore: "",
},
ActionRow{ // S35
Accept: 55,
Accept: 56,
Ignore: "",
},
ActionRow{ // S36
Accept: 32,
Accept: 33,
Ignore: "",
},
ActionRow{ // S37
@ -173,7 +173,7 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S38
Accept: 59,
Accept: 60,
Ignore: "",
},
ActionRow{ // S39
@ -181,23 +181,23 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S40
Accept: 8,
Accept: 9,
Ignore: "",
},
ActionRow{ // S41
Accept: 6,
Ignore: "",
},
ActionRow{ // S42
Accept: 6,
Ignore: "",
},
ActionRow{ // S43
Accept: 7,
Ignore: "",
},
ActionRow{ // S42
Accept: 7,
Ignore: "",
},
ActionRow{ // S43
Accept: 8,
Ignore: "",
},
ActionRow{ // S44
Accept: 6,
Accept: 7,
Ignore: "",
},
ActionRow{ // S45
@ -205,43 +205,43 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S46
Accept: 22,
Accept: 23,
Ignore: "",
},
ActionRow{ // S47
Accept: 28,
Accept: 29,
Ignore: "",
},
ActionRow{ // S48
Accept: 56,
Accept: 57,
Ignore: "",
},
ActionRow{ // S49
Accept: 19,
Accept: 20,
Ignore: "",
},
ActionRow{ // S50
Accept: 16,
Accept: 17,
Ignore: "",
},
ActionRow{ // S51
Accept: 18,
Accept: 19,
Ignore: "",
},
ActionRow{ // S52
Accept: 51,
Accept: 52,
Ignore: "",
},
ActionRow{ // S53
Accept: 44,
Ignore: "",
},
ActionRow{ // S54
Accept: 45,
Ignore: "",
},
ActionRow{ // S54
Accept: 46,
Ignore: "",
},
ActionRow{ // S55
Accept: 52,
Accept: 53,
Ignore: "",
},
ActionRow{ // S56
@ -249,19 +249,19 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S57
Accept: 17,
Accept: 18,
Ignore: "",
},
ActionRow{ // S58
Accept: 49,
Accept: 50,
Ignore: "",
},
ActionRow{ // S59
Accept: 20,
Accept: 21,
Ignore: "",
},
ActionRow{ // S60
Accept: 61,
Accept: 62,
Ignore: "",
},
ActionRow{ // S61
@ -273,27 +273,27 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S63
Accept: 40,
Accept: 41,
Ignore: "",
},
ActionRow{ // S64
Accept: 36,
Accept: 37,
Ignore: "",
},
ActionRow{ // S65
Accept: 31,
Accept: 32,
Ignore: "",
},
ActionRow{ // S66
Accept: 29,
Accept: 30,
Ignore: "",
},
ActionRow{ // S67
Accept: 34,
Accept: 35,
Ignore: "",
},
ActionRow{ // S68
Accept: 41,
Accept: 42,
Ignore: "",
},
ActionRow{ // S69
@ -317,11 +317,11 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S74
Accept: 70,
Accept: 71,
Ignore: "",
},
ActionRow{ // S75
Accept: 71,
Accept: 72,
Ignore: "",
},
ActionRow{ // S76
@ -337,11 +337,11 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S79
Accept: 13,
Accept: 14,
Ignore: "",
},
ActionRow{ // S80
Accept: 27,
Accept: 28,
Ignore: "",
},
ActionRow{ // S81
@ -353,19 +353,19 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S83
Accept: 12,
Accept: 13,
Ignore: "",
},
ActionRow{ // S84
Accept: 26,
Accept: 27,
Ignore: "",
},
ActionRow{ // S85
Accept: 30,
Accept: 31,
Ignore: "",
},
ActionRow{ // S86
Accept: 59,
Accept: 60,
Ignore: "",
},
ActionRow{ // S87
@ -373,15 +373,15 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S88
Accept: 11,
Accept: 12,
Ignore: "",
},
ActionRow{ // S89
Accept: 23,
Accept: 24,
Ignore: "",
},
ActionRow{ // S90
Accept: 53,
Accept: 54,
Ignore: "",
},
ActionRow{ // S91
@ -389,11 +389,11 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S92
Accept: 21,
Accept: 22,
Ignore: "",
},
ActionRow{ // S93
Accept: 61,
Accept: 62,
Ignore: "",
},
ActionRow{ // S94
@ -405,19 +405,19 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S96
Accept: 61,
Accept: 62,
Ignore: "",
},
ActionRow{ // S97
Accept: 60,
Accept: 61,
Ignore: "",
},
ActionRow{ // S98
Accept: 14,
Accept: 15,
Ignore: "",
},
ActionRow{ // S99
Accept: 15,
Accept: 16,
Ignore: "",
},
ActionRow{ // S100
@ -425,35 +425,35 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S101
Accept: 72,
Accept: 73,
Ignore: "",
},
ActionRow{ // S102
Accept: 65,
Ignore: "",
},
ActionRow{ // S103
Accept: 64,
Ignore: "",
},
ActionRow{ // S104
Accept: 66,
Ignore: "",
},
ActionRow{ // S105
Accept: 68,
ActionRow{ // S103
Accept: 65,
Ignore: "",
},
ActionRow{ // S106
ActionRow{ // S104
Accept: 67,
Ignore: "",
},
ActionRow{ // S107
ActionRow{ // S105
Accept: 69,
Ignore: "",
},
ActionRow{ // S106
Accept: 68,
Ignore: "",
},
ActionRow{ // S107
Accept: 70,
Ignore: "",
},
ActionRow{ // S108
Accept: 10,
Accept: 11,
Ignore: "",
},
ActionRow{ // S109
@ -465,7 +465,7 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S111
Accept: 9,
Accept: 10,
Ignore: "",
},
ActionRow{ // S112
@ -477,7 +477,7 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S114
Accept: 61,
Accept: 62,
Ignore: "",
},
ActionRow{ // S115
@ -489,11 +489,11 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S117
Accept: 61,
Accept: 62,
Ignore: "",
},
ActionRow{ // S118
Accept: 61,
Accept: 62,
Ignore: "",
},
ActionRow{ // S119
@ -505,7 +505,7 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S121
Accept: 62,
Accept: 63,
Ignore: "",
},
ActionRow{ // S122
@ -513,7 +513,7 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S123
Accept: 61,
Accept: 62,
Ignore: "",
},
ActionRow{ // S124
@ -521,11 +521,11 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S125
Accept: 61,
Accept: 62,
Ignore: "",
},
ActionRow{ // S126
Accept: 61,
Accept: 62,
Ignore: "",
},
ActionRow{ // S127
@ -533,7 +533,7 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S128
Accept: 62,
Accept: 63,
Ignore: "",
},
ActionRow{ // S129
@ -541,7 +541,7 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S130
Accept: 61,
Accept: 62,
Ignore: "",
},
ActionRow{ // S131
@ -561,7 +561,7 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S135
Accept: 74,
Accept: 75,
Ignore: "",
},
ActionRow{ // S136
@ -569,7 +569,7 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S137
Accept: 73,
Accept: 74,
Ignore: "",
},
ActionRow{ // S138
@ -581,7 +581,7 @@ var ActTab = ActionTable{
Ignore: "",
},
ActionRow{ // S140
Accept: 63,
Accept: 64,
Ignore: "",
},
}

5
go/src/miller/parsing/mlr.bnf
View file

@ -275,7 +275,10 @@ Root :
<<dsl.NewAST($0) >> ;
StatementBlock
: Statement
: empty
<< dsl.NewASTNodeEmpty(dsl.NodeTypeEmptyStatement) >>
| Statement
<< dsl.NewASTNodeUnary(nil, $0, dsl.NodeTypeStatementBlock) >>
| StatementBlock ";"

23103
go/src/miller/parsing/parser/actiontable.go
View file

File diff suppressed because it is too large Load diff

4
go/src/miller/parsing/parser/parser.go
View file

@ -11,9 +11,9 @@ import (
)
const (
numProductions = 125
numProductions = 126
numStates = 1051
numSymbols = 115
numSymbols = 116
)
// Stack

636
go/src/miller/parsing/parser/productionstable.go
View file

File diff suppressed because it is too large Load diff

154
go/src/miller/parsing/token/token.go
View file

@ -121,6 +121,7 @@ var TokMap = TokenMap{
typeMap: []string{
"INVALID",
"$",
"empty",
";",
"=",
"[",
@ -202,81 +203,82 @@ var TokMap = TokenMap{
idMap: map[string]Type{
"INVALID": 0,
"$": 1,
";": 2,
"=": 3,
"[": 4,
"]": 5,
"md_token_field_name": 6,
"$[": 7,
"md_token_full_srec": 8,
"||=": 9,
"^^=": 10,
"&&=": 11,
"|=": 12,
"^=": 13,
"<<=": 14,
">>=": 15,
"+=": 16,
".=": 17,
"-=": 18,
"*=": 19,
"/=": 20,
"//=": 21,
"%=": 22,
"**=": 23,
"?": 24,
":": 25,
"||": 26,
"^^": 27,
"&&": 28,
"=~": 29,
"!=~": 30,
"==": 31,
"!=": 32,
">": 33,
">=": 34,
"<": 35,
"<=": 36,
"|": 37,
"^": 38,
"&": 39,
"<<": 40,
">>": 41,
"+": 42,
"-": 43,
".+": 44,
".-": 45,
".": 46,
"*": 47,
"/": 48,
"//": 49,
"%": 50,
".*": 51,
"./": 52,
".//": 53,
"!": 54,
"~": 55,
"**": 56,
"(": 57,
")": 58,
"md_token_string_literal": 59,
"md_token_int_literal": 60,
"md_token_float_literal": 61,
"md_token_boolean_literal": 62,
"md_token_panic": 63,
"md_token_IPS": 64,
"md_token_IFS": 65,
"md_token_IRS": 66,
"md_token_OPS": 67,
"md_token_OFS": 68,
"md_token_ORS": 69,
"md_token_NF": 70,
"md_token_NR": 71,
"md_token_FNR": 72,
"md_token_FILENAME": 73,
"md_token_FILENUM": 74,
",": 75,
"{": 76,
"}": 77,
"empty": 2,
";": 3,
"=": 4,
"[": 5,
"]": 6,
"md_token_field_name": 7,
"$[": 8,
"md_token_full_srec": 9,
"||=": 10,
"^^=": 11,
"&&=": 12,
"|=": 13,
"^=": 14,
"<<=": 15,
">>=": 16,
"+=": 17,
".=": 18,
"-=": 19,
"*=": 20,
"/=": 21,
"//=": 22,
"%=": 23,
"**=": 24,
"?": 25,
":": 26,
"||": 27,
"^^": 28,
"&&": 29,
"=~": 30,
"!=~": 31,
"==": 32,
"!=": 33,
">": 34,
">=": 35,
"<": 36,
"<=": 37,
"|": 38,
"^": 39,
"&": 40,
"<<": 41,
">>": 42,
"+": 43,
"-": 44,
".+": 45,
".-": 46,
".": 47,
"*": 48,
"/": 49,
"//": 50,
"%": 51,
".*": 52,
"./": 53,
".//": 54,
"!": 55,
"~": 56,
"**": 57,
"(": 58,
")": 59,
"md_token_string_literal": 60,
"md_token_int_literal": 61,
"md_token_float_literal": 62,
"md_token_boolean_literal": 63,
"md_token_panic": 64,
"md_token_IPS": 65,
"md_token_IFS": 66,
"md_token_IRS": 67,
"md_token_OPS": 68,
"md_token_OFS": 69,
"md_token_ORS": 70,
"md_token_NF": 71,
"md_token_NR": 72,
"md_token_FNR": 73,
"md_token_FILENAME": 74,
"md_token_FILENUM": 75,
",": 76,
"{": 77,
"}": 78,
},
}

2
go/todo.txt
View file

@ -100,9 +100,7 @@ PARSER/LEXER PLAN:
----------------------------------------------------------------
NITS/NON-IMMEDIATE:
* "...\"..." into string-literal parsing ...
* address all manner of xxx and TODO comments
* support whitespace-only DSL strings (as NOPs), either in the parser or outside ...
* AST insertions: make a simple NodeFromToken & have all interface{} be *ASTNode, not *token.Token
* mlr --help-for w/ stdout redirect for manpage -- ?
* mlr verb -h -> stdout & exit 0