mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-21 10:29:02 +00:00
b06c449384
* incorporate reg-test into "go test" framework * reg-test -> regtest everywhere * update .github/workflows/go.yml
298 lines
10 KiB
BNF
298 lines
10 KiB
BNF
|
|
// ================================================================
|
|
// LEXICAL ELEMENTS
|
|
// ================================================================
|
|
|
|
// ----------------------------------------------------------------
|
|
// CHARACTER CLASSES
|
|
// ----------------------------------------------------------------
|
|
|
|
_letter : 'a'-'z' | 'A'-'Z' ;
|
|
_decdig : '0'-'9' ;
|
|
_hexdig : '0'-'9' | 'a'-'f' | 'A'-'F';
|
|
_bindig : '0'-'1' ;
|
|
_idchar : _letter | _decdig | '_' ;
|
|
!whitespace : ' ' | '\t' | '\n' | '\r' ;
|
|
|
|
// ----------------------------------------------------------------
|
|
// STRING/INT/FLOAT/BOOLEAN LITERALS
|
|
// ----------------------------------------------------------------
|
|
|
|
// Notes on string literals:
|
|
// * " isn't included here -- need \" handling to put that inside strings
|
|
// * GOCC seems to lack a '[^"] notation ...
|
|
_string_literal_element
|
|
: 'A'-'Z' | 'a'-'z' | '0'-'9'
|
|
| ' ' | '!' | '#' | '$' | '%' | '&' | '\'' | '\\'
|
|
| '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/'
|
|
| ':' | ';' | '<' | '=' | '>' | '?' | '@' | '['
|
|
| ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~'
|
|
| ( '\\' '"' )
|
|
| '\u0100'-'\U0010FFFF'
|
|
;
|
|
md_token_string_literal : '"' {_string_literal_element} '"' ;
|
|
|
|
// md_token_regexi : \"([^\\\"]|\\.)*\"\i ;
|
|
|
|
// Notes on int literals:
|
|
// * Leading minus sign is handled via the unary-minus operator, not here.
|
|
md_token_int_literal
|
|
: _decdig { _decdig }
|
|
| '0' 'x' _hexdig { _hexdig }
|
|
| '0' 'b' _bindig { _bindig }
|
|
;
|
|
|
|
// Notes on float literals:
|
|
// * Leading minus sign is handled via the unary-minus operator, not here.
|
|
// * The various shapes are for scientific notation. Examples:
|
|
// 123
|
|
// 123.
|
|
// 123.4
|
|
// .234
|
|
// 1e2
|
|
// 1e-2
|
|
// 1.2e3 1.e3
|
|
// 1.2e-3 1.e-3
|
|
// .2e3
|
|
// .2e-3 1.e-3
|
|
_scinotE : 'e' | 'E' ;
|
|
md_token_float_literal
|
|
: _decdig { _decdig} '.' { _decdig }
|
|
| _decdig { _decdig} _scinotE _decdig { _decdig}
|
|
| _decdig { _decdig} _scinotE '-' _decdig { _decdig}
|
|
| _decdig { _decdig} '.' { _decdig} _scinotE _decdig { _decdig}
|
|
| _decdig { _decdig} '.' { _decdig} _scinotE '-' _decdig { _decdig}
|
|
| { _decdig} '.' _decdig { _decdig} _scinotE _decdig { _decdig}
|
|
| { _decdig} '.' _decdig { _decdig} _scinotE '-' _decdig { _decdig}
|
|
;
|
|
|
|
md_token_M_PI : 'M' '_' 'P' 'I' ;
|
|
md_token_M_E : 'M' '_' 'E' ;
|
|
|
|
// Notes on boolean literals:
|
|
// * true and false should be defined here rather than as "true" / "false"
|
|
// within the grammar below -- this forces them to be keywords, not legal as
|
|
// variable names. We want them as keywords -- we don't want to allow things
|
|
// like 'true = 3'.
|
|
_md_token_true : 't' 'r' 'u' 'e' ;
|
|
_md_token_false : 'f' 'a' 'l' 's' 'e';
|
|
md_token_boolean_literal : ( _md_token_true | _md_token_false );
|
|
|
|
// ----------------------------------------------------------------
|
|
// MILLER CONTEXT VARIABLES
|
|
// ----------------------------------------------------------------
|
|
|
|
md_token_IPS : 'I' 'P' 'S' ;
|
|
md_token_IFS : 'I' 'F' 'S' ;
|
|
md_token_IRS : 'I' 'R' 'S' ;
|
|
|
|
md_token_OPS : 'O' 'P' 'S' ;
|
|
md_token_OFS : 'O' 'F' 'S' ;
|
|
md_token_ORS : 'O' 'R' 'S' ;
|
|
|
|
md_token_NF : 'N' 'F' ;
|
|
md_token_NR : 'N' 'R' ;
|
|
md_token_FNR : 'F' 'N' 'R' ;
|
|
|
|
md_token_FILENAME : 'F' 'I' 'L' 'E' 'N' 'A' 'M' 'E' ;
|
|
md_token_FILENUM : 'F' 'I' 'L' 'E' 'N' 'U' 'M' ;
|
|
|
|
// md_token_ENV : 'E' 'N' 'V' ;
|
|
|
|
// ----------------------------------------------------------------
|
|
// MILLER KEYWORDS
|
|
// ----------------------------------------------------------------
|
|
|
|
// Notes on keywords:
|
|
// * Any new keywords defined here should also be documented
|
|
// in dsl/mlr_dsl_cst.c's mlr_dsl_keyword_usage() et al.
|
|
// * true and false (boolean literals) are also keywords, defined above.
|
|
|
|
md_token_begin : 'b' 'e' 'g' 'i' 'n' ;
|
|
md_token_do : 'd' 'o' ;
|
|
md_token_dump : 'd' 'u' 'm' 'p' ;
|
|
md_token_edump : 'e' 'd' 'u' 'm' 'p' ;
|
|
md_token_elif : 'e' 'l' 'i' 'f' ;
|
|
md_token_else : 'e' 'l' 's' 'e' ;
|
|
md_token_emit : 'e' 'm' 'i' 't' ;
|
|
md_token_end : 'e' 'n' 'd' ;
|
|
md_token_filter : 'f' 'i' 'l' 't' 'e' 'r' ;
|
|
md_token_for : 'f' 'o' 'r' ;
|
|
md_token_if : 'i' 'f' ;
|
|
md_token_in : 'i' 'n' ;
|
|
md_token_while : 'w' 'h' 'i' 'l' 'e' ;
|
|
|
|
md_token_func : 'f' 'u' 'n' 'c' ;
|
|
//md_token_subr : 's' 'u' 'b' 'r' ;
|
|
|
|
//md_token_break : 'b' 'r' 'e' 'a' 'k' ;
|
|
//md_token_call : 'c' 'a' 'l' 'l' ;
|
|
//md_token_continue : 'c' 'o' 'n' 't' 'i' 'n' 'u' 'e' ;
|
|
//md_token_return : 'r' 'e' 't' 'u' 'r' 'n' ;
|
|
|
|
//md_token_unset : 'u' 'n' 's' 'e' 't' ;
|
|
|
|
//md_token_eprint : 'e' 'p' 'r' 'i' 'n' 't' ;
|
|
//md_token_eprintn : 'e' 'p' 'r' 'i' 'n' 't' 'n' ;
|
|
//md_token_print : 'p' 'r' 'i' 'n' 't' ;
|
|
//md_token_printn : 'p' 'r' 'i' 'n' 't' 'n' ;
|
|
//md_token_stderr : 's' 't' 'd' 'e' 'r' 'r' ;
|
|
//md_token_stdout : 's' 't' 'd' 'o' 'u' 't' ;
|
|
//md_token_tee : 't' 'e' 'e' ;
|
|
|
|
//md_token_all : 'a' 'l' 'l' ;
|
|
|
|
md_token_int_type : 'i' 'n' 't' ;
|
|
md_token_float_type : 'f' 'l' 'o' 'a' 't' ;
|
|
//md_token_num_type : 'n' 'u' 'm' ;
|
|
//md_token_bool_type : 'b' 'o' 'o' 'l' ;
|
|
//md_token_str_type : 's' 't' 'r' ;
|
|
//md_token_map_type : 'm' 'a' 'p' ;
|
|
//md_token_array_type : 'a' 'r' 'r' 'a' 'y';
|
|
|
|
// ----------------------------------------------------------------
|
|
// FIELD NAMES, OUT-OF-STREAM VARIABLES, LOCAL VARIABLES
|
|
// ----------------------------------------------------------------
|
|
|
|
// Note: the parser depends on the dollar sign being here. If this is changed,
|
|
// that needs to be changed as well.
|
|
//
|
|
// Also note: if we omit the '$' here and include it in the parser section
|
|
// below as "$", then we get an LR-1 conflict. So this must be dealt with at
|
|
// the AST level.
|
|
md_token_field_name : '$' _idchar { _idchar } ;
|
|
|
|
// This is for literal strings but where the field name might have spaces in it
|
|
// or somesuch.
|
|
_braced_char
|
|
: 'A'-'Z' | 'a'-'z' | '0'-'9'
|
|
| ' ' | '!' | '#' | '$' | '%' | '&' | '\'' | '\\'
|
|
| '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/'
|
|
| ':' | ';' | '<' | '=' | '>' | '?' | '@' | '['
|
|
| ']' | '^' | '_' | '`' | '|' | '~'
|
|
| ( '\\' '{' ) | ( '\\' '}' )
|
|
| '\u0100'-'\U0010FFFF'
|
|
;
|
|
md_token_braced_field_name: '$' '{' _braced_char { _braced_char } '}' ;
|
|
|
|
md_token_full_srec : '$' '*' ;
|
|
|
|
md_token_oosvar_name : '@' _idchar { _idchar } ;
|
|
|
|
// This is for literal strings but where the oosvar name might have spaces in it
|
|
// or somesuch.
|
|
md_token_braced_oosvar_name: '@' '{' _braced_char { _braced_char } '}' ;
|
|
|
|
md_token_full_oosvar : '@' '*' ;
|
|
|
|
// ----------------------------------------------------------------
|
|
// FUNCTIONS AND LOCAL VARIABLES
|
|
|
|
md_token_non_sigil_name : _idchar { _idchar } ;
|
|
|
|
// ----------------------------------------------------------------
|
|
// PANIC TOKEN
|
|
// ----------------------------------------------------------------
|
|
|
|
// This is for testing short-circuiting of "&&", "||", etc in the CST. The
|
|
// sole job of the CST evaluator for this token is to panic the process -- so
|
|
// we'll know if we're evaluating something we should not.
|
|
md_token_panic : '%' '%' '%' 'p' 'a' 'n' 'i' 'c' '%' '%' '%' ;
|
|
|
|
|
|
// ================================================================
|
|
// SYNTAX ELEMENTS
|
|
// ================================================================
|
|
|
|
// ================================================================
|
|
// Parsing goes through three formats:
|
|
//
|
|
// (1) Source code which is a string of characters.
|
|
//
|
|
// (2) Abstract syntax tree (AST):
|
|
//
|
|
// * Parentheses, commas, semicolons, line endings, whitespace are all stripped away
|
|
// * Variable names and literal values remain as leaf nodes of the AST
|
|
// * = + - * / ** {function names} remain as non-leaf nodes of the AST
|
|
//
|
|
// (3) Concrete syntax tree (CST): a reshaping of the AST with pre-processed
|
|
// setup of function pointers to handle each type of statement on a
|
|
// per-record basis. The if/else and/or switch statements to decide what to
|
|
// do with each AST node are done at CST-build time, so they don't need to
|
|
// be re-done when the syntax tree is executed once on every data record.
|
|
//
|
|
// The job of this parser is to turn (1) into (2).
|
|
//
|
|
// Note: This parser accepts many things that are invalid, e.g.
|
|
// * begin{end{}} -- begin/end not at top level
|
|
// * begin{$x=1} -- references to stream records at begin/end (there is no $x when
|
|
// there is no input record yet)
|
|
// * break/continue outside of for/while/do-while
|
|
// * return outside of a function definition
|
|
// * $x=x -- boundvars outside of for-loop variable bindings
|
|
//
|
|
// All of the above are enforced by the CST builder's semantic-analysis logic,
|
|
// which takes this parser's output AST as input. This is done (a) to keep
|
|
// this grammar from being overly complex, and (b) so we can get more
|
|
// informative error messages.
|
|
//
|
|
// For clearer visuals on what the ASTs look like, you can do
|
|
//
|
|
// mlr -n put -v 'your expression goes here'
|
|
//
|
|
// Also see expected outputs from 'mlr regtest', e.g. in regtest/.../*.expout.
|
|
// ================================================================
|
|
|
|
// Import the AST/ASTNode types and functions
|
|
<< import "miller/dsl" >>
|
|
|
|
// ================================================================
|
|
// TOP-LEVEL PRODUCTION RULE FOR THE MILLER DSL
|
|
|
|
Root :
|
|
FuncBlock
|
|
;
|
|
|
|
// ================================================================
|
|
StatementBlockInBraces
|
|
// Defining StatementBlock to be emptyable would have been more elegant ...
|
|
// I got LR-1 conflicts though. :^/
|
|
: "{" "}"
|
|
<< dsl.NewASTNodeZary(nil, dsl.NodeTypeStatementBlock) >>
|
|
|
|
| "{" StatementBlock "}"
|
|
<< dsl.Wrap($1) >>
|
|
;
|
|
|
|
StatementBlock: "fooblock";
|
|
|
|
// ================================================================
|
|
// FUNCTION AND SUBROUTINE DEFINITIONS
|
|
|
|
// ----------------------------------------------------------------
|
|
FuncBlock
|
|
: md_token_func
|
|
md_token_non_sigil_name
|
|
"("
|
|
FuncOrSubrParameterList
|
|
")"
|
|
StatementBlockInBraces
|
|
<<
|
|
dsl.NewASTNodeTernary(
|
|
$0, // "func"
|
|
$1, // function name
|
|
$3, // parameter list
|
|
$5, // { ... }
|
|
dsl.NodeTypeFunctionDefinition,
|
|
);
|
|
>>
|
|
;
|
|
|
|
// ----------------------------------------------------------------
|
|
FuncOrSubrParameterList
|
|
: empty
|
|
| FuncOrSubrParameterList ","
|
|
| FuncOrSubrParameterList "," FuncOrSubrParameter
|
|
;
|
|
|
|
FuncOrSubrParameter : md_token_non_sigil_name;
|