Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-08-02 12:32:21 +00:00
Code Issues Projects Releases Wiki Activity

AST-reorg iterate

Browse source
This commit is contained in:
John Kerl 2016-01-24 13:22:44 -05:00
parent 3dcc8a967c
commit 554ab8fd93
11 changed files with 80 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

24
c/containers/mlr_dsl_ast.c
View file

@ -3,12 +3,12 @@
#include "containers/mlr_dsl_ast.h"
// ----------------------------------------------------------------
mlr_dsl_ast_root_t* mlr_dsl_ast_root_alloc() {
mlr_dsl_ast_root_t* proot = mlr_malloc_or_die(sizeof(mlr_dsl_ast_root_t));
proot->pbegin_statements = sllv_alloc();
proot->pmain_statements = sllv_alloc();
proot->pend_statements = sllv_alloc();
return proot;
mlr_dsl_ast_t* mlr_dsl_ast_alloc() {
mlr_dsl_ast_t* past = mlr_malloc_or_die(sizeof(mlr_dsl_ast_t));
past->pbegin_statements = sllv_alloc();
past->pmain_statements = sllv_alloc();
past->pend_statements = sllv_alloc();
return past;
}
// ----------------------------------------------------------------
@ -131,17 +131,17 @@ char* mlr_dsl_ast_node_describe_type(int type) {
}
// ----------------------------------------------------------------
static void mlr_dsl_ast_root_free_statement_list(sllv_t* plist) {
static void mlr_dsl_ast_free_statement_list(sllv_t* plist) {
for (sllve_t* pe = plist->phead; pe != NULL; pe = pe->pnext)
mlr_dsl_ast_node_free(pe->pvvalue);
sllv_free(plist);
}
void mlr_dsl_ast_root_free(mlr_dsl_ast_root_t* proot) {
mlr_dsl_ast_root_free_statement_list(proot->pbegin_statements);
mlr_dsl_ast_root_free_statement_list(proot->pmain_statements);
mlr_dsl_ast_root_free_statement_list(proot->pend_statements);
free(proot);
void mlr_dsl_ast_free(mlr_dsl_ast_t* past) {
mlr_dsl_ast_free_statement_list(past->pbegin_statements);
mlr_dsl_ast_free_statement_list(past->pmain_statements);
mlr_dsl_ast_free_statement_list(past->pend_statements);
free(past);
}
// ----------------------------------------------------------------

8
c/containers/mlr_dsl_ast.h
View file

@ -23,11 +23,11 @@
#define MLR_DSL_AST_NODE_TYPE_BEGIN 0xff22
#define MLR_DSL_AST_NODE_TYPE_END 0xff44
typedef struct _mlr_dsl_ast_root_t {
typedef struct _mlr_dsl_ast_t {
sllv_t* pbegin_statements;
sllv_t* pmain_statements;
sllv_t* pend_statements;
} mlr_dsl_ast_root_t;
} mlr_dsl_ast_t;
typedef struct _mlr_dsl_ast_node_t {
char* text;
@ -35,7 +35,7 @@ typedef struct _mlr_dsl_ast_node_t {
sllv_t* pchildren;
} mlr_dsl_ast_node_t;
mlr_dsl_ast_root_t* mlr_dsl_ast_root_alloc();
mlr_dsl_ast_t* mlr_dsl_ast_alloc();
mlr_dsl_ast_node_t* mlr_dsl_ast_node_alloc(char* text, int type);
@ -64,6 +64,6 @@ char* mlr_dsl_ast_node_describe_type(int type);
void mlr_dsl_ast_node_free(mlr_dsl_ast_node_t* pnode);
void mlr_dsl_ast_root_free(mlr_dsl_ast_root_t* proot);
void mlr_dsl_ast_free(mlr_dsl_ast_t* past);
#endif // MLR_DSL_AST_H

10
c/containers/parse_trie.c
View file

@ -10,7 +10,7 @@ static void parse_trie_add_string_aux(parse_trie_node_t* pnode, char* string, in
// ----------------------------------------------------------------
parse_trie_t* parse_trie_alloc() {
parse_trie_t* ptrie = mlr_malloc_or_die(sizeof(parse_trie_t));
ptrie->proot = parse_trie_node_alloc(0);
ptrie->past = parse_trie_node_alloc(0);
ptrie->maxlen = 0;
return ptrie;
}
@ -37,14 +37,14 @@ static void parse_trie_free_node_descendants(parse_trie_node_t* pnode) {
}
void parse_trie_free(parse_trie_t* ptrie) {
parse_trie_free_node_descendants(ptrie->proot);
free(ptrie->proot);
parse_trie_free_node_descendants(ptrie->past);
free(ptrie->past);
free(ptrie);
}
// ----------------------------------------------------------------
void parse_trie_print(parse_trie_t* ptrie) {
parse_trie_node_t* pnode = ptrie->proot;
parse_trie_node_t* pnode = ptrie->past;
printf("PARSE TRIE DUMP START\n");
if (pnode != NULL) {
parse_trie_print_aux(pnode, 0);
@ -70,7 +70,7 @@ static void parse_trie_print_aux(parse_trie_node_t* pnode, int depth) {
// ----------------------------------------------------------------
void parse_trie_add_string(parse_trie_t* ptrie, char* string, int stridx) {
int len = strlen(string);
parse_trie_add_string_aux(ptrie->proot, string, stridx, strlen(string));
parse_trie_add_string_aux(ptrie->past, string, stridx, strlen(string));
if (len > ptrie->maxlen)
ptrie->maxlen = len;
}

6
c/containers/parse_trie.h
View file

@ -17,7 +17,7 @@ typedef struct _parse_trie_node_t {
int strlen; // length of string stored ending here; -1 if not end of string.
} parse_trie_node_t;
typedef struct _parse_trie_t {
parse_trie_node_t* proot;
parse_trie_node_t* past;
int maxlen;
} parse_trie_t;
@ -51,7 +51,7 @@ void parse_trie_add_string(parse_trie_t* ptrie, char* string, int stridx);
static inline int parse_trie_ring_match(parse_trie_t* ptrie, char* buf, int sob, int buflen, int mask,
int* pstridx, int* pmatchlen)
{
parse_trie_node_t* pnode = ptrie->proot;
parse_trie_node_t* pnode = ptrie->past;
parse_trie_node_t* pnext;
parse_trie_node_t* pterm = NULL;
for (int i = 0; i < buflen; i++) {
@ -74,7 +74,7 @@ static inline int parse_trie_ring_match(parse_trie_t* ptrie, char* buf, int sob,
}
static inline int parse_trie_match(parse_trie_t* ptrie, char* p, char* e, int* pstridx, int* pmatchlen) {
parse_trie_node_t* pnode = ptrie->proot;
parse_trie_node_t* pnode = ptrie->past;
parse_trie_node_t* pnext;
parse_trie_node_t* pterm = NULL;
for ( ; p < e; p++) {

39
c/dsls/mlr_dsl_parse.y
View file

@ -20,7 +20,7 @@
%token_type {mlr_dsl_ast_node_t*}
%default_type {mlr_dsl_ast_node_t*}
%extra_argument {sllv_t* pasts}
%extra_argument {mlr_dsl_ast_t* past}
//void token_destructor(mlr_dsl_ast_node_t t) {
// printf("In token_destructor t->text(%s)=t->type(%lf)\n", t->text, t->type);
@ -30,8 +30,8 @@
%parse_accept {
//printf("End of parse.\n");
//printf("End of parse: proot=%p\n", past->proot);
//mlr_dsl_ast_node_print(past->proot);
//printf("End of parse: past=%p\n", past->past);
//mlr_dsl_ast_node_print(past->past);
//printf("End of parse: tree end\n");
}
@ -68,12 +68,13 @@ mlr_dsl_statement ::= mlr_dsl_end_emit.
// ================================================================
mlr_dsl_srec_assignment(A) ::= mlr_dsl_field_name(B) MLR_DSL_ASSIGN(O) mlr_dsl_ternary(C). {
A = mlr_dsl_ast_node_alloc_binary(O->text, MLR_DSL_AST_NODE_TYPE_SREC_ASSIGNMENT, B, C);
sllv_add(pasts, A);
// xxx temp
sllv_add(past->pmain_statements, A);
}
mlr_dsl_top_level_oosvar_assignment(A) ::= mlr_dsl_oosvar_assignment(B). {
A = B;
sllv_add(pasts, A);
sllv_add(past->pmain_statements, A);
}
mlr_dsl_oosvar_assignment(A) ::= mlr_dsl_oosvar_name(B) MLR_DSL_ASSIGN(O) mlr_dsl_ternary(C). {
@ -82,21 +83,21 @@ mlr_dsl_oosvar_assignment(A) ::= mlr_dsl_oosvar_name(B) MLR_DSL_ASSIGN(O) mlr_d
mlr_dsl_bare_boolean(A) ::= mlr_dsl_ternary(B). {
A = B;
sllv_add(pasts, A);
sllv_add(past->pmain_statements, A);
}
mlr_dsl_record_filter(A) ::= MLR_DSL_FILTER(O) mlr_dsl_ternary(B). {
A = mlr_dsl_ast_node_alloc_unary(O->text, MLR_DSL_AST_NODE_TYPE_FILTER, B);
sllv_add(pasts, A);
sllv_add(past->pmain_statements, A);
}
mlr_dsl_expression_gate(A) ::= MLR_DSL_GATE(O) mlr_dsl_ternary(B). {
A = mlr_dsl_ast_node_alloc_unary(O->text, MLR_DSL_AST_NODE_TYPE_GATE, B);
sllv_add(pasts, A);
sllv_add(past->pmain_statements, A);
}
mlr_dsl_top_level_emit(A) ::= mlr_dsl_emit(B). {
A = B;
sllv_add(pasts, A);
sllv_add(past->pmain_statements, A);
}
// Given "emit @a,@b,@c": since this is a bottom-up parser, we get first the "@a",
@ -122,57 +123,57 @@ mlr_dsl_emit_args(A) ::= mlr_dsl_emit_args(B) MLR_DSL_COMMA mlr_dsl_oosvar_name(
// ================================================================
mlr_dsl_begin_oosvar_assignment(A) ::= MLR_DSL_BEGIN(X) mlr_dsl_oosvar_assignment(B). {
A = mlr_dsl_ast_node_alloc_unary(X->text, MLR_DSL_AST_NODE_TYPE_BEGIN, B);
sllv_add(pasts, A);
sllv_add(past->pmain_statements, A);
}
mlr_dsl_begin_bare_boolean(A) ::= MLR_DSL_BEGIN(X) mlr_dsl_ternary(B). {
A = mlr_dsl_ast_node_alloc_unary(X->text, MLR_DSL_AST_NODE_TYPE_BEGIN, B);
sllv_add(pasts, A);
sllv_add(past->pmain_statements, A);
}
mlr_dsl_begin_filter(A) ::= MLR_DSL_BEGIN(X) MLR_DSL_FILTER(O) mlr_dsl_ternary(B). {
B = mlr_dsl_ast_node_alloc_unary(O->text, MLR_DSL_AST_NODE_TYPE_FILTER, B);
A = mlr_dsl_ast_node_alloc_unary(X->text, MLR_DSL_AST_NODE_TYPE_BEGIN, B);
sllv_add(pasts, A);
sllv_add(past->pmain_statements, A);
}
mlr_dsl_begin_gate(A) ::= MLR_DSL_BEGIN(X) MLR_DSL_GATE(O) mlr_dsl_ternary(B). {
B = mlr_dsl_ast_node_alloc_unary(O->text, MLR_DSL_AST_NODE_TYPE_GATE, B);
A = mlr_dsl_ast_node_alloc_unary(X->text, MLR_DSL_AST_NODE_TYPE_BEGIN, B);
sllv_add(pasts, A);
sllv_add(past->pmain_statements, A);
}
mlr_dsl_begin_emit(A) ::= MLR_DSL_BEGIN(X) mlr_dsl_emit(B). {
A = mlr_dsl_ast_node_alloc_unary(X->text, MLR_DSL_AST_NODE_TYPE_BEGIN, B);
sllv_add(pasts, A);
sllv_add(past->pmain_statements, A);
}
mlr_dsl_end_oosvar_assignment(A) ::= MLR_DSL_END(X) mlr_dsl_oosvar_assignment(B). {
A = mlr_dsl_ast_node_alloc_unary(X->text, MLR_DSL_AST_NODE_TYPE_END, B);
sllv_add(pasts, A);
sllv_add(past->pmain_statements, A);
}
mlr_dsl_end_bare_boolean(A) ::= MLR_DSL_END(X) mlr_dsl_ternary(B). {
A = mlr_dsl_ast_node_alloc_unary(X->text, MLR_DSL_AST_NODE_TYPE_END, B);
sllv_add(pasts, A);
sllv_add(past->pmain_statements, A);
}
mlr_dsl_end_filter(A) ::= MLR_DSL_END(X) MLR_DSL_FILTER(O) mlr_dsl_ternary(B). {
B = mlr_dsl_ast_node_alloc_unary(O->text, MLR_DSL_AST_NODE_TYPE_FILTER, B);
A = mlr_dsl_ast_node_alloc_unary(X->text, MLR_DSL_AST_NODE_TYPE_END, B);
sllv_add(pasts, A);
sllv_add(past->pmain_statements, A);
}
mlr_dsl_end_gate(A) ::= MLR_DSL_END(X) MLR_DSL_GATE(O) mlr_dsl_ternary(B). {
B = mlr_dsl_ast_node_alloc_unary(O->text, MLR_DSL_AST_NODE_TYPE_GATE, B);
A = mlr_dsl_ast_node_alloc_unary(X->text, MLR_DSL_AST_NODE_TYPE_END, B);
sllv_add(pasts, A);
sllv_add(past->pmain_statements, A);
}
mlr_dsl_end_emit(A) ::= MLR_DSL_END(X) mlr_dsl_emit(B). {
A = mlr_dsl_ast_node_alloc_unary(X->text, MLR_DSL_AST_NODE_TYPE_END, B);
sllv_add(pasts, A);
sllv_add(past->pmain_statements, A);
}
// ================================================================

18
c/dsls/mlr_dsl_wrapper.c
View file

@ -17,7 +17,7 @@ int mlr_dsl_lemon_parser_parse_token(
void *pvparser, /* The parser */
int yymajor, /* The major token code number */
mlr_dsl_ast_node_t* yyminor, /* The value for the token */
sllv_t* pasts); /* Optional %extra_argument parameter */
mlr_dsl_ast_t* past); /* Optional %extra_argument parameter */
void mlr_dsl_lemon_parser_free(
void *pvparser, /* The parser to be deleted */
void (*freeProc)(void*)); /* Function used to reclaim memory */
@ -27,14 +27,14 @@ void mlr_dsl_lemon_parser_free(
// http://flex.sourceforge.net/manual/Extra-Data.html
// Returns linked list of mlr_dsl_ast_node_t*.
static sllv_t* mlr_dsl_parse_inner(yyscan_t scanner, void* pvparser, mlr_dsl_ast_node_t** ppnode) {
static mlr_dsl_ast_t* mlr_dsl_parse_inner(yyscan_t scanner, void* pvparser, mlr_dsl_ast_node_t** ppnode) {
int lex_code;
int parse_code;
sllv_t* pasts = sllv_alloc();
mlr_dsl_ast_t* past = mlr_dsl_ast_alloc();
do {
lex_code = mlr_dsl_lexer_lex(scanner);
mlr_dsl_ast_node_t* plexed_node = *ppnode;
parse_code = mlr_dsl_lemon_parser_parse_token(pvparser, lex_code, plexed_node, pasts);
parse_code = mlr_dsl_lemon_parser_parse_token(pvparser, lex_code, plexed_node, past);
if (parse_code == 0)
return NULL;
} while (lex_code > 0);
@ -42,16 +42,16 @@ static sllv_t* mlr_dsl_parse_inner(yyscan_t scanner, void* pvparser, mlr_dsl_ast
fprintf(stderr, "The scanner encountered an error.\n");
return NULL;
}
parse_code = mlr_dsl_lemon_parser_parse_token(pvparser, 0, NULL, pasts);
parse_code = mlr_dsl_lemon_parser_parse_token(pvparser, 0, NULL, past);
if (parse_code == 0)
return NULL;
return pasts;
return past;
}
// ----------------------------------------------------------------
// Returns linked list of mlr_dsl_ast_node_t*.
sllv_t* mlr_dsl_parse(char* string) {
mlr_dsl_ast_t* mlr_dsl_parse(char* string) {
mlr_dsl_ast_node_t* pnode = NULL;
yyscan_t scanner;
mlr_dsl_lexer_lex_init_extra(&pnode, &scanner);
@ -65,7 +65,7 @@ sllv_t* mlr_dsl_parse(char* string) {
mlr_dsl_lexer__switch_to_buffer (buf, scanner);
}
sllv_t* pasts = mlr_dsl_parse_inner(scanner, pvparser, &pnode);
mlr_dsl_ast_t* past = mlr_dsl_parse_inner(scanner, pvparser, &pnode);
if (buf != NULL)
mlr_dsl_lexer__delete_buffer(buf, scanner);
@ -73,7 +73,7 @@ sllv_t* mlr_dsl_parse(char* string) {
mlr_dsl_lexer_lex_destroy(scanner);
mlr_dsl_lemon_parser_free(pvparser, free);
return pasts;
return past;
}
// ----------------------------------------------------------------

4
c/dsls/mlr_dsl_wrapper.h
View file

@ -1,9 +1,7 @@
#ifndef MLR_DSL_WRAPPER_H
#define MLR_DSL_WRAPPER_H
#include "../containers/mlr_dsl_ast.h"
#include "../containers/sllv.h"
// Returns linked list of mlr_dsl_ast_node_t*.
sllv_t* mlr_dsl_parse(char* string);
mlr_dsl_ast_t* mlr_dsl_parse(char* string);
#endif // MLR_DSL_WRAPPER_H

2
c/mapping/lrec_evaluators.h
View file

@ -8,7 +8,7 @@
#define TYPE_INFER_STRING_FLOAT 0xce09
#define TYPE_INFER_STRING_ONLY 0xce0a
lrec_evaluator_t* lrec_evaluator_alloc_from_ast(mlr_dsl_ast_node_t* proot, int type_inferencing);
lrec_evaluator_t* lrec_evaluator_alloc_from_ast(mlr_dsl_ast_node_t* past, int type_inferencing);
void lrec_evaluator_list_functions(FILE* output_stream, char* leader);
// Pass function_name == NULL to get usage for all functions:
void lrec_evaluator_function_usage(FILE* output_stream, char* function_name);

10
c/mapping/mapper_filter.c
View file

@ -88,7 +88,9 @@ static mapper_t* mapper_filter_parse_cli(int* pargi, int argc, char** argv) {
}
mlr_dsl_expression = argv[(*pargi)++];
sllv_t* pasts = mlr_dsl_parse(mlr_dsl_expression);
// xxx temp iterate
mlr_dsl_ast_t* past = mlr_dsl_parse(mlr_dsl_expression);
sllv_t* pasts = past->pmain_statements;
if (pasts == NULL) {
fprintf(stderr, "%s %s: syntax error on DSL parse of '%s'\n",
argv[0], verb, mlr_dsl_expression);
@ -98,16 +100,16 @@ static mapper_t* mapper_filter_parse_cli(int* pargi, int argc, char** argv) {
fprintf(stderr, "%s %s: multiple expressions are unsupported.\n", argv[0], verb);
return NULL;
}
mlr_dsl_ast_node_t* past = sllv_pop(pasts);
mlr_dsl_ast_node_t* past1 = sllv_pop(pasts);
sllv_free(pasts);
// For just dev-testing the parser, you can do
// mlr filter -v 'expression goes here' /dev/null
if (print_asts) {
mlr_dsl_ast_node_print(past);
mlr_dsl_ast_node_print(past1);
}
return mapper_filter_alloc(pstate, past, type_inferencing, do_exclude);
return mapper_filter_alloc(pstate, past1, type_inferencing, do_exclude);
}
// ----------------------------------------------------------------

29
c/mapping/mapper_put.c
View file

@ -10,13 +10,13 @@
typedef struct _mapper_put_state_t {
ap_state_t* pargp;
sllv_t* pasts;
mlr_dsl_ast_t* past;
mlr_dsl_cst_t* pcst;
int at_begin;
lhmsv_t* poosvars;
} mapper_put_state_t;
static mapper_t* mapper_put_alloc(ap_state_t* pargp, sllv_t* pasts, int type_inferencing);
static mapper_t* mapper_put_alloc(ap_state_t* pargp, mlr_dsl_ast_t* past, int type_inferencing);
static void mapper_put_usage(FILE* o, char* argv0, char* verb);
static mapper_t* mapper_put_parse_cli(int* pargi, int argc, char** argv);
static sllv_t* mapper_put_process(lrec_t* pinrec, context_t* pctx, void* pvstate);
@ -103,7 +103,9 @@ static mapper_t* mapper_put_parse_cli(int* pargi, int argc, char** argv) {
mlr_dsl_expression = argv[(*pargi)++];
// Linked list of mlr_dsl_ast_node_t*.
sllv_t* pasts = mlr_dsl_parse(mlr_dsl_expression);
// xxx temp iterate
mlr_dsl_ast_t* past = mlr_dsl_parse(mlr_dsl_expression);
sllv_t* pasts = past->pmain_statements;
if (pasts == NULL) {
fprintf(stderr, "%s %s: syntax error on DSL parse of '%s'\n",
argv[0], verb, mlr_dsl_expression);
@ -117,14 +119,16 @@ static mapper_t* mapper_put_parse_cli(int* pargi, int argc, char** argv) {
mlr_dsl_ast_node_print(pe->pvvalue);
}
return mapper_put_alloc(pstate, pasts, type_inferencing);
return mapper_put_alloc(pstate, past, type_inferencing);
}
// ----------------------------------------------------------------
static mapper_t* mapper_put_alloc(ap_state_t* pargp, sllv_t* pasts, int type_inferencing) {
static mapper_t* mapper_put_alloc(ap_state_t* pargp, mlr_dsl_ast_t* past, int type_inferencing) {
mapper_put_state_t* pstate = mlr_malloc_or_die(sizeof(mapper_put_state_t));
pstate->pargp = pargp;
pstate->pasts = pasts;
pstate->past = past;
// xxx temp iterate
sllv_t* pasts = past->pmain_statements;
pstate->pcst = cst_alloc(pasts, type_inferencing);
pstate->at_begin = TRUE;
pstate->poosvars = lhmsv_alloc();
@ -140,11 +144,14 @@ static mapper_t* mapper_put_alloc(ap_state_t* pargp, sllv_t* pasts, int type_inf
static void mapper_put_free(mapper_t* pmapper) {
mapper_put_state_t* pstate = pmapper->pvstate;
for (sllve_t* pe = pstate->pasts->phead; pe != NULL; pe = pe->pnext) {
mlr_dsl_ast_node_t* past = pe->pvvalue;
mlr_dsl_ast_node_free(past);
}
sllv_free(pstate->pasts);
// xxx temp iterate
// for (sllve_t* pe = pstate->pasts->phead; pe != NULL; pe = pe->pnext) {
// mlr_dsl_ast_node_t* past = pe->pvvalue;
// mlr_dsl_ast_node_free(past);
// }
// sllv_free(pstate->pasts);
cst_free(pstate->pcst);
for (lhmsve_t* pe = pstate->poosvars->phead; pe != NULL; pe = pe->pnext)
mv_free(pe->pvvalue);

1
c/todo.txt
View file

@ -21,6 +21,7 @@ TOP OF LIST
o next up:
!! begin{}/end{} into the grammar
- renmae ast_root to ast
!! accumulate begin/end/main statements within the parser itself; root-noded AST ...
!! filter is also gating. it should not.