From 356e775e37be1b6070bf0ff6028a20501ea2f30e Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 5 Sep 2016 09:28:40 -0400 Subject: [PATCH] UDF iterate --- c/mapping/rval_evaluators.h | 16 +--- c/mapping/rval_expr_evaluators.c | 109 +----------------------- c/mapping/rval_func_evaluators.c | 139 +++++++++++++++++++++++++++++-- 3 files changed, 140 insertions(+), 124 deletions(-) diff --git a/c/mapping/rval_evaluators.h b/c/mapping/rval_evaluators.h index 1fdfe950e..93aa4fb8c 100644 --- a/c/mapping/rval_evaluators.h +++ b/c/mapping/rval_evaluators.h @@ -62,6 +62,8 @@ extern function_lookup_t FUNCTION_LOOKUP_TABLE[]; // Topmost function: rval_evaluator_t* rval_evaluator_alloc_from_ast(mlr_dsl_ast_node_t* past, int type_inferencing, int context_flags); +rval_evaluator_t* rval_evaluator_alloc_from_ast_aux(mlr_dsl_ast_node_t* pnode, int type_inferencing, int context_flags, + function_lookup_t* fcn_lookup_table); // Next level: rval_evaluator_t* rval_evaluator_alloc_from_field_name(char* field_name, int type_inferencing); @@ -108,20 +110,10 @@ void rval_evaluator_function_usage(FILE* output_stream, char* function_name); void rval_evaluator_list_all_functions_raw(FILE* output_stream); void check_arity_with_report(function_lookup_t fcn_lookup_table[], char* function_name, int user_provided_arity); - -rval_evaluator_t* rval_evaluator_alloc_from_zary_func_name(char* function_name); -rval_evaluator_t* rval_evaluator_alloc_from_unary_func_name(char* fnnm, rval_evaluator_t* parg1); -rval_evaluator_t* rval_evaluator_alloc_from_binary_func_name(char* fnnm, - rval_evaluator_t* parg1, rval_evaluator_t* parg2); -rval_evaluator_t* rval_evaluator_alloc_from_binary_regex_arg2_func_name(char* fnnm, - rval_evaluator_t* parg1, char* regex_string, int ignore_case); -rval_evaluator_t* rval_evaluator_alloc_from_ternary_func_name(char* fnnm, - rval_evaluator_t* parg1, rval_evaluator_t* parg2, rval_evaluator_t* parg3); -rval_evaluator_t* rval_evaluator_alloc_from_ternary_regex_arg2_func_name(char* fnnm, - rval_evaluator_t* parg1, char* regex_string, int ignore_case, rval_evaluator_t* parg3); +rval_evaluator_t* rval_evaluator_alloc_from_operator_or_function(mlr_dsl_ast_node_t* pnode, + int type_inferencing, int context_flags, function_lookup_t* fcn_lookup_table); // These have some shared code that would otherwise be duplicated per-function in containers/mlrval.c. - rval_evaluator_t* rval_evaluator_alloc_from_b_b_func(mv_unary_func_t* pfunc, rval_evaluator_t* parg1); rval_evaluator_t* rval_evaluator_alloc_from_b_bb_and_func(rval_evaluator_t* parg1, rval_evaluator_t* parg2); rval_evaluator_t* rval_evaluator_alloc_from_b_bb_or_func(rval_evaluator_t* parg1, rval_evaluator_t* parg2); diff --git a/c/mapping/rval_expr_evaluators.c b/c/mapping/rval_expr_evaluators.c index 3b18c49dd..5150a1810 100644 --- a/c/mapping/rval_expr_evaluators.c +++ b/c/mapping/rval_expr_evaluators.c @@ -14,9 +14,6 @@ // See comments in rval_evaluators.h // ================================================================ -static rval_evaluator_t* rval_evaluator_alloc_from_ast_aux(mlr_dsl_ast_node_t* pnode, - int type_inferencing, int context_flags, function_lookup_t* fcn_lookup_table); - // ================================================================ // The grammar permits certain statements which are syntactically invalid, (a) because it's awkward to handle // there, and (b) because we get far better control over error messages here (vs. 'syntax error'). @@ -26,10 +23,9 @@ rval_evaluator_t* rval_evaluator_alloc_from_ast(mlr_dsl_ast_node_t* pnode, int t return rval_evaluator_alloc_from_ast_aux(pnode, type_inferencing, context_flags, FUNCTION_LOOKUP_TABLE); } -static rval_evaluator_t* rval_evaluator_alloc_from_ast_aux(mlr_dsl_ast_node_t* pnode, +rval_evaluator_t* rval_evaluator_alloc_from_ast_aux(mlr_dsl_ast_node_t* pnode, int type_inferencing, int context_flags, function_lookup_t* fcn_lookup_table) { - if (pnode->pchildren == NULL) { // leaf node if (pnode->type == MD_AST_NODE_TYPE_FIELD_NAME) { if (context_flags & IN_BEGIN_OR_END) { @@ -87,109 +83,10 @@ static rval_evaluator_t* rval_evaluator_alloc_from_ast_aux(mlr_dsl_ast_node_t* p } else if (pnode->type == MD_AST_NODE_TYPE_ENV) { return rval_evaluator_alloc_from_environment(pnode, type_inferencing, context_flags); - } else { // operator/function + } else { - if ((pnode->type != MD_AST_NODE_TYPE_NON_SIGIL_NAME) - && (pnode->type != MD_AST_NODE_TYPE_OPERATOR)) { + return rval_evaluator_alloc_from_operator_or_function(pnode, type_inferencing, context_flags, fcn_lookup_table); - if (context_flags & IN_MLR_FILTER) { - fprintf(stderr, - "%s: statements in %s filter should only be single expressions evaluating to boolean.\n", - MLR_GLOBALS.bargv0, MLR_GLOBALS.bargv0); - exit(1); - } - - fprintf(stderr, "%s: internal coding error detected in file %s at line %d (node type %s).\n", - MLR_GLOBALS.bargv0, __FILE__, __LINE__, mlr_dsl_ast_node_describe_type(pnode->type)); - exit(1); - } - char* func_name = pnode->text; - - int user_provided_arity = pnode->pchildren->length; - - check_arity_with_report(fcn_lookup_table, func_name, user_provided_arity); - - // xxx move all to rval_func_evaluators.c - rval_evaluator_t* pevaluator = NULL; - if (user_provided_arity == 0) { - pevaluator = rval_evaluator_alloc_from_zary_func_name(func_name); - } else if (user_provided_arity == 1) { - mlr_dsl_ast_node_t* parg1_node = pnode->pchildren->phead->pvvalue; - rval_evaluator_t* parg1 = rval_evaluator_alloc_from_ast_aux(parg1_node, type_inferencing, - context_flags, fcn_lookup_table); - pevaluator = rval_evaluator_alloc_from_unary_func_name(func_name, parg1); - } else if (user_provided_arity == 2) { - mlr_dsl_ast_node_t* parg1_node = pnode->pchildren->phead->pvvalue; - mlr_dsl_ast_node_t* parg2_node = pnode->pchildren->phead->pnext->pvvalue; - int type2 = parg2_node->type; - - if ((streq(func_name, "=~") || streq(func_name, "!=~")) && type2 == MD_AST_NODE_TYPE_STRNUM_LITERAL) { - rval_evaluator_t* parg1 = rval_evaluator_alloc_from_ast_aux(parg1_node, type_inferencing, - context_flags, fcn_lookup_table); - pevaluator = rval_evaluator_alloc_from_binary_regex_arg2_func_name(func_name, - parg1, parg2_node->text, FALSE); - } else if ((streq(func_name, "=~") || streq(func_name, "!=~")) && type2 == MD_AST_NODE_TYPE_REGEXI) { - rval_evaluator_t* parg1 = rval_evaluator_alloc_from_ast_aux(parg1_node, type_inferencing, - context_flags, fcn_lookup_table); - pevaluator = rval_evaluator_alloc_from_binary_regex_arg2_func_name(func_name, parg1, parg2_node->text, - TYPE_INFER_STRING_FLOAT_INT); - } else { - // regexes can still be applied here, e.g. if the 2nd argument is a non-terminal AST: however - // the regexes will be compiled record-by-record rather than once at alloc time, which will - // be slower. - rval_evaluator_t* parg1 = rval_evaluator_alloc_from_ast_aux(parg1_node, type_inferencing, - context_flags, fcn_lookup_table); - rval_evaluator_t* parg2 = rval_evaluator_alloc_from_ast_aux(parg2_node, type_inferencing, - context_flags, fcn_lookup_table); - pevaluator = rval_evaluator_alloc_from_binary_func_name(func_name, parg1, parg2); - } - - } else if (user_provided_arity == 3) { - mlr_dsl_ast_node_t* parg1_node = pnode->pchildren->phead->pvvalue; - mlr_dsl_ast_node_t* parg2_node = pnode->pchildren->phead->pnext->pvvalue; - mlr_dsl_ast_node_t* parg3_node = pnode->pchildren->phead->pnext->pnext->pvvalue; - int type2 = parg2_node->type; - - if ((streq(func_name, "sub") || streq(func_name, "gsub")) && type2 == MD_AST_NODE_TYPE_STRNUM_LITERAL) { - // sub/gsub-regex special case: - rval_evaluator_t* parg1 = rval_evaluator_alloc_from_ast_aux(parg1_node, type_inferencing, - context_flags, fcn_lookup_table); - rval_evaluator_t* parg3 = rval_evaluator_alloc_from_ast_aux(parg3_node, type_inferencing, - context_flags, fcn_lookup_table); - pevaluator = rval_evaluator_alloc_from_ternary_regex_arg2_func_name(func_name, parg1, parg2_node->text, - FALSE, parg3); - - } else if ((streq(func_name, "sub") || streq(func_name, "gsub")) && type2 == MD_AST_NODE_TYPE_REGEXI) { - // sub/gsub-regex special case: - rval_evaluator_t* parg1 = rval_evaluator_alloc_from_ast_aux(parg1_node, type_inferencing, - context_flags, fcn_lookup_table); - rval_evaluator_t* parg3 = rval_evaluator_alloc_from_ast_aux(parg3_node, type_inferencing, - context_flags, fcn_lookup_table); - pevaluator = rval_evaluator_alloc_from_ternary_regex_arg2_func_name(func_name, parg1, parg2_node->text, - TYPE_INFER_STRING_FLOAT_INT, parg3); - - } else { - // regexes can still be applied here, e.g. if the 2nd argument is a non-terminal AST: however - // the regexes will be compiled record-by-record rather than once at alloc time, which will - // be slower. - rval_evaluator_t* parg1 = rval_evaluator_alloc_from_ast_aux(parg1_node, type_inferencing, - context_flags, fcn_lookup_table); - rval_evaluator_t* parg2 = rval_evaluator_alloc_from_ast_aux(parg2_node, type_inferencing, - context_flags, fcn_lookup_table); - rval_evaluator_t* parg3 = rval_evaluator_alloc_from_ast_aux(parg3_node, type_inferencing, - context_flags, fcn_lookup_table); - pevaluator = rval_evaluator_alloc_from_ternary_func_name(func_name, parg1, parg2, parg3); - } - } else { - fprintf(stderr, "Miller: internal coding error: arity for function name \"%s\" misdetected.\n", - func_name); - exit(1); - } - if (pevaluator == NULL) { - fprintf(stderr, "Miller: unrecognized function name \"%s\".\n", func_name); - exit(1); - } - return pevaluator; } } diff --git a/c/mapping/rval_func_evaluators.c b/c/mapping/rval_func_evaluators.c index 8256ac4c5..05e59fa99 100644 --- a/c/mapping/rval_func_evaluators.c +++ b/c/mapping/rval_func_evaluators.c @@ -7,12 +7,33 @@ #include "lib/mlrregex.h" #include "lib/mtrand.h" #include "mapping/mapper.h" +#include "mapping/context_flags.h" #include "mapping/rval_evaluators.h" // ================================================================ // See comments in rval_evaluators.h // ================================================================ +static rval_evaluator_t* rval_evaluator_alloc_from_zary_func_name(char* function_name); + +static rval_evaluator_t* rval_evaluator_alloc_from_unary_func_name(char* fnnm, rval_evaluator_t* parg1); + +static rval_evaluator_t* rval_evaluator_alloc_from_binary_func_name(char* fnnm, + + rval_evaluator_t* parg1, rval_evaluator_t* parg2); + +static rval_evaluator_t* rval_evaluator_alloc_from_binary_regex_arg2_func_name(char* fnnm, + + rval_evaluator_t* parg1, char* regex_string, int ignore_case); + +static rval_evaluator_t* rval_evaluator_alloc_from_ternary_func_name(char* fnnm, + + rval_evaluator_t* parg1, rval_evaluator_t* parg2, rval_evaluator_t* parg3); + +static rval_evaluator_t* rval_evaluator_alloc_from_ternary_regex_arg2_func_name(char* fnnm, + + rval_evaluator_t* parg1, char* regex_string, int ignore_case, rval_evaluator_t* parg3); + // ================================================================ function_lookup_t FUNCTION_LOOKUP_TABLE[] = { @@ -302,7 +323,113 @@ void rval_evaluator_list_all_functions_raw(FILE* output_stream) { } // ================================================================ -rval_evaluator_t* rval_evaluator_alloc_from_zary_func_name(char* function_name) { +rval_evaluator_t* rval_evaluator_alloc_from_operator_or_function(mlr_dsl_ast_node_t* pnode, + int type_inferencing, int context_flags, function_lookup_t* fcn_lookup_table) +{ + + if ((pnode->type != MD_AST_NODE_TYPE_NON_SIGIL_NAME) && (pnode->type != MD_AST_NODE_TYPE_OPERATOR)) { + + // xxx use error code & let the caller fatal it + if (context_flags & IN_MLR_FILTER) { + fprintf(stderr, + "%s: statements in %s filter should only be single expressions evaluating to boolean.\n", + MLR_GLOBALS.bargv0, MLR_GLOBALS.bargv0); + exit(1); + } + + fprintf(stderr, "%s: internal coding error detected in file %s at line %d (node type %s).\n", + MLR_GLOBALS.bargv0, __FILE__, __LINE__, mlr_dsl_ast_node_describe_type(pnode->type)); + exit(1); + } + char* func_name = pnode->text; + + int user_provided_arity = pnode->pchildren->length; + + check_arity_with_report(fcn_lookup_table, func_name, user_provided_arity); + + rval_evaluator_t* pevaluator = NULL; + if (user_provided_arity == 0) { + pevaluator = rval_evaluator_alloc_from_zary_func_name(func_name); + } else if (user_provided_arity == 1) { + mlr_dsl_ast_node_t* parg1_node = pnode->pchildren->phead->pvvalue; + rval_evaluator_t* parg1 = rval_evaluator_alloc_from_ast_aux(parg1_node, type_inferencing, + context_flags, fcn_lookup_table); + pevaluator = rval_evaluator_alloc_from_unary_func_name(func_name, parg1); + } else if (user_provided_arity == 2) { + mlr_dsl_ast_node_t* parg1_node = pnode->pchildren->phead->pvvalue; + mlr_dsl_ast_node_t* parg2_node = pnode->pchildren->phead->pnext->pvvalue; + int type2 = parg2_node->type; + + if ((streq(func_name, "=~") || streq(func_name, "!=~")) && type2 == MD_AST_NODE_TYPE_STRNUM_LITERAL) { + rval_evaluator_t* parg1 = rval_evaluator_alloc_from_ast_aux(parg1_node, type_inferencing, + context_flags, fcn_lookup_table); + pevaluator = rval_evaluator_alloc_from_binary_regex_arg2_func_name(func_name, + parg1, parg2_node->text, FALSE); + } else if ((streq(func_name, "=~") || streq(func_name, "!=~")) && type2 == MD_AST_NODE_TYPE_REGEXI) { + rval_evaluator_t* parg1 = rval_evaluator_alloc_from_ast_aux(parg1_node, type_inferencing, + context_flags, fcn_lookup_table); + pevaluator = rval_evaluator_alloc_from_binary_regex_arg2_func_name(func_name, parg1, parg2_node->text, + TYPE_INFER_STRING_FLOAT_INT); + } else { + // regexes can still be applied here, e.g. if the 2nd argument is a non-terminal AST: however + // the regexes will be compiled record-by-record rather than once at alloc time, which will + // be slower. + rval_evaluator_t* parg1 = rval_evaluator_alloc_from_ast_aux(parg1_node, type_inferencing, + context_flags, fcn_lookup_table); + rval_evaluator_t* parg2 = rval_evaluator_alloc_from_ast_aux(parg2_node, type_inferencing, + context_flags, fcn_lookup_table); + pevaluator = rval_evaluator_alloc_from_binary_func_name(func_name, parg1, parg2); + } + + } else if (user_provided_arity == 3) { + mlr_dsl_ast_node_t* parg1_node = pnode->pchildren->phead->pvvalue; + mlr_dsl_ast_node_t* parg2_node = pnode->pchildren->phead->pnext->pvvalue; + mlr_dsl_ast_node_t* parg3_node = pnode->pchildren->phead->pnext->pnext->pvvalue; + int type2 = parg2_node->type; + + if ((streq(func_name, "sub") || streq(func_name, "gsub")) && type2 == MD_AST_NODE_TYPE_STRNUM_LITERAL) { + // sub/gsub-regex special case: + rval_evaluator_t* parg1 = rval_evaluator_alloc_from_ast_aux(parg1_node, type_inferencing, + context_flags, fcn_lookup_table); + rval_evaluator_t* parg3 = rval_evaluator_alloc_from_ast_aux(parg3_node, type_inferencing, + context_flags, fcn_lookup_table); + pevaluator = rval_evaluator_alloc_from_ternary_regex_arg2_func_name(func_name, parg1, parg2_node->text, + FALSE, parg3); + + } else if ((streq(func_name, "sub") || streq(func_name, "gsub")) && type2 == MD_AST_NODE_TYPE_REGEXI) { + // sub/gsub-regex special case: + rval_evaluator_t* parg1 = rval_evaluator_alloc_from_ast_aux(parg1_node, type_inferencing, + context_flags, fcn_lookup_table); + rval_evaluator_t* parg3 = rval_evaluator_alloc_from_ast_aux(parg3_node, type_inferencing, + context_flags, fcn_lookup_table); + pevaluator = rval_evaluator_alloc_from_ternary_regex_arg2_func_name(func_name, parg1, parg2_node->text, + TYPE_INFER_STRING_FLOAT_INT, parg3); + + } else { + // regexes can still be applied here, e.g. if the 2nd argument is a non-terminal AST: however + // the regexes will be compiled record-by-record rather than once at alloc time, which will + // be slower. + rval_evaluator_t* parg1 = rval_evaluator_alloc_from_ast_aux(parg1_node, type_inferencing, + context_flags, fcn_lookup_table); + rval_evaluator_t* parg2 = rval_evaluator_alloc_from_ast_aux(parg2_node, type_inferencing, + context_flags, fcn_lookup_table); + rval_evaluator_t* parg3 = rval_evaluator_alloc_from_ast_aux(parg3_node, type_inferencing, + context_flags, fcn_lookup_table); + pevaluator = rval_evaluator_alloc_from_ternary_func_name(func_name, parg1, parg2, parg3); + } + } else { + fprintf(stderr, "Miller: internal coding error: arity for function name \"%s\" misdetected.\n", + func_name); + exit(1); + } + if (pevaluator == NULL) { + fprintf(stderr, "Miller: unrecognized function name \"%s\".\n", func_name); + exit(1); + } + return pevaluator; +} +// ================================================================ +static rval_evaluator_t* rval_evaluator_alloc_from_zary_func_name(char* function_name) { if (streq(function_name, "urand")) { return rval_evaluator_alloc_from_x_z_func(f_z_urand_func); } else if (streq(function_name, "urand32")) { @@ -315,7 +442,7 @@ rval_evaluator_t* rval_evaluator_alloc_from_zary_func_name(char* function_name) } // ================================================================ -rval_evaluator_t* rval_evaluator_alloc_from_unary_func_name(char* fnnm, rval_evaluator_t* parg1) { +static rval_evaluator_t* rval_evaluator_alloc_from_unary_func_name(char* fnnm, rval_evaluator_t* parg1) { if (streq(fnnm, "!")) { return rval_evaluator_alloc_from_b_b_func(b_b_not_func, parg1); } else if (streq(fnnm, "+")) { return rval_evaluator_alloc_from_x_x_func(x_x_upos_func, parg1); } else if (streq(fnnm, "-")) { return rval_evaluator_alloc_from_x_x_func(x_x_uneg_func, parg1); @@ -385,7 +512,7 @@ rval_evaluator_t* rval_evaluator_alloc_from_unary_func_name(char* fnnm, rval_eva } // ================================================================ -rval_evaluator_t* rval_evaluator_alloc_from_binary_func_name(char* fnnm, +static rval_evaluator_t* rval_evaluator_alloc_from_binary_func_name(char* fnnm, rval_evaluator_t* parg1, rval_evaluator_t* parg2) { if (streq(fnnm, "&&")) { return rval_evaluator_alloc_from_b_bb_and_func(parg1, parg2); @@ -425,7 +552,7 @@ rval_evaluator_t* rval_evaluator_alloc_from_binary_func_name(char* fnnm, } else { return NULL; } } -rval_evaluator_t* rval_evaluator_alloc_from_binary_regex_arg2_func_name(char* fnnm, +static rval_evaluator_t* rval_evaluator_alloc_from_binary_regex_arg2_func_name(char* fnnm, rval_evaluator_t* parg1, char* regex_string, int ignore_case) { if (streq(fnnm, "=~")) { @@ -436,7 +563,7 @@ rval_evaluator_t* rval_evaluator_alloc_from_binary_regex_arg2_func_name(char* fn } // ================================================================ -rval_evaluator_t* rval_evaluator_alloc_from_ternary_func_name(char* fnnm, +static rval_evaluator_t* rval_evaluator_alloc_from_ternary_func_name(char* fnnm, rval_evaluator_t* parg1, rval_evaluator_t* parg2, rval_evaluator_t* parg3) { if (streq(fnnm, "sub")) { @@ -458,7 +585,7 @@ rval_evaluator_t* rval_evaluator_alloc_from_ternary_func_name(char* fnnm, } else { return NULL; } } -rval_evaluator_t* rval_evaluator_alloc_from_ternary_regex_arg2_func_name(char* fnnm, +static rval_evaluator_t* rval_evaluator_alloc_from_ternary_regex_arg2_func_name(char* fnnm, rval_evaluator_t* parg1, char* regex_string, int ignore_case, rval_evaluator_t* parg3) { if (streq(fnnm, "sub")) {