From 362c0cd643eff094c6332ff36ca0189857df88a5 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Tue, 10 Jan 2017 09:05:00 -0500 Subject: [PATCH] put JSON-quoting flags under put/filter CLI control --- c/dsl/mlr_dsl_cst.h | 2 + c/mapping/mapper_put_or_filter.c | 70 +++++++++++++++++++++----------- c/todo.txt | 13 ++++-- 3 files changed, 58 insertions(+), 27 deletions(-) diff --git a/c/dsl/mlr_dsl_cst.h b/c/dsl/mlr_dsl_cst.h index 80d4e6dcd..db64d9017 100644 --- a/c/dsl/mlr_dsl_cst.h +++ b/c/dsl/mlr_dsl_cst.h @@ -69,6 +69,8 @@ typedef struct _cst_outputs_t { int* pshould_emit_rec; sllv_t* poutrecs; char* oosvar_flatten_separator; + int json_quote_int_keys; + int json_quote_non_string_values; cli_writer_opts_t* pwriter_opts; } cst_outputs_t; diff --git a/c/mapping/mapper_put_or_filter.c b/c/mapping/mapper_put_or_filter.c index 9da41d1fe..6cf969443 100644 --- a/c/mapping/mapper_put_or_filter.c +++ b/c/mapping/mapper_put_or_filter.c @@ -21,10 +21,12 @@ typedef struct _mapper_put_or_filter_state_t { mlr_dsl_cst_t* pcst; int at_begin; - mlhmmv_root_t* poosvars; + mlhmmv_root_t* poosvars; int trace_execution; char* oosvar_flatten_separator; + int json_quote_int_keys; + int json_quote_non_string_values; int flush_every_record; cli_writer_opts_t* pwriter_opts; @@ -66,6 +68,8 @@ static mapper_t* mapper_put_or_filter_alloc( int negate_final_filter, // mlr filter -x int type_inferencing, char* oosvar_flatten_separator, + int json_quote_int_keys, + int json_quote_non_string_values, int flush_every_record, cli_writer_opts_t* pwriter_opts, cli_writer_opts_t* pmain_writer_opts); @@ -182,6 +186,8 @@ static void shared_usage(FILE* o, char* argv0, char* verb) { fprintf(o, " with no inference to int.\n"); fprintf(o, "--oflatsep {string}: Separator to use when flattening multi-level @-variables\n"); fprintf(o, " to output records for emit. Default \"%s\".\n", DEFAULT_OOSVAR_FLATTEN_SEPARATOR); + fprintf(o, "--jknquoteint: For dump output (JSON-formatted), do not quote map keys if non-string.\n"); + fprintf(o, "--jvquoteall: For dump output (JSON-formatted), quote map values even if non-string.\n"); fprintf(o, "-f {filename}: the DSL expression is taken from the specified file rather\n"); fprintf(o, " than from the command line. Outer single quotes wrapping the expression\n"); fprintf(o, " should not be placed in the file. If -f is specified more than once,\n"); @@ -239,6 +245,8 @@ static mapper_t* shared_parse_cli(int* pargi, int argc, char** argv, int trace_parse = FALSE; int trace_execution = FALSE; char* oosvar_flatten_separator = DEFAULT_OOSVAR_FLATTEN_SEPARATOR; + int json_quote_int_keys = TRUE; // JSON standard + int json_quote_non_string_values = FALSE; // JSON standard int flush_every_record = TRUE; cli_writer_opts_t* pwriter_opts = mlr_malloc_or_die(sizeof(cli_writer_opts_t)); @@ -318,6 +326,12 @@ static mapper_t* shared_parse_cli(int* pargi, int argc, char** argv, } oosvar_flatten_separator = argv[argi+1]; argi += 2; + } else if (streq(argv[argi], "--jknquoteint")) { + json_quote_int_keys = FALSE; + argi += 1; + } else if (streq(argv[argi], "--jvquoteall")) { + json_quote_non_string_values = TRUE; + argi += 1; } else if (streq(argv[argi], "--no-fflush") || streq(argv[argi], "--no-flush")) { flush_every_record = FALSE; argi += 1; @@ -372,7 +386,7 @@ static mapper_t* shared_parse_cli(int* pargi, int argc, char** argv, *pargi = argi; return mapper_put_or_filter_alloc(mlr_dsl_expression, print_ast, trace_stack_allocation, trace_execution, past, put_output_disabled, do_final_filter, negate_final_filter, type_inferencing, oosvar_flatten_separator, - flush_every_record, pwriter_opts, pmain_writer_opts); + json_quote_int_keys, json_quote_non_string_values, flush_every_record, pwriter_opts, pmain_writer_opts); } // ---------------------------------------------------------------- @@ -387,6 +401,8 @@ static mapper_t* mapper_put_or_filter_alloc( int negate_final_filter, // mlr filter -x int type_inferencing, char* oosvar_flatten_separator, + int json_quote_int_keys, + int json_quote_non_string_values, int flush_every_record, cli_writer_opts_t* pwriter_opts, cli_writer_opts_t* pmain_writer_opts) @@ -397,15 +413,17 @@ static mapper_t* mapper_put_or_filter_alloc( pstate->past = past; pstate->pcst = mlr_dsl_cst_alloc(past, print_ast, trace_stack_allocation, type_inferencing, flush_every_record, do_final_filter, negate_final_filter); - pstate->at_begin = TRUE; - pstate->put_output_disabled = put_output_disabled; - pstate->poosvars = mlhmmv_root_alloc(); - pstate->trace_execution = trace_execution; - pstate->oosvar_flatten_separator = oosvar_flatten_separator; - pstate->flush_every_record = flush_every_record; - pstate->plocal_stack = local_stack_alloc(); - pstate->ploop_stack = loop_stack_alloc(); - pstate->pwriter_opts = pwriter_opts; + pstate->at_begin = TRUE; + pstate->put_output_disabled = put_output_disabled; + pstate->poosvars = mlhmmv_root_alloc(); + pstate->trace_execution = trace_execution; + pstate->oosvar_flatten_separator = oosvar_flatten_separator; + pstate->json_quote_int_keys = json_quote_int_keys; + pstate->json_quote_non_string_values = json_quote_non_string_values; + pstate->flush_every_record = flush_every_record; + pstate->plocal_stack = local_stack_alloc(); + pstate->ploop_stack = loop_stack_alloc(); + pstate->pwriter_opts = pwriter_opts; cli_merge_writer_opts(pstate->pwriter_opts, pmain_writer_opts); @@ -507,10 +525,12 @@ static sllv_t* mapper_put_or_filter_process(lrec_t* pinrec, context_t* pctx, voi .trace_execution = pstate->trace_execution, }; cst_outputs_t cst_outputs = (cst_outputs_t) { - .pshould_emit_rec = &should_emit_rec, - .poutrecs = poutrecs, - .oosvar_flatten_separator = pstate->oosvar_flatten_separator, - .pwriter_opts = pstate->pwriter_opts, + .pshould_emit_rec = &should_emit_rec, + .poutrecs = poutrecs, + .oosvar_flatten_separator = pstate->oosvar_flatten_separator, + .json_quote_int_keys = pstate->json_quote_int_keys, + .json_quote_non_string_values = pstate->json_quote_non_string_values, + .pwriter_opts = pstate->pwriter_opts, }; string_array_free(pregex_captures); @@ -536,10 +556,12 @@ static sllv_t* mapper_put_or_filter_process(lrec_t* pinrec, context_t* pctx, voi .trace_execution = pstate->trace_execution, }; cst_outputs_t cst_outputs = (cst_outputs_t) { - .pshould_emit_rec = &should_emit_rec, - .poutrecs = poutrecs, - .oosvar_flatten_separator = pstate->oosvar_flatten_separator, - .pwriter_opts = pstate->pwriter_opts, + .pshould_emit_rec = &should_emit_rec, + .poutrecs = poutrecs, + .oosvar_flatten_separator = pstate->oosvar_flatten_separator, + .json_quote_int_keys = pstate->json_quote_int_keys, + .json_quote_non_string_values = pstate->json_quote_non_string_values, + .pwriter_opts = pstate->pwriter_opts, }; mlr_dsl_cst_handle_top_level_statement_blocks(pstate->pcst->pend_blocks, &variables, &cst_outputs); @@ -569,10 +591,12 @@ static sllv_t* mapper_put_or_filter_process(lrec_t* pinrec, context_t* pctx, voi .trace_execution = pstate->trace_execution, }; cst_outputs_t cst_outputs = (cst_outputs_t) { - .pshould_emit_rec = &should_emit_rec, - .poutrecs = poutrecs, - .oosvar_flatten_separator = pstate->oosvar_flatten_separator, - .pwriter_opts = pstate->pwriter_opts, + .pshould_emit_rec = &should_emit_rec, + .poutrecs = poutrecs, + .oosvar_flatten_separator = pstate->oosvar_flatten_separator, + .json_quote_int_keys = pstate->json_quote_int_keys, + .json_quote_non_string_values = pstate->json_quote_non_string_values, + .pwriter_opts = pstate->pwriter_opts, }; mlr_dsl_cst_handle_top_level_statement_block(pstate->pcst->pmain_block, &variables, &cst_outputs); diff --git a/c/todo.txt b/c/todo.txt index 9590b0de9..542af8dda 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -6,12 +6,19 @@ BUGFIXES ================================================================ FEATURE REQUESTS -* strptime w/ fractional seconds. maybe %OS? -* strftime: %OS1-%OS9? +* strptime/strftime: + - strptime w/ fractional seconds. maybe %OS? + - strftime: %OS1-%OS9? + - use stddlib vers w/ strpokes/hacks, or reimpl stdlib vers? + - 1st write a mlrlib vers?!? ================================================================ OTHER: +* option to inhibit dquot of int map keys on dump + - hook dump into jflags + - doc as incompact @ relnotes + * expose strcmp & strcasecmp for lhm sorting * expose movefirst/movelast funcs * mlr sort --fold! @@ -24,8 +31,6 @@ OTHER: there is a DSL so you *can* loop things out w/ variables etc etc etc but also verbs for often-used idioms so you don't *have* to. then show a concise and convincing example. -* option to inhibit dquot of int map keys on dump - ================================================================ UT FOR 5.0.0: