diff --git a/c/cli/mlrcli.c b/c/cli/mlrcli.c index c1b7aa084..a4abc9d96 100644 --- a/c/cli/mlrcli.c +++ b/c/cli/mlrcli.c @@ -197,6 +197,8 @@ static char* rebackslash(char* sep) { #define DEFAULT_OQUOTING QUOTE_MINIMAL +#define DEFAULT_JSON_FLATTEN_SEPARATOR ":" + // ---------------------------------------------------------------- // The main_usage() function is split out into subroutines in support of the // manpage autogenerator. @@ -323,6 +325,11 @@ static void main_usage_data_format_options(FILE* o, char* argv0) { fprintf(o, " --jlistwrap Wrap JSON output in outermost [ ].\n"); fprintf(o, " --jquoteall Quote map keys in JSON output, even if they're\n"); fprintf(o, " numeric.\n"); + fprintf(o, " --jflatsep {string} Separator for flattening multi-level JSON keys,\n"); + fprintf(o, " e.g. '{\"a\":{\"b\":3}}' becomes a:b => 3 for\n"); + fprintf(o, " non-JSON formats. Defaults to %s.\n", + DEFAULT_JSON_FLATTEN_SEPARATOR); + // xxx fix this cmt fprintf(o, " NOTE: --json and --ijson are currently under construction, but --ojson works.\n"); fprintf(o, "\n"); fprintf(o, " -p is a keystroke-saver for --nidx --fs space --repifs\n"); @@ -561,10 +568,12 @@ cli_opts_t* parse_command_line(int argc, char** argv) { popts->ors = NULL; popts->ofs = NULL; popts->ops = NULL; + popts->right_justify_xtab_value = FALSE; popts->stack_json_output_vertically = FALSE; popts->wrap_json_output_in_outer_list = FALSE; popts->quote_json_values_always = FALSE; + popts->json_flatten_separator = DEFAULT_JSON_FLATTEN_SEPARATOR; popts->ofmt = DEFAULT_OFMT; popts->oquoting = DEFAULT_OQUOTING; @@ -734,6 +743,10 @@ cli_opts_t* parse_command_line(int argc, char** argv) { popts->wrap_json_output_in_outer_list = TRUE; } else if (streq(argv[argi], "--jquoteall")) { popts->quote_json_values_always = TRUE; + } else if (streq(argv[argi], "--jflatsep")) { + check_arg_count(argv, argi, argc, 2); + popts->json_flatten_separator = argv[argi+1]; + argi++; } else if (streq(argv[argi], "--csv")) { popts->ifile_fmt = popts->ofile_fmt = "csv"; } else if (streq(argv[argi], "--icsv")) { popts->ifile_fmt = "csv"; @@ -881,7 +894,7 @@ cli_opts_t* parse_command_line(int argc, char** argv) { popts->plrec_writer = lrec_writer_dkvp_alloc(popts->ors, popts->ofs, popts->ops); else if (streq(popts->ofile_fmt, "json")) popts->plrec_writer = lrec_writer_json_alloc(popts->stack_json_output_vertically, - popts->wrap_json_output_in_outer_list, popts->quote_json_values_always); + popts->wrap_json_output_in_outer_list, popts->quote_json_values_always, popts->json_flatten_separator); else if (streq(popts->ofile_fmt, "csv")) popts->plrec_writer = lrec_writer_csv_alloc(popts->ors, popts->ofs, popts->oquoting, popts->headerless_csv_output); @@ -942,7 +955,7 @@ cli_opts_t* parse_command_line(int argc, char** argv) { popts->plrec_reader = lrec_reader_alloc(popts->ifile_fmt, popts->use_mmap_for_read, popts->irs, popts->ifs, popts->allow_repeat_ifs, popts->ips, popts->allow_repeat_ips, - popts->use_implicit_csv_header); + popts->use_implicit_csv_header, popts->json_flatten_separator); if (popts->plrec_reader == NULL) { main_usage(stderr, argv[0]); exit(1); diff --git a/c/cli/mlrcli.h b/c/cli/mlrcli.h index 4ffc65916..c9841abe9 100644 --- a/c/cli/mlrcli.h +++ b/c/cli/mlrcli.h @@ -31,6 +31,7 @@ typedef struct _cli_opts_t { int stack_json_output_vertically; int wrap_json_output_in_outer_list; int quote_json_values_always; + char* json_flatten_separator; char* ofmt; int oquoting; diff --git a/c/containers/join_bucket_keeper.c b/c/containers/join_bucket_keeper.c index b31a3f2fb..a93a9b794 100644 --- a/c/containers/join_bucket_keeper.c +++ b/c/containers/join_bucket_keeper.c @@ -150,10 +150,11 @@ join_bucket_keeper_t* join_bucket_keeper_alloc( char* ips, int allow_repeat_ips, int use_implicit_csv_header, + char* json_flatten_separator, slls_t* pleft_field_names ) { lrec_reader_t* plrec_reader = lrec_reader_alloc(input_file_format, use_mmap_for_read, - irs, ifs, allow_repeat_ifs, ips, allow_repeat_ips, use_implicit_csv_header); + irs, ifs, allow_repeat_ifs, ips, allow_repeat_ips, use_implicit_csv_header, json_flatten_separator); return join_bucket_keeper_alloc_from_reader(plrec_reader, prepipe, left_file_name, pleft_field_names); } diff --git a/c/containers/join_bucket_keeper.h b/c/containers/join_bucket_keeper.h index e3c7e6a6d..28412bc6c 100644 --- a/c/containers/join_bucket_keeper.h +++ b/c/containers/join_bucket_keeper.h @@ -45,6 +45,7 @@ join_bucket_keeper_t* join_bucket_keeper_alloc( char* ips, int allow_repeat_ips, int use_implicit_csv_header, + char* json_flatten_separator, slls_t* pleft_field_names); join_bucket_keeper_t* join_bucket_keeper_alloc_from_reader( diff --git a/c/containers/mixutil.h b/c/containers/mixutil.h index 821e08c0a..1dcbcbf05 100644 --- a/c/containers/mixutil.h +++ b/c/containers/mixutil.h @@ -23,7 +23,6 @@ slls_t* mlr_copy_keys_from_record(lrec_t* prec); // respect that and not corrupt the lrec. However, the slls values will be // invalid after the lrec is freed. slls_t* mlr_reference_selected_values_from_record(lrec_t* prec, slls_t* pselected_field_names); -// xxx rename this void mlr_reference_values_from_record_into_string_array(lrec_t* prec, string_array_t* pselected_field_names, string_array_t* pvalues); int record_has_all_keys(lrec_t* prec, slls_t* pselected_field_names); diff --git a/c/input/json_parser.c b/c/input/json_parser.c index 9e84ed5ef..ea9b3cfcc 100644 --- a/c/input/json_parser.c +++ b/c/input/json_parser.c @@ -134,7 +134,7 @@ static int new_value( return 0; } - value->_reserved.p.pobject_mem = (*(char **) &value->u.object.p.mem) + values_size; // xxx pun + value->_reserved.p.pobject_mem = (*(char **) &value->u.object.p.mem) + values_size; value->u.object.length = 0; break; @@ -394,14 +394,14 @@ json_value_t * json_parse_ex( case JSON_OBJECT: if (state.first_pass) { - (*(json_char **) &top->u.object.p.mem) += string_length + 1; // xxx pun + (*(json_char **) &top->u.object.p.mem) += string_length + 1; } else { top->u.object.p.values [top->u.object.length].name = (json_char *) top->_reserved.p.pobject_mem; top->u.object.p.values [top->u.object.length].name_length = string_length; - (*(json_char **) &top->_reserved.p.pobject_mem) += string_length + 1; // xxx pun + (*(json_char **) &top->_reserved.p.pobject_mem) += string_length + 1; } flags |= flag_seek_value | flag_need_colon; diff --git a/c/input/lrec_reader_mmap_json.c b/c/input/lrec_reader_mmap_json.c index d98f3e70b..36817287f 100644 --- a/c/input/lrec_reader_mmap_json.c +++ b/c/input/lrec_reader_mmap_json.c @@ -32,8 +32,7 @@ typedef struct _lrec_reader_mmap_json_state_t { // manipulate pointers to strings rather than copying strings.) sllv_t* ptop_level_json_objects; sllv_t* precords; - // xxx parameterize - char* flatten_sep; + char* json_flatten_separator; } lrec_reader_mmap_json_state_t; static void lrec_reader_mmap_json_free(lrec_reader_t* preader); @@ -41,13 +40,13 @@ static void lrec_reader_mmap_json_sof(void* pvstate, void* pvhandle); static lrec_t* lrec_reader_mmap_json_process(void* pvstate, void* pvhandle, context_t* pctx); // ---------------------------------------------------------------- -lrec_reader_t* lrec_reader_mmap_json_alloc(char* irs, char* ifs, char* ips, int allow_repeat_ifs) { +lrec_reader_t* lrec_reader_mmap_json_alloc(char* json_flatten_separator) { lrec_reader_t* plrec_reader = mlr_malloc_or_die(sizeof(lrec_reader_t)); lrec_reader_mmap_json_state_t* pstate = mlr_malloc_or_die(sizeof(lrec_reader_mmap_json_state_t)); pstate->ptop_level_json_objects = NULL; pstate->precords = NULL; - pstate->flatten_sep = ":"; + pstate->json_flatten_separator = json_flatten_separator; plrec_reader->pvstate = (void*)pstate; plrec_reader->popen_func = file_reader_mmap_vopen; @@ -100,8 +99,6 @@ static void lrec_reader_mmap_json_sof(void* pvstate, void* pvhandle) { .max_memory = 0 }; - // xxx make an sllv_free_with_callback & use it throughout - if (pstate->ptop_level_json_objects != NULL) { for (sllve_t* pe = pstate->ptop_level_json_objects->phead; pe != NULL; pe = pe->pnext) { json_value_t* top_level_json_object = pe->pvvalue; @@ -152,7 +149,7 @@ static void lrec_reader_mmap_json_sof(void* pvstate, void* pvhandle) { // The lrecs have their string pointers pointing into the parsed-JSON objects (for // efficiency) so it's important we not free the latter until our free method. - reference_json_objects_as_lrecs(pstate->precords, parsed_top_level_json, pstate->flatten_sep); + reference_json_objects_as_lrecs(pstate->precords, parsed_top_level_json, pstate->json_flatten_separator); if (item_start == NULL) break; diff --git a/c/input/lrec_readers.c b/c/input/lrec_readers.c index e643d92d6..b66a76775 100644 --- a/c/input/lrec_readers.c +++ b/c/input/lrec_readers.c @@ -4,7 +4,7 @@ #include "input/byte_readers.h" lrec_reader_t* lrec_reader_alloc(char* fmtdesc, int use_mmap, char* irs, char* ifs, int allow_repeat_ifs, - char* ips, int allow_repeat_ips, int use_implicit_csv_header) + char* ips, int allow_repeat_ips, int use_implicit_csv_header, char* json_flatten_separator) { if (streq(fmtdesc, "dkvp")) { if (use_mmap) @@ -33,8 +33,9 @@ lrec_reader_t* lrec_reader_alloc(char* fmtdesc, int use_mmap, char* irs, char* return lrec_reader_stdio_xtab_alloc(ifs, ips, allow_repeat_ips); } else if (streq(fmtdesc, "json")) { if (use_mmap) - return lrec_reader_mmap_json_alloc(); + return lrec_reader_mmap_json_alloc(json_flatten_separator); else + fprintf(stderr, "The JSON stdio reader is not yet implemented.\n"); return NULL; // xxx stub } else { return NULL; diff --git a/c/input/lrec_readers.h b/c/input/lrec_readers.h index 1b9b7dd0a..55678242e 100644 --- a/c/input/lrec_readers.h +++ b/c/input/lrec_readers.h @@ -7,7 +7,7 @@ // Factory method. fmtdesc: "dkvp", "nidx", "csv", "csvlite", "nidx", "xtab". lrec_reader_t* lrec_reader_alloc(char* fmtdesc, int use_mmap, char* irs, char* ifs, int allow_repeat_ifs, - char* ips, int allow_repeat_ips, int use_implicit_csv_header); + char* ips, int allow_repeat_ips, int use_implicit_csv_header, char* json_flatten_separator); lrec_reader_t* lrec_reader_stdio_csvlite_alloc(char* irs, char* ifs, int allow_repeat_ifs, int use_implicit_header); lrec_reader_t* lrec_reader_stdio_csv_alloc(char* irs, char* ifs, int use_implicit_header); @@ -20,7 +20,7 @@ lrec_reader_t* lrec_reader_mmap_csvlite_alloc(char* irs, char* ifs, int allow_re lrec_reader_t* lrec_reader_mmap_dkvp_alloc(char* irs, char* ifs, char* ips, int allow_repeat_ifs); lrec_reader_t* lrec_reader_mmap_nidx_alloc(char* irs, char* ifs, int allow_repeat_ifs); lrec_reader_t* lrec_reader_mmap_xtab_alloc(char* ifs, char* ips, int allow_repeat_ips); -lrec_reader_t* lrec_reader_mmap_json_alloc(); +lrec_reader_t* lrec_reader_mmap_json_alloc(char* json_flatten_separator); lrec_reader_t* lrec_reader_in_memory_alloc(sllv_t* precords); diff --git a/c/input/mlr_json_adapter.h b/c/input/mlr_json_adapter.h index 0145abe0e..b3a6427f5 100644 --- a/c/input/mlr_json_adapter.h +++ b/c/input/mlr_json_adapter.h @@ -1,19 +1,12 @@ -// Transfers data from the JSON parser to Miller records - #ifndef MLR_JSON_ADAPTER_H #define MLR_JSON_ADAPTER_H #include "input/json_parser.h" #include "containers/lrec.h" - -// ---------------------------------------------------------------- -// xxx fix cmt: -// input: current sllv of lrecs -// input: top-level json value -// output: appended sllv -// xxx define pointer-ownership ... the sllv should not free the strings. - +// Given parsed JSON, constructs a list of lrecs with string values pointing into the parsed JSON. +// This is done for efficiency, to avoid data copying. It also means the parsed JSON should not be +// freed until the lrecs are freed. int reference_json_objects_as_lrecs(sllv_t* precords, json_value_t* ptop_level_json, char* flatten_sep); #endif // MLR_JSON_ADAPTER_H diff --git a/c/mapping/mapper_join.c b/c/mapping/mapper_join.c index 8b80f148e..1048c4751 100644 --- a/c/mapping/mapper_join.c +++ b/c/mapping/mapper_join.c @@ -38,6 +38,7 @@ typedef struct _mapper_join_opts_t { int use_implicit_csv_header; char* ifile_fmt; int use_mmap_for_read; + char* json_flatten_separator; } mapper_join_opts_t; typedef struct _mapper_join_state_t { @@ -142,6 +143,7 @@ static mapper_t* mapper_join_parse_cli(int* pargi, int argc, char** argv) { popts->allow_repeat_ips = OPTION_UNSPECIFIED; popts->use_implicit_csv_header = OPTION_UNSPECIFIED; popts->use_mmap_for_read = OPTION_UNSPECIFIED; + popts->json_flatten_separator = NULL; char* verb = argv[(*pargi)++]; @@ -167,6 +169,7 @@ static mapper_t* mapper_join_parse_cli(int* pargi, int argc, char** argv) { ap_define_true_flag(pstate, "--implicit-csv-header", &popts->use_implicit_csv_header); ap_define_true_flag(pstate, "--use-mmap", &popts->use_mmap_for_read); ap_define_false_flag(pstate, "--no-mmap", &popts->use_mmap_for_read); + ap_define_string_flag(pstate, "--jflatsep", &popts->json_flatten_separator); if (!ap_parse(pstate, verb, pargi, argc, argv)) { mapper_join_usage(stderr, argv[0], verb); @@ -291,6 +294,7 @@ static sllv_t* mapper_join_process_sorted(lrec_t* pright_rec, context_t* pctx, v popts->ips, popts->allow_repeat_ips, popts->use_implicit_csv_header, + popts->json_flatten_separator, popts->pleft_join_field_names); } join_bucket_keeper_t* pkeeper = pstate->pjoin_bucket_keeper; // keystroke-saver @@ -493,6 +497,8 @@ static void merge_options(mapper_join_opts_t* popts) { popts->use_implicit_csv_header = MLR_GLOBALS.popts->use_implicit_csv_header; if (popts->use_mmap_for_read == OPTION_UNSPECIFIED) popts->use_mmap_for_read = MLR_GLOBALS.popts->use_mmap_for_read; + if (popts->json_flatten_separator == NULL) + popts->json_flatten_separator = MLR_GLOBALS.popts->json_flatten_separator; } static void ingest_left_file(mapper_join_state_t* pstate) { @@ -501,7 +507,7 @@ static void ingest_left_file(mapper_join_state_t* pstate) { lrec_reader_t* plrec_reader = lrec_reader_alloc(popts->input_file_format, popts->use_mmap_for_read, popts->irs, popts->ifs, popts->allow_repeat_ifs, popts->ips, popts->allow_repeat_ips, - popts->use_implicit_csv_header); + popts->use_implicit_csv_header, popts->json_flatten_separator); void* pvhandle = plrec_reader->popen_func(plrec_reader->pvstate, pstate->popts->prepipe, pstate->popts->left_file_name); diff --git a/c/output/lrec_writer_json.c b/c/output/lrec_writer_json.c index 9a3198b59..2fb192849 100644 --- a/c/output/lrec_writer_json.c +++ b/c/output/lrec_writer_json.c @@ -6,6 +6,7 @@ typedef struct _lrec_writer_json_state_t { unsigned long long counter; + char* json_flatten_separator; int quote_json_values_always; char* before_records_at_start_of_stream; @@ -20,13 +21,14 @@ static void lrec_writer_json_process(FILE* output_stream, lrec_t* prec, void* pv // ---------------------------------------------------------------- lrec_writer_t* lrec_writer_json_alloc(int stack_vertically, int wrap_json_output_in_outer_list, - int quote_json_values_always) + int quote_json_values_always, char* json_flatten_separator) { lrec_writer_t* plrec_writer = mlr_malloc_or_die(sizeof(lrec_writer_t)); lrec_writer_json_state_t* pstate = mlr_malloc_or_die(sizeof(lrec_writer_json_state_t)); pstate->quote_json_values_always = quote_json_values_always; pstate->counter = 0; + pstate->json_flatten_separator = json_flatten_separator; pstate->before_records_at_start_of_stream = wrap_json_output_in_outer_list ? "[\n" : ""; pstate->between_records_after_start_of_stream = wrap_json_output_in_outer_list ? "," : ""; @@ -55,14 +57,14 @@ static void lrec_writer_json_process(FILE* output_stream, lrec_t* prec, void* pv printf("%s", pstate->between_records_after_start_of_stream); mlhmmv_t* pmap = mlhmmv_alloc(); - char* flatten_sep = ":"; // xxx temp; needs to be parameterized + char* sep = pstate->json_flatten_separator; for (lrece_t* pe = prec->phead; pe != NULL; pe = pe->pnext) { char* lkey = pe->key; char* lvalue = pe->value; sllmv_t* pmvkeys = sllmv_alloc(); - for (char* piece = strtok(lkey, flatten_sep); piece != NULL; piece = strtok(NULL, flatten_sep)) { + for (char* piece = strtok(lkey, sep); piece != NULL; piece = strtok(NULL, sep)) { mv_t mvkey = mv_from_string(piece, NO_FREE); sllmv_add(pmvkeys, &mvkey); } diff --git a/c/output/lrec_writers.h b/c/output/lrec_writers.h index 749c97db2..19cc566c5 100644 --- a/c/output/lrec_writers.h +++ b/c/output/lrec_writers.h @@ -6,7 +6,7 @@ lrec_writer_t* lrec_writer_csv_alloc(char* ors, char* ofs, int oquoting, int hea lrec_writer_t* lrec_writer_csvlite_alloc(char* ors, char* ofs, int headerless_csv_output); lrec_writer_t* lrec_writer_dkvp_alloc(char* ors, char* ofs, char* ops); lrec_writer_t* lrec_writer_json_alloc(int stack_vertically, int wrap_json_output_in_outer_list, - int quote_json_values_always); + int quote_json_values_always, char* json_flatten_separator); lrec_writer_t* lrec_writer_nidx_alloc(char* ors, char* ofs); lrec_writer_t* lrec_writer_pprint_alloc(char* ors, char ofs, int left_align); lrec_writer_t* lrec_writer_xtab_alloc(char* ofs, char* ops, int right_justify_value); diff --git a/c/todo.txt b/c/todo.txt index 315d94b7c..659898b13 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -62,7 +62,7 @@ TOP-OF-LIST DETAILS * JSON I/O: ! acknowledgement to mld: https://github.com/udp/json-parser ! stdio version. & prefer stremable version with fgetc rather than pointer-math. - k output: mlr-native json output is trivial & just do it. + - on output, check 1st if mlhmmv is necessary? or just always? - cover x 2 - define 'tabular JSON' by example @ mld ? flatten/unflatten option? ":" syntax? @@ -71,6 +71,7 @@ TOP-OF-LIST DETAILS - programmable flatten-separator ? or support multi-level lrecs?? o hp re :) + ! JSON/join UT. both/left/right json cases. o bug: @@ -291,6 +292,7 @@ COOKBOOK/FAQ/ETC.: ? wiki quickselect ? +* sllv_free option with callback for void-star-payload free; likewise other void-star-payload containers * double-check for off-by-one buflen in cline/sline * hash-collision ifdef instrumentation -> maybe find a better hash function out there * pprint join?