Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-27 01:36:10 +00:00
Code Issues Projects Releases Wiki Activity

JSON neaten

Browse source
This commit is contained in:
John Kerl 2016-02-04 10:07:02 -05:00
parent b0342a6f14
commit f78c12fa68
14 changed files with 50 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

17
c/cli/mlrcli.c
View file

@ -197,6 +197,8 @@ static char* rebackslash(char* sep) {
#define DEFAULT_OQUOTING QUOTE_MINIMAL
#define DEFAULT_JSON_FLATTEN_SEPARATOR ":"
// ----------------------------------------------------------------
// The main_usage() function is split out into subroutines in support of the
// manpage autogenerator.
@ -323,6 +325,11 @@ static void main_usage_data_format_options(FILE* o, char* argv0) {
fprintf(o, " --jlistwrap Wrap JSON output in outermost [ ].\n");
fprintf(o, " --jquoteall Quote map keys in JSON output, even if they're\n");
fprintf(o, " numeric.\n");
fprintf(o, " --jflatsep {string} Separator for flattening multi-level JSON keys,\n");
fprintf(o, " e.g. '{\"a\":{\"b\":3}}' becomes a:b => 3 for\n");
fprintf(o, " non-JSON formats. Defaults to %s.\n",
DEFAULT_JSON_FLATTEN_SEPARATOR);
// xxx fix this cmt
fprintf(o, " NOTE: --json and --ijson are currently under construction, but --ojson works.\n");
fprintf(o, "\n");
fprintf(o, " -p is a keystroke-saver for --nidx --fs space --repifs\n");
@ -561,10 +568,12 @@ cli_opts_t* parse_command_line(int argc, char** argv) {
popts->ors = NULL;
popts->ofs = NULL;
popts->ops = NULL;
popts->right_justify_xtab_value = FALSE;
popts->stack_json_output_vertically = FALSE;
popts->wrap_json_output_in_outer_list = FALSE;
popts->quote_json_values_always = FALSE;
popts->json_flatten_separator = DEFAULT_JSON_FLATTEN_SEPARATOR;
popts->ofmt = DEFAULT_OFMT;
popts->oquoting = DEFAULT_OQUOTING;
@ -734,6 +743,10 @@ cli_opts_t* parse_command_line(int argc, char** argv) {
popts->wrap_json_output_in_outer_list = TRUE;
} else if (streq(argv[argi], "--jquoteall")) {
popts->quote_json_values_always = TRUE;
} else if (streq(argv[argi], "--jflatsep")) {
check_arg_count(argv, argi, argc, 2);
popts->json_flatten_separator = argv[argi+1];
argi++;
} else if (streq(argv[argi], "--csv")) { popts->ifile_fmt = popts->ofile_fmt = "csv";
} else if (streq(argv[argi], "--icsv")) { popts->ifile_fmt = "csv";
@ -881,7 +894,7 @@ cli_opts_t* parse_command_line(int argc, char** argv) {
popts->plrec_writer = lrec_writer_dkvp_alloc(popts->ors, popts->ofs, popts->ops);
else if (streq(popts->ofile_fmt, "json"))
popts->plrec_writer = lrec_writer_json_alloc(popts->stack_json_output_vertically,
popts->wrap_json_output_in_outer_list, popts->quote_json_values_always);
popts->wrap_json_output_in_outer_list, popts->quote_json_values_always, popts->json_flatten_separator);
else if (streq(popts->ofile_fmt, "csv"))
popts->plrec_writer = lrec_writer_csv_alloc(popts->ors, popts->ofs, popts->oquoting,
popts->headerless_csv_output);
@ -942,7 +955,7 @@ cli_opts_t* parse_command_line(int argc, char** argv) {
popts->plrec_reader = lrec_reader_alloc(popts->ifile_fmt, popts->use_mmap_for_read,
popts->irs, popts->ifs, popts->allow_repeat_ifs, popts->ips, popts->allow_repeat_ips,
popts->use_implicit_csv_header);
popts->use_implicit_csv_header, popts->json_flatten_separator);
if (popts->plrec_reader == NULL) {
main_usage(stderr, argv[0]);
exit(1);

1
c/cli/mlrcli.h
View file

@ -31,6 +31,7 @@ typedef struct _cli_opts_t {
int stack_json_output_vertically;
int wrap_json_output_in_outer_list;
int quote_json_values_always;
char* json_flatten_separator;
char* ofmt;
int oquoting;

3
c/containers/join_bucket_keeper.c
View file

@ -150,10 +150,11 @@ join_bucket_keeper_t* join_bucket_keeper_alloc(
char* ips,
int allow_repeat_ips,
int use_implicit_csv_header,
char* json_flatten_separator,
slls_t* pleft_field_names
) {
lrec_reader_t* plrec_reader = lrec_reader_alloc(input_file_format, use_mmap_for_read,
irs, ifs, allow_repeat_ifs, ips, allow_repeat_ips, use_implicit_csv_header);
irs, ifs, allow_repeat_ifs, ips, allow_repeat_ips, use_implicit_csv_header, json_flatten_separator);
return join_bucket_keeper_alloc_from_reader(plrec_reader, prepipe, left_file_name, pleft_field_names);
}

1
c/containers/join_bucket_keeper.h
View file

@ -45,6 +45,7 @@ join_bucket_keeper_t* join_bucket_keeper_alloc(
char* ips,
int allow_repeat_ips,
int use_implicit_csv_header,
char* json_flatten_separator,
slls_t* pleft_field_names);
join_bucket_keeper_t* join_bucket_keeper_alloc_from_reader(

1
c/containers/mixutil.h
View file

@ -23,7 +23,6 @@ slls_t* mlr_copy_keys_from_record(lrec_t* prec);
// respect that and not corrupt the lrec. However, the slls values will be
// invalid after the lrec is freed.
slls_t* mlr_reference_selected_values_from_record(lrec_t* prec, slls_t* pselected_field_names);
// xxx rename this
void mlr_reference_values_from_record_into_string_array(lrec_t* prec, string_array_t* pselected_field_names,
string_array_t* pvalues);
int record_has_all_keys(lrec_t* prec, slls_t* pselected_field_names);

6
c/input/json_parser.c
View file

@ -134,7 +134,7 @@ static int new_value(
return 0;
}
value->_reserved.p.pobject_mem = (*(char **) &value->u.object.p.mem) + values_size; // xxx pun
value->_reserved.p.pobject_mem = (*(char **) &value->u.object.p.mem) + values_size;
value->u.object.length = 0;
break;
@ -394,14 +394,14 @@ json_value_t * json_parse_ex(
case JSON_OBJECT:
if (state.first_pass) {
(*(json_char **) &top->u.object.p.mem) += string_length + 1; // xxx pun
(*(json_char **) &top->u.object.p.mem) += string_length + 1;
} else {
top->u.object.p.values [top->u.object.length].name
= (json_char *) top->_reserved.p.pobject_mem;
top->u.object.p.values [top->u.object.length].name_length = string_length;
(*(json_char **) &top->_reserved.p.pobject_mem) += string_length + 1; // xxx pun
(*(json_char **) &top->_reserved.p.pobject_mem) += string_length + 1;
}
flags |= flag_seek_value | flag_need_colon;

11
c/input/lrec_reader_mmap_json.c
View file

@ -32,8 +32,7 @@ typedef struct _lrec_reader_mmap_json_state_t {
// manipulate pointers to strings rather than copying strings.)
sllv_t* ptop_level_json_objects;
sllv_t* precords;
// xxx parameterize
char* flatten_sep;
char* json_flatten_separator;
} lrec_reader_mmap_json_state_t;
static void lrec_reader_mmap_json_free(lrec_reader_t* preader);
@ -41,13 +40,13 @@ static void lrec_reader_mmap_json_sof(void* pvstate, void* pvhandle);
static lrec_t* lrec_reader_mmap_json_process(void* pvstate, void* pvhandle, context_t* pctx);
// ----------------------------------------------------------------
lrec_reader_t* lrec_reader_mmap_json_alloc(char* irs, char* ifs, char* ips, int allow_repeat_ifs) {
lrec_reader_t* lrec_reader_mmap_json_alloc(char* json_flatten_separator) {
lrec_reader_t* plrec_reader = mlr_malloc_or_die(sizeof(lrec_reader_t));
lrec_reader_mmap_json_state_t* pstate = mlr_malloc_or_die(sizeof(lrec_reader_mmap_json_state_t));
pstate->ptop_level_json_objects = NULL;
pstate->precords = NULL;
pstate->flatten_sep = ":";
pstate->json_flatten_separator = json_flatten_separator;
plrec_reader->pvstate = (void*)pstate;
plrec_reader->popen_func = file_reader_mmap_vopen;
@ -100,8 +99,6 @@ static void lrec_reader_mmap_json_sof(void* pvstate, void* pvhandle) {
.max_memory = 0
};
// xxx make an sllv_free_with_callback & use it throughout
if (pstate->ptop_level_json_objects != NULL) {
for (sllve_t* pe = pstate->ptop_level_json_objects->phead; pe != NULL; pe = pe->pnext) {
json_value_t* top_level_json_object = pe->pvvalue;
@ -152,7 +149,7 @@ static void lrec_reader_mmap_json_sof(void* pvstate, void* pvhandle) {
// The lrecs have their string pointers pointing into the parsed-JSON objects (for
// efficiency) so it's important we not free the latter until our free method.
reference_json_objects_as_lrecs(pstate->precords, parsed_top_level_json, pstate->flatten_sep);
reference_json_objects_as_lrecs(pstate->precords, parsed_top_level_json, pstate->json_flatten_separator);
if (item_start == NULL)
break;

5
c/input/lrec_readers.c
View file

@ -4,7 +4,7 @@
#include "input/byte_readers.h"
lrec_reader_t* lrec_reader_alloc(char* fmtdesc, int use_mmap, char* irs, char* ifs, int allow_repeat_ifs,
char* ips, int allow_repeat_ips, int use_implicit_csv_header)
char* ips, int allow_repeat_ips, int use_implicit_csv_header, char* json_flatten_separator)
{
if (streq(fmtdesc, "dkvp")) {
if (use_mmap)
@ -33,8 +33,9 @@ lrec_reader_t* lrec_reader_alloc(char* fmtdesc, int use_mmap, char* irs, char*
return lrec_reader_stdio_xtab_alloc(ifs, ips, allow_repeat_ips);
} else if (streq(fmtdesc, "json")) {
if (use_mmap)
return lrec_reader_mmap_json_alloc();
return lrec_reader_mmap_json_alloc(json_flatten_separator);
else
fprintf(stderr, "The JSON stdio reader is not yet implemented.\n");
return NULL; // xxx stub
} else {
return NULL;

4
c/input/lrec_readers.h
View file

@ -7,7 +7,7 @@
// Factory method. fmtdesc: "dkvp", "nidx", "csv", "csvlite", "nidx", "xtab".
lrec_reader_t* lrec_reader_alloc(char* fmtdesc, int use_mmap, char* irs, char* ifs, int allow_repeat_ifs,
char* ips, int allow_repeat_ips, int use_implicit_csv_header);
char* ips, int allow_repeat_ips, int use_implicit_csv_header, char* json_flatten_separator);
lrec_reader_t* lrec_reader_stdio_csvlite_alloc(char* irs, char* ifs, int allow_repeat_ifs, int use_implicit_header);
lrec_reader_t* lrec_reader_stdio_csv_alloc(char* irs, char* ifs, int use_implicit_header);
@ -20,7 +20,7 @@ lrec_reader_t* lrec_reader_mmap_csvlite_alloc(char* irs, char* ifs, int allow_re
lrec_reader_t* lrec_reader_mmap_dkvp_alloc(char* irs, char* ifs, char* ips, int allow_repeat_ifs);
lrec_reader_t* lrec_reader_mmap_nidx_alloc(char* irs, char* ifs, int allow_repeat_ifs);
lrec_reader_t* lrec_reader_mmap_xtab_alloc(char* ifs, char* ips, int allow_repeat_ips);
lrec_reader_t* lrec_reader_mmap_json_alloc();
lrec_reader_t* lrec_reader_mmap_json_alloc(char* json_flatten_separator);
lrec_reader_t* lrec_reader_in_memory_alloc(sllv_t* precords);

13
c/input/mlr_json_adapter.h
View file

@ -1,19 +1,12 @@
// Transfers data from the JSON parser to Miller records
#ifndef MLR_JSON_ADAPTER_H
#define MLR_JSON_ADAPTER_H
#include "input/json_parser.h"
#include "containers/lrec.h"
// ----------------------------------------------------------------
// xxx fix cmt:
// input: current sllv of lrecs
// input: top-level json value
// output: appended sllv
// xxx define pointer-ownership ... the sllv should not free the strings.
// Given parsed JSON, constructs a list of lrecs with string values pointing into the parsed JSON.
// This is done for efficiency, to avoid data copying. It also means the parsed JSON should not be
// freed until the lrecs are freed.
int reference_json_objects_as_lrecs(sllv_t* precords, json_value_t* ptop_level_json, char* flatten_sep);
#endif // MLR_JSON_ADAPTER_H

8
c/mapping/mapper_join.c
View file

@ -38,6 +38,7 @@ typedef struct _mapper_join_opts_t {
int use_implicit_csv_header;
char* ifile_fmt;
int use_mmap_for_read;
char* json_flatten_separator;
} mapper_join_opts_t;
typedef struct _mapper_join_state_t {
@ -142,6 +143,7 @@ static mapper_t* mapper_join_parse_cli(int* pargi, int argc, char** argv) {
popts->allow_repeat_ips = OPTION_UNSPECIFIED;
popts->use_implicit_csv_header = OPTION_UNSPECIFIED;
popts->use_mmap_for_read = OPTION_UNSPECIFIED;
popts->json_flatten_separator = NULL;
char* verb = argv[(*pargi)++];
@ -167,6 +169,7 @@ static mapper_t* mapper_join_parse_cli(int* pargi, int argc, char** argv) {
ap_define_true_flag(pstate, "--implicit-csv-header", &popts->use_implicit_csv_header);
ap_define_true_flag(pstate, "--use-mmap", &popts->use_mmap_for_read);
ap_define_false_flag(pstate, "--no-mmap", &popts->use_mmap_for_read);
ap_define_string_flag(pstate, "--jflatsep", &popts->json_flatten_separator);
if (!ap_parse(pstate, verb, pargi, argc, argv)) {
mapper_join_usage(stderr, argv[0], verb);
@ -291,6 +294,7 @@ static sllv_t* mapper_join_process_sorted(lrec_t* pright_rec, context_t* pctx, v
popts->ips,
popts->allow_repeat_ips,
popts->use_implicit_csv_header,
popts->json_flatten_separator,
popts->pleft_join_field_names);
}
join_bucket_keeper_t* pkeeper = pstate->pjoin_bucket_keeper; // keystroke-saver
@ -493,6 +497,8 @@ static void merge_options(mapper_join_opts_t* popts) {
popts->use_implicit_csv_header = MLR_GLOBALS.popts->use_implicit_csv_header;
if (popts->use_mmap_for_read == OPTION_UNSPECIFIED)
popts->use_mmap_for_read = MLR_GLOBALS.popts->use_mmap_for_read;
if (popts->json_flatten_separator == NULL)
popts->json_flatten_separator = MLR_GLOBALS.popts->json_flatten_separator;
}
static void ingest_left_file(mapper_join_state_t* pstate) {
@ -501,7 +507,7 @@ static void ingest_left_file(mapper_join_state_t* pstate) {
lrec_reader_t* plrec_reader = lrec_reader_alloc(popts->input_file_format, popts->use_mmap_for_read,
popts->irs, popts->ifs, popts->allow_repeat_ifs, popts->ips, popts->allow_repeat_ips,
popts->use_implicit_csv_header);
popts->use_implicit_csv_header, popts->json_flatten_separator);
void* pvhandle = plrec_reader->popen_func(plrec_reader->pvstate, pstate->popts->prepipe,
pstate->popts->left_file_name);

8
c/output/lrec_writer_json.c
View file

@ -6,6 +6,7 @@
typedef struct _lrec_writer_json_state_t {
unsigned long long counter;
char* json_flatten_separator;
int quote_json_values_always;
char* before_records_at_start_of_stream;
@ -20,13 +21,14 @@ static void lrec_writer_json_process(FILE* output_stream, lrec_t* prec, void* pv
// ----------------------------------------------------------------
lrec_writer_t* lrec_writer_json_alloc(int stack_vertically, int wrap_json_output_in_outer_list,
int quote_json_values_always)
int quote_json_values_always, char* json_flatten_separator)
{
lrec_writer_t* plrec_writer = mlr_malloc_or_die(sizeof(lrec_writer_t));
lrec_writer_json_state_t* pstate = mlr_malloc_or_die(sizeof(lrec_writer_json_state_t));
pstate->quote_json_values_always = quote_json_values_always;
pstate->counter = 0;
pstate->json_flatten_separator = json_flatten_separator;
pstate->before_records_at_start_of_stream = wrap_json_output_in_outer_list ? "[\n" : "";
pstate->between_records_after_start_of_stream = wrap_json_output_in_outer_list ? "," : "";
@ -55,14 +57,14 @@ static void lrec_writer_json_process(FILE* output_stream, lrec_t* prec, void* pv
printf("%s", pstate->between_records_after_start_of_stream);
mlhmmv_t* pmap = mlhmmv_alloc();
char* flatten_sep = ":"; // xxx temp; needs to be parameterized
char* sep = pstate->json_flatten_separator;
for (lrece_t* pe = prec->phead; pe != NULL; pe = pe->pnext) {
char* lkey = pe->key;
char* lvalue = pe->value;
sllmv_t* pmvkeys = sllmv_alloc();
for (char* piece = strtok(lkey, flatten_sep); piece != NULL; piece = strtok(NULL, flatten_sep)) {
for (char* piece = strtok(lkey, sep); piece != NULL; piece = strtok(NULL, sep)) {
mv_t mvkey = mv_from_string(piece, NO_FREE);
sllmv_add(pmvkeys, &mvkey);
}

2
c/output/lrec_writers.h
View file

@ -6,7 +6,7 @@ lrec_writer_t* lrec_writer_csv_alloc(char* ors, char* ofs, int oquoting, int hea
lrec_writer_t* lrec_writer_csvlite_alloc(char* ors, char* ofs, int headerless_csv_output);
lrec_writer_t* lrec_writer_dkvp_alloc(char* ors, char* ofs, char* ops);
lrec_writer_t* lrec_writer_json_alloc(int stack_vertically, int wrap_json_output_in_outer_list,
int quote_json_values_always);
int quote_json_values_always, char* json_flatten_separator);
lrec_writer_t* lrec_writer_nidx_alloc(char* ors, char* ofs);
lrec_writer_t* lrec_writer_pprint_alloc(char* ors, char ofs, int left_align);
lrec_writer_t* lrec_writer_xtab_alloc(char* ofs, char* ops, int right_justify_value);

4
c/todo.txt
View file

@ -62,7 +62,7 @@ TOP-OF-LIST DETAILS
* JSON I/O:
! acknowledgement to mld: https://github.com/udp/json-parser
! stdio version. & prefer stremable version with fgetc rather than pointer-math.
k output: mlr-native json output is trivial & just do it.
- on output, check 1st if mlhmmv is necessary? or just always?
- cover x 2
- define 'tabular JSON' by example @ mld
? flatten/unflatten option? ":" syntax?
@ -71,6 +71,7 @@ TOP-OF-LIST DETAILS
- programmable flatten-separator
? or support multi-level lrecs??
o hp re :)
! JSON/join UT. both/left/right json cases.
o bug:
@ -291,6 +292,7 @@ COOKBOOK/FAQ/ETC.:
? wiki quickselect ?
* sllv_free option with callback for void-star-payload free; likewise other void-star-payload containers
* double-check for off-by-one buflen in cline/sline
* hash-collision ifdef instrumentation -> maybe find a better hash function out there
* pprint join?