Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-28 18:21:52 +00:00
Code Issues Projects Releases Wiki Activity

json-arrays-as-map: now the default

Browse source
This commit is contained in:
John Kerl 2017-03-17 08:55:28 -04:00
parent 9414995ab3
commit dafa792fb3
8 changed files with 78 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

21
c/cli/mlrcli.c
View file

@ -723,11 +723,12 @@ static void main_usage_data_format_options(FILE* o, char* argv0) {
fprintf(o, "\n");
fprintf(o, " --ijson --ojson --json JSON tabular: sequence or list of one-level\n");
fprintf(o, " maps: {...}{...} or [{...},{...}].\n");
// xxx
fprintf(o, " --json-skip-arrays-on-input JSON arrays are unmillerable, and by default they\n");
fprintf(o, " cause a fatal error when read. With this option,\n");
fprintf(o, " they are ignored. Please use the jq tool for full\n");
fprintf(o, " --json-map-arrays-on-input JSON arrays are unmillerable. --json-map-arrays-on-input\n");
fprintf(o, " --json-skip-arrays-on-input is the default: arrays are converted to integer-indexed\n");
fprintf(o, " --json-fatal-arrays-on-input maps. The other two options cause them to be skipped, or\n");
fprintf(o, " to be treated as errors. Please use the jq tool for full\n");
fprintf(o, " JSON (pre)processing.\n");
fprintf(o, " --jvstack Put one key-value pair per line for JSON\n");
fprintf(o, " output.\n");
fprintf(o, " --jlistwrap Wrap JSON output in outermost [ ].\n");
@ -1010,8 +1011,8 @@ void cli_writer_opts_init(cli_writer_opts_t* pwriter_opts) {
pwriter_opts->pprint_barred = NEITHER_TRUE_NOR_FALSE;
pwriter_opts->stack_json_output_vertically = NEITHER_TRUE_NOR_FALSE;
pwriter_opts->wrap_json_output_in_outer_list = NEITHER_TRUE_NOR_FALSE;
pwriter_opts->json_quote_int_keys = NEITHER_TRUE_NOR_FALSE;
pwriter_opts->json_quote_non_string_values = NEITHER_TRUE_NOR_FALSE;
pwriter_opts->json_quote_int_keys = NEITHER_TRUE_NOR_FALSE;
pwriter_opts->json_quote_non_string_values = NEITHER_TRUE_NOR_FALSE;
pwriter_opts->output_json_flatten_separator = NULL;
pwriter_opts->oosvar_flatten_separator = NULL;
@ -1034,7 +1035,7 @@ void cli_apply_reader_defaults(cli_reader_opts_t* preader_opts) {
preader_opts->ifile_fmt = "dkvp";
if (preader_opts->json_array_ingest == JSON_ARRAY_INGEST_UNSPECIFIED)
preader_opts->json_array_ingest = JSON_ARRAY_INGEST_FATAL; // xxx temp
preader_opts->json_array_ingest = JSON_ARRAY_INGEST_AS_MAP;
if (preader_opts->use_implicit_csv_header == NEITHER_TRUE_NOR_FALSE)
preader_opts->use_implicit_csv_header = FALSE;
@ -1339,13 +1340,13 @@ int cli_handle_reader_options(char** argv, int argc, int *pargi, cli_reader_opts
preader_opts->allow_repeat_ifs = TRUE;
argi += 1;
} else if (streq(argv[argi], "--json-fatal-arrays-on-input")) { // xxx
} else if (streq(argv[argi], "--json-fatal-arrays-on-input")) {
preader_opts->json_array_ingest = JSON_ARRAY_INGEST_FATAL;
argi += 1;
} else if (streq(argv[argi], "--json-skip-arrays-on-input")) { // xxx
} else if (streq(argv[argi], "--json-skip-arrays-on-input")) {
preader_opts->json_array_ingest = JSON_ARRAY_INGEST_SKIP;
argi += 1;
} else if (streq(argv[argi], "--json-map-arrays-on-input")) { // xxx
} else if (streq(argv[argi], "--json-map-arrays-on-input")) {
preader_opts->json_array_ingest = JSON_ARRAY_INGEST_AS_MAP;
argi += 1;

9
c/input/mlr_json_adapter.c
View file

@ -79,7 +79,8 @@ lrec_t* validate_millerable_object(json_value_t* pjson, char* flatten_sep, json_
case JSON_ARRAY_INGEST_FATAL:
fprintf(stderr,
"%s: found array item within JSON object. This is valid but unmillerable JSON.\n"
"Use --json-skip-arrays-on-input to exclude these from input without fataling.\n",
"Use --json-skip-arrays-on-input to exclude these from input without fataling.\n"
"Or, --json-map-arrays-on-input to convert them to integer-indexed maps.\n",
MLR_GLOBALS.bargv0);
return NULL;
break;
@ -156,7 +157,8 @@ static int populate_from_nested_object(lrec_t* prec, json_value_t* pjson_object,
case JSON_ARRAY_INGEST_FATAL:
fprintf(stderr,
"%s: found array item within JSON object. This is valid but unmillerable JSON.\n"
"Use --json-skip-arrays-on-input to exclude these from input without fataling.\n",
"Use --json-skip-arrays-on-input to exclude these from input without fataling.\n"
"Or, --json-map-arrays-on-input to convert them to integer-indexed maps.\n",
MLR_GLOBALS.bargv0);
return FALSE;
break;
@ -225,7 +227,8 @@ static int populate_from_nested_array(lrec_t* prec, json_value_t* pjson_array, c
case JSON_ARRAY_INGEST_FATAL:
fprintf(stderr,
"%s: found array item within JSON object. This is valid but unmillerable JSON.\n"
"Use --json-skip-arrays-on-input to exclude these from input without fataling.\n",
"Use --json-skip-arrays-on-input to exclude these from input without fataling.\n"
"Or, --json-map-arrays-on-input to convert them to integer-indexed maps.\n",
MLR_GLOBALS.bargv0);
return FALSE;
break;

48
c/reg_test/expected/out
View file

@ -45848,24 +45848,30 @@ pan wye 10 0.5026260055412137 0.9526183602969864
mlr --icsv --ojson --rs lf cat
{ "col": "abc \"def\" \\ghi" }
mlr --ijson --oxtab cat
mlr: found array item within JSON object. This is valid but unmillerable JSON.
Use --json-skip-arrays-on-input to exclude these from input without fataling.
mlr: Unable to parse JSON data.
mlr --ijson --oxtab --json-skip-arrays-on-input cat ./reg_test/input/arrays.json
mlr --ijson --oxtab cat ./reg_test/input/arrays.json
a 1
b 1
a 2
a 2
b:0 2
b:1 3
a 3
a 3
b:c:0 4
b:c:1 5
a 4
a 4
b:c:d:0 4
b:c:d:1 5
a 5
a 5
b:0 6
b:1 7
b:2:0 8
b:2:1 9
a 6
a 6
b:0:0:0 11
mlr --ijson --oxtab --json-map-arrays-on-input cat ./reg_test/input/arrays.json
a 1
@ -45892,6 +45898,26 @@ b:2:1 9
a 6
b:0:0:0 11
mlr --ijson --oxtab --json-skip-arrays-on-input cat ./reg_test/input/arrays.json
a 1
b 1
a 2
a 3
a 4
a 5
a 6
mlr --ijson --oxtab --json-fatal-arrays-on-input cat ./reg_test/input/arrays.json
mlr: found array item within JSON object. This is valid but unmillerable JSON.
Use --json-skip-arrays-on-input to exclude these from input without fataling.
Or, --json-map-arrays-on-input to convert them to integer-indexed maps.
mlr: Unable to parse JSON data.
================================================================
FORMAT-CONVERSION KEYSTROKE-SAVERS

10
c/reg_test/run
View file

@ -5229,12 +5229,10 @@ col
"abc ""def"" \ghi"
EOF
mlr_expect_fail --ijson --oxtab cat <<EOF
{ "a": 1, "b": [2, 3], "c": 4}
EOF
run_mlr --ijson --oxtab --json-skip-arrays-on-input cat $indir/arrays.json
run_mlr --ijson --oxtab --json-map-arrays-on-input cat $indir/arrays.json
run_mlr --ijson --oxtab cat $indir/arrays.json
run_mlr --ijson --oxtab --json-map-arrays-on-input cat $indir/arrays.json
run_mlr --ijson --oxtab --json-skip-arrays-on-input cat $indir/arrays.json
mlr_expect_fail --ijson --oxtab --json-fatal-arrays-on-input cat $indir/arrays.json
# ----------------------------------------------------------------
announce FORMAT-CONVERSION KEYSTROKE-SAVERS

9
doc/manpage.html
View file

@ -369,9 +369,10 @@ OPTIONS
--ijson --ojson --json JSON tabular: sequence or list of one-level
maps: {...}{...} or [{...},{...}].
--json-skip-arrays-on-input JSON arrays are unmillerable, and by default they
cause a fatal error when read. With this option,
they are ignored. Please use the jq tool for full
--json-map-arrays-on-input JSON arrays are unmillerable. --json-map-arrays-on-input
--json-skip-arrays-on-input is the default: arrays are converted to integer-indexed
--json-fatal-arrays-on-input maps. The other two options cause them to be skipped, or
to be treated as errors. Please use the jq tool for full
JSON (pre)processing.
--jvstack Put one key-value pair per line for JSON
output.
@ -2154,7 +2155,7 @@ SEE ALSO
2017-03-16 MILLER(1)
2017-03-17 MILLER(1)
</pre>
</div>
<p/>

9
doc/manpage.txt
View file

@ -176,9 +176,10 @@ OPTIONS
--ijson --ojson --json JSON tabular: sequence or list of one-level
maps: {...}{...} or [{...},{...}].
--json-skip-arrays-on-input JSON arrays are unmillerable, and by default they
cause a fatal error when read. With this option,
they are ignored. Please use the jq tool for full
--json-map-arrays-on-input JSON arrays are unmillerable. --json-map-arrays-on-input
--json-skip-arrays-on-input is the default: arrays are converted to integer-indexed
--json-fatal-arrays-on-input maps. The other two options cause them to be skipped, or
to be treated as errors. Please use the jq tool for full
JSON (pre)processing.
--jvstack Put one key-value pair per line for JSON
output.
@ -1961,4 +1962,4 @@ SEE ALSO
2017-03-16 MILLER(1)
2017-03-17 MILLER(1)

11
doc/mlr.1
View file

@ -2,12 +2,12 @@
.\" Title: mlr
.\" Author: [see the "AUTHOR" section]
.\" Generator: ./mkman.rb
.\" Date: 2017-03-16
.\" Date: 2017-03-17
.\" Manual: \ \&
.\" Source: \ \&
.\" Language: English
.\"
.TH "MILLER" "1" "2017-03-16" "\ \&" "\ \&"
.TH "MILLER" "1" "2017-03-17" "\ \&" "\ \&"
.\" -----------------------------------------------------------------
.\" * Portability definitions
.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -231,9 +231,10 @@ Please use "mlr --help-function {function name}" for function-specific help.
--ijson --ojson --json JSON tabular: sequence or list of one-level
maps: {...}{...} or [{...},{...}].
--json-skip-arrays-on-input JSON arrays are unmillerable, and by default they
cause a fatal error when read. With this option,
they are ignored. Please use the jq tool for full
--json-map-arrays-on-input JSON arrays are unmillerable. --json-map-arrays-on-input
--json-skip-arrays-on-input is the default: arrays are converted to integer-indexed
--json-fatal-arrays-on-input maps. The other two options cause them to be skipped, or
to be treated as errors. Please use the jq tool for full
JSON (pre)processing.
--jvstack Put one key-value pair per line for JSON
output.

7
doc/reference.html
View file

@ -1346,9 +1346,10 @@ Data-format options, for input, output, or both:
--ijson --ojson --json JSON tabular: sequence or list of one-level
maps: {...}{...} or [{...},{...}].
--json-skip-arrays-on-input JSON arrays are unmillerable, and by default they
cause a fatal error when read. With this option,
they are ignored. Please use the jq tool for full
--json-map-arrays-on-input JSON arrays are unmillerable. --json-map-arrays-on-input
--json-skip-arrays-on-input is the default: arrays are converted to integer-indexed
--json-fatal-arrays-on-input maps. The other two options cause them to be skipped, or
to be treated as errors. Please use the jq tool for full
JSON (pre)processing.
--jvstack Put one key-value pair per line for JSON
output.