diff --git a/c/cli/mlrcli.c b/c/cli/mlrcli.c index d73e56482..8f7b42c84 100644 --- a/c/cli/mlrcli.c +++ b/c/cli/mlrcli.c @@ -723,11 +723,12 @@ static void main_usage_data_format_options(FILE* o, char* argv0) { fprintf(o, "\n"); fprintf(o, " --ijson --ojson --json JSON tabular: sequence or list of one-level\n"); fprintf(o, " maps: {...}{...} or [{...},{...}].\n"); - // xxx - fprintf(o, " --json-skip-arrays-on-input JSON arrays are unmillerable, and by default they\n"); - fprintf(o, " cause a fatal error when read. With this option,\n"); - fprintf(o, " they are ignored. Please use the jq tool for full\n"); + fprintf(o, " --json-map-arrays-on-input JSON arrays are unmillerable. --json-map-arrays-on-input\n"); + fprintf(o, " --json-skip-arrays-on-input is the default: arrays are converted to integer-indexed\n"); + fprintf(o, " --json-fatal-arrays-on-input maps. The other two options cause them to be skipped, or\n"); + fprintf(o, " to be treated as errors. Please use the jq tool for full\n"); fprintf(o, " JSON (pre)processing.\n"); + fprintf(o, " --jvstack Put one key-value pair per line for JSON\n"); fprintf(o, " output.\n"); fprintf(o, " --jlistwrap Wrap JSON output in outermost [ ].\n"); @@ -1010,8 +1011,8 @@ void cli_writer_opts_init(cli_writer_opts_t* pwriter_opts) { pwriter_opts->pprint_barred = NEITHER_TRUE_NOR_FALSE; pwriter_opts->stack_json_output_vertically = NEITHER_TRUE_NOR_FALSE; pwriter_opts->wrap_json_output_in_outer_list = NEITHER_TRUE_NOR_FALSE; - pwriter_opts->json_quote_int_keys = NEITHER_TRUE_NOR_FALSE; - pwriter_opts->json_quote_non_string_values = NEITHER_TRUE_NOR_FALSE; + pwriter_opts->json_quote_int_keys = NEITHER_TRUE_NOR_FALSE; + pwriter_opts->json_quote_non_string_values = NEITHER_TRUE_NOR_FALSE; pwriter_opts->output_json_flatten_separator = NULL; pwriter_opts->oosvar_flatten_separator = NULL; @@ -1034,7 +1035,7 @@ void cli_apply_reader_defaults(cli_reader_opts_t* preader_opts) { preader_opts->ifile_fmt = "dkvp"; if (preader_opts->json_array_ingest == JSON_ARRAY_INGEST_UNSPECIFIED) - preader_opts->json_array_ingest = JSON_ARRAY_INGEST_FATAL; // xxx temp + preader_opts->json_array_ingest = JSON_ARRAY_INGEST_AS_MAP; if (preader_opts->use_implicit_csv_header == NEITHER_TRUE_NOR_FALSE) preader_opts->use_implicit_csv_header = FALSE; @@ -1339,13 +1340,13 @@ int cli_handle_reader_options(char** argv, int argc, int *pargi, cli_reader_opts preader_opts->allow_repeat_ifs = TRUE; argi += 1; - } else if (streq(argv[argi], "--json-fatal-arrays-on-input")) { // xxx + } else if (streq(argv[argi], "--json-fatal-arrays-on-input")) { preader_opts->json_array_ingest = JSON_ARRAY_INGEST_FATAL; argi += 1; - } else if (streq(argv[argi], "--json-skip-arrays-on-input")) { // xxx + } else if (streq(argv[argi], "--json-skip-arrays-on-input")) { preader_opts->json_array_ingest = JSON_ARRAY_INGEST_SKIP; argi += 1; - } else if (streq(argv[argi], "--json-map-arrays-on-input")) { // xxx + } else if (streq(argv[argi], "--json-map-arrays-on-input")) { preader_opts->json_array_ingest = JSON_ARRAY_INGEST_AS_MAP; argi += 1; diff --git a/c/input/mlr_json_adapter.c b/c/input/mlr_json_adapter.c index 0a253eaa5..1db870284 100644 --- a/c/input/mlr_json_adapter.c +++ b/c/input/mlr_json_adapter.c @@ -79,7 +79,8 @@ lrec_t* validate_millerable_object(json_value_t* pjson, char* flatten_sep, json_ case JSON_ARRAY_INGEST_FATAL: fprintf(stderr, "%s: found array item within JSON object. This is valid but unmillerable JSON.\n" - "Use --json-skip-arrays-on-input to exclude these from input without fataling.\n", + "Use --json-skip-arrays-on-input to exclude these from input without fataling.\n" + "Or, --json-map-arrays-on-input to convert them to integer-indexed maps.\n", MLR_GLOBALS.bargv0); return NULL; break; @@ -156,7 +157,8 @@ static int populate_from_nested_object(lrec_t* prec, json_value_t* pjson_object, case JSON_ARRAY_INGEST_FATAL: fprintf(stderr, "%s: found array item within JSON object. This is valid but unmillerable JSON.\n" - "Use --json-skip-arrays-on-input to exclude these from input without fataling.\n", + "Use --json-skip-arrays-on-input to exclude these from input without fataling.\n" + "Or, --json-map-arrays-on-input to convert them to integer-indexed maps.\n", MLR_GLOBALS.bargv0); return FALSE; break; @@ -225,7 +227,8 @@ static int populate_from_nested_array(lrec_t* prec, json_value_t* pjson_array, c case JSON_ARRAY_INGEST_FATAL: fprintf(stderr, "%s: found array item within JSON object. This is valid but unmillerable JSON.\n" - "Use --json-skip-arrays-on-input to exclude these from input without fataling.\n", + "Use --json-skip-arrays-on-input to exclude these from input without fataling.\n" + "Or, --json-map-arrays-on-input to convert them to integer-indexed maps.\n", MLR_GLOBALS.bargv0); return FALSE; break; diff --git a/c/reg_test/expected/out b/c/reg_test/expected/out index bae1ff649..3f2eee619 100644 --- a/c/reg_test/expected/out +++ b/c/reg_test/expected/out @@ -45848,24 +45848,30 @@ pan wye 10 0.5026260055412137 0.9526183602969864 mlr --icsv --ojson --rs lf cat { "col": "abc \"def\" \\ghi" } -mlr --ijson --oxtab cat -mlr: found array item within JSON object. This is valid but unmillerable JSON. -Use --json-skip-arrays-on-input to exclude these from input without fataling. -mlr: Unable to parse JSON data. - -mlr --ijson --oxtab --json-skip-arrays-on-input cat ./reg_test/input/arrays.json +mlr --ijson --oxtab cat ./reg_test/input/arrays.json a 1 b 1 -a 2 +a 2 +b:0 2 +b:1 3 -a 3 +a 3 +b:c:0 4 +b:c:1 5 -a 4 +a 4 +b:c:d:0 4 +b:c:d:1 5 -a 5 +a 5 +b:0 6 +b:1 7 +b:2:0 8 +b:2:1 9 -a 6 +a 6 +b:0:0:0 11 mlr --ijson --oxtab --json-map-arrays-on-input cat ./reg_test/input/arrays.json a 1 @@ -45892,6 +45898,26 @@ b:2:1 9 a 6 b:0:0:0 11 +mlr --ijson --oxtab --json-skip-arrays-on-input cat ./reg_test/input/arrays.json +a 1 +b 1 + +a 2 + +a 3 + +a 4 + +a 5 + +a 6 + +mlr --ijson --oxtab --json-fatal-arrays-on-input cat ./reg_test/input/arrays.json +mlr: found array item within JSON object. This is valid but unmillerable JSON. +Use --json-skip-arrays-on-input to exclude these from input without fataling. +Or, --json-map-arrays-on-input to convert them to integer-indexed maps. +mlr: Unable to parse JSON data. + ================================================================ FORMAT-CONVERSION KEYSTROKE-SAVERS diff --git a/c/reg_test/run b/c/reg_test/run index 0a7b96b07..7b3d5813b 100755 --- a/c/reg_test/run +++ b/c/reg_test/run @@ -5229,12 +5229,10 @@ col "abc ""def"" \ghi" EOF -mlr_expect_fail --ijson --oxtab cat <

diff --git a/doc/manpage.txt b/doc/manpage.txt index 8dced457d..7cb43ccaa 100644 --- a/doc/manpage.txt +++ b/doc/manpage.txt @@ -176,9 +176,10 @@ OPTIONS --ijson --ojson --json JSON tabular: sequence or list of one-level maps: {...}{...} or [{...},{...}]. - --json-skip-arrays-on-input JSON arrays are unmillerable, and by default they - cause a fatal error when read. With this option, - they are ignored. Please use the jq tool for full + --json-map-arrays-on-input JSON arrays are unmillerable. --json-map-arrays-on-input + --json-skip-arrays-on-input is the default: arrays are converted to integer-indexed + --json-fatal-arrays-on-input maps. The other two options cause them to be skipped, or + to be treated as errors. Please use the jq tool for full JSON (pre)processing. --jvstack Put one key-value pair per line for JSON output. @@ -1961,4 +1962,4 @@ SEE ALSO - 2017-03-16 MILLER(1) + 2017-03-17 MILLER(1) diff --git a/doc/mlr.1 b/doc/mlr.1 index 08d18ae57..b56e92c49 100644 --- a/doc/mlr.1 +++ b/doc/mlr.1 @@ -2,12 +2,12 @@ .\" Title: mlr .\" Author: [see the "AUTHOR" section] .\" Generator: ./mkman.rb -.\" Date: 2017-03-16 +.\" Date: 2017-03-17 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "MILLER" "1" "2017-03-16" "\ \&" "\ \&" +.TH "MILLER" "1" "2017-03-17" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Portability definitions .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -231,9 +231,10 @@ Please use "mlr --help-function {function name}" for function-specific help. --ijson --ojson --json JSON tabular: sequence or list of one-level maps: {...}{...} or [{...},{...}]. - --json-skip-arrays-on-input JSON arrays are unmillerable, and by default they - cause a fatal error when read. With this option, - they are ignored. Please use the jq tool for full + --json-map-arrays-on-input JSON arrays are unmillerable. --json-map-arrays-on-input + --json-skip-arrays-on-input is the default: arrays are converted to integer-indexed + --json-fatal-arrays-on-input maps. The other two options cause them to be skipped, or + to be treated as errors. Please use the jq tool for full JSON (pre)processing. --jvstack Put one key-value pair per line for JSON output. diff --git a/doc/reference.html b/doc/reference.html index 97bf3bfec..e1e424cee 100644 --- a/doc/reference.html +++ b/doc/reference.html @@ -1346,9 +1346,10 @@ Data-format options, for input, output, or both: --ijson --ojson --json JSON tabular: sequence or list of one-level maps: {...}{...} or [{...},{...}]. - --json-skip-arrays-on-input JSON arrays are unmillerable, and by default they - cause a fatal error when read. With this option, - they are ignored. Please use the jq tool for full + --json-map-arrays-on-input JSON arrays are unmillerable. --json-map-arrays-on-input + --json-skip-arrays-on-input is the default: arrays are converted to integer-indexed + --json-fatal-arrays-on-input maps. The other two options cause them to be skipped, or + to be treated as errors. Please use the jq tool for full JSON (pre)processing. --jvstack Put one key-value pair per line for JSON output.