diff --git a/docs/src/data/quoted-numeric.csv b/docs/src/data/quoted-numeric.csv new file mode 100644 index 000000000..5cba089bc --- /dev/null +++ b/docs/src/data/quoted-numeric.csv @@ -0,0 +1,2 @@ +a,b +"hello","004.56" diff --git a/docs/src/reference-main-data-types.md b/docs/src/reference-main-data-types.md index 0ef616fd9..aa6bce487 100644 --- a/docs/src/reference-main-data-types.md +++ b/docs/src/reference-main-data-types.md @@ -62,6 +62,54 @@ will produce `x=1,y=2,z=3`. Numbers retain their original string representation, so if `x` is `1.2` on one record and `1.200` on another, they'll print out that way on output (unless of course they've been modified during processing, e.g. `mlr put '$x = $x + 10`). +One exception: on JSON output, numbers whose original text isn't valid in the +JSON grammar -- e.g. `004.56`, whose leading zeros JSON disallows -- are +re-rendered (here, as `4.56`) so that Miller always writes valid JSON. + +Note that double quotes in CSV input don't affect type inference. In CSV, +quoting exists to allow field content containing commas, newlines, and/or +double quotes -- unlike in JSON, quoting doesn't distinguish strings from +numbers. So `"4.56"` in a CSV file scans as a float, just as `4.56` does. If +you want values kept as strings, you can use `mlr -S` (or, synonymously, `mlr +--infer-none`) to disable type inference entirely, or use the +[`string` DSL function](reference-dsl-builtin-functions.md#string) to cast +specific fields: + +
+mlr --icsv --ojson cat data/quoted-numeric.csv
+
+
+[
+{
+  "a": "hello",
+  "b": 4.56
+}
+]
+
+ +
+mlr --icsv --ojson -S cat data/quoted-numeric.csv
+
+
+[
+{
+  "a": "hello",
+  "b": "004.56"
+}
+]
+
+ +
+mlr --icsv --ojson put '$b = string($b)' data/quoted-numeric.csv
+
+
+[
+{
+  "a": "hello",
+  "b": "004.56"
+}
+]
+
Generally strings, numbers, and booleans don't mix; use type-casting like `string($x)` to convert. However, the dot (string-concatenation) operator has diff --git a/docs/src/reference-main-data-types.md.in b/docs/src/reference-main-data-types.md.in index 485dece20..b7b1172b2 100644 --- a/docs/src/reference-main-data-types.md.in +++ b/docs/src/reference-main-data-types.md.in @@ -46,6 +46,30 @@ will produce `x=1,y=2,z=3`. Numbers retain their original string representation, so if `x` is `1.2` on one record and `1.200` on another, they'll print out that way on output (unless of course they've been modified during processing, e.g. `mlr put '$x = $x + 10`). +One exception: on JSON output, numbers whose original text isn't valid in the +JSON grammar -- e.g. `004.56`, whose leading zeros JSON disallows -- are +re-rendered (here, as `4.56`) so that Miller always writes valid JSON. + +Note that double quotes in CSV input don't affect type inference. In CSV, +quoting exists to allow field content containing commas, newlines, and/or +double quotes -- unlike in JSON, quoting doesn't distinguish strings from +numbers. So `"4.56"` in a CSV file scans as a float, just as `4.56` does. If +you want values kept as strings, you can use `mlr -S` (or, synonymously, `mlr +--infer-none`) to disable type inference entirely, or use the +[`string` DSL function](reference-dsl-builtin-functions.md#string) to cast +specific fields: + +GENMD-RUN-COMMAND +mlr --icsv --ojson cat data/quoted-numeric.csv +GENMD-EOF + +GENMD-RUN-COMMAND +mlr --icsv --ojson -S cat data/quoted-numeric.csv +GENMD-EOF + +GENMD-RUN-COMMAND +mlr --icsv --ojson put '$b = string($b)' data/quoted-numeric.csv +GENMD-EOF Generally strings, numbers, and booleans don't mix; use type-casting like `string($x)` to convert. However, the dot (string-concatenation) operator has diff --git a/pkg/mlrval/mlrval_json.go b/pkg/mlrval/mlrval_json.go index a96228e6c..113f99eeb 100644 --- a/pkg/mlrval/mlrval_json.go +++ b/pkg/mlrval/mlrval_json.go @@ -413,7 +413,79 @@ func (mv *Mlrval) marshalJSONInt(outputIsStdout bool) (string, error) { func (mv *Mlrval) marshalJSONFloat(outputIsStdout bool) (string, error) { lib.InternalCodingErrorIf(mv.mvtype != MT_FLOAT) - return colorizer.MaybeColorizeValue(mv.String(), outputIsStdout), nil + s := mv.String() + // Miller type-inference accepts various number formats which the JSON + // grammar disallows: leading zeros like 004.56, leading '+', bare leading + // or trailing decimal points like .56 or 4., etc. Normally we preserve the + // user's original formatting on output; however, for JSON, we are required + // to disrespect the user's formatting whenever it would produce invalid + // JSON, and re-render the number. This parallels marshalJSONInt. + // See also https://github.com/johnkerl/miller/issues/1114 + // and https://github.com/johnkerl/miller/issues/1293. + if !isValidJSONNumber(s) { + fval, ok := mv.GetFloatValue() + if !ok { + panic("Internal coding error: float-typed mlrval denied float access") + } + s = strconv.FormatFloat(fval, 'f', -1, 64) + } + return colorizer.MaybeColorizeValue(s, outputIsStdout), nil +} + +// isValidJSONNumber determines whether a string lies within the JSON grammar +// for numbers, namely, -?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?. +func isValidJSONNumber(s string) bool { + n := len(s) + i := 0 + + if i >= n { + return false + } + if s[i] == '-' { + i++ + } + + // Integer part: '0' alone, or a nonzero digit followed by any digits. + if i >= n { + return false + } + if s[i] == '0' { + i++ + } else if '1' <= s[i] && s[i] <= '9' { + i++ + for i < n && '0' <= s[i] && s[i] <= '9' { + i++ + } + } else { + return false + } + + // Optional fractional part: '.' followed by one or more digits. + if i < n && s[i] == '.' { + i++ + if i >= n || s[i] < '0' || s[i] > '9' { + return false + } + for i < n && '0' <= s[i] && s[i] <= '9' { + i++ + } + } + + // Optional exponent part: 'e'/'E', optional sign, one or more digits. + if i < n && (s[i] == 'e' || s[i] == 'E') { + i++ + if i < n && (s[i] == '+' || s[i] == '-') { + i++ + } + if i >= n || s[i] < '0' || s[i] > '9' { + return false + } + for i < n && '0' <= s[i] && s[i] <= '9' { + i++ + } + } + + return i == n } func (mv *Mlrval) marshalJSONBool(outputIsStdout bool) (string, error) { diff --git a/pkg/mlrval/mlrval_json_test.go b/pkg/mlrval/mlrval_json_test.go new file mode 100644 index 000000000..5bb34c54b --- /dev/null +++ b/pkg/mlrval/mlrval_json_test.go @@ -0,0 +1,101 @@ +package mlrval + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsValidJSONNumber(t *testing.T) { + validCases := []string{ + "0", + "-0", + "1", + "-1", + "123", + "-123", + "0.5", + "-0.5", + "4.56", + "-4.56", + "123.456", + "1e2", + "1E2", + "1e+2", + "1e-2", + "0.5e2", + "-0.5E-2", + "1234567890123456789", + } + for _, input := range validCases { + assert.True(t, isValidJSONNumber(input), "expected valid JSON number: %q", input) + } + + invalidCases := []string{ + "", + "-", + "+", + "+1", // leading plus + "+4.56", // leading plus + "004.56", // leading zeros + "00.56", // leading zeros + "-004.56", // leading zeros + "007.5", // leading zeros + "01", // leading zero + ".56", // no digit before decimal point + "-.5", // no digit before decimal point + "4.", // no digit after decimal point + "1.e3", // no digit after decimal point + ".5e2", // no digit before decimal point + "00.5e2", // leading zeros + "1e", // no exponent digits + "1e+", // no exponent digits + "1.2.3", // multiple decimal points + "0x1f", // hex is not JSON + "0b101", // binary is not JSON + "NaN", // not in the JSON grammar + "+Inf", // not in the JSON grammar + "-Inf", // not in the JSON grammar + "Infinity", // not in the JSON grammar + "abc", + "1abc", + } + for _, input := range invalidCases { + assert.False(t, isValidJSONNumber(input), "expected invalid JSON number: %q", input) + } +} + +// TestMarshalJSONFloatValidity checks that float-typed mlrvals whose original +// string representations are not within the JSON number grammar -- e.g. +// leading zeros like 004.56 -- are re-rendered as valid JSON numbers. +// See https://github.com/johnkerl/miller/issues/1114 +// and https://github.com/johnkerl/miller/issues/1293. +func TestMarshalJSONFloatValidity(t *testing.T) { + cases := []struct { + input string + expected string + }{ + // Invalid-for-JSON originals are re-rendered: + {"004.56", "4.56"}, + {"-004.56", "-4.56"}, + {"+4.56", "4.56"}, + {".56", "0.56"}, + {"4.", "4"}, + {"1.e3", "1000"}, + {".5e2", "50"}, + {"00.5e2", "50"}, + // Valid-for-JSON originals are passed through as-is: + {"4.56", "4.56"}, + {"0.5", "0.5"}, + {"1e-2", "1e-2"}, + {"1E3", "1E3"}, + {"123.456", "123.456"}, + } + for _, c := range cases { + mv := FromInferredType(c.input) + assert.True(t, mv.IsFloat(), "expected float inference for %q", c.input) + actual, err := mv.marshalJSONFloat(false) + assert.Nil(t, err) + assert.Equal(t, c.expected, actual, "JSON marshal of %q", c.input) + } +} diff --git a/test/cases/io-json-io/0040/cmd b/test/cases/io-json-io/0040/cmd new file mode 100644 index 000000000..915a41119 --- /dev/null +++ b/test/cases/io-json-io/0040/cmd @@ -0,0 +1 @@ +mlr --icsv --ojson --ofmt '' cat ${CASEDIR}/input diff --git a/test/cases/io-json-io/0040/experr b/test/cases/io-json-io/0040/experr new file mode 100644 index 000000000..e69de29bb diff --git a/test/cases/io-json-io/0040/expout b/test/cases/io-json-io/0040/expout new file mode 100644 index 000000000..58821df8a --- /dev/null +++ b/test/cases/io-json-io/0040/expout @@ -0,0 +1,12 @@ +[ +{ + "a": "hello", + "b": 4.56, + "c": "0123456789", + "d": "0012AS4", + "e": 0.56, + "f": 4, + "g": 4.56, + "h": 1e-2 +} +] diff --git a/test/cases/io-json-io/0040/input b/test/cases/io-json-io/0040/input new file mode 100644 index 000000000..200c305a8 --- /dev/null +++ b/test/cases/io-json-io/0040/input @@ -0,0 +1,2 @@ +a,b,c,d,e,f,g,h +"hello","004.56","0123456789","0012AS4",".56","4.","+4.56","1e-2"