diff --git a/docs/src/data/quoted-numeric.csv b/docs/src/data/quoted-numeric.csv new file mode 100644 index 000000000..5cba089bc --- /dev/null +++ b/docs/src/data/quoted-numeric.csv @@ -0,0 +1,2 @@ +a,b +"hello","004.56" diff --git a/docs/src/reference-main-data-types.md b/docs/src/reference-main-data-types.md index 0ef616fd9..aa6bce487 100644 --- a/docs/src/reference-main-data-types.md +++ b/docs/src/reference-main-data-types.md @@ -62,6 +62,54 @@ will produce `x=1,y=2,z=3`. Numbers retain their original string representation, so if `x` is `1.2` on one record and `1.200` on another, they'll print out that way on output (unless of course they've been modified during processing, e.g. `mlr put '$x = $x + 10`). +One exception: on JSON output, numbers whose original text isn't valid in the +JSON grammar -- e.g. `004.56`, whose leading zeros JSON disallows -- are +re-rendered (here, as `4.56`) so that Miller always writes valid JSON. + +Note that double quotes in CSV input don't affect type inference. In CSV, +quoting exists to allow field content containing commas, newlines, and/or +double quotes -- unlike in JSON, quoting doesn't distinguish strings from +numbers. So `"4.56"` in a CSV file scans as a float, just as `4.56` does. If +you want values kept as strings, you can use `mlr -S` (or, synonymously, `mlr +--infer-none`) to disable type inference entirely, or use the +[`string` DSL function](reference-dsl-builtin-functions.md#string) to cast +specific fields: + +
+mlr --icsv --ojson cat data/quoted-numeric.csv ++
+[
+{
+ "a": "hello",
+ "b": 4.56
+}
+]
+
+
++mlr --icsv --ojson -S cat data/quoted-numeric.csv ++
+[
+{
+ "a": "hello",
+ "b": "004.56"
+}
+]
+
+
++mlr --icsv --ojson put '$b = string($b)' data/quoted-numeric.csv ++
+[
+{
+ "a": "hello",
+ "b": "004.56"
+}
+]
+
Generally strings, numbers, and booleans don't mix; use type-casting like
`string($x)` to convert. However, the dot (string-concatenation) operator has
diff --git a/docs/src/reference-main-data-types.md.in b/docs/src/reference-main-data-types.md.in
index 485dece20..b7b1172b2 100644
--- a/docs/src/reference-main-data-types.md.in
+++ b/docs/src/reference-main-data-types.md.in
@@ -46,6 +46,30 @@ will produce `x=1,y=2,z=3`.
Numbers retain their original string representation, so if `x` is `1.2` on one
record and `1.200` on another, they'll print out that way on output (unless of
course they've been modified during processing, e.g. `mlr put '$x = $x + 10`).
+One exception: on JSON output, numbers whose original text isn't valid in the
+JSON grammar -- e.g. `004.56`, whose leading zeros JSON disallows -- are
+re-rendered (here, as `4.56`) so that Miller always writes valid JSON.
+
+Note that double quotes in CSV input don't affect type inference. In CSV,
+quoting exists to allow field content containing commas, newlines, and/or
+double quotes -- unlike in JSON, quoting doesn't distinguish strings from
+numbers. So `"4.56"` in a CSV file scans as a float, just as `4.56` does. If
+you want values kept as strings, you can use `mlr -S` (or, synonymously, `mlr
+--infer-none`) to disable type inference entirely, or use the
+[`string` DSL function](reference-dsl-builtin-functions.md#string) to cast
+specific fields:
+
+GENMD-RUN-COMMAND
+mlr --icsv --ojson cat data/quoted-numeric.csv
+GENMD-EOF
+
+GENMD-RUN-COMMAND
+mlr --icsv --ojson -S cat data/quoted-numeric.csv
+GENMD-EOF
+
+GENMD-RUN-COMMAND
+mlr --icsv --ojson put '$b = string($b)' data/quoted-numeric.csv
+GENMD-EOF
Generally strings, numbers, and booleans don't mix; use type-casting like
`string($x)` to convert. However, the dot (string-concatenation) operator has
diff --git a/pkg/mlrval/mlrval_json.go b/pkg/mlrval/mlrval_json.go
index a96228e6c..113f99eeb 100644
--- a/pkg/mlrval/mlrval_json.go
+++ b/pkg/mlrval/mlrval_json.go
@@ -413,7 +413,79 @@ func (mv *Mlrval) marshalJSONInt(outputIsStdout bool) (string, error) {
func (mv *Mlrval) marshalJSONFloat(outputIsStdout bool) (string, error) {
lib.InternalCodingErrorIf(mv.mvtype != MT_FLOAT)
- return colorizer.MaybeColorizeValue(mv.String(), outputIsStdout), nil
+ s := mv.String()
+ // Miller type-inference accepts various number formats which the JSON
+ // grammar disallows: leading zeros like 004.56, leading '+', bare leading
+ // or trailing decimal points like .56 or 4., etc. Normally we preserve the
+ // user's original formatting on output; however, for JSON, we are required
+ // to disrespect the user's formatting whenever it would produce invalid
+ // JSON, and re-render the number. This parallels marshalJSONInt.
+ // See also https://github.com/johnkerl/miller/issues/1114
+ // and https://github.com/johnkerl/miller/issues/1293.
+ if !isValidJSONNumber(s) {
+ fval, ok := mv.GetFloatValue()
+ if !ok {
+ panic("Internal coding error: float-typed mlrval denied float access")
+ }
+ s = strconv.FormatFloat(fval, 'f', -1, 64)
+ }
+ return colorizer.MaybeColorizeValue(s, outputIsStdout), nil
+}
+
+// isValidJSONNumber determines whether a string lies within the JSON grammar
+// for numbers, namely, -?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?.
+func isValidJSONNumber(s string) bool {
+ n := len(s)
+ i := 0
+
+ if i >= n {
+ return false
+ }
+ if s[i] == '-' {
+ i++
+ }
+
+ // Integer part: '0' alone, or a nonzero digit followed by any digits.
+ if i >= n {
+ return false
+ }
+ if s[i] == '0' {
+ i++
+ } else if '1' <= s[i] && s[i] <= '9' {
+ i++
+ for i < n && '0' <= s[i] && s[i] <= '9' {
+ i++
+ }
+ } else {
+ return false
+ }
+
+ // Optional fractional part: '.' followed by one or more digits.
+ if i < n && s[i] == '.' {
+ i++
+ if i >= n || s[i] < '0' || s[i] > '9' {
+ return false
+ }
+ for i < n && '0' <= s[i] && s[i] <= '9' {
+ i++
+ }
+ }
+
+ // Optional exponent part: 'e'/'E', optional sign, one or more digits.
+ if i < n && (s[i] == 'e' || s[i] == 'E') {
+ i++
+ if i < n && (s[i] == '+' || s[i] == '-') {
+ i++
+ }
+ if i >= n || s[i] < '0' || s[i] > '9' {
+ return false
+ }
+ for i < n && '0' <= s[i] && s[i] <= '9' {
+ i++
+ }
+ }
+
+ return i == n
}
func (mv *Mlrval) marshalJSONBool(outputIsStdout bool) (string, error) {
diff --git a/pkg/mlrval/mlrval_json_test.go b/pkg/mlrval/mlrval_json_test.go
new file mode 100644
index 000000000..5bb34c54b
--- /dev/null
+++ b/pkg/mlrval/mlrval_json_test.go
@@ -0,0 +1,101 @@
+package mlrval
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestIsValidJSONNumber(t *testing.T) {
+ validCases := []string{
+ "0",
+ "-0",
+ "1",
+ "-1",
+ "123",
+ "-123",
+ "0.5",
+ "-0.5",
+ "4.56",
+ "-4.56",
+ "123.456",
+ "1e2",
+ "1E2",
+ "1e+2",
+ "1e-2",
+ "0.5e2",
+ "-0.5E-2",
+ "1234567890123456789",
+ }
+ for _, input := range validCases {
+ assert.True(t, isValidJSONNumber(input), "expected valid JSON number: %q", input)
+ }
+
+ invalidCases := []string{
+ "",
+ "-",
+ "+",
+ "+1", // leading plus
+ "+4.56", // leading plus
+ "004.56", // leading zeros
+ "00.56", // leading zeros
+ "-004.56", // leading zeros
+ "007.5", // leading zeros
+ "01", // leading zero
+ ".56", // no digit before decimal point
+ "-.5", // no digit before decimal point
+ "4.", // no digit after decimal point
+ "1.e3", // no digit after decimal point
+ ".5e2", // no digit before decimal point
+ "00.5e2", // leading zeros
+ "1e", // no exponent digits
+ "1e+", // no exponent digits
+ "1.2.3", // multiple decimal points
+ "0x1f", // hex is not JSON
+ "0b101", // binary is not JSON
+ "NaN", // not in the JSON grammar
+ "+Inf", // not in the JSON grammar
+ "-Inf", // not in the JSON grammar
+ "Infinity", // not in the JSON grammar
+ "abc",
+ "1abc",
+ }
+ for _, input := range invalidCases {
+ assert.False(t, isValidJSONNumber(input), "expected invalid JSON number: %q", input)
+ }
+}
+
+// TestMarshalJSONFloatValidity checks that float-typed mlrvals whose original
+// string representations are not within the JSON number grammar -- e.g.
+// leading zeros like 004.56 -- are re-rendered as valid JSON numbers.
+// See https://github.com/johnkerl/miller/issues/1114
+// and https://github.com/johnkerl/miller/issues/1293.
+func TestMarshalJSONFloatValidity(t *testing.T) {
+ cases := []struct {
+ input string
+ expected string
+ }{
+ // Invalid-for-JSON originals are re-rendered:
+ {"004.56", "4.56"},
+ {"-004.56", "-4.56"},
+ {"+4.56", "4.56"},
+ {".56", "0.56"},
+ {"4.", "4"},
+ {"1.e3", "1000"},
+ {".5e2", "50"},
+ {"00.5e2", "50"},
+ // Valid-for-JSON originals are passed through as-is:
+ {"4.56", "4.56"},
+ {"0.5", "0.5"},
+ {"1e-2", "1e-2"},
+ {"1E3", "1E3"},
+ {"123.456", "123.456"},
+ }
+ for _, c := range cases {
+ mv := FromInferredType(c.input)
+ assert.True(t, mv.IsFloat(), "expected float inference for %q", c.input)
+ actual, err := mv.marshalJSONFloat(false)
+ assert.Nil(t, err)
+ assert.Equal(t, c.expected, actual, "JSON marshal of %q", c.input)
+ }
+}
diff --git a/test/cases/io-json-io/0040/cmd b/test/cases/io-json-io/0040/cmd
new file mode 100644
index 000000000..915a41119
--- /dev/null
+++ b/test/cases/io-json-io/0040/cmd
@@ -0,0 +1 @@
+mlr --icsv --ojson --ofmt '' cat ${CASEDIR}/input
diff --git a/test/cases/io-json-io/0040/experr b/test/cases/io-json-io/0040/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-json-io/0040/expout b/test/cases/io-json-io/0040/expout
new file mode 100644
index 000000000..58821df8a
--- /dev/null
+++ b/test/cases/io-json-io/0040/expout
@@ -0,0 +1,12 @@
+[
+{
+ "a": "hello",
+ "b": 4.56,
+ "c": "0123456789",
+ "d": "0012AS4",
+ "e": 0.56,
+ "f": 4,
+ "g": 4.56,
+ "h": 1e-2
+}
+]
diff --git a/test/cases/io-json-io/0040/input b/test/cases/io-json-io/0040/input
new file mode 100644
index 000000000..200c305a8
--- /dev/null
+++ b/test/cases/io-json-io/0040/input
@@ -0,0 +1,2 @@
+a,b,c,d,e,f,g,h
+"hello","004.56","0123456789","0012AS4",".56","4.","+4.56","1e-2"