Emit valid JSON numbers for floats like 004.56; document CSV quoting vs. type inference (#2151)

Miller type-inference accepts float formats which the JSON grammar
disallows -- leading zeros like 004.56, leading plus signs, bare leading
or trailing decimal points like .56 or 4., etc. -- and the JSON writer
passed the original text through verbatim, producing invalid JSON.
Leading-zero integers such as 0123456789 were already handled (inferred
as strings), and hex/binary/octal ints were already re-rendered as
decimal for JSON per #1761; this does the same for floats: when a
float string representation is not a valid JSON number, re-render it
(e.g. 004.56 -> 4.56).

Also documents in reference-main-data-types that CSV double-quoting
does not affect type inference, with -S / --infer-none and string()
as the ways to keep such values as strings.

Addresses #1114 and #1293.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

docs/src/data/quoted-numeric.csv

@@ -0,0 +1,2 @@
+a,b
+"hello","004.56"

docs/src/reference-main-data-types.md

@@ -62,6 +62,54 @@ will produce `x=1,y=2,z=3`.
 Numbers retain their original string representation, so if `x` is `1.2` on one
 record and `1.200` on another, they'll print out that way on output (unless of
 course they've been modified during processing, e.g. `mlr put '$x = $x + 10`).
+One exception: on JSON output, numbers whose original text isn't valid in the
+JSON grammar -- e.g. `004.56`, whose leading zeros JSON disallows -- are
+re-rendered (here, as `4.56`) so that Miller always writes valid JSON.
+
+Note that double quotes in CSV input don't affect type inference. In CSV,
+quoting exists to allow field content containing commas, newlines, and/or
+double quotes -- unlike in JSON, quoting doesn't distinguish strings from
+numbers. So `"4.56"` in a CSV file scans as a float, just as `4.56` does. If
+you want values kept as strings, you can use `mlr -S` (or, synonymously, `mlr
+--infer-none`) to disable type inference entirely, or use the
+[`string` DSL function](reference-dsl-builtin-functions.md#string) to cast
+specific fields:
+
+<pre class="pre-highlight-in-pair">
+<b>mlr --icsv --ojson cat data/quoted-numeric.csv</b>
+</pre>
+<pre class="pre-non-highlight-in-pair">
+[
+{
+  "a": "hello",
+  "b": 4.56
+}
+]
+</pre>
+
+<pre class="pre-highlight-in-pair">
+<b>mlr --icsv --ojson -S cat data/quoted-numeric.csv</b>
+</pre>
+<pre class="pre-non-highlight-in-pair">
+[
+{
+  "a": "hello",
+  "b": "004.56"
+}
+]
+</pre>
+
+<pre class="pre-highlight-in-pair">
+<b>mlr --icsv --ojson put '$b = string($b)' data/quoted-numeric.csv</b>
+</pre>
+<pre class="pre-non-highlight-in-pair">
+[
+{
+  "a": "hello",
+  "b": "004.56"
+}
+]
+</pre>
 
 Generally strings, numbers, and booleans don't mix; use type-casting like
 `string($x)` to convert. However, the dot (string-concatenation) operator has

docs/src/reference-main-data-types.md.in

@@ -46,6 +46,30 @@ will produce `x=1,y=2,z=3`.
 Numbers retain their original string representation, so if `x` is `1.2` on one
 record and `1.200` on another, they'll print out that way on output (unless of
 course they've been modified during processing, e.g. `mlr put '$x = $x + 10`).
+One exception: on JSON output, numbers whose original text isn't valid in the
+JSON grammar -- e.g. `004.56`, whose leading zeros JSON disallows -- are
+re-rendered (here, as `4.56`) so that Miller always writes valid JSON.
+
+Note that double quotes in CSV input don't affect type inference. In CSV,
+quoting exists to allow field content containing commas, newlines, and/or
+double quotes -- unlike in JSON, quoting doesn't distinguish strings from
+numbers. So `"4.56"` in a CSV file scans as a float, just as `4.56` does. If
+you want values kept as strings, you can use `mlr -S` (or, synonymously, `mlr
+--infer-none`) to disable type inference entirely, or use the
+[`string` DSL function](reference-dsl-builtin-functions.md#string) to cast
+specific fields:
+
+GENMD-RUN-COMMAND
+mlr --icsv --ojson cat data/quoted-numeric.csv
+GENMD-EOF
+
+GENMD-RUN-COMMAND
+mlr --icsv --ojson -S cat data/quoted-numeric.csv
+GENMD-EOF
+
+GENMD-RUN-COMMAND
+mlr --icsv --ojson put '$b = string($b)' data/quoted-numeric.csv
+GENMD-EOF
 
 Generally strings, numbers, and booleans don't mix; use type-casting like
 `string($x)` to convert. However, the dot (string-concatenation) operator has

pkg/mlrval/mlrval_json.go

@@ -413,7 +413,79 @@ func (mv *Mlrval) marshalJSONInt(outputIsStdout bool) (string, error) {
 
 func (mv *Mlrval) marshalJSONFloat(outputIsStdout bool) (string, error) {
 	lib.InternalCodingErrorIf(mv.mvtype != MT_FLOAT)
-	return colorizer.MaybeColorizeValue(mv.String(), outputIsStdout), nil
+	s := mv.String()
+	// Miller type-inference accepts various number formats which the JSON
+	// grammar disallows: leading zeros like 004.56, leading '+', bare leading
+	// or trailing decimal points like .56 or 4., etc. Normally we preserve the
+	// user's original formatting on output; however, for JSON, we are required
+	// to disrespect the user's formatting whenever it would produce invalid
+	// JSON, and re-render the number. This parallels marshalJSONInt.
+	// See also https://github.com/johnkerl/miller/issues/1114
+	// and https://github.com/johnkerl/miller/issues/1293.
+	if !isValidJSONNumber(s) {
+		fval, ok := mv.GetFloatValue()
+		if !ok {
+			panic("Internal coding error: float-typed mlrval denied float access")
+		}
+		s = strconv.FormatFloat(fval, 'f', -1, 64)
+	}
+	return colorizer.MaybeColorizeValue(s, outputIsStdout), nil
+}
+
+// isValidJSONNumber determines whether a string lies within the JSON grammar
+// for numbers, namely, -?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?.
+func isValidJSONNumber(s string) bool {
+	n := len(s)
+	i := 0
+
+	if i >= n {
+		return false
+	}
+	if s[i] == '-' {
+		i++
+	}
+
+	// Integer part: '0' alone, or a nonzero digit followed by any digits.
+	if i >= n {
+		return false
+	}
+	if s[i] == '0' {
+		i++
+	} else if '1' <= s[i] && s[i] <= '9' {
+		i++
+		for i < n && '0' <= s[i] && s[i] <= '9' {
+			i++
+		}
+	} else {
+		return false
+	}
+
+	// Optional fractional part: '.' followed by one or more digits.
+	if i < n && s[i] == '.' {
+		i++
+		if i >= n || s[i] < '0' || s[i] > '9' {
+			return false
+		}
+		for i < n && '0' <= s[i] && s[i] <= '9' {
+			i++
+		}
+	}
+
+	// Optional exponent part: 'e'/'E', optional sign, one or more digits.
+	if i < n && (s[i] == 'e' || s[i] == 'E') {
+		i++
+		if i < n && (s[i] == '+' || s[i] == '-') {
+			i++
+		}
+		if i >= n || s[i] < '0' || s[i] > '9' {
+			return false
+		}
+		for i < n && '0' <= s[i] && s[i] <= '9' {
+			i++
+		}
+	}
+
+	return i == n
 }
 
 func (mv *Mlrval) marshalJSONBool(outputIsStdout bool) (string, error) {

pkg/mlrval/mlrval_json_test.go

@@ -0,0 +1,101 @@
+package mlrval
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsValidJSONNumber(t *testing.T) {
+	validCases := []string{
+		"0",
+		"-0",
+		"1",
+		"-1",
+		"123",
+		"-123",
+		"0.5",
+		"-0.5",
+		"4.56",
+		"-4.56",
+		"123.456",
+		"1e2",
+		"1E2",
+		"1e+2",
+		"1e-2",
+		"0.5e2",
+		"-0.5E-2",
+		"1234567890123456789",
+	}
+	for _, input := range validCases {
+		assert.True(t, isValidJSONNumber(input), "expected valid JSON number: %q", input)
+	}
+
+	invalidCases := []string{
+		"",
+		"-",
+		"+",
+		"+1",       // leading plus
+		"+4.56",    // leading plus
+		"004.56",   // leading zeros
+		"00.56",    // leading zeros
+		"-004.56",  // leading zeros
+		"007.5",    // leading zeros
+		"01",       // leading zero
+		".56",      // no digit before decimal point
+		"-.5",      // no digit before decimal point
+		"4.",       // no digit after decimal point
+		"1.e3",     // no digit after decimal point
+		".5e2",     // no digit before decimal point
+		"00.5e2",   // leading zeros
+		"1e",       // no exponent digits
+		"1e+",      // no exponent digits
+		"1.2.3",    // multiple decimal points
+		"0x1f",     // hex is not JSON
+		"0b101",    // binary is not JSON
+		"NaN",      // not in the JSON grammar
+		"+Inf",     // not in the JSON grammar
+		"-Inf",     // not in the JSON grammar
+		"Infinity", // not in the JSON grammar
+		"abc",
+		"1abc",
+	}
+	for _, input := range invalidCases {
+		assert.False(t, isValidJSONNumber(input), "expected invalid JSON number: %q", input)
+	}
+}
+
+// TestMarshalJSONFloatValidity checks that float-typed mlrvals whose original
+// string representations are not within the JSON number grammar -- e.g.
+// leading zeros like 004.56 -- are re-rendered as valid JSON numbers.
+// See https://github.com/johnkerl/miller/issues/1114
+// and https://github.com/johnkerl/miller/issues/1293.
+func TestMarshalJSONFloatValidity(t *testing.T) {
+	cases := []struct {
+		input    string
+		expected string
+	}{
+		// Invalid-for-JSON originals are re-rendered:
+		{"004.56", "4.56"},
+		{"-004.56", "-4.56"},
+		{"+4.56", "4.56"},
+		{".56", "0.56"},
+		{"4.", "4"},
+		{"1.e3", "1000"},
+		{".5e2", "50"},
+		{"00.5e2", "50"},
+		// Valid-for-JSON originals are passed through as-is:
+		{"4.56", "4.56"},
+		{"0.5", "0.5"},
+		{"1e-2", "1e-2"},
+		{"1E3", "1E3"},
+		{"123.456", "123.456"},
+	}
+	for _, c := range cases {
+		mv := FromInferredType(c.input)
+		assert.True(t, mv.IsFloat(), "expected float inference for %q", c.input)
+		actual, err := mv.marshalJSONFloat(false)
+		assert.Nil(t, err)
+		assert.Equal(t, c.expected, actual, "JSON marshal of %q", c.input)
+	}
+}

test/cases/io-json-io/0040/cmd

@@ -0,0 +1 @@
+mlr --icsv --ojson --ofmt '' cat ${CASEDIR}/input

test/cases/io-json-io/0040/expout

@@ -0,0 +1,12 @@
+[
+{
+  "a": "hello",
+  "b": 4.56,
+  "c": "0123456789",
+  "d": "0012AS4",
+  "e": 0.56,
+  "f": 4,
+  "g": 4.56,
+  "h": 1e-2
+}
+]

test/cases/io-json-io/0040/input

@@ -0,0 +1,2 @@
+a,b,c,d,e,f,g,h
+"hello","004.56","0123456789","0012AS4",".56","4.","+4.56","1e-2"