mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-18 00:45:47 +00:00
78363f1cc3
The DKVPX record reader set dkvpxReader.Comma = ',' unconditionally,
ignoring the resolved --ifs value; the key-value separator '=' was
likewise hard-coded in the dkvpx parser, ignoring --ips. So e.g.
printf 'a=1;b=2\n' | mlr -i dkvpx --ifs semicolon --ojson cat
produced a single field {"a": "1;b=2"} instead of {"a": 1, "b": 2}.
Fixes:
* pkg/dkvpx.Reader gains an Equals field (default '=') alongside Comma.
* The DKVPX record reader now passes the resolved IFS/IPS through,
validating that each is a single character (mirroring the CSV
reader's "IFS can only be a single character" error) rather than
silently ignoring or truncating them.
* The DKVPX record writer already emitted OFS/OPS, but its quoting
decisions were based on the default separators, so with --ofs
semicolon a value containing ';' went out unquoted (ambiguous) while
a value containing ',' was quoted needlessly. Quoting now keys off
the actual OFS/OPS.
Adds unit tests, regression cases under test/cases/io-dkvpx/, and a
note in the separators reference table.
Discovered while investigating #369; tracked in #2170.
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
134 lines
3.7 KiB
Go
134 lines
3.7 KiB
Go
package dkvpx
|
|
|
|
import (
|
|
"io"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/johnkerl/miller/v6/pkg/lib"
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
// orderedMapToMap converts OrderedMap to map for easier assertion (order-agnostic).
|
|
func orderedMapToMap(om *lib.OrderedMap[string]) map[string]string {
|
|
if om == nil {
|
|
return nil
|
|
}
|
|
m := make(map[string]string)
|
|
for pe := om.Head; pe != nil; pe = pe.Next {
|
|
m[pe.Key] = pe.Value
|
|
}
|
|
return m
|
|
}
|
|
|
|
func TestRead_Basic(t *testing.T) {
|
|
r := NewReader(strings.NewReader("x=1,y=2,z=3\n"))
|
|
rec, err := r.Read()
|
|
assert.NoError(t, err)
|
|
assert.NotNil(t, rec)
|
|
assert.Equal(t, map[string]string{"x": "1", "y": "2", "z": "3"}, orderedMapToMap(rec))
|
|
|
|
rec, err = r.Read()
|
|
assert.Nil(t, rec)
|
|
assert.ErrorIs(t, err, io.EOF)
|
|
}
|
|
|
|
func TestRead_ImplicitKeys(t *testing.T) {
|
|
r := NewReader(strings.NewReader("x=1,2,z=3\n"))
|
|
rec, err := r.Read()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, map[string]string{"x": "1", "2": "2", "z": "3"}, orderedMapToMap(rec))
|
|
}
|
|
|
|
func TestRead_QuotedKeyAndValue(t *testing.T) {
|
|
r := NewReader(strings.NewReader(`"x,y"="a,b,c",z=3` + "\n"))
|
|
rec, err := r.Read()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, map[string]string{"x,y": "a,b,c", "z": "3"}, orderedMapToMap(rec))
|
|
}
|
|
|
|
func TestRead_EscapedQuotes(t *testing.T) {
|
|
r := NewReader(strings.NewReader(`x="the ""word""",y=2` + "\n"))
|
|
rec, err := r.Read()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, map[string]string{"x": `the "word"`, "y": "2"}, orderedMapToMap(rec))
|
|
}
|
|
|
|
func TestRead_MultilineValue(t *testing.T) {
|
|
r := NewReader(strings.NewReader("x=\"a\n b\",y=2\n"))
|
|
rec, err := r.Read()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, map[string]string{"x": "a\n b", "y": "2"}, orderedMapToMap(rec))
|
|
}
|
|
|
|
func TestRead_EmptyLine(t *testing.T) {
|
|
r := NewReader(strings.NewReader("\n"))
|
|
rec, err := r.Read()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, map[string]string{}, orderedMapToMap(rec))
|
|
}
|
|
|
|
func TestRead_EmptyEOF(t *testing.T) {
|
|
r := NewReader(strings.NewReader(""))
|
|
rec, err := r.Read()
|
|
assert.Nil(t, rec)
|
|
assert.ErrorIs(t, err, io.EOF)
|
|
}
|
|
|
|
func TestRead_ReadAll(t *testing.T) {
|
|
r := NewReader(strings.NewReader("a=1,b=2\nx=10,y=20\n"))
|
|
rec1, err := r.Read()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, map[string]string{"a": "1", "b": "2"}, orderedMapToMap(rec1))
|
|
|
|
rec2, err := r.Read()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, map[string]string{"x": "10", "y": "20"}, orderedMapToMap(rec2))
|
|
|
|
rec3, err := r.Read()
|
|
assert.Nil(t, rec3)
|
|
assert.ErrorIs(t, err, io.EOF)
|
|
}
|
|
|
|
func TestRead_ValueWithEquals(t *testing.T) {
|
|
r := NewReader(strings.NewReader("a=b=c,d=1\n"))
|
|
rec, err := r.Read()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, map[string]string{"a": "b=c", "d": "1"}, orderedMapToMap(rec))
|
|
}
|
|
|
|
func TestRead_CommentSkipped(t *testing.T) {
|
|
r := NewReader(strings.NewReader("# comment\na=1,b=2\n"))
|
|
r.Comment = '#'
|
|
rec, err := r.Read()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, map[string]string{"a": "1", "b": "2"}, orderedMapToMap(rec))
|
|
}
|
|
|
|
func TestRead_NonDefaultComma(t *testing.T) {
|
|
r := NewReader(strings.NewReader("a=1;b=2;c=\"x;y\"\n"))
|
|
r.Comma = ';'
|
|
rec, err := r.Read()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, map[string]string{"a": "1", "b": "2", "c": "x;y"}, orderedMapToMap(rec))
|
|
}
|
|
|
|
func TestRead_NonDefaultEquals(t *testing.T) {
|
|
r := NewReader(strings.NewReader("a:1,b:2,c:\"x:y\"\n"))
|
|
r.Equals = ':'
|
|
rec, err := r.Read()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, map[string]string{"a": "1", "b": "2", "c": "x:y"}, orderedMapToMap(rec))
|
|
}
|
|
|
|
func TestRead_OrderPreserved(t *testing.T) {
|
|
r := NewReader(strings.NewReader("z=3,x=1,y=2\n"))
|
|
rec, err := r.Read()
|
|
assert.NoError(t, err)
|
|
// OrderedMap should preserve insertion order
|
|
keys := make([]string, 0)
|
|
for pe := rec.Head; pe != nil; pe = pe.Next {
|
|
keys = append(keys, pe.Key)
|
|
}
|
|
assert.Equal(t, []string{"z", "x", "y"}, keys)
|
|
}
|