mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-17 16:38:54 +00:00
12c96298b9
* Add MT_BYTES mlrval type: foundation and disposition tables First step toward a first-class bytes type in the DSL (#1231). Adds MT_BYTES (payload []byte, rendered as lowercase hex in all output formats, JSON-encoded as a hex string), extends every disposition matrix/vector with the new row/column -- real cells for comparison, sorting, and dot-concat of bytes with bytes; type-error stubs elsewhere -- and adds sweep tests asserting no table has nil cells, since Go zero-fills short array literals when MT_DIM grows. Bytes values are not yet constructible from the DSL; b"..." literals and constructor/codec functions follow in subsequent commits. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Add b"..." bytes-literal syntax to the DSL Adds a bytes_literal token to the grammar (regenerating the PGPG lexer and parser) and a BytesLiteralNode in the CST which evaluates to an MT_BYTES mlrval. Escape handling reuses UnbackslashStringLiteral, which is already byte-oriented: b"\xff" is the single byte 0xff. Unlike string literals, bytes literals never participate in regex-capture replacement. A bare identifier b is unaffected. Part of #1231. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Add bytes DSL functions: conversions, codecs, and bytes-aware built-ins - bytes(x) converts strings to bytes; string(b) reinterprets raw bytes as UTF-8 text (the reverse) - base64_decode now always returns bytes (superseding the interim string-or-hex behavior); base64_encode accepts string or bytes - New hex_encode/hex_decode functions - is_bytes and asserting_bytes predicates - md5/sha1/sha256/sha512 accept bytes, hashing the raw payload - strlen of bytes is the byte count; substr/substr0/substr1 on bytes slice by byte position and return bytes The Cyrillic-LDAP scenario from #1231 now works without exec workarounds: string(base64_decode($x)) recovers the text, and binary payloads survive undamaged as bytes. Closes #1231. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Add bytes-type docs and regression cases Documents the bytes type on the data-types page, regenerates the function-reference/man-page material, and adds regression coverage: literal escape forms, operators (concat/compare/slice/sort and type errors), conversions and codec round-trips, and CSV-to-JSON output rendering of bytes fields. Part of #1231. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Reposition MT_BYTES to sort adjacent to MT_STRING in the type enum MT_BYTES was appended after MT_ABSENT for index stability; move it right after MT_STRING instead, since that's where it conceptually belongs and where it already sorts in the cmp disposition matrices. Mechanically re-derive all ~40 disposition tables in pkg/bifs and pkg/mlrval accordingly. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> * fix windows CI * fix merge --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
61 lines
1.7 KiB
Go
61 lines
1.7 KiB
Go
package mlrval
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func TestFromBytes(t *testing.T) {
|
|
mv := FromBytes([]byte{0xff, 0x01})
|
|
assert.Equal(t, MT_BYTES, mv.Type())
|
|
assert.Equal(t, "bytes", mv.GetTypeName())
|
|
assert.True(t, mv.IsBytes())
|
|
assert.Equal(t, []byte{0xff, 0x01}, mv.AcquireBytesValue())
|
|
|
|
empty := FromBytes([]byte{})
|
|
assert.Equal(t, MT_BYTES, empty.Type())
|
|
assert.Equal(t, 0, len(empty.AcquireBytesValue()))
|
|
}
|
|
|
|
func TestBytesStringIsHex(t *testing.T) {
|
|
mv := FromBytes([]byte{0xff, 0x01})
|
|
assert.Equal(t, "ff01", mv.String())
|
|
|
|
assert.Equal(t, "", FromBytes([]byte{}).String())
|
|
|
|
// StringMaybeQuoted: bytes are unquoted, like numbers
|
|
assert.Equal(t, "ff01", FromBytes([]byte{0xff, 0x01}).StringMaybeQuoted())
|
|
}
|
|
|
|
func TestBytesJSON(t *testing.T) {
|
|
mv := FromBytes([]byte{0xff, 0x01})
|
|
s, err := mv.FormatAsJSON(JSON_SINGLE_LINE, false)
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, `"ff01"`, s)
|
|
}
|
|
|
|
func TestBytesCopyIsIndependent(t *testing.T) {
|
|
original := []byte{1, 2, 3}
|
|
mv := FromBytes(original)
|
|
other := mv.Copy()
|
|
original[0] = 99
|
|
assert.Equal(t, []byte{99, 2, 3}, mv.AcquireBytesValue())
|
|
assert.Equal(t, []byte{1, 2, 3}, other.AcquireBytesValue())
|
|
}
|
|
|
|
func TestBytesCmp(t *testing.T) {
|
|
a := FromBytes([]byte{0x01})
|
|
b := FromBytes([]byte{0x02})
|
|
assert.Equal(t, -1, Cmp(a, b))
|
|
assert.Equal(t, 1, Cmp(b, a))
|
|
assert.Equal(t, 0, Cmp(a, FromBytes([]byte{0x01})))
|
|
assert.True(t, Equals(a, FromBytes([]byte{0x01})))
|
|
assert.False(t, Equals(a, b))
|
|
|
|
// Mixed-type sort order: string < bytes < array
|
|
assert.Equal(t, -1, Cmp(FromString("zzz"), a))
|
|
assert.Equal(t, 1, Cmp(a, FromString("zzz")))
|
|
assert.Equal(t, -1, Cmp(a, FromArray([]*Mlrval{})))
|
|
assert.Equal(t, 1, Cmp(FromArray([]*Mlrval{}), a))
|
|
}
|