mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-17 16:38:54 +00:00
* Add MT_BYTES mlrval type: foundation and disposition tables First step toward a first-class bytes type in the DSL (#1231). Adds MT_BYTES (payload []byte, rendered as lowercase hex in all output formats, JSON-encoded as a hex string), extends every disposition matrix/vector with the new row/column -- real cells for comparison, sorting, and dot-concat of bytes with bytes; type-error stubs elsewhere -- and adds sweep tests asserting no table has nil cells, since Go zero-fills short array literals when MT_DIM grows. Bytes values are not yet constructible from the DSL; b"..." literals and constructor/codec functions follow in subsequent commits. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Add b"..." bytes-literal syntax to the DSL Adds a bytes_literal token to the grammar (regenerating the PGPG lexer and parser) and a BytesLiteralNode in the CST which evaluates to an MT_BYTES mlrval. Escape handling reuses UnbackslashStringLiteral, which is already byte-oriented: b"\xff" is the single byte 0xff. Unlike string literals, bytes literals never participate in regex-capture replacement. A bare identifier b is unaffected. Part of #1231. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Add bytes DSL functions: conversions, codecs, and bytes-aware built-ins - bytes(x) converts strings to bytes; string(b) reinterprets raw bytes as UTF-8 text (the reverse) - base64_decode now always returns bytes (superseding the interim string-or-hex behavior); base64_encode accepts string or bytes - New hex_encode/hex_decode functions - is_bytes and asserting_bytes predicates - md5/sha1/sha256/sha512 accept bytes, hashing the raw payload - strlen of bytes is the byte count; substr/substr0/substr1 on bytes slice by byte position and return bytes The Cyrillic-LDAP scenario from #1231 now works without exec workarounds: string(base64_decode($x)) recovers the text, and binary payloads survive undamaged as bytes. Closes #1231. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Add bytes-type docs and regression cases Documents the bytes type on the data-types page, regenerates the function-reference/man-page material, and adds regression coverage: literal escape forms, operators (concat/compare/slice/sort and type errors), conversions and codec round-trips, and CSV-to-JSON output rendering of bytes fields. Part of #1231. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Reposition MT_BYTES to sort adjacent to MT_STRING in the type enum MT_BYTES was appended after MT_ABSENT for index stability; move it right after MT_STRING instead, since that's where it conceptually belongs and where it already sorts in the cmp disposition matrices. Mechanically re-derive all ~40 disposition tables in pkg/bifs and pkg/mlrval accordingly. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> * fix windows CI * fix merge --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
145 lines
4.2 KiB
Go
145 lines
4.2 KiB
Go
package mlrval
|
|
|
|
import (
|
|
"encoding/hex"
|
|
"fmt"
|
|
"os"
|
|
"reflect"
|
|
"strconv"
|
|
)
|
|
|
|
// Must have non-pointer receiver in order to implement the fmt.Stringer
|
|
// interface to make this printable via fmt.Println et al. However, that
|
|
// results in a needless copy of the Mlrval. So, we intentionally use pointer
|
|
// receiver, and if we need to print to stdout, we can fmt.Printf with "%s" and
|
|
// mv.String().
|
|
func (mv *Mlrval) String() string {
|
|
// TODO: comment re deferral -- important perf effect!
|
|
// if mv.IsFloat() && floatOutputFormatter != nil
|
|
// if mv.mvtype == MT_FLOAT && floatOutputFormatter != nil {
|
|
//if floatOutputFormatter != nil && (mv.mvtype == MT_FLOAT || mv.mvtype == MT_PENDING) {
|
|
if floatOutputFormatter != nil && mv.Type() == MT_FLOAT {
|
|
// Use the format string from global --ofmt, if supplied
|
|
return floatOutputFormatter.FormatFloat(mv.intf.(float64))
|
|
}
|
|
|
|
// TODO: track dirty-flag checking / somesuch.
|
|
// At present it's cumbersome to check if an array or map has been modified
|
|
// and it's safest to always recompute the string-rep.
|
|
if mv.IsArrayOrMap() {
|
|
mv.printrepValid = false
|
|
}
|
|
|
|
mv.setPrintRep()
|
|
return mv.printrep
|
|
}
|
|
|
|
// OriginalString gets the field value as a string regardless of --ofmt specification.
|
|
// E.g if the ofmt is "%.4f" and input is 3.1415926535, OriginalString() will return
|
|
// "3.1415926535" while String() will return "3.1416".
|
|
func (mv *Mlrval) OriginalString() string {
|
|
if mv.printrepValid {
|
|
return mv.printrep
|
|
}
|
|
return mv.String()
|
|
}
|
|
|
|
// StringMaybeQuoted Returns strings double-quoted; all else not.
|
|
func (mv *Mlrval) StringMaybeQuoted() string {
|
|
output := mv.String()
|
|
if mv.mvtype == MT_VOID || mv.mvtype == MT_STRING {
|
|
return `"` + output + `"`
|
|
}
|
|
return output
|
|
}
|
|
|
|
// See mlrval.go for more about JIT-formatting of string backings
|
|
func (mv *Mlrval) setPrintRep() {
|
|
if !mv.printrepValid {
|
|
switch mv.mvtype {
|
|
|
|
case MT_PENDING:
|
|
// Should not have gotten outside of the JSON decoder, so flag this
|
|
// clearly visually if it should (buggily) slip through to
|
|
// user-level visibility.
|
|
mv.printrep = "(bug-if-you-see-this:case=3)" // xxx constdef at top of file
|
|
|
|
case MT_ERROR:
|
|
mv.printrep = "(error)" // xxx constdef at top of file
|
|
|
|
case MT_ABSENT:
|
|
// Callsites should be using absence to do non-assigns, so flag
|
|
// this clearly visually if it should (buggily) slip through to
|
|
// user-level visibility.
|
|
mv.printrep = "(bug-if-you-see-this:case=4)" // xxx constdef at top of file
|
|
|
|
case MT_VOID:
|
|
mv.printrep = "" // xxx constdef at top of file
|
|
|
|
case MT_STRING:
|
|
break
|
|
|
|
case MT_INT:
|
|
mv.printrep = strconv.FormatInt(mv.intf.(int64), 10)
|
|
|
|
case MT_FLOAT:
|
|
mv.printrep = strconv.FormatFloat(mv.intf.(float64), 'f', -1, 64)
|
|
|
|
case MT_BOOL:
|
|
if mv.intf.(bool) {
|
|
mv.printrep = "true"
|
|
} else {
|
|
mv.printrep = "false"
|
|
}
|
|
|
|
case MT_BYTES:
|
|
mv.printrep = hex.EncodeToString(mv.intf.([]byte))
|
|
|
|
case MT_ARRAY:
|
|
bytes, err := mv.FormatAsJSON(JSON_MULTILINE, false)
|
|
// maybe just InternalCodingErrorIf(err != nil)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "mlr: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
mv.printrep = string(bytes)
|
|
|
|
case MT_MAP:
|
|
bytes, err := mv.FormatAsJSON(JSON_MULTILINE, false)
|
|
// maybe just InternalCodingErrorIf(err != nil)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "mlr: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
mv.printrep = string(bytes)
|
|
}
|
|
mv.printrepValid = true
|
|
}
|
|
}
|
|
|
|
// StringifyValuesRecursively is nominally for the `--jvquoteall` flag.
|
|
func (mv *Mlrval) StringifyValuesRecursively() {
|
|
switch mv.mvtype {
|
|
|
|
case MT_ARRAY:
|
|
for i := range mv.intf.([]*Mlrval) {
|
|
mv.intf.([]*Mlrval)[i].StringifyValuesRecursively()
|
|
}
|
|
|
|
case MT_MAP:
|
|
for pe := mv.intf.(*Mlrmap).Head; pe != nil; pe = pe.Next {
|
|
pe.Value.StringifyValuesRecursively()
|
|
}
|
|
|
|
default:
|
|
mv.SetFromString(mv.String())
|
|
}
|
|
}
|
|
|
|
func (mv *Mlrval) ShowSizes() {
|
|
fmt.Printf("TOTAL %p %d\n", mv, reflect.TypeFor[Mlrval]().Size())
|
|
//fmt.Printf("mv.intf %p %d\n", &mv.intf, reflect.TypeOf(mv.intf).Size())
|
|
fmt.Printf("mv.printrep %p %d\n", &mv.printrep, reflect.TypeFor[string]().Size())
|
|
fmt.Printf("mv.printrepValid %p %d\n", &mv.printrepValid, reflect.TypeFor[bool]().Size())
|
|
fmt.Printf("mv.mvtype %p %d\n", &mv.mvtype, reflect.TypeFor[MVType]().Size())
|
|
}
|