miller/pkg/mlrval/mlrval_output.go
John Kerl 12c96298b9
Add a first-class bytes type to the DSL, with b"..." literals and base64/hex codecs (#2122)
* Add MT_BYTES mlrval type: foundation and disposition tables

First step toward a first-class bytes type in the DSL (#1231).
Adds MT_BYTES (payload []byte, rendered as lowercase hex in all output
formats, JSON-encoded as a hex string), extends every disposition
matrix/vector with the new row/column -- real cells for comparison,
sorting, and dot-concat of bytes with bytes; type-error stubs
elsewhere -- and adds sweep tests asserting no table has nil cells,
since Go zero-fills short array literals when MT_DIM grows.

Bytes values are not yet constructible from the DSL; b"..." literals
and constructor/codec functions follow in subsequent commits.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Add b"..." bytes-literal syntax to the DSL

Adds a bytes_literal token to the grammar (regenerating the PGPG lexer
and parser) and a BytesLiteralNode in the CST which evaluates to an
MT_BYTES mlrval. Escape handling reuses UnbackslashStringLiteral,
which is already byte-oriented: b"\xff" is the single byte 0xff.
Unlike string literals, bytes literals never participate in
regex-capture replacement. A bare identifier b is unaffected.

Part of #1231.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Add bytes DSL functions: conversions, codecs, and bytes-aware built-ins

- bytes(x) converts strings to bytes; string(b) reinterprets raw bytes
  as UTF-8 text (the reverse)
- base64_decode now always returns bytes (superseding the interim
  string-or-hex behavior); base64_encode accepts string or bytes
- New hex_encode/hex_decode functions
- is_bytes and asserting_bytes predicates
- md5/sha1/sha256/sha512 accept bytes, hashing the raw payload
- strlen of bytes is the byte count; substr/substr0/substr1 on bytes
  slice by byte position and return bytes

The Cyrillic-LDAP scenario from #1231 now works without exec
workarounds: string(base64_decode($x)) recovers the text, and binary
payloads survive undamaged as bytes.

Closes #1231.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Add bytes-type docs and regression cases

Documents the bytes type on the data-types page, regenerates the
function-reference/man-page material, and adds regression coverage:
literal escape forms, operators (concat/compare/slice/sort and
type errors), conversions and codec round-trips, and CSV-to-JSON
output rendering of bytes fields.

Part of #1231.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Reposition MT_BYTES to sort adjacent to MT_STRING in the type enum

MT_BYTES was appended after MT_ABSENT for index stability; move it
right after MT_STRING instead, since that's where it conceptually
belongs and where it already sorts in the cmp disposition matrices.
Mechanically re-derive all ~40 disposition tables in pkg/bifs and
pkg/mlrval accordingly.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* fix windows CI

* fix merge

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 11:58:44 -04:00

145 lines
4.2 KiB
Go

package mlrval
import (
"encoding/hex"
"fmt"
"os"
"reflect"
"strconv"
)
// Must have non-pointer receiver in order to implement the fmt.Stringer
// interface to make this printable via fmt.Println et al. However, that
// results in a needless copy of the Mlrval. So, we intentionally use pointer
// receiver, and if we need to print to stdout, we can fmt.Printf with "%s" and
// mv.String().
func (mv *Mlrval) String() string {
// TODO: comment re deferral -- important perf effect!
// if mv.IsFloat() && floatOutputFormatter != nil
// if mv.mvtype == MT_FLOAT && floatOutputFormatter != nil {
//if floatOutputFormatter != nil && (mv.mvtype == MT_FLOAT || mv.mvtype == MT_PENDING) {
if floatOutputFormatter != nil && mv.Type() == MT_FLOAT {
// Use the format string from global --ofmt, if supplied
return floatOutputFormatter.FormatFloat(mv.intf.(float64))
}
// TODO: track dirty-flag checking / somesuch.
// At present it's cumbersome to check if an array or map has been modified
// and it's safest to always recompute the string-rep.
if mv.IsArrayOrMap() {
mv.printrepValid = false
}
mv.setPrintRep()
return mv.printrep
}
// OriginalString gets the field value as a string regardless of --ofmt specification.
// E.g if the ofmt is "%.4f" and input is 3.1415926535, OriginalString() will return
// "3.1415926535" while String() will return "3.1416".
func (mv *Mlrval) OriginalString() string {
if mv.printrepValid {
return mv.printrep
}
return mv.String()
}
// StringMaybeQuoted Returns strings double-quoted; all else not.
func (mv *Mlrval) StringMaybeQuoted() string {
output := mv.String()
if mv.mvtype == MT_VOID || mv.mvtype == MT_STRING {
return `"` + output + `"`
}
return output
}
// See mlrval.go for more about JIT-formatting of string backings
func (mv *Mlrval) setPrintRep() {
if !mv.printrepValid {
switch mv.mvtype {
case MT_PENDING:
// Should not have gotten outside of the JSON decoder, so flag this
// clearly visually if it should (buggily) slip through to
// user-level visibility.
mv.printrep = "(bug-if-you-see-this:case=3)" // xxx constdef at top of file
case MT_ERROR:
mv.printrep = "(error)" // xxx constdef at top of file
case MT_ABSENT:
// Callsites should be using absence to do non-assigns, so flag
// this clearly visually if it should (buggily) slip through to
// user-level visibility.
mv.printrep = "(bug-if-you-see-this:case=4)" // xxx constdef at top of file
case MT_VOID:
mv.printrep = "" // xxx constdef at top of file
case MT_STRING:
break
case MT_INT:
mv.printrep = strconv.FormatInt(mv.intf.(int64), 10)
case MT_FLOAT:
mv.printrep = strconv.FormatFloat(mv.intf.(float64), 'f', -1, 64)
case MT_BOOL:
if mv.intf.(bool) {
mv.printrep = "true"
} else {
mv.printrep = "false"
}
case MT_BYTES:
mv.printrep = hex.EncodeToString(mv.intf.([]byte))
case MT_ARRAY:
bytes, err := mv.FormatAsJSON(JSON_MULTILINE, false)
// maybe just InternalCodingErrorIf(err != nil)
if err != nil {
fmt.Fprintf(os.Stderr, "mlr: %v\n", err)
os.Exit(1)
}
mv.printrep = string(bytes)
case MT_MAP:
bytes, err := mv.FormatAsJSON(JSON_MULTILINE, false)
// maybe just InternalCodingErrorIf(err != nil)
if err != nil {
fmt.Fprintf(os.Stderr, "mlr: %v\n", err)
os.Exit(1)
}
mv.printrep = string(bytes)
}
mv.printrepValid = true
}
}
// StringifyValuesRecursively is nominally for the `--jvquoteall` flag.
func (mv *Mlrval) StringifyValuesRecursively() {
switch mv.mvtype {
case MT_ARRAY:
for i := range mv.intf.([]*Mlrval) {
mv.intf.([]*Mlrval)[i].StringifyValuesRecursively()
}
case MT_MAP:
for pe := mv.intf.(*Mlrmap).Head; pe != nil; pe = pe.Next {
pe.Value.StringifyValuesRecursively()
}
default:
mv.SetFromString(mv.String())
}
}
func (mv *Mlrval) ShowSizes() {
fmt.Printf("TOTAL %p %d\n", mv, reflect.TypeFor[Mlrval]().Size())
//fmt.Printf("mv.intf %p %d\n", &mv.intf, reflect.TypeOf(mv.intf).Size())
fmt.Printf("mv.printrep %p %d\n", &mv.printrep, reflect.TypeFor[string]().Size())
fmt.Printf("mv.printrepValid %p %d\n", &mv.printrepValid, reflect.TypeFor[bool]().Size())
fmt.Printf("mv.mvtype %p %d\n", &mv.mvtype, reflect.TypeFor[MVType]().Size())
}