mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-17 16:38:54 +00:00
* Add MT_BYTES mlrval type: foundation and disposition tables First step toward a first-class bytes type in the DSL (#1231). Adds MT_BYTES (payload []byte, rendered as lowercase hex in all output formats, JSON-encoded as a hex string), extends every disposition matrix/vector with the new row/column -- real cells for comparison, sorting, and dot-concat of bytes with bytes; type-error stubs elsewhere -- and adds sweep tests asserting no table has nil cells, since Go zero-fills short array literals when MT_DIM grows. Bytes values are not yet constructible from the DSL; b"..." literals and constructor/codec functions follow in subsequent commits. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Add b"..." bytes-literal syntax to the DSL Adds a bytes_literal token to the grammar (regenerating the PGPG lexer and parser) and a BytesLiteralNode in the CST which evaluates to an MT_BYTES mlrval. Escape handling reuses UnbackslashStringLiteral, which is already byte-oriented: b"\xff" is the single byte 0xff. Unlike string literals, bytes literals never participate in regex-capture replacement. A bare identifier b is unaffected. Part of #1231. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Add bytes DSL functions: conversions, codecs, and bytes-aware built-ins - bytes(x) converts strings to bytes; string(b) reinterprets raw bytes as UTF-8 text (the reverse) - base64_decode now always returns bytes (superseding the interim string-or-hex behavior); base64_encode accepts string or bytes - New hex_encode/hex_decode functions - is_bytes and asserting_bytes predicates - md5/sha1/sha256/sha512 accept bytes, hashing the raw payload - strlen of bytes is the byte count; substr/substr0/substr1 on bytes slice by byte position and return bytes The Cyrillic-LDAP scenario from #1231 now works without exec workarounds: string(base64_decode($x)) recovers the text, and binary payloads survive undamaged as bytes. Closes #1231. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Add bytes-type docs and regression cases Documents the bytes type on the data-types page, regenerates the function-reference/man-page material, and adds regression coverage: literal escape forms, operators (concat/compare/slice/sort and type errors), conversions and codec round-trips, and CSV-to-JSON output rendering of bytes fields. Part of #1231. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Reposition MT_BYTES to sort adjacent to MT_STRING in the type enum MT_BYTES was appended after MT_ABSENT for index stability; move it right after MT_STRING instead, since that's where it conceptually belongs and where it already sorts in the cmp disposition matrices. Mechanically re-derive all ~40 disposition tables in pkg/bifs and pkg/mlrval accordingly. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> * fix windows CI * fix merge --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
348 lines
8.2 KiB
Go
348 lines
8.2 KiB
Go
// Constructors
|
|
|
|
package mlrval
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
|
|
"github.com/johnkerl/miller/v6/pkg/lib"
|
|
)
|
|
|
|
// TODO: comment for JSON-scanner context.
|
|
func FromPending() *Mlrval {
|
|
return &Mlrval{
|
|
mvtype: MT_PENDING,
|
|
printrep: "(bug-if-you-see-this:case-1)",
|
|
printrepValid: false,
|
|
}
|
|
}
|
|
|
|
// TODO: comment JIT context. Some things we already know are typed -- DSL
|
|
// things, or JSON contents. Others are deferred, e.g. items from any file
|
|
// format except JSON.
|
|
// TODO: comment re inferBool.
|
|
func FromDeferredType(input string) *Mlrval {
|
|
return &Mlrval{
|
|
mvtype: MT_PENDING,
|
|
printrep: input,
|
|
printrepValid: true,
|
|
}
|
|
}
|
|
|
|
func FromError(err error) *Mlrval {
|
|
return &Mlrval{
|
|
mvtype: MT_ERROR,
|
|
err: err,
|
|
printrep: ERROR_PRINTREP,
|
|
printrepValid: true,
|
|
}
|
|
}
|
|
|
|
func FromErrorString(err string) *Mlrval {
|
|
return &Mlrval{
|
|
mvtype: MT_ERROR,
|
|
err: errors.New(err),
|
|
printrep: ERROR_PRINTREP,
|
|
printrepValid: true,
|
|
}
|
|
}
|
|
|
|
func FromAnonymousError() *Mlrval {
|
|
return &Mlrval{
|
|
mvtype: MT_ERROR,
|
|
printrep: ERROR_PRINTREP,
|
|
printrepValid: true,
|
|
}
|
|
}
|
|
|
|
func FromTypeErrorUnary(funcname string, v *Mlrval) *Mlrval {
|
|
return FromError(
|
|
fmt.Errorf(
|
|
"%s: unacceptable type %s with value %s",
|
|
funcname,
|
|
v.GetTypeName(),
|
|
v.StringMaybeQuoted(),
|
|
),
|
|
)
|
|
}
|
|
|
|
func FromTypeErrorBinary(funcname string, v, input2 *Mlrval) *Mlrval {
|
|
return FromError(
|
|
fmt.Errorf(
|
|
"%s: unacceptable types %s, %s with values %s, %s",
|
|
funcname,
|
|
v.GetTypeName(),
|
|
input2.GetTypeName(),
|
|
v.StringMaybeQuoted(),
|
|
input2.StringMaybeQuoted(),
|
|
),
|
|
)
|
|
}
|
|
|
|
func FromTypeErrorTernary(funcname string, v, input2, input3 *Mlrval) *Mlrval {
|
|
return FromError(
|
|
fmt.Errorf(
|
|
"%s: unacceptable types %s, %s, %s with values %s, %s, %s",
|
|
funcname,
|
|
v.GetTypeName(),
|
|
input2.GetTypeName(),
|
|
input3.GetTypeName(),
|
|
v.StringMaybeQuoted(),
|
|
input2.StringMaybeQuoted(),
|
|
input3.StringMaybeQuoted(),
|
|
),
|
|
)
|
|
}
|
|
|
|
func FromNotStringError(funcname string, v *Mlrval) *Mlrval {
|
|
return FromNotNamedTypeError(funcname, v, "string")
|
|
}
|
|
|
|
func FromNotBooleanError(funcname string, v *Mlrval) *Mlrval {
|
|
return FromNotNamedTypeError(funcname, v, "boolean")
|
|
}
|
|
|
|
func FromNotIntError(funcname string, v *Mlrval) *Mlrval {
|
|
return FromNotNamedTypeError(funcname, v, "int")
|
|
}
|
|
|
|
func FromNotNumericError(funcname string, v *Mlrval) *Mlrval {
|
|
return FromNotNamedTypeError(funcname, v, "int or float")
|
|
}
|
|
|
|
func FromNotArrayError(funcname string, v *Mlrval) *Mlrval {
|
|
return FromNotNamedTypeError(funcname, v, "array")
|
|
}
|
|
|
|
func FromNotMapError(funcname string, v *Mlrval) *Mlrval {
|
|
return FromNotNamedTypeError(funcname, v, "map")
|
|
}
|
|
|
|
func FromNotCollectionError(funcname string, v *Mlrval) *Mlrval {
|
|
return FromNotNamedTypeError(funcname, v, "array or map")
|
|
}
|
|
|
|
func FromNotNamedTypeError(funcname string, v *Mlrval, expectedTypeName string) *Mlrval {
|
|
return FromError(
|
|
fmt.Errorf(
|
|
"%s: unacceptable value %s with type %s; needed type %s",
|
|
funcname,
|
|
v.StringMaybeQuoted(),
|
|
v.GetTypeName(),
|
|
expectedTypeName,
|
|
),
|
|
)
|
|
}
|
|
|
|
// TODO: comment non-JIT context like mlr put -s.
|
|
// TODO: comment re inferBool.
|
|
func FromInferredType(input string) *Mlrval {
|
|
mv := &Mlrval{
|
|
mvtype: MT_PENDING,
|
|
printrep: input,
|
|
printrepValid: true,
|
|
}
|
|
// TODO: comment re data files vs literals context -- this is for the latter
|
|
switch input {
|
|
case "true":
|
|
return TRUE
|
|
case "false":
|
|
return FALSE
|
|
}
|
|
packageLevelInferrer(mv)
|
|
return mv
|
|
}
|
|
|
|
func FromString(input string) *Mlrval {
|
|
if input == "" {
|
|
return VOID
|
|
}
|
|
return &Mlrval{
|
|
mvtype: MT_STRING,
|
|
printrep: input,
|
|
printrepValid: true,
|
|
}
|
|
}
|
|
|
|
func (mv *Mlrval) SetFromString(input string) *Mlrval {
|
|
mv.printrep = input
|
|
mv.printrepValid = true
|
|
if input == "" {
|
|
mv.mvtype = MT_VOID
|
|
} else {
|
|
mv.mvtype = MT_STRING
|
|
}
|
|
return mv
|
|
}
|
|
|
|
func FromBytes(input []byte) *Mlrval {
|
|
return &Mlrval{
|
|
mvtype: MT_BYTES,
|
|
printrepValid: false,
|
|
intf: input,
|
|
}
|
|
}
|
|
|
|
func FromInt(input int64) *Mlrval {
|
|
return &Mlrval{
|
|
mvtype: MT_INT,
|
|
printrepValid: false,
|
|
intf: input,
|
|
}
|
|
}
|
|
|
|
func FromIntShowingOctal(input int64) *Mlrval {
|
|
return &Mlrval{
|
|
mvtype: MT_INT,
|
|
printrepValid: true,
|
|
printrep: fmt.Sprintf("0%o", input),
|
|
intf: input,
|
|
}
|
|
}
|
|
|
|
// TryFromIntString is used by the mlrval Formatter (fmtnum DSL function,
|
|
// format-values verb, etc). Each mlrval has printrep and a printrepValid for
|
|
// its original string, then a type-code like MT_INT or MT_FLOAT, and
|
|
// type-specific storage like intval or floatval.
|
|
//
|
|
// If the user has taken a mlrval with original string "314" and formatted it
|
|
// with "0x%04x" then its printrep will be "0x013a" but its type should still
|
|
// be MT_INT.
|
|
//
|
|
// If on the other hand the user has formatted the same mlrval with
|
|
// "[[%0x04x]]" then its printrep will be "[[0x013a]]" and it will be
|
|
// MT_STRING. This function supports that.
|
|
func TryFromIntString(input string) *Mlrval {
|
|
intval, ok := lib.TryIntFromString(input)
|
|
if ok {
|
|
return FromPrevalidatedIntString(input, intval)
|
|
}
|
|
return FromString(input)
|
|
}
|
|
|
|
// TODO: comment
|
|
func (mv *Mlrval) SetFromPrevalidatedIntString(input string, intval int64) *Mlrval {
|
|
mv.printrep = input
|
|
mv.printrepValid = true
|
|
mv.intf = intval
|
|
mv.mvtype = MT_INT
|
|
return mv
|
|
}
|
|
|
|
// TODO: comment
|
|
func FromPrevalidatedIntString(input string, intval int64) *Mlrval {
|
|
mv := &Mlrval{}
|
|
mv.SetFromPrevalidatedIntString(input, intval)
|
|
return mv
|
|
}
|
|
|
|
func FromFloat(input float64) *Mlrval {
|
|
return &Mlrval{
|
|
mvtype: MT_FLOAT,
|
|
printrepValid: false,
|
|
intf: input,
|
|
}
|
|
}
|
|
|
|
// TryFromFloatString is used by the mlrval Formatter (fmtnum DSL function,
|
|
// format-values verb, etc). Each mlrval has printrep and a printrepValid for
|
|
// its original string, then a type-code like MT_INT or MT_FLOAT, and
|
|
// type-specific storage like intval or floatval.
|
|
//
|
|
// If the user has taken a mlrval with original string "3.14" and formatted it
|
|
// with "%.4f" then its printrep will be "3.1400" but its type should still be
|
|
// MT_FLOAT.
|
|
//
|
|
// If on the other hand the user has formatted the same mlrval with "[[%.4f]]"
|
|
// then its printrep will be "[[3.1400]]" and it will be MT_STRING. This
|
|
// function supports that.
|
|
func TryFromFloatString(input string) *Mlrval {
|
|
floatval, ok := lib.TryFloatFromString(input)
|
|
if ok {
|
|
return FromPrevalidatedFloatString(input, floatval)
|
|
}
|
|
return FromString(input)
|
|
}
|
|
|
|
// TODO: comment
|
|
func (mv *Mlrval) SetFromPrevalidatedFloatString(input string, floatval float64) *Mlrval {
|
|
mv.printrep = input
|
|
mv.printrepValid = true
|
|
mv.intf = floatval
|
|
mv.mvtype = MT_FLOAT
|
|
return mv
|
|
}
|
|
|
|
// TODO: comment
|
|
func FromPrevalidatedFloatString(input string, floatval float64) *Mlrval {
|
|
mv := &Mlrval{}
|
|
mv.SetFromPrevalidatedFloatString(input, floatval)
|
|
return mv
|
|
}
|
|
|
|
func FromBool(input bool) *Mlrval {
|
|
if input {
|
|
return TRUE
|
|
}
|
|
return FALSE
|
|
}
|
|
|
|
func FromBoolString(input string) *Mlrval {
|
|
switch input {
|
|
case "true":
|
|
return TRUE
|
|
case "false":
|
|
return FALSE
|
|
}
|
|
lib.InternalCodingErrorIf(true)
|
|
return nil // not reached
|
|
}
|
|
|
|
// The user-defined function is of type 'interface{}' here to avoid what would
|
|
// otherwise be a package-dependency cycle between this package and
|
|
// github.com/johnkerl/miller/v6/pkg/dsl/cst.
|
|
//
|
|
// Nominally the name argument is the user-specified name if `func f(a, b) {
|
|
// ... }`, or some autogenerated UUID like `fl0052` if `func (a, b) { ... }`.
|
|
|
|
func FromFunction(funcval interface{}, name string) *Mlrval {
|
|
return &Mlrval{
|
|
mvtype: MT_FUNC,
|
|
printrep: name,
|
|
printrepValid: true,
|
|
intf: funcval,
|
|
}
|
|
}
|
|
|
|
func FromArray(arrayval []*Mlrval) *Mlrval {
|
|
return &Mlrval{
|
|
mvtype: MT_ARRAY,
|
|
printrep: "(bug-if-you-see-this:case-4)", // INVALID_PRINTREP,
|
|
printrepValid: false,
|
|
intf: CopyMlrvalArray(arrayval),
|
|
}
|
|
}
|
|
|
|
func FromSingletonArray(element *Mlrval) *Mlrval {
|
|
a := make([]*Mlrval, 1)
|
|
a[0] = element
|
|
return FromArray(a)
|
|
}
|
|
|
|
func FromEmptyArray() *Mlrval {
|
|
return FromArray([]*Mlrval{})
|
|
}
|
|
|
|
func FromMap(mapval *Mlrmap) *Mlrval {
|
|
return &Mlrval{
|
|
mvtype: MT_MAP,
|
|
printrep: "(bug-if-you-see-this:case-5)", // INVALID_PRINTREP,
|
|
printrepValid: false,
|
|
intf: mapval.Copy(),
|
|
}
|
|
}
|
|
|
|
func FromEmptyMap() *Mlrval {
|
|
return FromMap(NewMlrmap())
|
|
}
|