mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-17 16:38:54 +00:00
* Add MT_BYTES mlrval type: foundation and disposition tables First step toward a first-class bytes type in the DSL (#1231). Adds MT_BYTES (payload []byte, rendered as lowercase hex in all output formats, JSON-encoded as a hex string), extends every disposition matrix/vector with the new row/column -- real cells for comparison, sorting, and dot-concat of bytes with bytes; type-error stubs elsewhere -- and adds sweep tests asserting no table has nil cells, since Go zero-fills short array literals when MT_DIM grows. Bytes values are not yet constructible from the DSL; b"..." literals and constructor/codec functions follow in subsequent commits. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Add b"..." bytes-literal syntax to the DSL Adds a bytes_literal token to the grammar (regenerating the PGPG lexer and parser) and a BytesLiteralNode in the CST which evaluates to an MT_BYTES mlrval. Escape handling reuses UnbackslashStringLiteral, which is already byte-oriented: b"\xff" is the single byte 0xff. Unlike string literals, bytes literals never participate in regex-capture replacement. A bare identifier b is unaffected. Part of #1231. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Add bytes DSL functions: conversions, codecs, and bytes-aware built-ins - bytes(x) converts strings to bytes; string(b) reinterprets raw bytes as UTF-8 text (the reverse) - base64_decode now always returns bytes (superseding the interim string-or-hex behavior); base64_encode accepts string or bytes - New hex_encode/hex_decode functions - is_bytes and asserting_bytes predicates - md5/sha1/sha256/sha512 accept bytes, hashing the raw payload - strlen of bytes is the byte count; substr/substr0/substr1 on bytes slice by byte position and return bytes The Cyrillic-LDAP scenario from #1231 now works without exec workarounds: string(base64_decode($x)) recovers the text, and binary payloads survive undamaged as bytes. Closes #1231. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Add bytes-type docs and regression cases Documents the bytes type on the data-types page, regenerates the function-reference/man-page material, and adds regression coverage: literal escape forms, operators (concat/compare/slice/sort and type errors), conversions and codec round-trips, and CSV-to-JSON output rendering of bytes fields. Part of #1231. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Reposition MT_BYTES to sort adjacent to MT_STRING in the type enum MT_BYTES was appended after MT_ABSENT for index stability; move it right after MT_STRING instead, since that's where it conceptually belongs and where it already sorts in the cmp disposition matrices. Mechanically re-derive all ~40 disposition tables in pkg/bifs and pkg/mlrval accordingly. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> * fix windows CI * fix merge --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
160 lines
5.2 KiB
Go
160 lines
5.2 KiB
Go
// Boolean expressions for ==, !=, >, >=, <, <=, <=> on Mlrvals.
|
|
//
|
|
// Note that in bifs/boolean.go we have similar functions which take pairs of
|
|
// Mlrvals as input and return Mlrval as output. Those are for use in the
|
|
// Miller DSL. The functions here are primarily for 'mlr sort'. Their benefit
|
|
// is they don't allocate memory, and so are more efficient for sort we don't
|
|
// want to trigger lots of allocations, nor garbage collection, if we can avoid
|
|
// it.
|
|
|
|
// TODO: comment about mvtype; deferral; copying of deferrence.
|
|
|
|
package mlrval
|
|
|
|
import (
|
|
"bytes"
|
|
|
|
"github.com/johnkerl/miller/v6/pkg/lib"
|
|
)
|
|
|
|
type CmpFuncBool func(input1, input2 *Mlrval) bool
|
|
|
|
// The Go sort API is just a bool a<b, not triple a<b, a==b, a>b. Miller does the latter since when
|
|
// we sort primarily on field 1, then secondarily on field 2, etc., we need to be able to detect
|
|
// ties on field 1 so we can know whether to compare on field 2 or not.
|
|
type CmpFuncInt func(input1, input2 *Mlrval) int // -1, 0, 1 for <=>
|
|
|
|
// Exported methods
|
|
|
|
func Equals(input1, input2 *Mlrval) bool {
|
|
return cmp_dispositions[input1.Type()][input2.Type()](input1, input2) == 0
|
|
}
|
|
func GreaterThan(input1, input2 *Mlrval) bool {
|
|
return cmp_dispositions[input1.Type()][input2.Type()](input1, input2) > 0
|
|
}
|
|
func GreaterThanOrEquals(input1, input2 *Mlrval) bool {
|
|
return cmp_dispositions[input1.Type()][input2.Type()](input1, input2) >= 0
|
|
}
|
|
func LessThan(input1, input2 *Mlrval) bool {
|
|
return cmp_dispositions[input1.Type()][input2.Type()](input1, input2) < 0
|
|
}
|
|
func LessThanOrEquals(input1, input2 *Mlrval) bool {
|
|
return cmp_dispositions[input1.Type()][input2.Type()](input1, input2) <= 0
|
|
}
|
|
func Cmp(input1, input2 *Mlrval) int {
|
|
return cmp_dispositions[input1.Type()][input2.Type()](input1, input2)
|
|
}
|
|
|
|
// Support routines for disposition-matrix entries
|
|
|
|
// _same returns int 0 as a binary-input function
|
|
func _same(input1, input2 *Mlrval) int {
|
|
return 0
|
|
}
|
|
|
|
// _less returns int -1 as a binary-input function
|
|
func _less(input1, input2 *Mlrval) int {
|
|
return -1
|
|
}
|
|
|
|
// _more returns int 1 as a binary-input function
|
|
func _more(input1, input2 *Mlrval) int {
|
|
return 1
|
|
}
|
|
|
|
// int_cmp implements the spaceship operator for ints.
|
|
func int_cmp(a, b int64) int {
|
|
if a < b {
|
|
return -1
|
|
}
|
|
if a > b {
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// float_cmp implements the spaceship operator for floats.
|
|
func float_cmp(a, b float64) int {
|
|
if a < b {
|
|
return -1
|
|
}
|
|
if a > b {
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// string_cmp implements the spaceship operator for strings.
|
|
func string_cmp(a, b string) int {
|
|
if a < b {
|
|
return -1
|
|
}
|
|
if a > b {
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// Disposition-matrix entries
|
|
|
|
func cmp_b_ss(input1, input2 *Mlrval) int {
|
|
return string_cmp(input1.printrep, input2.printrep)
|
|
}
|
|
func cmp_b_ii(input1, input2 *Mlrval) int {
|
|
return int_cmp(input1.intf.(int64), input2.intf.(int64))
|
|
}
|
|
func cmp_b_if(input1, input2 *Mlrval) int {
|
|
return float_cmp(float64(input1.intf.(int64)), input2.intf.(float64))
|
|
}
|
|
func cmp_b_fi(input1, input2 *Mlrval) int {
|
|
return float_cmp(input1.intf.(float64), float64(input2.intf.(int64)))
|
|
}
|
|
func cmp_b_ff(input1, input2 *Mlrval) int {
|
|
return float_cmp(input1.intf.(float64), input2.intf.(float64))
|
|
}
|
|
func cmp_b_bb(input1, input2 *Mlrval) int {
|
|
return int_cmp(int64(lib.BoolToInt(input1.intf.(bool))), int64(lib.BoolToInt(input2.intf.(bool))))
|
|
}
|
|
func cmp_b_yy(input1, input2 *Mlrval) int {
|
|
return bytes.Compare(input1.intf.([]byte), input2.intf.([]byte))
|
|
}
|
|
|
|
// TODO: cmp on array & map
|
|
//func eq_b_aa(input1, input2 *Mlrval) bool {
|
|
// a := input1.arrayval
|
|
// b := input2.arrayval
|
|
//
|
|
// // Different-length arrays are not equal
|
|
// if len(a) != len(b) {
|
|
// return false
|
|
// }
|
|
//
|
|
// // Same-length arrays: return false if any slot is not equal, else true.
|
|
// for i := range a {
|
|
// if !Equals(&a[i], &b[i]) {
|
|
// return false
|
|
// }
|
|
// }
|
|
//
|
|
// return true
|
|
//}
|
|
|
|
//func eq_b_mm(input1, input2 *Mlrval) bool {
|
|
// return input1.mapval.Equals(input2.mapval)
|
|
//}
|
|
|
|
var cmp_dispositions = [MT_DIM][MT_DIM]CmpFuncInt{
|
|
// . INT FLOAT BOOL VOID STRING BYTES ARRAY MAP FUNC ERROR NULL ABSENT
|
|
/*INT */ {cmp_b_ii, cmp_b_if, _less, _less, _less, _less, _less, _less, _less, _less, _less, _less},
|
|
/*FLOAT */ {cmp_b_fi, cmp_b_ff, _less, _less, _less, _less, _less, _less, _less, _less, _less, _less},
|
|
/*BOOL */ {_more, _more, cmp_b_bb, _less, _less, _less, _less, _less, _less, _less, _less, _less},
|
|
/*VOID */ {_more, _more, _more, cmp_b_ss, cmp_b_ss, _less, _less, _less, _less, _less, _less, _less},
|
|
/*STRING */ {_more, _more, _more, cmp_b_ss, cmp_b_ss, _less, _less, _less, _less, _less, _less, _less},
|
|
/*BYTES */ {_more, _more, _more, _more, _more, cmp_b_yy, _less, _less, _less, _less, _less, _less},
|
|
/*ARRAY */ {_more, _more, _more, _more, _more, _more, _same, _less, _less, _less, _less, _less},
|
|
/*MAP */ {_more, _more, _more, _more, _more, _more, _more, _same, _less, _less, _less, _less},
|
|
/*func */ {_more, _more, _more, _more, _more, _more, _more, _more, _same, _less, _less, _less},
|
|
/*ERROR */ {_more, _more, _more, _more, _more, _more, _more, _more, _more, _same, _less, _less},
|
|
/*NULL */ {_more, _more, _more, _more, _more, _more, _more, _more, _more, _more, _same, _less},
|
|
/*ABSENT */ {_more, _more, _more, _more, _more, _more, _more, _more, _more, _more, _more, _same},
|
|
}
|