From eae73fe7cf961b56fe5cecbc9f13335d2acd6c36 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Tue, 30 Nov 2021 22:21:08 -0500 Subject: [PATCH 01/19] JIT mlrval type-interfence: mlrval package --- .vimrc | 3 +- Makefile | 71 +- go.mod | 1 + go.sum | 5 + .../arithmetic.go} | 4 +- .../mlrval_functions_base.go => bifs/base.go} | 0 .../bifs_test.go} | 0 .../mlrval_functions_bits.go => bifs/bits.go} | 0 .../booleans.go} | 6 +- .../collections.go} | 60 +- .../datetime.go} | 2 +- .../hashing.go} | 0 .../mlrval_functions_math.go => bifs/math.go} | 0 .../mathlib.go} | 0 .../random.go} | 0 .../regex.go} | 2 +- .../relative_time.go} | 0 .../mlrval_functions_sort.go => bifs/sort.go} | 4 +- .../stats.go} | 0 .../strings.go} | 20 +- .../system.go} | 0 .../types.go} | 34 +- .../constants.go} | 64 +- internal/pkg/mlrval/get.go | 81 + internal/pkg/mlrval/get_test.go | 163 ++ internal/pkg/mlrval/infer.go | 105 ++ internal/pkg/mlrval/is.go | 104 ++ internal/pkg/mlrval/is_test.go | 149 ++ internal/pkg/mlrval/new.go | 161 ++ internal/pkg/mlrval/new_test.go | 122 ++ internal/pkg/mlrval/tbf.go | 249 +++ .../pkg/{types/mlrval.go => mlrval/type.go} | 107 +- internal/pkg/types/mlrmap.go | 10 +- internal/pkg/types/mlrmap_accessors.go | 1403 +++++++++-------- internal/pkg/types/mlrval_accessors.go | 196 +-- internal/pkg/types/mlrval_collections.go | 44 +- internal/pkg/types/mlrval_new.go | 199 --- internal/pkg/types/mlrval_output.go | 2 +- todo.txt | 49 + 39 files changed, 2148 insertions(+), 1272 deletions(-) rename internal/pkg/{types/mlrval_functions_arithmetic.go => bifs/arithmetic.go} (99%) rename internal/pkg/{types/mlrval_functions_base.go => bifs/base.go} (100%) rename internal/pkg/{types/mlrval_functions_test.go => bifs/bifs_test.go} (100%) rename internal/pkg/{types/mlrval_functions_bits.go => bifs/bits.go} (100%) rename internal/pkg/{types/mlrval_functions_booleans.go => bifs/booleans.go} (99%) rename internal/pkg/{types/mlrval_functions_collections.go => bifs/collections.go} (94%) rename internal/pkg/{types/mlrval_functions_datetime.go => bifs/datetime.go} (99%) rename internal/pkg/{types/mlrval_function_hashing.go => bifs/hashing.go} (100%) rename internal/pkg/{types/mlrval_functions_math.go => bifs/math.go} (100%) rename internal/pkg/{types/mlrval_functions_mathlib.go => bifs/mathlib.go} (100%) rename internal/pkg/{types/mlrval_functions_random.go => bifs/random.go} (100%) rename internal/pkg/{types/mlrval_functions_regex.go => bifs/regex.go} (99%) rename internal/pkg/{types/mlrval_functions_relative_time.go => bifs/relative_time.go} (100%) rename internal/pkg/{types/mlrval_functions_sort.go => bifs/sort.go} (98%) rename internal/pkg/{types/mlrval_functions_stats.go => bifs/stats.go} (100%) rename internal/pkg/{types/mlrval_functions_strings.go => bifs/strings.go} (96%) rename internal/pkg/{types/mlrval_functions_system.go => bifs/system.go} (100%) rename internal/pkg/{types/mlrval_functions_types.go => bifs/types.go} (89%) rename internal/pkg/{types/mlrval_constants.go => mlrval/constants.go} (61%) create mode 100644 internal/pkg/mlrval/get.go create mode 100644 internal/pkg/mlrval/get_test.go create mode 100644 internal/pkg/mlrval/infer.go create mode 100644 internal/pkg/mlrval/is.go create mode 100644 internal/pkg/mlrval/is_test.go create mode 100644 internal/pkg/mlrval/new.go create mode 100644 internal/pkg/mlrval/new_test.go create mode 100644 internal/pkg/mlrval/tbf.go rename internal/pkg/{types/mlrval.go => mlrval/type.go} (54%) delete mode 100644 internal/pkg/types/mlrval_new.go diff --git a/.vimrc b/.vimrc index 18c0048c3..a2083dc8e 100644 --- a/.vimrc +++ b/.vimrc @@ -1,2 +1,3 @@ map \d :w:!clear;echo Building ...; echo; make mlr -map \f :w:!clear;echo Building ...; echo; make mall +"map \f :w:!clear;echo Building ...; echo; make mlrval-tests +map \f :w:!clear;echo Building ...; echo; make mlrmap-tests diff --git a/Makefile b/Makefile index 276dcae21..9873d3aed 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ PREFIX=/usr/local INSTALLDIR=$(PREFIX)/bin +# ---------------------------------------------------------------- +# General-use targets + # This must remain the first target in this file, which is what 'make' with no # arguments will run. build: @@ -14,19 +17,62 @@ check: unit_test regression_test @echo "by './configure --prefix=/your/install/path' if you wish to install to" @echo "somewhere other than /usr/local/bin -- the default prefix is /usr/local." +# DESTDIR is for package installs; nominally blank when this is run interactively. +# See also https://www.gnu.org/prep/standards/html_node/DESTDIR.html +install: build + cp mlr $(DESTDIR)/$(INSTALLDIR) + make -C man install + +# ---------------------------------------------------------------- +# Dev targets + # Unit tests (small number) -unit_test: +unit-test ut: go test github.com/johnkerl/miller/internal/pkg/... # Keystroke-savers -unbackslash_test: - go test $(ls internal/pkg/lib/*.go|grep -v test) internal/pkg/lib/unbackslash_test.go -mlrval_functions_test: - go test internal/pkg/types/mlrval_functions_test.go $(ls internal/pkg/types/*.go | grep -v test) -mlrval_format_test: - go test internal/pkg/types/mlrval_format_test.go $(ls internal/pkg/types/*.go|grep -v test) -regex_test: +lib-unbackslash-test: + go test internal/pkg/lib/unbackslash_test.go internal/pkg/lib/unbackslash.go +lib_regex_test: go test internal/pkg/lib/regex_test.go internal/pkg/lib/regex.go +lib_tests: lib_unbackslash_test lib_regex_test + +mlrval-new-test: + go test internal/pkg/mlrval/new_test.go \ + internal/pkg/mlrval/type.go \ + internal/pkg/mlrval/constants.go \ + internal/pkg/mlrval/new.go \ + internal/pkg/mlrval/infer.go +mlrval-is-test: + go test internal/pkg/mlrval/is_test.go \ + internal/pkg/mlrval/type.go \ + internal/pkg/mlrval/constants.go \ + internal/pkg/mlrval/new.go \ + internal/pkg/mlrval/infer.go \ + internal/pkg/mlrval/is.go +mlrval-get-test: + go test internal/pkg/mlrval/get_test.go \ + internal/pkg/mlrval/type.go \ + internal/pkg/mlrval/constants.go \ + internal/pkg/mlrval/new.go \ + internal/pkg/mlrval/infer.go \ + internal/pkg/mlrval/is.go \ + internal/pkg/mlrval/get.go +mlrval-tests: mlrval-new-test mlrval-is-test mlrval-get-test + +mlrmap-new-test: + go test internal/pkg/types/mlrmap_new_test.go \ + internal/pkg/types/mlrmap.go +mlrmap-accessors-test: + go test internal/pkg/types/mlrmap_accessors_test.go \ + internal/pkg/types/mlrmap.go \ + internal/pkg/types/mlrmap_accessors.go +mlrmap-tests: mlrmap-new-test mlrmap-accessors-test + +#mlrval_functions_test: +# go test internal/pkg/types/mlrval_functions_test.go $(ls internal/pkg/types/*.go | grep -v test) +#mlrval_format_test: +# go test internal/pkg/types/mlrval_format_test.go $(ls internal/pkg/types/*.go|grep -v test) # Regression tests (large number) # @@ -34,15 +80,9 @@ regex_test: # for debugging. TL;DR is for CI jobs, we have 'go test -v'; for # interactive use, instead of 'go test -v' simply use 'mlr regtest # -vvv' or 'mlr regtest -s 20'. See also internal/pkg/auxents/regtest. -regression_test: +regression-test: go test -v regression_test.go -# DESTDIR is for package installs; nominally blank when this is run interactively. -# See also https://www.gnu.org/prep/standards/html_node/DESTDIR.html -install: build - cp mlr $(DESTDIR)/$(INSTALLDIR) - make -C man install - fmt: -go fmt ./... @@ -90,5 +130,6 @@ mall: mprof5 mprof4 mprof3 mprof2 mprof mlr release_tarball: build check ./create-release-tarball +# ---------------------------------------------------------------- # Go does its own dependency management, outside of make. .PHONY: build mlr mprof mprof2 mprof3 mprof4 mprof5 check unit_test regression_test fmt dev diff --git a/go.mod b/go.mod index 6348401c6..65acb8d29 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/lestrrat-go/strftime v1.0.4 github.com/mattn/go-isatty v0.0.12 github.com/pkg/profile v1.6.0 // indirect + github.com/stretchr/testify v1.7.0 // indirect golang.org/x/sys v0.0.0-20210326220804-49726bf1d181 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf ) diff --git a/go.sum b/go.sum index 04746f2e1..268f50836 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -49,3 +51,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/pkg/types/mlrval_functions_arithmetic.go b/internal/pkg/bifs/arithmetic.go similarity index 99% rename from internal/pkg/types/mlrval_functions_arithmetic.go rename to internal/pkg/bifs/arithmetic.go index 239e18934..21312d80e 100644 --- a/internal/pkg/types/mlrval_functions_arithmetic.go +++ b/internal/pkg/bifs/arithmetic.go @@ -58,7 +58,7 @@ func BIF_minus_unary(input1 *Mlrval) *Mlrval { // Logical NOT operator func BIF_logicalnot(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_BOOL { + if input1.IsBool() { return MlrvalFromBool(!input1.boolval) } else { return MLRVAL_ERROR @@ -738,7 +738,7 @@ func BIF_mod_mul(input1, input2, input3 *Mlrval) *Mlrval { func BIF_mod_exp(input1, input2, input3 *Mlrval) *Mlrval { // Pre-check for negative exponent - if input2.mvtype == MT_INT && input2.intval < 0 { + if input2.IsInt() && input2.intval < 0 { return MLRVAL_ERROR } return imodop(input1, input2, input3, imodexp) diff --git a/internal/pkg/types/mlrval_functions_base.go b/internal/pkg/bifs/base.go similarity index 100% rename from internal/pkg/types/mlrval_functions_base.go rename to internal/pkg/bifs/base.go diff --git a/internal/pkg/types/mlrval_functions_test.go b/internal/pkg/bifs/bifs_test.go similarity index 100% rename from internal/pkg/types/mlrval_functions_test.go rename to internal/pkg/bifs/bifs_test.go diff --git a/internal/pkg/types/mlrval_functions_bits.go b/internal/pkg/bifs/bits.go similarity index 100% rename from internal/pkg/types/mlrval_functions_bits.go rename to internal/pkg/bifs/bits.go diff --git a/internal/pkg/types/mlrval_functions_booleans.go b/internal/pkg/bifs/booleans.go similarity index 99% rename from internal/pkg/types/mlrval_functions_booleans.go rename to internal/pkg/bifs/booleans.go index 3d00230ea..1e704c6ce 100644 --- a/internal/pkg/types/mlrval_functions_booleans.go +++ b/internal/pkg/bifs/booleans.go @@ -447,7 +447,7 @@ func MlrvalEqualsAsBool(input1, input2 *Mlrval) bool { // ---------------------------------------------------------------- func MlrvalLogicalAND(input1, input2 *Mlrval) *Mlrval { - if input1.mvtype == MT_BOOL && input2.mvtype == MT_BOOL { + if input1.IsBool() && input2.IsBool() { return MlrvalFromBool(input1.boolval && input2.boolval) } else { return MLRVAL_ERROR @@ -455,7 +455,7 @@ func MlrvalLogicalAND(input1, input2 *Mlrval) *Mlrval { } func MlrvalLogicalOR(input1, input2 *Mlrval) *Mlrval { - if input1.mvtype == MT_BOOL && input2.mvtype == MT_BOOL { + if input1.IsBool() && input2.IsBool() { return MlrvalFromBool(input1.boolval || input2.boolval) } else { return MLRVAL_ERROR @@ -463,7 +463,7 @@ func MlrvalLogicalOR(input1, input2 *Mlrval) *Mlrval { } func BIF_logicalxor(input1, input2 *Mlrval) *Mlrval { - if input1.mvtype == MT_BOOL && input2.mvtype == MT_BOOL { + if input1.IsBool() && input2.IsBool() { return MlrvalFromBool(input1.boolval != input2.boolval) } else { return MLRVAL_ERROR diff --git a/internal/pkg/types/mlrval_functions_collections.go b/internal/pkg/bifs/collections.go similarity index 94% rename from internal/pkg/types/mlrval_functions_collections.go rename to internal/pkg/bifs/collections.go index 1fe9fd0fb..dca279999 100644 --- a/internal/pkg/types/mlrval_functions_collections.go +++ b/internal/pkg/bifs/collections.go @@ -92,9 +92,9 @@ func leafcount_from_array(input1 *Mlrval) *Mlrval { // childLeafCount := BIF_leafcount(&child) childLeafCount := MlrvalFromInt(1) - if child.mvtype == MT_ARRAY { + if child.IsArray() { childLeafCount = leafcount_from_array(&child) - } else if child.mvtype == MT_MAP { + } else if child.IsMap() { childLeafCount = leafcount_from_map(&child) } @@ -114,9 +114,9 @@ func leafcount_from_map(input1 *Mlrval) *Mlrval { // childLeafCount := BIF_leafcount(child) childLeafCount := MlrvalFromInt(1) - if child.mvtype == MT_ARRAY { + if child.IsArray() { childLeafCount = leafcount_from_array(child) - } else if child.mvtype == MT_MAP { + } else if child.IsMap() { childLeafCount = leafcount_from_map(child) } @@ -151,7 +151,7 @@ func BIF_leafcount(input1 *Mlrval) *Mlrval { // ---------------------------------------------------------------- func has_key_in_array(input1, input2 *Mlrval) *Mlrval { - if input2.mvtype == MT_STRING { + if input2.IsString() { return MLRVAL_FALSE } if input2.mvtype != MT_INT { @@ -162,7 +162,7 @@ func has_key_in_array(input1, input2 *Mlrval) *Mlrval { } func has_key_in_map(input1, input2 *Mlrval) *Mlrval { - if input2.mvtype == MT_STRING || input2.mvtype == MT_INT { + if input2.IsString() || input2.IsInt() { return MlrvalFromBool(input1.mapval.Has(input2.String())) } else { return MLRVAL_ERROR @@ -170,9 +170,9 @@ func has_key_in_map(input1, input2 *Mlrval) *Mlrval { } func BIF_haskey(input1, input2 *Mlrval) *Mlrval { - if input1.mvtype == MT_ARRAY { + if input1.IsArray() { return has_key_in_array(input1, input2) - } else if input1.mvtype == MT_MAP { + } else if input1.IsMap() { return has_key_in_map(input1, input2) } else { return MLRVAL_ERROR @@ -192,13 +192,13 @@ func BIF_mapselect(mlrvals []*Mlrval) *Mlrval { newKeys := make(map[string]bool) for _, selectArg := range mlrvals[1:] { - if selectArg.mvtype == MT_STRING { + if selectArg.IsString() { newKeys[selectArg.printrep] = true - } else if selectArg.mvtype == MT_INT { + } else if selectArg.IsInt() { newKeys[selectArg.String()] = true - } else if selectArg.mvtype == MT_ARRAY { + } else if selectArg.IsArray() { for _, element := range selectArg.arrayval { - if element.mvtype == MT_STRING { + if element.IsString() { newKeys[element.printrep] = true } else { return MLRVAL_ERROR @@ -231,13 +231,13 @@ func BIF_mapexcept(mlrvals []*Mlrval) *Mlrval { newMap := mlrvals[0].mapval.Copy() for _, exceptArg := range mlrvals[1:] { - if exceptArg.mvtype == MT_STRING { + if exceptArg.IsString() { newMap.Remove(exceptArg.printrep) - } else if exceptArg.mvtype == MT_INT { + } else if exceptArg.IsInt() { newMap.Remove(exceptArg.String()) - } else if exceptArg.mvtype == MT_ARRAY { + } else if exceptArg.IsArray() { for _, element := range exceptArg.arrayval { - if element.mvtype == MT_STRING { + if element.IsString() { newMap.Remove(element.printrep) } else { return MLRVAL_ERROR @@ -311,7 +311,7 @@ func BIF_joink(input1, input2 *Mlrval) *Mlrval { return MLRVAL_ERROR } fieldSeparator := input2.printrep - if input1.mvtype == MT_MAP { + if input1.IsMap() { var buffer bytes.Buffer for pe := input1.mapval.Head; pe != nil; pe = pe.Next { @@ -322,7 +322,7 @@ func BIF_joink(input1, input2 *Mlrval) *Mlrval { } return MlrvalFromString(buffer.String()) - } else if input1.mvtype == MT_ARRAY { + } else if input1.IsArray() { var buffer bytes.Buffer for i := range input1.arrayval { @@ -348,7 +348,7 @@ func BIF_joinv(input1, input2 *Mlrval) *Mlrval { } fieldSeparator := input2.printrep - if input1.mvtype == MT_MAP { + if input1.IsMap() { var buffer bytes.Buffer for pe := input1.mapval.Head; pe != nil; pe = pe.Next { @@ -359,7 +359,7 @@ func BIF_joinv(input1, input2 *Mlrval) *Mlrval { } return MlrvalFromString(buffer.String()) - } else if input1.mvtype == MT_ARRAY { + } else if input1.IsArray() { var buffer bytes.Buffer for i, element := range input1.arrayval { @@ -388,7 +388,7 @@ func BIF_joinkv(input1, input2, input3 *Mlrval) *Mlrval { } fieldSeparator := input3.printrep - if input1.mvtype == MT_MAP { + if input1.IsMap() { var buffer bytes.Buffer for pe := input1.mapval.Head; pe != nil; pe = pe.Next { @@ -401,7 +401,7 @@ func BIF_joinkv(input1, input2, input3 *Mlrval) *Mlrval { } return MlrvalFromString(buffer.String()) - } else if input1.mvtype == MT_ARRAY { + } else if input1.IsArray() { var buffer bytes.Buffer for i, element := range input1.arrayval { @@ -590,7 +590,7 @@ func mlrvalSplitAXHelper(input string, separator string) *Mlrval { // ---------------------------------------------------------------- func BIF_get_keys(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_MAP { + if input1.IsMap() { // TODO: make a ReferenceFrom with comments output := NewSizedMlrvalArray(input1.mapval.FieldCount) i := 0 @@ -600,7 +600,7 @@ func BIF_get_keys(input1 *Mlrval) *Mlrval { } return output - } else if input1.mvtype == MT_ARRAY { + } else if input1.IsArray() { output := NewSizedMlrvalArray(int(len(input1.arrayval))) for i := range input1.arrayval { output.arrayval[i] = *MlrvalFromInt(int(i + 1)) // Miller user-space indices are 1-up @@ -614,7 +614,7 @@ func BIF_get_keys(input1 *Mlrval) *Mlrval { // ---------------------------------------------------------------- func BIF_get_values(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_MAP { + if input1.IsMap() { // TODO: make a ReferenceFrom with commenbs output := NewSizedMlrvalArray(input1.mapval.FieldCount) i := 0 @@ -624,7 +624,7 @@ func BIF_get_values(input1 *Mlrval) *Mlrval { } return output - } else if input1.mvtype == MT_ARRAY { + } else if input1.IsArray() { output := NewSizedMlrvalArray(int(len(input1.arrayval))) for i, value := range input1.arrayval { output.arrayval[i] = *value.Copy() @@ -654,7 +654,7 @@ func BIF_append(input1, input2 *Mlrval) *Mlrval { // flatten("a", ".", {"b": { "c": 4 }}) is {"a.b.c" : 4}. // flatten("", ".", {"a": { "b": 3 }}) is {"a.b" : 3}. func BIF_flatten(input1, input2, input3 *Mlrval) *Mlrval { - if input3.mvtype == MT_MAP || input3.mvtype == MT_ARRAY { + if input3.IsMap() || input3.IsArray() { if input1.mvtype != MT_STRING && input1.mvtype != MT_VOID { return MLRVAL_ERROR } @@ -696,7 +696,7 @@ func BIF_unflatten(input1, input2 *Mlrval) *Mlrval { // ---------------------------------------------------------------- // Converts maps with "1", "2", ... keys into arrays. Recurses nested data structures. func BIF_arrayify(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_MAP { + if input1.IsMap() { if input1.mapval.IsEmpty() { return input1 } @@ -725,7 +725,7 @@ func BIF_arrayify(input1 *Mlrval) *Mlrval { return input1 } - } else if input1.mvtype == MT_ARRAY { + } else if input1.IsArray() { // TODO: comment (or rethink) that this modifies its inputs!! output := input1.Copy() for i := range input1.arrayval { @@ -740,7 +740,7 @@ func BIF_arrayify(input1 *Mlrval) *Mlrval { // ---------------------------------------------------------------- func BIF_json_parse(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_VOID { + if input1.IsVoid() { return input1 } else if input1.mvtype != MT_STRING { return MLRVAL_ERROR diff --git a/internal/pkg/types/mlrval_functions_datetime.go b/internal/pkg/bifs/datetime.go similarity index 99% rename from internal/pkg/types/mlrval_functions_datetime.go rename to internal/pkg/bifs/datetime.go index 9ae6e244c..65c946e6d 100644 --- a/internal/pkg/types/mlrval_functions_datetime.go +++ b/internal/pkg/bifs/datetime.go @@ -194,7 +194,7 @@ func BIF_strftime_local_ternary(input1, input2, input3 *Mlrval) *Mlrval { } func strftimeHelper(input1, input2 *Mlrval, doLocal bool, location *time.Location) *Mlrval { - if input1.mvtype == MT_VOID { + if input1.IsVoid() { return input1 } epochSeconds, ok := input1.GetNumericToFloatValue() diff --git a/internal/pkg/types/mlrval_function_hashing.go b/internal/pkg/bifs/hashing.go similarity index 100% rename from internal/pkg/types/mlrval_function_hashing.go rename to internal/pkg/bifs/hashing.go diff --git a/internal/pkg/types/mlrval_functions_math.go b/internal/pkg/bifs/math.go similarity index 100% rename from internal/pkg/types/mlrval_functions_math.go rename to internal/pkg/bifs/math.go diff --git a/internal/pkg/types/mlrval_functions_mathlib.go b/internal/pkg/bifs/mathlib.go similarity index 100% rename from internal/pkg/types/mlrval_functions_mathlib.go rename to internal/pkg/bifs/mathlib.go diff --git a/internal/pkg/types/mlrval_functions_random.go b/internal/pkg/bifs/random.go similarity index 100% rename from internal/pkg/types/mlrval_functions_random.go rename to internal/pkg/bifs/random.go diff --git a/internal/pkg/types/mlrval_functions_regex.go b/internal/pkg/bifs/regex.go similarity index 99% rename from internal/pkg/types/mlrval_functions_regex.go rename to internal/pkg/bifs/regex.go index 72c9af669..20ff71920 100644 --- a/internal/pkg/types/mlrval_functions_regex.go +++ b/internal/pkg/bifs/regex.go @@ -118,7 +118,7 @@ func BIF_string_matches_regexp(input1, input2 *Mlrval) (retval *Mlrval, captures // BIF_string_matches_regexp implements the !=~ operator. func BIF_string_does_not_match_regexp(input1, input2 *Mlrval) (retval *Mlrval, captures []string) { output, captures := BIF_string_matches_regexp(input1, input2) - if output.mvtype == MT_BOOL { + if output.IsBool() { return MlrvalFromBool(!output.boolval), captures } else { // else leave it as error, absent, etc. diff --git a/internal/pkg/types/mlrval_functions_relative_time.go b/internal/pkg/bifs/relative_time.go similarity index 100% rename from internal/pkg/types/mlrval_functions_relative_time.go rename to internal/pkg/bifs/relative_time.go diff --git a/internal/pkg/types/mlrval_functions_sort.go b/internal/pkg/bifs/sort.go similarity index 98% rename from internal/pkg/types/mlrval_functions_sort.go rename to internal/pkg/bifs/sort.go index d452bed44..def08da45 100644 --- a/internal/pkg/types/mlrval_functions_sort.go +++ b/internal/pkg/bifs/sort.go @@ -35,10 +35,10 @@ func LexicalDescendingComparator(input1 *Mlrval, input2 *Mlrval) int { func CaseFoldAscendingComparator(input1 *Mlrval, input2 *Mlrval) int { sa := input1.String() sb := input2.String() - if input1.mvtype == MT_STRING { + if input1.IsString() { sa = strings.ToLower(sa) } - if input2.mvtype == MT_STRING { + if input2.IsString() { sb = strings.ToLower(sb) } if sa < sb { diff --git a/internal/pkg/types/mlrval_functions_stats.go b/internal/pkg/bifs/stats.go similarity index 100% rename from internal/pkg/types/mlrval_functions_stats.go rename to internal/pkg/bifs/stats.go diff --git a/internal/pkg/types/mlrval_functions_strings.go b/internal/pkg/bifs/strings.go similarity index 96% rename from internal/pkg/types/mlrval_functions_strings.go rename to internal/pkg/bifs/strings.go index 81dd4e548..d57d04c00 100644 --- a/internal/pkg/types/mlrval_functions_strings.go +++ b/internal/pkg/bifs/strings.go @@ -191,7 +191,7 @@ func BIF_truncate(input1, input2 *Mlrval) *Mlrval { // ================================================================ func BIF_lstrip(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_STRING { + if input1.IsString() { return MlrvalFromString(strings.TrimLeft(input1.printrep, " \t")) } else { return input1 @@ -199,7 +199,7 @@ func BIF_lstrip(input1 *Mlrval) *Mlrval { } func BIF_rstrip(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_STRING { + if input1.IsString() { return MlrvalFromString(strings.TrimRight(input1.printrep, " \t")) } else { return input1 @@ -207,7 +207,7 @@ func BIF_rstrip(input1 *Mlrval) *Mlrval { } func BIF_strip(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_STRING { + if input1.IsString() { return MlrvalFromString(strings.Trim(input1.printrep, " \t")) } else { return input1 @@ -220,7 +220,7 @@ func BIF_collapse_whitespace(input1 *Mlrval) *Mlrval { } func MlrvalCollapseWhitespaceRegexp(input1 *Mlrval, whitespaceRegexp *regexp.Regexp) *Mlrval { - if input1.mvtype == MT_STRING { + if input1.IsString() { return MlrvalFromString(whitespaceRegexp.ReplaceAllString(input1.printrep, " ")) } else { return input1 @@ -233,9 +233,9 @@ func WhitespaceRegexp() *regexp.Regexp { // ================================================================ func BIF_toupper(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_STRING { + if input1.IsString() { return MlrvalFromString(strings.ToUpper(input1.printrep)) - } else if input1.mvtype == MT_VOID { + } else if input1.IsVoid() { return input1 } else { return input1 @@ -243,9 +243,9 @@ func BIF_toupper(input1 *Mlrval) *Mlrval { } func BIF_tolower(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_STRING { + if input1.IsString() { return MlrvalFromString(strings.ToLower(input1.printrep)) - } else if input1.mvtype == MT_VOID { + } else if input1.IsVoid() { return input1 } else { return input1 @@ -253,7 +253,7 @@ func BIF_tolower(input1 *Mlrval) *Mlrval { } func BIF_capitalize(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_STRING { + if input1.IsString() { if input1.printrep == "" { return input1 } else { @@ -280,7 +280,7 @@ func BIF_clean_whitespace(input1 *Mlrval) *Mlrval { // ================================================================ func BIF_hexfmt(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_INT { + if input1.IsInt() { return MlrvalFromString("0x" + strconv.FormatUint(uint64(input1.intval), 16)) } else { return input1 diff --git a/internal/pkg/types/mlrval_functions_system.go b/internal/pkg/bifs/system.go similarity index 100% rename from internal/pkg/types/mlrval_functions_system.go rename to internal/pkg/bifs/system.go diff --git a/internal/pkg/types/mlrval_functions_types.go b/internal/pkg/bifs/types.go similarity index 89% rename from internal/pkg/types/mlrval_functions_types.go rename to internal/pkg/bifs/types.go index c06d06d44..0dbed3d34 100644 --- a/internal/pkg/types/mlrval_functions_types.go +++ b/internal/pkg/bifs/types.go @@ -130,21 +130,21 @@ func BIF_boolean(input1 *Mlrval) *Mlrval { // ---------------------------------------------------------------- func BIF_is_absent(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_ABSENT) + return MlrvalFromBool(input1.IsAbsent()) } func BIF_is_error(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_ERROR) + return MlrvalFromBool(input1.IsError()) } func BIF_is_bool(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_BOOL) + return MlrvalFromBool(input1.IsBool()) } func BIF_is_boolean(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_BOOL) + return MlrvalFromBool(input1.IsBool()) } func BIF_is_empty(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_VOID { + if input1.IsVoid() { return MLRVAL_TRUE - } else if input1.mvtype == MT_STRING { + } else if input1.IsString() { if input1.printrep == "" { return MLRVAL_TRUE } else { @@ -155,27 +155,27 @@ func BIF_is_empty(input1 *Mlrval) *Mlrval { } } func BIF_is_emptymap(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_MAP && input1.mapval.IsEmpty()) + return MlrvalFromBool(input1.IsMap() && input1.mapval.IsEmpty()) } func BIF_is_float(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_FLOAT) + return MlrvalFromBool(input1.IsFloat()) } func BIF_is_int(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_INT) + return MlrvalFromBool(input1.IsInt()) } func BIF_is_map(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_MAP) + return MlrvalFromBool(input1.IsMap()) } func BIF_is_array(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_ARRAY) + return MlrvalFromBool(input1.IsArray()) } func BIF_is_nonemptymap(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_MAP && input1.mapval.FieldCount != 0) + return MlrvalFromBool(input1.IsMap() && input1.mapval.FieldCount != 0) } func BIF_is_notempty(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_VOID { + if input1.IsVoid() { return MLRVAL_FALSE - } else if input1.mvtype == MT_STRING { + } else if input1.IsString() { if input1.printrep == "" { return MLRVAL_FALSE } else { @@ -195,16 +195,16 @@ func BIF_is_notnull(input1 *Mlrval) *Mlrval { return MlrvalFromBool(input1.mvtype != MT_ABSENT && input1.mvtype != MT_VOID) } func BIF_is_null(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_ABSENT || input1.mvtype == MT_VOID) + return MlrvalFromBool(input1.IsAbsent() || input1.IsVoid()) } func BIF_is_numeric(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_INT || input1.mvtype == MT_FLOAT) + return MlrvalFromBool(input1.IsInt() || input1.IsFloat()) } func BIF_is_present(input1 *Mlrval) *Mlrval { return MlrvalFromBool(input1.mvtype != MT_ABSENT) } func BIF_is_string(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_STRING || input1.mvtype == MT_VOID) + return MlrvalFromBool(input1.IsString() || input1.IsVoid()) } // ---------------------------------------------------------------- diff --git a/internal/pkg/types/mlrval_constants.go b/internal/pkg/mlrval/constants.go similarity index 61% rename from internal/pkg/types/mlrval_constants.go rename to internal/pkg/mlrval/constants.go index ff903497a..1fe36ebc1 100644 --- a/internal/pkg/types/mlrval_constants.go +++ b/internal/pkg/mlrval/constants.go @@ -2,92 +2,58 @@ // Constants/singletons of various types // ================================================================ -package types +package mlrval + +// MlrvalFromPending is designed solely for the JSON API, for something +// intended to be mutated after construction once its type is (later) known. +// Whereas MLRVAL_ERROR, MLRVAL_ABSENT, etc are all singletons, this one +// must be mutable and therefor non-singleton. + +func MlrvalFromPending() Mlrval { + return Mlrval{ + mvtype: MT_PENDING, + printrep: INVALID_PRINTREP, + } +} // These are made singletons as part of a copy-reduction effort. They're not // marked const (I haven't figured out the right way to get that to compile; // just using `const` isn't enough) but the gentelpersons' agreement is that // the caller should never modify these. -// MlrvalFromPending is designed solely for the JSON API, for something -// intended to be mutated after construction once its type is (later) known. -// Whereas MLRVAL_ERROR, MLRVAL_ABSENT, etc are all singletons, this one -// must be mutable and therefor non-singleton. -func MlrvalFromPending() Mlrval { - return Mlrval{ - mvtype: MT_PENDING, - printrep: "(bug-if-you-see-this-pending-type)", - printrepValid: true, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: nil, - mapval: nil, - } -} - var MLRVAL_ERROR = &Mlrval{ mvtype: MT_ERROR, - printrep: "(error)", // xxx const somewhere + printrep: ERROR_PRINTREP, printrepValid: true, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: nil, - mapval: nil, } var MLRVAL_ABSENT = &Mlrval{ mvtype: MT_ABSENT, - printrep: "(absent)", + printrep: ABSENT_PRINTREP, printrepValid: true, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: nil, - mapval: nil, } var MLRVAL_NULL = &Mlrval{ mvtype: MT_NULL, printrep: "null", printrepValid: true, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: nil, - mapval: nil, } var MLRVAL_VOID = &Mlrval{ mvtype: MT_VOID, printrep: "", printrepValid: true, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: nil, - mapval: nil, } var MLRVAL_TRUE = &Mlrval{ mvtype: MT_BOOL, printrep: "true", printrepValid: true, - intval: 0, - floatval: 0.0, boolval: true, - arrayval: nil, - mapval: nil, } var MLRVAL_FALSE = &Mlrval{ mvtype: MT_BOOL, printrep: "false", printrepValid: true, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: nil, - mapval: nil, } diff --git a/internal/pkg/mlrval/get.go b/internal/pkg/mlrval/get.go new file mode 100644 index 000000000..57759c288 --- /dev/null +++ b/internal/pkg/mlrval/get.go @@ -0,0 +1,81 @@ +package mlrval + +import ( + "github.com/johnkerl/miller/internal/pkg/lib" +) + +// It's essential that we use mv.Type() not mv.mvtype, or use an Is...() +// predicate, or another Get...(), since types are JIT-computed on first access +// for most data-file values. See type.go for more information. + +func (mv *Mlrval) GetString() (stringValue string, isString bool) { + if mv.Type() == MT_STRING || mv.Type() == MT_VOID { + return mv.printrep, true + } else { + return "", false + } +} + +func (mv *Mlrval) GetIntValue() (intValue int, isInt bool) { + if mv.Type() == MT_INT { + return mv.intval, true + } else { + return -999, false + } +} + +func (mv *Mlrval) GetFloatValue() (floatValue float64, isFloat bool) { + if mv.Type() == MT_FLOAT { + return mv.floatval, true + } else { + return -777.0, false + } +} + +func (mv *Mlrval) GetNumericToFloatValue() (floatValue float64, isFloat bool) { + if mv.Type() == MT_FLOAT { + return mv.floatval, true + } else if mv.Type() == MT_INT { + return float64(mv.intval), true + } else { + return -888.0, false + } +} + +func (mv *Mlrval) GetNumericNegativeorDie() bool { + floatValue, ok := mv.GetNumericToFloatValue() + lib.InternalCodingErrorIf(!ok) + return floatValue < 0.0 +} + +func (mv *Mlrval) GetBoolValue() (boolValue bool, isBool bool) { + if mv.Type() == MT_BOOL { + return mv.boolval, true + } else { + return false, false + } +} + +func (mv *Mlrval) GetArray() interface{} { + if mv.IsArray() { + return mv.arrayval + } else { + return nil + } +} + +func (mv *Mlrval) GetMap() interface{} { + if mv.IsMap() { + return mv.mapval + } else { + return nil + } +} + +func (mv *Mlrval) GetFunction() interface{} { + if mv.Type() == MT_FUNC { + return mv.funcval + } else { + return nil + } +} diff --git a/internal/pkg/mlrval/get_test.go b/internal/pkg/mlrval/get_test.go new file mode 100644 index 000000000..7fdbf8287 --- /dev/null +++ b/internal/pkg/mlrval/get_test.go @@ -0,0 +1,163 @@ +// ================================================================ +// Tests mlrval typed-value extractors +// ================================================================ + +package mlrval + +import ( + "testing" + + "github.com/stretchr/testify/assert" + //"github.com/johnkerl/miller/internal/pkg/lib" +) + +func TestGetString(t *testing.T) { + mv := FromInferredType("234") + stringval, ok := mv.GetString() + assert.False(t, ok) + + mv = FromDeferredType("234") + stringval, ok = mv.GetString() + assert.False(t, ok) + + mv = FromInferredType("234.5") + stringval, ok = mv.GetString() + assert.False(t, ok) + + mv = FromDeferredType("234.5") + stringval, ok = mv.GetString() + assert.False(t, ok) + + mv = FromInferredType("abc") + stringval, ok = mv.GetString() + assert.Equal(t, "abc", stringval) + assert.True(t, ok) + + mv = FromDeferredType("abc") + stringval, ok = mv.GetString() + assert.Equal(t, "abc", stringval) + assert.True(t, ok) + + mv = FromInferredType("") + stringval, ok = mv.GetString() + assert.Equal(t, "", stringval) + assert.True(t, ok) + + mv = FromDeferredType("") + stringval, ok = mv.GetString() + assert.Equal(t, "", stringval) + assert.True(t, ok) +} + +func TestGetIntValue(t *testing.T) { + mv := FromInferredType("123") + intval, ok := mv.GetIntValue() + assert.Equal(t, 123, intval) + assert.True(t, ok) + + mv = FromDeferredType("123") + intval, ok = mv.GetIntValue() + assert.Equal(t, 123, intval) + assert.True(t, ok) + + mv = FromInferredType("123.4") + intval, ok = mv.GetIntValue() + assert.False(t, ok) + + mv = FromDeferredType("123.4") + intval, ok = mv.GetIntValue() + assert.False(t, ok) + + mv = FromInferredType("abc") + intval, ok = mv.GetIntValue() + assert.False(t, ok) + + mv = FromDeferredType("abc") + intval, ok = mv.GetIntValue() + assert.False(t, ok) +} + +func TestGetFloatValue(t *testing.T) { + mv := FromInferredType("234") + floatval, ok := mv.GetFloatValue() + assert.False(t, ok) + + mv = FromDeferredType("234") + floatval, ok = mv.GetFloatValue() + assert.False(t, ok) + + mv = FromInferredType("234.5") + floatval, ok = mv.GetFloatValue() + assert.Equal(t, 234.5, floatval) + assert.True(t, ok) + + mv = FromDeferredType("234.5") + floatval, ok = mv.GetFloatValue() + assert.Equal(t, 234.5, floatval) + assert.True(t, ok) + + mv = FromInferredType("abc") + floatval, ok = mv.GetFloatValue() + assert.False(t, ok) + + mv = FromDeferredType("abc") + floatval, ok = mv.GetFloatValue() + assert.False(t, ok) +} + +func TestGetNumericToFloatValue(t *testing.T) { + mv := FromInferredType("234") + floatval, ok := mv.GetNumericToFloatValue() + assert.Equal(t, 234.0, floatval) + assert.True(t, ok) + + mv = FromDeferredType("234") + floatval, ok = mv.GetNumericToFloatValue() + assert.Equal(t, 234.0, floatval) + assert.True(t, ok) + + mv = FromInferredType("234.5") + floatval, ok = mv.GetNumericToFloatValue() + assert.Equal(t, 234.5, floatval) + assert.True(t, ok) + + mv = FromDeferredType("234.5") + floatval, ok = mv.GetNumericToFloatValue() + assert.Equal(t, 234.5, floatval) + assert.True(t, ok) + + mv = FromInferredType("abc") + floatval, ok = mv.GetNumericToFloatValue() + assert.False(t, ok) + + mv = FromDeferredType("abc") + floatval, ok = mv.GetNumericToFloatValue() + assert.False(t, ok) +} + +func TestGetBoolValue(t *testing.T) { + mv := FromInferredType("234") + boolval, ok := mv.GetBoolValue() + assert.False(t, ok) + + mv = FromDeferredType("234") + boolval, ok = mv.GetBoolValue() + assert.False(t, ok) + + mv = FromInferredType("abc") + boolval, ok = mv.GetBoolValue() + assert.False(t, ok) + + mv = FromDeferredType("abc") + boolval, ok = mv.GetBoolValue() + assert.False(t, ok) + + mv = FromInferredType("true") + boolval, ok = mv.GetBoolValue() + assert.True(t, boolval) + assert.True(t, ok) + + mv = FromDeferredType("false") + boolval, ok = mv.GetBoolValue() + assert.False(t, ok, "from-data-file \"false\" should infer to string") +} diff --git a/internal/pkg/mlrval/infer.go b/internal/pkg/mlrval/infer.go new file mode 100644 index 000000000..682ad3171 --- /dev/null +++ b/internal/pkg/mlrval/infer.go @@ -0,0 +1,105 @@ +package mlrval + +import ( + "strings" + + "github.com/johnkerl/miller/internal/pkg/lib" +) + +// TODO: no infer bool. Always false in this path. + +// It's essential that we use mv.Type() not mv.mvtype since types are +// JIT-computed on first access for most data-file values. See type.go for more +// information. + +func (mv *Mlrval) Type() MVType { + if mv.mvtype == MT_PENDING { + packageLevelInferrer(mv, mv.printrep, false) + } + return mv.mvtype +} + +// Support for mlr -S, mlr -A, mlr -O. +type tInferrer func(mv *Mlrval, input string, inferBool bool) *Mlrval + +var packageLevelInferrer tInferrer = inferNormally + +func SetInferrerNoOctal() { + packageLevelInferrer = inferWithOctalSuppress +} +func SetInferrerIntAsFloat() { + packageLevelInferrer = inferWithIntAsFloat +} +func SetInferrerStringOnly() { + packageLevelInferrer = inferStringOnly +} + +// When loading data files, don't scan these words into floats -- even though +// the Go library is willing to do so. +var downcasedFloatNamesToNotInfer = map[string]bool{ + "inf": true, + "+inf": true, + "-inf": true, + "infinity": true, + "+infinity": true, + "-infinity": true, + "nan": true, +} + +func inferNormally(mv *Mlrval, input string, inferBool bool) *Mlrval { + if input == "" { + return mv.SetFromVoid() + } + + intval, iok := lib.TryIntFromString(input) + if iok { + return mv.SetFromPrevalidatedIntString(input, intval) + } + + if downcasedFloatNamesToNotInfer[strings.ToLower(input)] == false { + floatval, fok := lib.TryFloat64FromString(input) + if fok { + return mv.SetFromPrevalidatedFloatString(input, floatval) + } + } + + if inferBool { + boolval, bok := lib.TryBoolFromBoolString(input) + if bok { + return mv.SetFromPrevalidatedBoolString(input, boolval) + } + } + return mv.SetFromString(input) +} + +func inferWithOctalSuppress(mv *Mlrval, input string, inferBool bool) *Mlrval { + inferNormally(mv, input, inferBool) + if mv.mvtype == MT_INT { + if input[0] == '0' && len(input) > 1 { + c := input[1] + if c != 'x' && c != 'X' && c != 'b' && c != 'B' { + return mv.SetFromString(input) + } + } + if strings.HasPrefix(input, "-0") && len(input) > 2 { + c := input[2] + if c != 'x' && c != 'X' && c != 'b' && c != 'B' { + return mv.SetFromString(input) + } + } + } + return mv +} + +func inferWithIntAsFloat(mv *Mlrval, input string, inferBool bool) *Mlrval { + inferNormally(mv, input, inferBool) + if mv.mvtype == MT_INT { + mv.floatval = float64(mv.intval) + mv.mvtype = MT_FLOAT + } + return mv +} + +func inferStringOnly(mv *Mlrval, input string, inferBool bool) *Mlrval { + return mv.SetFromString(input) +} diff --git a/internal/pkg/mlrval/is.go b/internal/pkg/mlrval/is.go new file mode 100644 index 000000000..e40529246 --- /dev/null +++ b/internal/pkg/mlrval/is.go @@ -0,0 +1,104 @@ +package mlrval + +// It's essential that we use mv.Type() not mv.mvtype since types are +// JIT-computed on first access for most data-file values. See type.go for more +// information. + +func (mv *Mlrval) IsLegit() bool { + return mv.Type() >= MT_VOID +} + +func (mv *Mlrval) IsErrorOrAbsent() bool { + t := mv.Type() + return t == MT_ERROR || t == MT_ABSENT +} + +func (mv *Mlrval) IsError() bool { + return mv.Type() == MT_ERROR +} + +func (mv *Mlrval) IsAbsent() bool { + return mv.Type() == MT_ABSENT +} + +func (mv *Mlrval) IsNull() bool { + return mv.Type() == MT_NULL +} + +func (mv *Mlrval) IsVoid() bool { + return mv.Type() == MT_VOID +} + +func (mv *Mlrval) IsErrorOrVoid() bool { + t := mv.Type() + return t == MT_ERROR || t == MT_VOID +} + +// * Error is non-empty +// * Absent is non-empty (shouldn't have been assigned in the first place; error should be surfaced) +// * Void is empty +// * Empty string is empty +// * Int/float/bool/array/map are all non-empty +func (mv *Mlrval) IsEmptyString() bool { + t := mv.Type() + if t == MT_VOID { + return true + } else if t == MT_STRING { + return mv.printrep == "" + } else { + return false + } +} + +func (mv *Mlrval) IsString() bool { + return mv.Type() == MT_STRING +} + +func (mv *Mlrval) IsStringOrVoid() bool { + t := mv.Type() + return t == MT_STRING || t == MT_VOID +} + +func (mv *Mlrval) IsInt() bool { + return mv.Type() == MT_INT +} + +func (mv *Mlrval) IsFloat() bool { + return mv.Type() == MT_FLOAT +} + +func (mv *Mlrval) IsNumeric() bool { + t := mv.Type() + return t == MT_INT || t == MT_FLOAT +} + +func (mv *Mlrval) IsIntZero() bool { + return mv.Type() == MT_INT && mv.intval == 0 +} + +func (mv *Mlrval) IsBool() bool { + return mv.Type() == MT_BOOL +} + +func (mv *Mlrval) IsTrue() bool { + return mv.Type() == MT_BOOL && mv.boolval == true +} +func (mv *Mlrval) IsFalse() bool { + return mv.Type() == MT_BOOL && mv.boolval == false +} + +func (mv *Mlrval) IsArray() bool { + return mv.Type() == MT_ARRAY +} +func (mv *Mlrval) IsMap() bool { + return mv.Type() == MT_MAP +} +func (mv *Mlrval) IsArrayOrMap() bool { + t := mv.Type() + return t == MT_ARRAY || t == MT_MAP +} + +func (mv *Mlrval) IsFunction() bool { + return mv.Type() == MT_FUNC +} + diff --git a/internal/pkg/mlrval/is_test.go b/internal/pkg/mlrval/is_test.go new file mode 100644 index 000000000..4e4eaec42 --- /dev/null +++ b/internal/pkg/mlrval/is_test.go @@ -0,0 +1,149 @@ +// ================================================================ +// Tests mlrval constructors. +// ================================================================ + +package mlrval + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsLegit(t *testing.T) { + assert.False(t, MLRVAL_ERROR.IsLegit()) + assert.False(t, MLRVAL_ABSENT.IsLegit()) + assert.False(t, MLRVAL_NULL.IsLegit()) + assert.True(t, FromString("").IsLegit()) + assert.True(t, FromString("abc").IsLegit()) + assert.True(t, FromInt(123).IsLegit()) + assert.True(t, FromFloat(123.4).IsLegit()) + assert.True(t, FromBool(true).IsLegit()) + assert.True(t, FromArray("test data").IsLegit()) + assert.True(t, FromMap("test data").IsLegit()) +} + +func TestIsErrorOrAbsent(t *testing.T) { + assert.True(t, MLRVAL_ERROR.IsErrorOrAbsent()) + assert.True(t, MLRVAL_ABSENT.IsErrorOrAbsent()) + assert.False(t, MLRVAL_NULL.IsErrorOrAbsent()) + assert.False(t, FromString("").IsErrorOrAbsent()) +} + +func TestIsError(t *testing.T) { + assert.True(t, MLRVAL_ERROR.IsError()) + assert.False(t, MLRVAL_ABSENT.IsError()) + assert.False(t, MLRVAL_NULL.IsError()) + assert.False(t, FromString("").IsError()) +} + +func TestIsAbsent(t *testing.T) { + assert.False(t, MLRVAL_ERROR.IsAbsent()) + assert.True(t, MLRVAL_ABSENT.IsAbsent()) + assert.False(t, MLRVAL_NULL.IsAbsent()) + assert.False(t, FromString("").IsAbsent()) +} + +func TestIsNull(t *testing.T) { + assert.False(t, MLRVAL_ERROR.IsNull()) + assert.False(t, MLRVAL_ABSENT.IsNull()) + assert.True(t, MLRVAL_NULL.IsNull()) + assert.False(t, FromString("").IsNull()) +} + +func TestIsVoid(t *testing.T) { + assert.False(t, MLRVAL_ERROR.IsVoid()) + assert.False(t, MLRVAL_ABSENT.IsVoid()) + assert.False(t, MLRVAL_NULL.IsVoid()) + assert.True(t, FromString("").IsVoid()) + assert.True(t, FromDeferredType("").IsVoid()) + assert.True(t, FromInferredType("").IsVoid()) + assert.False(t, FromDeferredType("abc").IsVoid()) + assert.False(t, FromInferredType("abc").IsVoid()) +} + +func TestIsEmptyString(t *testing.T) { + assert.False(t, MLRVAL_ERROR.IsEmptyString()) + assert.False(t, MLRVAL_ABSENT.IsEmptyString()) + assert.False(t, MLRVAL_NULL.IsEmptyString()) + assert.True(t, FromString("").IsEmptyString()) + assert.True(t, FromDeferredType("").IsEmptyString()) + assert.True(t, FromInferredType("").IsEmptyString()) + assert.False(t, FromDeferredType("abc").IsEmptyString()) + assert.False(t, FromInferredType("abc").IsEmptyString()) +} + +func TestIsString(t *testing.T) { + assert.False(t, MLRVAL_ERROR.IsString()) + assert.False(t, MLRVAL_ABSENT.IsString()) + assert.False(t, MLRVAL_NULL.IsString()) + assert.False(t, FromString("").IsString()) + assert.False(t, FromDeferredType("").IsString()) + assert.False(t, FromInferredType("").IsString()) + assert.True(t, FromDeferredType("abc").IsString()) + assert.True(t, FromInferredType("abc").IsString()) +} + +func TestIsStringOrVoid(t *testing.T) { + assert.False(t, MLRVAL_ERROR.IsStringOrVoid()) + assert.False(t, MLRVAL_ABSENT.IsStringOrVoid()) + assert.False(t, MLRVAL_NULL.IsStringOrVoid()) + assert.True(t, FromString("").IsStringOrVoid()) + assert.True(t, FromDeferredType("").IsStringOrVoid()) + assert.True(t, FromInferredType("").IsStringOrVoid()) + assert.True(t, FromDeferredType("abc").IsStringOrVoid()) + assert.True(t, FromInferredType("abc").IsStringOrVoid()) +} + +func TestIsInt(t *testing.T) { + assert.True(t, FromDeferredType("123").IsInt()) + assert.True(t, FromInferredType("123").IsInt()) + assert.False(t, FromDeferredType("123.4").IsInt()) + assert.False(t, FromInferredType("123.4").IsInt()) + assert.False(t, FromDeferredType("abc").IsInt()) + assert.False(t, FromInferredType("abc").IsInt()) +} + +func TestIsIntZero(t *testing.T) { + assert.True(t, FromDeferredType("0").IsIntZero()) + assert.True(t, FromInferredType("0").IsIntZero()) + assert.True(t, FromDeferredType("-0").IsIntZero()) + assert.True(t, FromInferredType("-0").IsIntZero()) + assert.False(t, FromDeferredType("123").IsIntZero()) + assert.False(t, FromInferredType("123").IsIntZero()) + assert.False(t, FromDeferredType("abc").IsIntZero()) + assert.False(t, FromInferredType("abc").IsIntZero()) +} + +func TestIsFloat(t *testing.T) { + assert.False(t, FromDeferredType("123").IsFloat()) + assert.False(t, FromInferredType("123").IsFloat()) + assert.True(t, FromDeferredType("123.4").IsFloat()) + assert.True(t, FromInferredType("123.4").IsFloat()) + assert.False(t, FromDeferredType("abc").IsFloat()) + assert.False(t, FromInferredType("abc").IsFloat()) +} + +func TestIsNumeric(t *testing.T) { + assert.True(t, FromDeferredType("123").IsNumeric()) + assert.True(t, FromInferredType("123").IsNumeric()) + assert.True(t, FromDeferredType("123.4").IsNumeric()) + assert.True(t, FromInferredType("123.4").IsNumeric()) + assert.False(t, FromDeferredType("abc").IsNumeric()) + assert.False(t, FromInferredType("abc").IsNumeric()) +} + +func TestIsBool(t *testing.T) { + assert.False(t, FromDeferredType("123").IsBool()) + assert.False(t, FromInferredType("123").IsBool()) + assert.False(t, FromDeferredType("123.4").IsBool()) + assert.False(t, FromInferredType("123.4").IsBool()) + assert.False(t, FromDeferredType("abc").IsBool()) + assert.False(t, FromInferredType("abc").IsBool()) + assert.False(t, FromDeferredType("true").IsBool(), "from-data-file \"true\" should infer to string") + assert.True(t, FromDeferredType("true").IsString(), "from-data-file \"true\" should infer to string") + assert.True(t, FromInferredType("true").IsBool()) + assert.False(t, FromDeferredType("false").IsBool(), "from-data-file \"false\" should infer to string") + assert.True(t, FromDeferredType("false").IsString(), "from-data-file \"false\" should infer to string") + assert.True(t, FromInferredType("false").IsBool()) +} diff --git a/internal/pkg/mlrval/new.go b/internal/pkg/mlrval/new.go new file mode 100644 index 000000000..585effc22 --- /dev/null +++ b/internal/pkg/mlrval/new.go @@ -0,0 +1,161 @@ +// ================================================================ +// Constructors +// ================================================================ + +package mlrval + +import ( + //"errors" + + "github.com/johnkerl/miller/internal/pkg/lib" +) + +// TODO: comment JIT context. Some things we already know are typed -- DSL +// things, or JSON contents. Others are deferred, e.g. items from any file +// format except JSON. +// TODO: comment re inferBool. +func FromDeferredType(input string) *Mlrval { + return &Mlrval{ + mvtype: MT_PENDING, + printrep: input, + printrepValid: true, + } +} + +// TODO: comment non-JIT context like mlr put -s. +// TODO: comment re inferBool. +func FromInferredType(input string) *Mlrval { + mv := &Mlrval{ + mvtype: MT_PENDING, + printrep: input, + printrepValid: true, + } + // TODO: comment re inferBool arg + packageLevelInferrer(mv, mv.printrep, true) + return mv +} + +func FromString(input string) *Mlrval { + if input == "" { + return MLRVAL_VOID + } + return &Mlrval{ + mvtype: MT_STRING, + printrep: input, + printrepValid: true, + } +} + +func (mv *Mlrval) SetFromString(input string) *Mlrval { + mv.printrep = input + mv.printrepValid = true + if input == "" { + mv.mvtype = MT_VOID + } else { + mv.mvtype = MT_STRING + } + return mv +} + +func (mv *Mlrval) SetFromVoid() *Mlrval { + mv.printrep = "" + mv.printrepValid = true + mv.mvtype = MT_VOID + return mv +} + +func FromInt(input int) *Mlrval { + return &Mlrval{ + mvtype: MT_INT, + printrepValid: false, + intval: input, + } +} + +// TODO: comment +func (mv *Mlrval) SetFromPrevalidatedIntString(input string, intval int) *Mlrval { + mv.printrep = input + mv.printrepValid = true + mv.intval = intval + mv.mvtype = MT_INT + return mv +} + +func FromFloat(input float64) *Mlrval { + return &Mlrval{ + mvtype: MT_FLOAT, + printrepValid: false, + floatval: input, + } +} + +// TODO: comment +func (mv *Mlrval) SetFromPrevalidatedFloatString(input string, floatval float64) *Mlrval { + mv.printrep = input + mv.printrepValid = true + mv.floatval = floatval + mv.mvtype = MT_FLOAT + return mv +} + +func FromBool(input bool) *Mlrval { + if input == true { + return MLRVAL_TRUE + } else { + return MLRVAL_FALSE + } +} + +func FromBoolString(input string) *Mlrval { + if input == "true" { + return MLRVAL_TRUE + } else if input == "false" { + return MLRVAL_FALSE + } else { + lib.InternalCodingErrorIf(true) + return nil // not reached + } +} + +// TODO: comment +func (mv *Mlrval) SetFromPrevalidatedBoolString(input string, boolval bool) *Mlrval { + mv.printrep = input + mv.printrepValid = true + mv.boolval = boolval + mv.mvtype = MT_BOOL + return mv +} + +// The user-defined function is of type 'interface{}' here to avoid what would +// otherwise be a package-dependency cycle between this package and +// github.com/johnkerl/miller/internal/pkg/dsl/cst. +// +// Nominally the name argument is the user-specified name if `func f(a, b) { +// ... }`, or some autogenerated UUID like `fl0052` if `func (a, b) { ... }`. + +func FromFunction(funcval interface{}, name string) *Mlrval { + return &Mlrval{ + mvtype: MT_FUNC, + printrep: name, + printrepValid: true, + funcval: funcval, + } +} + +func FromArray(arrayval interface{}) *Mlrval { + return &Mlrval{ + mvtype: MT_ARRAY, + printrep: INVALID_PRINTREP, + printrepValid: true, + arrayval: arrayval, + } +} + +func FromMap(mapval interface{}) *Mlrval { + return &Mlrval{ + mvtype: MT_MAP, + printrep: INVALID_PRINTREP, + printrepValid: true, + mapval: mapval, + } +} diff --git a/internal/pkg/mlrval/new_test.go b/internal/pkg/mlrval/new_test.go new file mode 100644 index 000000000..16b713b89 --- /dev/null +++ b/internal/pkg/mlrval/new_test.go @@ -0,0 +1,122 @@ +// ================================================================ +// Tests mlrval constructors. +// ================================================================ + +package mlrval + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFromDeferredType(t *testing.T) { + mv := FromDeferredType("123") + assert.Equal(t, MVType(MT_PENDING), MVType(mv.mvtype)) + assert.Equal(t, "123", mv.printrep) + assert.True(t, mv.printrepValid) + + mv = FromDeferredType("true") + assert.Equal(t, MVType(MT_PENDING), MVType(mv.mvtype)) + assert.Equal(t, "true", mv.printrep) + assert.True(t, mv.printrepValid) + + mv = FromDeferredType("abc") + assert.Equal(t, MVType(MT_PENDING), MVType(mv.mvtype)) + assert.Equal(t, "abc", mv.printrep) + assert.True(t, mv.printrepValid) + + mv = FromDeferredType("") + assert.Equal(t, MVType(MT_PENDING), MVType(mv.mvtype)) + assert.Equal(t, "", mv.printrep) + assert.True(t, mv.printrepValid) +} + +func TestFromInferredType(t *testing.T) { + mv := FromInferredType("123") + assert.Equal(t, MVType(MT_INT), MVType(mv.mvtype)) + assert.Equal(t, "123", mv.printrep) + assert.True(t, mv.printrepValid) + assert.Equal(t, mv.intval, 123) + + mv = FromInferredType("true") + assert.Equal(t, MVType(MT_BOOL), MVType(mv.mvtype)) + assert.Equal(t, "true", mv.printrep) + assert.True(t, mv.printrepValid) + assert.Equal(t, mv.boolval, true) + + mv = FromInferredType("abc") + assert.Equal(t, MVType(MT_STRING), MVType(mv.mvtype)) + assert.Equal(t, "abc", mv.printrep) + assert.True(t, mv.printrepValid) + + mv = FromInferredType("") + assert.Equal(t, MVType(MT_VOID), MVType(mv.mvtype)) + assert.Equal(t, "", mv.printrep) + assert.True(t, mv.printrepValid) +} + +func TestFromString(t *testing.T) { + mv := FromString("123") + assert.Equal(t, MVType(MT_STRING), MVType(mv.mvtype)) + assert.Equal(t, "123", mv.printrep) + assert.True(t, mv.printrepValid) + + mv = FromString("") + assert.Equal(t, MVType(MT_VOID), MVType(mv.mvtype)) + assert.Equal(t, "", mv.printrep) + assert.True(t, mv.printrepValid) +} + +func TestFromInt(t *testing.T) { + mv := FromInt(123) + assert.Equal(t, MVType(MT_INT), MVType(mv.mvtype)) + assert.False(t, mv.printrepValid, "printrep should not be computed yet") +} + +func TestFromFloat(t *testing.T) { + mv := FromFloat(123.4) + assert.Equal(t, MVType(MT_FLOAT), MVType(mv.mvtype)) + assert.False(t, mv.printrepValid, "printrep should not be computed yet") +} + +func TestFromBool(t *testing.T) { + mv := FromBool(true) + assert.Equal(t, MVType(MT_BOOL), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid) + + mv = FromBool(false) + assert.Equal(t, MVType(MT_BOOL), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid) +} + +func TestFromBoolString(t *testing.T) { + mv := FromBoolString("true") + assert.Equal(t, MVType(MT_BOOL), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid) + + mv = FromBoolString("false") + assert.Equal(t, MVType(MT_BOOL), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid) +} + +func TestFromFunction(t *testing.T) { + mv := FromFunction("test data", "f001") + assert.Equal(t, MVType(MT_FUNC), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid) + assert.Equal(t, "test data", mv.funcval.(string)) +} + +func TestFromArray(t *testing.T) { + mv := FromArray("test data") + assert.Equal(t, MVType(MT_ARRAY), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid) + assert.Equal(t, "test data", mv.arrayval.(string)) +} + +func TestFromMap(t *testing.T) { + mv := FromMap("test data") + assert.Equal(t, MVType(MT_MAP), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid) + assert.Equal(t, "test data", mv.mapval.(string)) +} diff --git a/internal/pkg/mlrval/tbf.go b/internal/pkg/mlrval/tbf.go new file mode 100644 index 000000000..290cc306f --- /dev/null +++ b/internal/pkg/mlrval/tbf.go @@ -0,0 +1,249 @@ +package mlrval + +// things to be filed + +//func (mv *Mlrval) GetTypeBit() int { +// return 1 << mv.mvtype +//} + +//// NewMlrvalForAutoDeepen is for auto-deepen of nested maps in things like +//// +//// $foo[1]["a"][2]["b"] = 3 +//// +//// Autocreated levels are maps. Array levels can be explicitly created e.g. +//// +//// $foo[1]["a"] ??= [] +//// $foo[1]["a"][2]["b"] = 3 +//func NewMlrvalForAutoDeepen(mvtype MVType) (*Mlrval, error) { +// if mvtype == MT_STRING || mvtype == MT_INT { +// empty := MlrvalFromEmptyMap() +// return empty, nil +// } else { +// return nil, errors.New( +// "mlr: indices must be string, int, or array thereof; got " + GetTypeName(mvtype), +// ) +// } +//} + + +//func MlrvalFromEmptyMap() *Mlrval { +// return &Mlrval{ +// mvtype: MT_MAP, +// printrep: "(bug-if-you-see-this-map-type)", +// printrepValid: false, +// mapval: NewMlrmap(), +// } +//} +// +//func MlrvalFromMap(mlrmap *Mlrmap) *Mlrval { +// mv := MlrvalFromEmptyMap() +// if mlrmap == nil { +// // TODO maybe return 2nd-arg error in the API +// return MLRVAL_ERROR +// } +// +// for pe := mlrmap.Head; pe != nil; pe = pe.Next { +// mv.mapval.PutCopy(pe.Key, pe.Value) +// } +// return mv +//} +// +//// Like previous but doesn't copy. Only safe when the argument's sole purpose +//// is to be passed into here. +//func MlrvalFromMapReferenced(mlrmap *Mlrmap) *Mlrval { +// mv := MlrvalFromEmptyMap() +// if mlrmap == nil { +// // xxx maybe return 2nd-arg error in the API +// return MLRVAL_ERROR +// } +// +// for pe := mlrmap.Head; pe != nil; pe = pe.Next { +// mv.mapval.PutReference(pe.Key, pe.Value) +// } +// return mv +//} +// +//// Does not copy the data. We can make a MlrvalFromArrayLiteralCopy if needed, +//// using values.CopyMlrvalArray(). +//func MlrvalEmptyArray() Mlrval { +// return Mlrval{ +// mvtype: MT_ARRAY, +// printrep: "(bug-if-you-see-this-array-type)", +// printrepValid: false, +// intval: 0, +// floatval: 0.0, +// boolval: false, +// arrayval: make([]Mlrval, 0, 10), +// mapval: nil, +// } +//} +// +//// Users can do things like '$new[1][2][3] = 4' even if '$new' isn't already +//// allocated. This function supports that. +//func NewSizedMlrvalArray(length int) *Mlrval { +// arrayval := make([]Mlrval, length, 2*length) +// +// for i := 0; i < int(length); i++ { +// arrayval[i] = *MLRVAL_VOID +// } +// +// return &Mlrval{ +// mvtype: MT_ARRAY, +// printrep: "(bug-if-you-see-this-array-type)", +// printrepValid: false, +// intval: 0, +// floatval: 0.0, +// boolval: false, +// arrayval: arrayval, +// mapval: nil, +// } +//} +// +//// Does not copy the data. We can make a SetFromArrayLiteralCopy if needed +//// using values.CopyMlrvalArray(). +//func MlrvalFromArrayReference(input []Mlrval) *Mlrval { +// return &Mlrval{ +// mvtype: MT_ARRAY, +// printrepValid: false, +// arrayval: input, +// } +//} +// +//func LengthenMlrvalArray(array *[]Mlrval, newLength64 int) { +// newLength := int(newLength64) +// lib.InternalCodingErrorIf(newLength <= len(*array)) +// +// if newLength <= cap(*array) { +// newArray := (*array)[:newLength] +// for zindex := len(*array); zindex < newLength; zindex++ { +// // TODO: comment why not MT_ABSENT or MT_VOID +// newArray[zindex] = *MLRVAL_NULL +// } +// *array = newArray +// } else { +// newArray := make([]Mlrval, newLength, 2*newLength) +// zindex := 0 +// for zindex = 0; zindex < len(*array); zindex++ { +// newArray[zindex] = (*array)[zindex] +// } +// for zindex = len(*array); zindex < newLength; zindex++ { +// // TODO: comment why not MT_ABSENT or MT_VOID +// newArray[zindex] = *MLRVAL_NULL +// } +// *array = newArray +// } +//} +// +//// NewMlrvalForAutoDeepen is for auto-deepen of nested maps in things like +//// +//// $foo[1]["a"][2]["b"] = 3 +//// +//// Autocreated levels are maps. Array levels can be explicitly created e.g. +//// +//// $foo[1]["a"] ??= [] +//// $foo[1]["a"][2]["b"] = 3 +//func NewMlrvalForAutoDeepen(mvtype MVType) (*Mlrval, error) { +// if mvtype == MT_STRING || mvtype == MT_INT { +// empty := MlrvalFromEmptyMap() +// return empty, nil +// } else { +// return nil, errors.New( +// "mlr: indices must be string, int, or array thereof; got " + GetTypeName(mvtype), +// ) +// } +//} + +//func TypeNameToMask(typeName string) (mask int, present bool) { +// retval := typeNameToMaskMap[typeName] +// if retval != 0 { +// return retval, true +// } else { +// return 0, false +// } +//} + +//// MlrvalTryPointerFromFloatString is used by MlrvalFormatter (fmtnum DSL +//// function, format-values verb, etc). Each mlrval has printrep and a +//// printrepValid for its original string, then a type-code like MT_INT or +//// MT_FLOAT, and type-specific storage like intval or floatval. +//// +//// If the user has taken a mlrval with original string "3.14" and formatted it +//// with "%.4f" then its printrep will be "3.1400" but its type should still be +//// MT_FLOAT. +//// +//// If on the other hand the user has formatted the same mlrval with "[[%.4f]]" +//// then its printrep will be "[[3.1400]]" and it will be MT_STRING. +//// This function supports that. +//func MlrvalTryPointerFromFloatString(input string) *Mlrval { +// _, fok := lib.TryFloat64FromString(input) +// if fok { +// return MlrvalFromPrevalidatedFloat64String(input) +// } else { +// return MlrvalFromString(input) +// } +//} +// +//// MlrvalTryPointerFromIntString is used by MlrvalFormatter (fmtnum DSL +//// function, format-values verb, etc). Each mlrval has printrep and a +//// printrepValid for its original string, then a type-code like MT_INT or +//// MT_FLOAT, and type-specific storage like intval or floatval. +//// +//// If the user has taken a mlrval with original string "314" and formatted it +//// with "0x%04x" then its printrep will be "0x013a" but its type should still be +//// MT_INT. +//// +//// If on the other hand the user has formatted the same mlrval with +//// "[[%0x04x]]" then its printrep will be "[[0x013a]]" and it will be +//// MT_STRING. This function supports that. +//func MlrvalTryPointerFromIntString(input string) *Mlrval { +// _, iok := lib.TryIntFromString(input) +// if iok { +// return MlrvalFromPrevalidatedIntString(input) +// } else { +// return MlrvalFromString(input) +// } +//} +// +//// ---------------------------------------------------------------- + +// TODO: FILE +// +//// MlrvalFromInferredTypeForDataFiles is for parsing field values directly from +//// data files (except JSON, which is typed -- "true" and true are distinct). +//// Mostly the same as MlrvalFromInferredType, except it doesn't auto-infer +//// true/false to bool; don't auto-infer NaN/Inf to float; etc. +//func MlrvalFromInferredTypeForDataFiles(input string) *Mlrval { +// return inferrer(input, false) +//} +// +//// MlrvalFromInferredType is for parsing field values not directly from data +//// files. Mostly the same as MlrvalFromInferredTypeForDataFiles, except it +//// auto-infers true/false to bool; don't auto-infer NaN/Inf to float; etc. +//func MlrvalFromInferredType(input string) *Mlrval { +// return inferrer(input, true) +//} + +//func (mv *Mlrval) GetNumericToFloatValueOrDie() (floatValue float64) { +// floatValue, ok := mv.GetNumericToFloatValue() +// if !ok { +// fmt.Fprintf( +// os.Stderr, +// "%s: couldn't parse \"%s\" as number.", +// "mlr", mv.String(), +// ) +// os.Exit(1) +// } +// return floatValue +//} + +//func (mv *Mlrval) AssertNumeric() { +// _ = mv.GetNumericToFloatValueOrDie() +//} + +//func (mv *Mlrval) GetArrayLength() (int, bool) { +// if mv.IsArray() { +// return len(mv.arrayval), true +// } else { +// return -999, false +// } +//} diff --git a/internal/pkg/types/mlrval.go b/internal/pkg/mlrval/type.go similarity index 54% rename from internal/pkg/types/mlrval.go rename to internal/pkg/mlrval/type.go index 9075637ab..8430c15fc 100644 --- a/internal/pkg/types/mlrval.go +++ b/internal/pkg/mlrval/type.go @@ -1,5 +1,5 @@ // ================================================================ -// The `types.Mlrval` structure includes **string, int, float, boolean, void, +// The `mlrval.Mlrval` structure includes **string, int, float, boolean, void, // absent, and error** types (not unlike PHP's `zval`) as well as // type-conversion logic for various operators. // @@ -8,7 +8,50 @@ // that is a bug. It would be great to be able to somehow lint for this. // ================================================================ -package types +// TODO: +// * why here carefully fenced +// * why interface for recursive/external items +// * why optimizing: +// o def string-to-type, e.g. 17-column-wide, million-row CSV file, +// '$z = $x + $y' kind of thing -- don't need to type-infer +// column $a "1234" as int only to write it back non-modified +// o defer type-to-string, e.g. '$z = a*b + c*d' w/ a,b,c,d ints, +// string-rep not needed -- defer string-formatting until write-out. +// * Careful fencing: want things to "just work" for as much of the +// code as possible. +// o Hide x.mvtype as x.Type() and that function can JIT-infer printrep +// "1234" and mvtype MT_PENDING to mvtype MT_INT and intval 1234. +// o Similarly MT_INT, intval 1235, printrepValid = false, only +// on to-string do printrep = "1235" and printrepValid=true. +// * For all but JSON: everything is string/int/float/bool. Any +// array-valued/map-valued are JSON-encoded _as strings_. +// * For JSON: no defer; everything is explicitly typed as part of +// the decode. + +// TODO: MERGE +// An int/float always starts from a string -- be it from record data from +// a file, or a literal within a DSL expression. The printrep is exactly +// that string, however the user formatted it, and the intval/floatval is +// computed from that -- and in sync with it -- at construction time. +// +// When a mlrval is computed from one or more others -- e.g. '$z = $x + 4' +// -- the printrep is not updated. That would be wasted CPU, since the +// string representation is not needed until when/if the value is printed +// as output. For computation methods the printrep is neglected and the +// printrepValid is set to false. +// +// In the String() method the printrep is computed from the intval/floatval +// and printrepValid is set back to true. +// +// Note that for MT_STRING, the printrep is always valid since it is the +// only payload for the mlrval. +// +// Thus we (a) keep user-specific input-formatting when possible, for the +// principle of least surprise; (b) avoid reformatting strings during +// intermediate arithmetic expressions; (c) resync arithmetic results to +// string formatting on a just-in-time basis when output is printed. + +package mlrval type Mlrval struct { @@ -16,46 +59,29 @@ type Mlrval struct { // I would call this "type" not "mvtype" but "type" is a keyword in Go. mvtype MVType - // An int/float always starts from a string -- be it from record data from - // a file, or a literal within a DSL expression. The printrep is exactly - // that string, however the user formatted it, and the intval/floatval is - // computed from that -- and in sync with it -- at construction time. - // - // When a mlrval is computed from one or more others -- e.g. '$z = $x + 4' - // -- the printrep is not updated. That would be wasted CPU, since the - // string representation is not needed until when/if the value is printed - // as output. For computation methods the printrep is neglected and the - // printrepValid is set to false. - // - // In the String() method the printrep is computed from the intval/floatval - // and printrepValid is set back to true. - // - // Note that for MT_STRING, the printrep is always valid since it is the - // only payload for the mlrval. - // - // Thus we (a) keep user-specific input-formatting when possible, for the - // principle of least surprise; (b) avoid reformatting strings during - // intermediate arithmetic expressions; (c) resync arithmetic results to - // string formatting on a just-in-time basis when output is printed. printrep string printrepValid bool intval int floatval float64 boolval bool - arrayval []Mlrval - mapval *Mlrmap - // These are first-class-function literals. Stored here as interface{} to - // avoid what would otherwise be a package-dependency cycle with the - // github.com/johnkerl/miller/internal/pkg/dsl/cst package. - funcval interface{} + + // Interfaced here to avoid package-dependency cycles: + arrayval interface{} // []Mlrval + mapval interface{} // *Mlrmap + funcval interface{} // first-class-function literals from internal/pkg/dsl/cst } +const INVALID_PRINTREP = "(bug-if-you-see-this)" +const ERROR_PRINTREP = "(error)" +const ABSENT_PRINTREP = "(error)" + // Enumeration for mlrval types // -// There are two kinds of null: ABSENT (key not present in a record) and VOID -// (key present with empty value). Note void is an acceptable string (empty -// string) but not an acceptable number. (In Javascript, similarly, there are -// undefined and null, respectively.) +// There are three kinds of null: ABSENT (key not present in a record) and VOID +// (key present with empty value); thirdly NULL for JSON null. Note void is an +// acceptable string (empty string) but not an acceptable number. (In +// JavaScript, similarly, there are undefined and null, respectively -- +// Miller's absent is more like JavaScript's undefined.) type MVType int @@ -63,7 +89,7 @@ type MVType int // matrices. If they are changed, it will break the disposition matrices, or // they will all need manual re-indexing. const ( - // Type not yet determined, during JSON decode. + // Type not yet determined: during JSON decode or TODO comment. MT_PENDING MVType = -1 // E.g. error encountered in one eval & it propagates up the AST at @@ -121,7 +147,6 @@ var TYPE_NAMES = [MT_DIM]string{ "funct", } -// ---------------------------------------------------------------- // For typed assignments in the DSL // TODO: comment more re typedecls @@ -158,15 +183,3 @@ var typeNameToMaskMap = map[string]int{ "funct": MT_TYPE_MASK_FUNC, } -func TypeNameToMask(typeName string) (mask int, present bool) { - retval := typeNameToMaskMap[typeName] - if retval != 0 { - return retval, true - } else { - return 0, false - } -} - -func (mv *Mlrval) GetTypeBit() int { - return 1 << mv.mvtype -} diff --git a/internal/pkg/types/mlrmap.go b/internal/pkg/types/mlrmap.go index be344050c..c91827b74 100644 --- a/internal/pkg/types/mlrmap.go +++ b/internal/pkg/types/mlrmap.go @@ -53,6 +53,10 @@ package types +import( + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + // For the C port having this off was a noticeable performance improvement (10-15%). // For the Go port having it off is a less-noticeable performance improvement (5%). // Both these figures are for just doing mlr cat. At the moment I'm leaving this @@ -78,7 +82,7 @@ type Mlrmap struct { type MlrmapEntry struct { Key string - Value *Mlrval + Value *mlrval.Mlrval Prev *MlrmapEntry Next *MlrmapEntry } @@ -86,7 +90,7 @@ type MlrmapEntry struct { // Only used for sorting, map-to-pairs-array and pairs-array-to-map contexts. type MlrmapPair struct { Key string - Value *Mlrval + Value *mlrval.Mlrval } // ---------------------------------------------------------------- @@ -137,7 +141,7 @@ func (mlrmap *Mlrmap) isHashed() bool { // ---------------------------------------------------------------- // Value-copy is up to the caller -- PutReference and PutCopy // are in the public Mlrmap API. -func newMlrmapEntry(key string, value *Mlrval) *MlrmapEntry { +func newMlrmapEntry(key string, value *mlrval.Mlrval) *MlrmapEntry { return &MlrmapEntry{ key, value, diff --git a/internal/pkg/types/mlrmap_accessors.go b/internal/pkg/types/mlrmap_accessors.go index 445d5865e..7bac89e99 100644 --- a/internal/pkg/types/mlrmap_accessors.go +++ b/internal/pkg/types/mlrmap_accessors.go @@ -1,10 +1,11 @@ package types import ( - "bytes" - "errors" + //"bytes" + //"errors" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // IsEmpty determines if an map is empty. @@ -12,40 +13,25 @@ func (mlrmap *Mlrmap) IsEmpty() bool { return mlrmap.Head == nil } -// ---------------------------------------------------------------- func (mlrmap *Mlrmap) Has(key string) bool { return mlrmap.findEntry(key) != nil } -// PutCpoy copies the key and value (deep-copying in case the value is array/map). -// This is safe for DSL use. See also PutReference. -func (mlrmap *Mlrmap) PutCopy(key string, value *Mlrval) { +func (mlrmap *Mlrmap) Get(key string) *mlrval.Mlrval { pe := mlrmap.findEntry(key) if pe == nil { - pe = newMlrmapEntry(key, value.Copy()) - if mlrmap.Head == nil { - mlrmap.Head = pe - mlrmap.Tail = pe - } else { - pe.Prev = mlrmap.Tail - pe.Next = nil - mlrmap.Tail.Next = pe - mlrmap.Tail = pe - } - if mlrmap.keysToEntries != nil { - mlrmap.keysToEntries[key] = pe - } - mlrmap.FieldCount++ + return nil } else { - pe.Value = value.Copy() + return pe.Value } + return nil } // PutReference copies the key but not the value. This is not safe for DSL use, // where we could create undesired references between different objects. Only // intended to be used at callsites which allocate a mlrval solely for the // purpose of putting into a map, e.g. input-record readers. -func (mlrmap *Mlrmap) PutReference(key string, value *Mlrval) { +func (mlrmap *Mlrmap) PutReference(key string, value *mlrval.Mlrval) { pe := mlrmap.findEntry(key) if pe == nil { pe = newMlrmapEntry(key, value) @@ -67,11 +53,57 @@ func (mlrmap *Mlrmap) PutReference(key string, value *Mlrval) { } } +//// PutCopy copies the key and value (deep-copying in case the value is array/map). +//// This is safe for DSL use. See also PutReference. +//func (mlrmap *Mlrmap) PutCopy(key string, value *mlrval.Mlrval) { +// pe := mlrmap.findEntry(key) +// if pe == nil { +// pe = newMlrmapEntry(key, value.Copy()) +// if mlrmap.Head == nil { +// mlrmap.Head = pe +// mlrmap.Tail = pe +// } else { +// pe.Prev = mlrmap.Tail +// pe.Next = nil +// mlrmap.Tail.Next = pe +// mlrmap.Tail = pe +// } +// if mlrmap.keysToEntries != nil { +// mlrmap.keysToEntries[key] = pe +// } +// mlrmap.FieldCount++ +// } else { +// pe.Value = value.Copy() +// } +//} + +func (mlrmap *Mlrmap) PrependReference(key string, value *mlrval.Mlrval) { + pe := mlrmap.findEntry(key) + if pe == nil { + pe = newMlrmapEntry(key, value) + if mlrmap.Tail == nil { + mlrmap.Head = pe + mlrmap.Tail = pe + } else { + pe.Prev = nil + pe.Next = mlrmap.Head + mlrmap.Head.Prev = pe + mlrmap.Head = pe + } + if mlrmap.keysToEntries != nil { + mlrmap.keysToEntries[key] = pe + } + mlrmap.FieldCount++ + } else { + pe.Value = value + } +} + // TODO: COMMENT func (mlrmap *Mlrmap) PutReferenceAfter( pe *MlrmapEntry, key string, - value *Mlrval, + value *mlrval.Mlrval, ) *MlrmapEntry { if pe == nil || pe.Next == nil { // New entry is supposed to go 'after' old, but there is no such old @@ -123,86 +155,24 @@ func (mlrmap *Mlrmap) PutReferenceAfter( } } -func (mlrmap *Mlrmap) PutCopyWithMlrvalIndex(key *Mlrval, value *Mlrval) error { - if key.mvtype == MT_STRING { - mlrmap.PutCopy(key.printrep, value) - return nil - } else if key.mvtype == MT_INT { - mlrmap.PutCopy(key.String(), value) - return nil +// findEntry is the basic hash-map accessor for Has, Put, Get, Remove, etc. +// Hashmaps in Go are a built-in type; Mlrmap is (a) +// insertion-order-preserving, and (b) supports either hashed or non-hashed +// access. +func (mlrmap *Mlrmap) findEntry(key string) *MlrmapEntry { + if mlrmap.keysToEntries != nil { + return mlrmap.keysToEntries[key] } else { - return errors.New( - "mlr: record/map indices must be string, int, or array thereof; got " + key.GetTypeName(), - ) - } -} - -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) PrependCopy(key string, value *Mlrval) { - mlrmap.PrependReference(key, value.Copy()) -} - -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) PrependReference(key string, value *Mlrval) { - pe := mlrmap.findEntry(key) - if pe == nil { - pe = newMlrmapEntry(key, value) - if mlrmap.Tail == nil { - mlrmap.Head = pe - mlrmap.Tail = pe - } else { - pe.Prev = nil - pe.Next = mlrmap.Head - mlrmap.Head.Prev = pe - mlrmap.Head = pe + for pe := mlrmap.Head; pe != nil; pe = pe.Next { + if pe.Key == key { + return pe + } } - if mlrmap.keysToEntries != nil { - mlrmap.keysToEntries[key] = pe - } - mlrmap.FieldCount++ - } else { - pe.Value = value - } -} - -// ---------------------------------------------------------------- -// Merges that into mlrmap. -func (mlrmap *Mlrmap) Merge(other *Mlrmap) { - for pe := other.Head; pe != nil; pe = pe.Next { - mlrmap.PutCopy(pe.Key, pe.Value) - } -} - -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) Get(key string) *Mlrval { - pe := mlrmap.findEntry(key) - if pe == nil { return nil - } else { - return pe.Value } - return nil } -// ---------------------------------------------------------------- -// Exposed for the 'nest' verb -func (mlrmap *Mlrmap) GetEntry(key string) *MlrmapEntry { - return mlrmap.findEntry(key) -} - -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) GetKeys() []string { - keys := make([]string, mlrmap.FieldCount) - i := 0 - for pe := mlrmap.Head; pe != nil; pe = pe.Next { - keys[i] = pe.Key - i++ - } - return keys -} - -// ---------------------------------------------------------------- -// For '$[[[1]]]' etc. in the DSL. +// findEntryByPositionalIndex is for '$[1]' etc. in the DSL. // // Notes: // * This is a linear search. @@ -211,64 +181,79 @@ func (mlrmap *Mlrmap) GetKeys() []string { // get the -1st field than the nth. // * Returns 0 on invalid index: 0, or < -n, or > n where n is the number of // fields. - -func (mlrmap *Mlrmap) GetWithPositionalIndex(position int) *Mlrval { - mapEntry := mlrmap.findEntryByPositionalIndex(position) - if mapEntry == nil { +func (mlrmap *Mlrmap) findEntryByPositionalIndex(position int) *MlrmapEntry { + if position > mlrmap.FieldCount || position < -mlrmap.FieldCount || position == 0 { return nil } - return mapEntry.Value -} - -func (mlrmap *Mlrmap) GetWithMlrvalIndex(index *Mlrval) (*Mlrval, error) { - if index.mvtype == MT_ARRAY { - return mlrmap.getWithMlrvalArrayIndex(index) - } else { - return mlrmap.getWithMlrvalSingleIndex(index) - } -} - -// This lets the user do '$y = $x[ ["a", "b", "c"] ]' in lieu of -// '$y = $x["a"]["b"]["c"]'. -func (mlrmap *Mlrmap) getWithMlrvalArrayIndex(index *Mlrval) (*Mlrval, error) { - current := mlrmap - var retval *Mlrval = nil - lib.InternalCodingErrorIf(index.mvtype != MT_ARRAY) - array := index.arrayval - n := len(array) - for i, piece := range array { - next, err := current.GetWithMlrvalIndex(&piece) - if err != nil { - return nil, err - } - if i < n-1 { - if !next.IsMap() { - return nil, errors.New( - "mlr: cannot multi-index non-map.", - ) + if position > 0 { + var i int = 1 + for pe := mlrmap.Head; pe != nil; pe = pe.Next { + if i == position { + return pe } - current = next.mapval - } else { - retval = next.Copy() + i++ } - } - lib.InternalCodingErrorIf(retval == nil) - return retval, nil -} - -func (mlrmap *Mlrmap) getWithMlrvalSingleIndex(index *Mlrval) (*Mlrval, error) { - if index.mvtype == MT_STRING { - return mlrmap.Get(index.printrep), nil - } else if index.mvtype == MT_INT { - return mlrmap.Get(index.String()), nil + lib.InternalCodingErrorIf(true) } else { - return nil, errors.New( - "Record/map indices must be string, int, or array thereof; got " + index.GetTypeName(), - ) + var i int = -1 + for pe := mlrmap.Tail; pe != nil; pe = pe.Prev { + if i == position { + return pe + } + i-- + } + lib.InternalCodingErrorIf(true) } + lib.InternalCodingErrorIf(true) + return nil } -// For '$[[1]]' etc. in the DSL. +//func (mlrmap *Mlrmap) PutCopyWithMlrvalIndex(key *mlrval.Mlrval, value *mlrval.Mlrval) error { +// if key.IsString() { +// mlrmap.PutCopy(key.printrep, value) +// return nil +// } else if key.IsInt() { +// mlrmap.PutCopy(key.String(), value) +// return nil +// } else { +// return errors.New( +// "mlr: record/map indices must be string, int, or array thereof; got " + key.GetTypeName(), +// ) +// } +//} + +//// ---------------------------------------------------------------- +//func (mlrmap *Mlrmap) PrependCopy(key string, value *mlrval.Mlrval) { +// mlrmap.PrependReference(key, value.Copy()) +//} + +//// ---------------------------------------------------------------- +//// Merges that into mlrmap. +//func (mlrmap *Mlrmap) Merge(other *Mlrmap) { +// for pe := other.Head; pe != nil; pe = pe.Next { +// mlrmap.PutCopy(pe.Key, pe.Value) +// } +//} + +//// ---------------------------------------------------------------- +//// Exposed for the 'nest' verb +//func (mlrmap *Mlrmap) GetEntry(key string) *MlrmapEntry { +// return mlrmap.findEntry(key) +//} + +//// ---------------------------------------------------------------- +//func (mlrmap *Mlrmap) GetKeys() []string { +// keys := make([]string, mlrmap.FieldCount) +// i := 0 +// for pe := mlrmap.Head; pe != nil; pe = pe.Next { +// keys[i] = pe.Key +// i++ +// } +// return keys +//} + +// ---------------------------------------------------------------- +// For '$[[[1]]]' etc. in the DSL. // // Notes: // * This is a linear search. @@ -324,156 +309,263 @@ func (mlrmap *Mlrmap) PutNameWithPositionalIndex(position int, name *Mlrval) { } } +//func (mlrmap *Mlrmap) GetWithPositionalIndex(position int) *mlrval.Mlrval { +// mapEntry := mlrmap.findEntryByPositionalIndex(position) +// if mapEntry == nil { +// return nil +// } +// return mapEntry.Value +//} + +//func (mlrmap *Mlrmap) GetWithMlrvalIndex(index *mlrval.Mlrval) (*mlrval.Mlrval, error) { +// if index.IsArray() { +// return mlrmap.getWithMlrvalArrayIndex(index) +// } else { +// return mlrmap.getWithMlrvalSingleIndex(index) +// } +//} + +//// This lets the user do '$y = $x[ ["a", "b", "c"] ]' in lieu of +//// '$y = $x["a"]["b"]["c"]'. +//func (mlrmap *Mlrmap) getWithMlrvalArrayIndex(index *mlrval.Mlrval) (*mlrval.Mlrval, error) { +// current := mlrmap +// var retval *mlrval.Mlrval = nil +// lib.InternalCodingErrorIf(!index.IsArray()) +// array := index.arrayval +// n := len(array) +// for i, piece := range array { +// next, err := current.GetWithMlrvalIndex(&piece) +// if err != nil { +// return nil, err +// } +// if i < n-1 { +// if !next.IsMap() { +// return nil, errors.New( +// "mlr: cannot multi-index non-map.", +// ) +// } +// current = next.mapval +// } else { +// retval = next.Copy() +// } +// } +// lib.InternalCodingErrorIf(retval == nil) +// return retval, nil +//} + +//func (mlrmap *Mlrmap) getWithMlrvalSingleIndex(index *mlrval.Mlrval) (*mlrval.Mlrval, error) { +// if index.IsString() { +// return mlrmap.Get(index.printrep), nil +// } else if index.IsInt() { +// return mlrmap.Get(index.String()), nil +// } else { +// return nil, errors.New( +// "Record/map indices must be string, int, or array thereof; got " + index.GetTypeName(), +// ) +// } +//} + +// For '$[[1]]' etc. in the DSL. +// +// Notes: +// * This is a linear search. +// * Indices are 1-up not 0-up +// * Indices -n..-1 are aliases for 1..n. In particular, it will be faster to +// get the -1st field than the nth. +// * Returns 0 on invalid index: 0, or < -n, or > n where n is the number of +// fields. + +//func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { +// mapEntry := mlrmap.findEntryByPositionalIndex(position) +// if mapEntry == nil { +// return "", false +// } +// return mapEntry.Key, true +//} + +//// ---------------------------------------------------------------- +//// TODO: put error-return into this API +//func (mlrmap *Mlrmap) PutNameWithPositionalIndex(position int, name *mlrval.Mlrval) { +// positionalEntry := mlrmap.findEntryByPositionalIndex(position) +// +// if positionalEntry == nil { +// // TODO: handle out-of-bounds accesses +// return +// } +// +// // TODO: rekey the hashmap +// s := "" +// if name.IsString() { +// s = name.printrep +// } else if name.IsInt() { +// s = name.String() +// } else { +// // TODO: return MlrvalFromError() +// return +// } +// +// // E.g. there are fields named 'a' and 'b', as positions 1 and 2, +// // and the user does '$[[1]] = $[[2]]'. Then there would be two b's. +// mapEntry := mlrmap.findEntry(s) +// if mapEntry != nil && mapEntry != positionalEntry { +// mlrmap.Unlink(mapEntry) +// } +// +// lib.InternalCodingErrorIf(s == "") +// positionalEntry.Key = s +//} + // ---------------------------------------------------------------- // Copies the key and value (deep-copying in case the value is array/map). // This is safe for DSL use. See also PutReference. // TODO: put error-return into this API -func (mlrmap *Mlrmap) PutCopyWithPositionalIndex(position int, value *Mlrval) { - mapEntry := mlrmap.findEntryByPositionalIndex(position) +//func (mlrmap *Mlrmap) PutCopyWithPositionalIndex(position int, value *mlrval.Mlrval) { +// mapEntry := mlrmap.findEntryByPositionalIndex(position) +// +// if mapEntry != nil { +// mapEntry.Value = value.Copy() +// } else { +// return +// } +//} - if mapEntry != nil { - mapEntry.Value = value.Copy() - } else { - return - } -} +//func (mlrmap *Mlrmap) RemoveWithPositionalIndex(position int) { +// mapEntry := mlrmap.findEntryByPositionalIndex(position) +// if mapEntry != nil { +// mlrmap.Unlink(mapEntry) +// } +//} -func (mlrmap *Mlrmap) RemoveWithPositionalIndex(position int) { - mapEntry := mlrmap.findEntryByPositionalIndex(position) - if mapEntry != nil { - mlrmap.Unlink(mapEntry) - } -} +//// ---------------------------------------------------------------- +//func (mlrmap *Mlrmap) Equals(other *Mlrmap) bool { +// if mlrmap.FieldCount != mlrmap.FieldCount { +// return false +// } +// if !mlrmap.Contains(other) { +// return false +// } +// if !other.Contains(mlrmap) { +// return false +// } +// return true +//} +// +//// True if this contains other, i.e. if other is contained by mlrmap. +//// * If any key of other is not a key of this, return false. +//// * If any key of other has a value unequal to this' value at the same key, return false. +//// * Else return true +//func (mlrmap *Mlrmap) Contains(other *Mlrmap) bool { +// for pe := other.Head; pe != nil; pe = pe.Next { +// if !mlrmap.Has(pe.Key) { +// return false +// } +// thisval := mlrmap.Get(pe.Key) +// thatval := pe.Value +// meq := BIF_equals(thisval, thatval) +// eq, ok := meq.GetBoolValue() +// lib.InternalCodingErrorIf(!ok) +// if !eq { +// return false +// } +// } +// return true +//} -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) Equals(other *Mlrmap) bool { - if mlrmap.FieldCount != mlrmap.FieldCount { - return false - } - if !mlrmap.Contains(other) { - return false - } - if !other.Contains(mlrmap) { - return false - } - return true -} +//// ---------------------------------------------------------------- +//func (mlrmap *Mlrmap) Clear() { +// mlrmap.FieldCount = 0 +// // Assuming everything unreferenced is getting GC'ed by the Go runtime +// mlrmap.Head = nil +// mlrmap.Tail = nil +// if mlrmap.keysToEntries != nil { +// mlrmap.keysToEntries = make(map[string]*MlrmapEntry) +// } +//} -// True if this contains other, i.e. if other is contained by mlrmap. -// * If any key of other is not a key of this, return false. -// * If any key of other has a value unequal to this' value at the same key, return false. -// * Else return true -func (mlrmap *Mlrmap) Contains(other *Mlrmap) bool { - for pe := other.Head; pe != nil; pe = pe.Next { - if !mlrmap.Has(pe.Key) { - return false - } - thisval := mlrmap.Get(pe.Key) - thatval := pe.Value - meq := BIF_equals(thisval, thatval) - eq, ok := meq.GetBoolValue() - lib.InternalCodingErrorIf(!ok) - if !eq { - return false - } - } - return true -} +//// ---------------------------------------------------------------- +//func (mlrmap *Mlrmap) Copy() *Mlrmap { +// other := NewMlrmapMaybeHashed(mlrmap.isHashed()) +// for pe := mlrmap.Head; pe != nil; pe = pe.Next { +// other.PutCopy(pe.Key, pe.Value) +// } +// return other +//} -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) Clear() { - mlrmap.FieldCount = 0 - // Assuming everything unreferenced is getting GC'ed by the Go runtime - mlrmap.Head = nil - mlrmap.Tail = nil - if mlrmap.keysToEntries != nil { - mlrmap.keysToEntries = make(map[string]*MlrmapEntry) - } -} +//// ---------------------------------------------------------------- +//// Returns true if it was found and removed +//func (mlrmap *Mlrmap) Remove(key string) bool { +// pe := mlrmap.findEntry(key) +// if pe == nil { +// return false +// } else { +// mlrmap.Unlink(pe) +// return true +// } +//} -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) Copy() *Mlrmap { - other := NewMlrmapMaybeHashed(mlrmap.isHashed()) - for pe := mlrmap.Head; pe != nil; pe = pe.Next { - other.PutCopy(pe.Key, pe.Value) - } - return other -} +//// ---------------------------------------------------------------- +//func (mlrmap *Mlrmap) MoveToHead(key string) { +// pe := mlrmap.findEntry(key) +// if pe != nil { +// mlrmap.Unlink(pe) +// mlrmap.linkAtHead(pe) +// } +//} -// ---------------------------------------------------------------- -// Returns true if it was found and removed -func (mlrmap *Mlrmap) Remove(key string) bool { - pe := mlrmap.findEntry(key) - if pe == nil { - return false - } else { - mlrmap.Unlink(pe) - return true - } -} - -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) MoveToHead(key string) { - pe := mlrmap.findEntry(key) - if pe != nil { - mlrmap.Unlink(pe) - mlrmap.linkAtHead(pe) - } -} - -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) MoveToTail(key string) { - pe := mlrmap.findEntry(key) - if pe != nil { - mlrmap.Unlink(pe) - mlrmap.linkAtTail(pe) - } -} +//// ---------------------------------------------------------------- +//func (mlrmap *Mlrmap) MoveToTail(key string) { +// pe := mlrmap.findEntry(key) +// if pe != nil { +// mlrmap.Unlink(pe) +// mlrmap.linkAtTail(pe) +// } +//} // ---------------------------------------------------------------- // E.g. '$name[1]["foo"] = "bar"' or '$*["foo"][1] = "bar"' // In the former case the indices are ["name", 1, "foo"] and in the latter case // the indices are ["foo", 1]. See also indexed-lvalues.md. // -// This is a Mlrmap (from string to Mlrval) so we handle the first level of -// indexing here, then pass the remaining indices to the Mlrval at the desired +// This is a Mlrmap (from string to mlrval.Mlrval) so we handle the first level of +// indexing here, then pass the remaining indices to the mlrval.Mlrval at the desired // slot. -func (mlrmap *Mlrmap) PutIndexed(indices []*Mlrval, rvalue *Mlrval) error { - return putIndexedOnMap(mlrmap, indices, rvalue) -} -func (mlrmap *Mlrmap) RemoveIndexed(indices []*Mlrval) error { - return removeIndexedOnMap(mlrmap, indices) -} +//func (mlrmap *Mlrmap) PutIndexed(indices []*mlrval.Mlrval, rvalue *mlrval.Mlrval) error { +// return putIndexedOnMap(mlrmap, indices, rvalue) +//} -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) GetKeysJoined() string { - var buffer bytes.Buffer - i := 0 - for pe := mlrmap.Head; pe != nil; pe = pe.Next { - if i > 0 { - buffer.WriteString(",") - } - i++ - buffer.WriteString(pe.Key) - } - return buffer.String() -} +//func (mlrmap *Mlrmap) RemoveIndexed(indices []*mlrval.Mlrval) error { +// return removeIndexedOnMap(mlrmap, indices) +//} -// For mlr reshape -func (mlrmap *Mlrmap) GetValuesJoined() string { - var buffer bytes.Buffer - i := 0 - for pe := mlrmap.Head; pe != nil; pe = pe.Next { - if i > 0 { - buffer.WriteString(",") - } - i++ - buffer.WriteString(pe.Value.String()) - } - return buffer.String() -} +//// ---------------------------------------------------------------- +//func (mlrmap *Mlrmap) GetKeysJoined() string { +// var buffer bytes.Buffer +// i := 0 +// for pe := mlrmap.Head; pe != nil; pe = pe.Next { +// if i > 0 { +// buffer.WriteString(",") +// } +// i++ +// buffer.WriteString(pe.Key) +// } +// return buffer.String() +//} + +//// For mlr reshape +//func (mlrmap *Mlrmap) GetValuesJoined() string { +// var buffer bytes.Buffer +// i := 0 +// for pe := mlrmap.Head; pe != nil; pe = pe.Next { +// if i > 0 { +// buffer.WriteString(",") +// } +// i++ +// buffer.WriteString(pe.Value.String()) +// } +// return buffer.String() +//} // ---------------------------------------------------------------- // For group-by in several transformers. If the record is 'a=x,b=y,c=3,d=4,e=5' and @@ -485,431 +577,382 @@ func (mlrmap *Mlrmap) GetValuesJoined() string { // transformers which support a -g option but are invoked without it (e.g. 'mlr tail // -n 1' vs 'mlr tail -n 1 -g a,b,c'). In this case the return value is simply // the empty string. -func (mlrmap *Mlrmap) GetSelectedValuesJoined(selectedFieldNames []string) (string, bool) { - if len(selectedFieldNames) == 0 { - // The fall-through is functionally correct, but this is quicker with - // skipping setting up an empty bytes-buffer and stringifying it. The - // non-grouped case is quite normal and is worth optimizing for. - return "", true - } - var buffer bytes.Buffer - for i, selectedFieldName := range selectedFieldNames { - entry := mlrmap.findEntry(selectedFieldName) - if entry == nil { - return "", false - } - if i > 0 { - buffer.WriteString(",") - } - // This may be an array or map, or just a string/int/etc. Regardless we - // stringify it. - buffer.WriteString(entry.Value.String()) - } - return buffer.String(), true -} +//func (mlrmap *Mlrmap) GetSelectedValuesJoined(selectedFieldNames []string) (string, bool) { +// if len(selectedFieldNames) == 0 { +// // The fall-through is functionally correct, but this is quicker with +// // skipping setting up an empty bytes-buffer and stringifying it. The +// // non-grouped case is quite normal and is worth optimizing for. +// return "", true +// } +// +// var buffer bytes.Buffer +// for i, selectedFieldName := range selectedFieldNames { +// entry := mlrmap.findEntry(selectedFieldName) +// if entry == nil { +// return "", false +// } +// if i > 0 { +// buffer.WriteString(",") +// } +// // This may be an array or map, or just a string/int/etc. Regardless we +// // stringify it. +// buffer.WriteString(entry.Value.String()) +// } +// return buffer.String(), true +//} -// As with GetSelectedValuesJoined but also returning the array of mlrvals. -// For sort. -// TODO: put 'Copy' into the method name -func (mlrmap *Mlrmap) GetSelectedValuesAndJoined(selectedFieldNames []string) ( - string, - []*Mlrval, - bool, -) { - mlrvals := make([]*Mlrval, 0, len(selectedFieldNames)) +//// As with GetSelectedValuesJoined but also returning the array of mlrvals. +//// For sort. +//// TODO: put 'Copy' into the method name +//func (mlrmap *Mlrmap) GetSelectedValuesAndJoined(selectedFieldNames []string) ( +// string, +// []*mlrval.Mlrval, +// bool, +//) { +// mlrvals := make([]*mlrval.Mlrval, 0, len(selectedFieldNames)) +// +// if len(selectedFieldNames) == 0 { +// // The fall-through is functionally correct, but this is quicker with +// // skipping setting up an empty bytes-buffer and stringifying it. The +// // non-grouped case is quite normal and is worth optimizing for. +// return "", mlrvals, true +// } +// +// var buffer bytes.Buffer +// for i, selectedFieldName := range selectedFieldNames { +// entry := mlrmap.findEntry(selectedFieldName) +// if entry == nil { +// return "", mlrvals, false +// } +// if i > 0 { +// buffer.WriteString(",") +// } +// // This may be an array or map, or just a string/int/etc. Regardless we +// // stringify it. +// buffer.WriteString(entry.Value.String()) +// mlrvals = append(mlrvals, entry.Value.Copy()) +// } +// return buffer.String(), mlrvals, true +//} - if len(selectedFieldNames) == 0 { - // The fall-through is functionally correct, but this is quicker with - // skipping setting up an empty bytes-buffer and stringifying it. The - // non-grouped case is quite normal and is worth optimizing for. - return "", mlrvals, true - } +//// As above but only returns the array. Also, these are references, NOT copies. +//// For step and join. +//func (mlrmap *Mlrmap) ReferenceSelectedValues(selectedFieldNames []string) ([]*mlrval.Mlrval, bool) { +// allFound := true +// mlrvals := make([]*mlrval.Mlrval, 0, len(selectedFieldNames)) +// +// for _, selectedFieldName := range selectedFieldNames { +// entry := mlrmap.findEntry(selectedFieldName) +// if entry != nil { +// mlrvals = append(mlrvals, entry.Value) +// } else { +// mlrvals = append(mlrvals, nil) +// allFound = false +// } +// } +// return mlrvals, allFound +//} - var buffer bytes.Buffer - for i, selectedFieldName := range selectedFieldNames { - entry := mlrmap.findEntry(selectedFieldName) - if entry == nil { - return "", mlrvals, false - } - if i > 0 { - buffer.WriteString(",") - } - // This may be an array or map, or just a string/int/etc. Regardless we - // stringify it. - buffer.WriteString(entry.Value.String()) - mlrvals = append(mlrvals, entry.Value.Copy()) - } - return buffer.String(), mlrvals, true -} +//// TODO: rename to CopySelectedValues +//// As previous but with copying. For stats1. +//func (mlrmap *Mlrmap) GetSelectedValues(selectedFieldNames []string) ([]*mlrval.Mlrval, bool) { +// allFound := true +// mlrvals := make([]*mlrval.Mlrval, 0, len(selectedFieldNames)) +// +// for _, selectedFieldName := range selectedFieldNames { +// entry := mlrmap.findEntry(selectedFieldName) +// if entry != nil { +// mlrvals = append(mlrvals, entry.Value.Copy()) +// } else { +// mlrvals = append(mlrvals, nil) +// allFound = false +// } +// } +// return mlrvals, allFound +//} -// As above but only returns the array. Also, these are references, NOT copies. -// For step and join. -func (mlrmap *Mlrmap) ReferenceSelectedValues(selectedFieldNames []string) ([]*Mlrval, bool) { - allFound := true - mlrvals := make([]*Mlrval, 0, len(selectedFieldNames)) +//// Similar to the above but only checks availability. For join. +//func (mlrmap *Mlrmap) HasSelectedKeys(selectedFieldNames []string) bool { +// for _, selectedFieldName := range selectedFieldNames { +// entry := mlrmap.findEntry(selectedFieldName) +// if entry == nil { +// return false +// } +// } +// return true +//} - for _, selectedFieldName := range selectedFieldNames { - entry := mlrmap.findEntry(selectedFieldName) - if entry != nil { - mlrvals = append(mlrvals, entry.Value) - } else { - mlrvals = append(mlrvals, nil) - allFound = false - } - } - return mlrvals, allFound -} +//// ---------------------------------------------------------------- +//// For mlr nest implode across records. +//func (mlrmap *Mlrmap) GetKeysJoinedExcept(px *MlrmapEntry) string { +// var buffer bytes.Buffer +// i := 0 +// for pe := mlrmap.Head; pe != nil; pe = pe.Next { +// if pe == px { +// continue +// } +// if i > 0 { +// buffer.WriteString(",") +// } +// i++ +// buffer.WriteString(pe.Key) +// } +// return buffer.String() +//} -// TODO: rename to CopySelectedValues -// As previous but with copying. For stats1. -func (mlrmap *Mlrmap) GetSelectedValues(selectedFieldNames []string) ([]*Mlrval, bool) { - allFound := true - mlrvals := make([]*Mlrval, 0, len(selectedFieldNames)) +//// For mlr nest implode across records. +//func (mlrmap *Mlrmap) GetValuesJoinedExcept(px *MlrmapEntry) string { +// var buffer bytes.Buffer +// i := 0 +// for pe := mlrmap.Head; pe != nil; pe = pe.Next { +// if pe == px { +// continue +// } +// if i > 0 { +// buffer.WriteString(",") +// } +// i++ +// // This may be an array or map, or just a string/int/etc. Regardless we +// // stringify it. +// buffer.WriteString(pe.Value.String()) +// } +// return buffer.String() +//} - for _, selectedFieldName := range selectedFieldNames { - entry := mlrmap.findEntry(selectedFieldName) - if entry != nil { - mlrvals = append(mlrvals, entry.Value.Copy()) - } else { - mlrvals = append(mlrvals, nil) - allFound = false - } - } - return mlrvals, allFound -} +//// ---------------------------------------------------------------- +//func (mlrmap *Mlrmap) Rename(oldKey string, newKey string) bool { +// entry := mlrmap.findEntry(oldKey) +// if entry == nil { +// // Rename field from 'a' to 'b' where there is no 'a': no-op +// return false +// } +// +// existing := mlrmap.findEntry(newKey) +// if existing == nil { +// // Rename field from 'a' to 'b' where there is no 'b': simple update +// entry.Key = newKey +// +// if mlrmap.keysToEntries != nil { +// delete(mlrmap.keysToEntries, oldKey) +// mlrmap.keysToEntries[newKey] = entry +// } +// } else { +// // Rename field from 'a' to 'b' where there are both 'a' and 'b': +// // remove old 'a' and put its value into the slot of 'b'. +// existing.Value = entry.Value +// if mlrmap.keysToEntries != nil { +// delete(mlrmap.keysToEntries, oldKey) +// } +// mlrmap.Unlink(entry) +// } +// +// return true +//} -// Similar to the above but only checks availability. For join. -func (mlrmap *Mlrmap) HasSelectedKeys(selectedFieldNames []string) bool { - for _, selectedFieldName := range selectedFieldNames { - entry := mlrmap.findEntry(selectedFieldName) - if entry == nil { - return false - } - } - return true -} +//// ---------------------------------------------------------------- +//func (mlrmap *Mlrmap) Label(newNames []string) { +// other := NewMlrmapAsRecord() +// +// i := 0 +// numNewNames := len(newNames) +// for { +// if i >= numNewNames { +// break +// } +// pe := mlrmap.pop() +// if pe == nil { +// break +// } +// // Old record will be GC'ed: just move pointers +// other.PutReference(newNames[i], pe.Value) +// i++ +// } +// +// for { +// pe := mlrmap.pop() +// if pe == nil { +// break +// } +// // Example: +// // * Input record has keys a,b,i,x,y +// // * Requested labeling is d,x,f +// // * The first three records a,b,i should be renamed to d,x,f +// // * The old x needs to disappear (for key-uniqueness) +// // * The y field is carried through +// if other.Has(pe.Key) { +// continue +// } +// other.PutReference(pe.Key, pe.Value) +// } +// +// *mlrmap = *other +//} -// ---------------------------------------------------------------- -// For mlr nest implode across records. -func (mlrmap *Mlrmap) GetKeysJoinedExcept(px *MlrmapEntry) string { - var buffer bytes.Buffer - i := 0 - for pe := mlrmap.Head; pe != nil; pe = pe.Next { - if pe == px { - continue - } - if i > 0 { - buffer.WriteString(",") - } - i++ - buffer.WriteString(pe.Key) - } - return buffer.String() -} +//// ---------------------------------------------------------------- +//func (mlrmap *Mlrmap) SortByKey() { +// keys := mlrmap.GetKeys() +// +// lib.SortStrings(keys) +// +// other := NewMlrmapAsRecord() +// +// for _, key := range keys { +// // Old record will be GC'ed: just move pointers +// other.PutReference(key, mlrmap.Get(key)) +// } +// +// *mlrmap = *other +//} -// For mlr nest implode across records. -func (mlrmap *Mlrmap) GetValuesJoinedExcept(px *MlrmapEntry) string { - var buffer bytes.Buffer - i := 0 - for pe := mlrmap.Head; pe != nil; pe = pe.Next { - if pe == px { - continue - } - if i > 0 { - buffer.WriteString(",") - } - i++ - // This may be an array or map, or just a string/int/etc. Regardless we - // stringify it. - buffer.WriteString(pe.Value.String()) - } - return buffer.String() -} +//// ---------------------------------------------------------------- +//func (mlrmap *Mlrmap) SortByKeyRecursively() { +// keys := mlrmap.GetKeys() +// +// lib.SortStrings(keys) +// +// other := NewMlrmapAsRecord() +// +// for _, key := range keys { +// // Old record will be GC'ed: just move pointers +// value := mlrmap.Get(key) +// if value.IsMap() { +// value.mapval.SortByKeyRecursively() +// } +// other.PutReference(key, value) +// } +// +// *mlrmap = *other +//} -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) Rename(oldKey string, newKey string) bool { - entry := mlrmap.findEntry(oldKey) - if entry == nil { - // Rename field from 'a' to 'b' where there is no 'a': no-op - return false - } - - existing := mlrmap.findEntry(newKey) - if existing == nil { - // Rename field from 'a' to 'b' where there is no 'b': simple update - entry.Key = newKey - - if mlrmap.keysToEntries != nil { - delete(mlrmap.keysToEntries, oldKey) - mlrmap.keysToEntries[newKey] = entry - } - } else { - // Rename field from 'a' to 'b' where there are both 'a' and 'b': - // remove old 'a' and put its value into the slot of 'b'. - existing.Value = entry.Value - if mlrmap.keysToEntries != nil { - delete(mlrmap.keysToEntries, oldKey) - } - mlrmap.Unlink(entry) - } - - return true -} - -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) Label(newNames []string) { - other := NewMlrmapAsRecord() - - i := 0 - numNewNames := len(newNames) - for { - if i >= numNewNames { - break - } - pe := mlrmap.pop() - if pe == nil { - break - } - // Old record will be GC'ed: just move pointers - other.PutReference(newNames[i], pe.Value) - i++ - } - - for { - pe := mlrmap.pop() - if pe == nil { - break - } - // Example: - // * Input record has keys a,b,i,x,y - // * Requested labeling is d,x,f - // * The first three records a,b,i should be renamed to d,x,f - // * The old x needs to disappear (for key-uniqueness) - // * The y field is carried through - if other.Has(pe.Key) { - continue - } - other.PutReference(pe.Key, pe.Value) - } - - *mlrmap = *other -} - -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) SortByKey() { - keys := mlrmap.GetKeys() - - lib.SortStrings(keys) - - other := NewMlrmapAsRecord() - - for _, key := range keys { - // Old record will be GC'ed: just move pointers - other.PutReference(key, mlrmap.Get(key)) - } - - *mlrmap = *other -} - -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) SortByKeyRecursively() { - keys := mlrmap.GetKeys() - - lib.SortStrings(keys) - - other := NewMlrmapAsRecord() - - for _, key := range keys { - // Old record will be GC'ed: just move pointers - value := mlrmap.Get(key) - if value.IsMap() { - value.mapval.SortByKeyRecursively() - } - other.PutReference(key, value) - } - - *mlrmap = *other -} - -// ---------------------------------------------------------------- -// Only checks to see if the first entry is a map. For emit/emitp. -func (mlrmap *Mlrmap) IsNested() bool { - if mlrmap.Head == nil { - return false - } else if mlrmap.Head.Value.GetMap() == nil { - return false - } else { - return true - } -} +//// ---------------------------------------------------------------- +//// Only checks to see if the first entry is a map. For emit/emitp. +//func (mlrmap *Mlrmap) IsNested() bool { +// if mlrmap.Head == nil { +// return false +// } else if mlrmap.Head.Value.GetMap() == nil { +// TODO: check IsArrayOrMap() +// return false +// } else { +// return true +// } +//} // ================================================================ // PRIVATE METHODS -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) findEntry(key string) *MlrmapEntry { - if mlrmap.keysToEntries != nil { - return mlrmap.keysToEntries[key] - } else { - for pe := mlrmap.Head; pe != nil; pe = pe.Next { - if pe.Key == key { - return pe - } - } - return nil - } -} +//// ---------------------------------------------------------------- +//func (mlrmap *Mlrmap) Unlink(pe *MlrmapEntry) { +// if pe == mlrmap.Head { +// if pe == mlrmap.Tail { +// mlrmap.Head = nil +// mlrmap.Tail = nil +// } else { +// mlrmap.Head = pe.Next +// pe.Next.Prev = nil +// } +// } else { +// pe.Prev.Next = pe.Next +// if pe == mlrmap.Tail { +// mlrmap.Tail = pe.Prev +// } else { +// pe.Next.Prev = pe.Prev +// } +// } +// if mlrmap.keysToEntries != nil { +// delete(mlrmap.keysToEntries, pe.Key) +// } +// mlrmap.FieldCount-- +//} -// ---------------------------------------------------------------- -// For '$[1]' etc. in the DSL. +//// ---------------------------------------------------------------- +//// Does not check for duplicate keys +//func (mlrmap *Mlrmap) linkAtHead(pe *MlrmapEntry) { +// if mlrmap.Head == nil { +// pe.Prev = nil +// pe.Next = nil +// mlrmap.Head = pe +// mlrmap.Tail = pe +// } else { +// pe.Prev = nil +// pe.Next = mlrmap.Head +// mlrmap.Head.Prev = pe +// mlrmap.Head = pe +// } +// if mlrmap.keysToEntries != nil { +// mlrmap.keysToEntries[pe.Key] = pe +// } +// mlrmap.FieldCount++ +//} + +//// Does not check for duplicate keys +//func (mlrmap *Mlrmap) linkAtTail(pe *MlrmapEntry) { +// if mlrmap.Head == nil { +// pe.Prev = nil +// pe.Next = nil +// mlrmap.Head = pe +// mlrmap.Tail = pe +// } else { +// pe.Prev = mlrmap.Tail +// pe.Next = nil +// mlrmap.Tail.Next = pe +// mlrmap.Tail = pe +// } +// if mlrmap.keysToEntries != nil { +// mlrmap.keysToEntries[pe.Key] = pe +// } +// mlrmap.FieldCount++ +//} + +//// ---------------------------------------------------------------- +//func (mlrmap *Mlrmap) pop() *MlrmapEntry { +// if mlrmap.Head == nil { +// return nil +// } else { +// pe := mlrmap.Head +// mlrmap.Unlink(pe) +// return pe +// } +//} // -// Notes: -// * This is a linear search. -// * Indices are 1-up not 0-up -// * Indices -n..-1 are aliases for 1..n. In particular, it will be faster to -// get the -1st field than the nth. -// * Returns 0 on invalid index: 0, or < -n, or > n where n is the number of -// fields. -func (mlrmap *Mlrmap) findEntryByPositionalIndex(position int) *MlrmapEntry { - if position > mlrmap.FieldCount || position < -mlrmap.FieldCount || position == 0 { - return nil - } - if position > 0 { - var i int = 1 - for pe := mlrmap.Head; pe != nil; pe = pe.Next { - if i == position { - return pe - } - i++ - } - lib.InternalCodingErrorIf(true) - } else { - var i int = -1 - for pe := mlrmap.Tail; pe != nil; pe = pe.Prev { - if i == position { - return pe - } - i-- - } - lib.InternalCodingErrorIf(true) - } - lib.InternalCodingErrorIf(true) - return nil -} +//// ---------------------------------------------------------------- -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) Unlink(pe *MlrmapEntry) { - if pe == mlrmap.Head { - if pe == mlrmap.Tail { - mlrmap.Head = nil - mlrmap.Tail = nil - } else { - mlrmap.Head = pe.Next - pe.Next.Prev = nil - } - } else { - pe.Prev.Next = pe.Next - if pe == mlrmap.Tail { - mlrmap.Tail = pe.Prev - } else { - pe.Next.Prev = pe.Prev - } - } - if mlrmap.keysToEntries != nil { - delete(mlrmap.keysToEntries, pe.Key) - } - mlrmap.FieldCount-- -} +//// ToPairsArray is used for sorting maps by key/value/etc, e.g. the sortmf DSL function. +//func (mlrmap *Mlrmap) ToPairsArray() []MlrmapPair { +// pairsArray := make([]MlrmapPair, mlrmap.FieldCount) +// i := 0 +// for pe := mlrmap.Head; pe != nil; pe = pe.Next { +// pairsArray[i].Key = pe.Key +// pairsArray[i].Value = pe.Value.Copy() +// i++ +// } +// +// return pairsArray +//} -// ---------------------------------------------------------------- -// Does not check for duplicate keys -func (mlrmap *Mlrmap) linkAtHead(pe *MlrmapEntry) { - if mlrmap.Head == nil { - pe.Prev = nil - pe.Next = nil - mlrmap.Head = pe - mlrmap.Tail = pe - } else { - pe.Prev = nil - pe.Next = mlrmap.Head - mlrmap.Head.Prev = pe - mlrmap.Head = pe - } - if mlrmap.keysToEntries != nil { - mlrmap.keysToEntries[pe.Key] = pe - } - mlrmap.FieldCount++ -} - -// Does not check for duplicate keys -func (mlrmap *Mlrmap) linkAtTail(pe *MlrmapEntry) { - if mlrmap.Head == nil { - pe.Prev = nil - pe.Next = nil - mlrmap.Head = pe - mlrmap.Tail = pe - } else { - pe.Prev = mlrmap.Tail - pe.Next = nil - mlrmap.Tail.Next = pe - mlrmap.Tail = pe - } - if mlrmap.keysToEntries != nil { - mlrmap.keysToEntries[pe.Key] = pe - } - mlrmap.FieldCount++ -} - -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) pop() *MlrmapEntry { - if mlrmap.Head == nil { - return nil - } else { - pe := mlrmap.Head - mlrmap.Unlink(pe) - return pe - } -} +//// MlrmapFromPairsArray is used for sorting maps by key/value/etc, e.g. the sortmf DSL function. +//func MlrmapFromPairsArray(pairsArray []MlrmapPair) *Mlrmap { +// mlrmap := NewMlrmap() +// for i := range pairsArray { +// mlrmap.PutCopy(pairsArray[i].Key, pairsArray[i].Value) +// } +// +// return mlrmap +//} // ---------------------------------------------------------------- -// ToPairsArray is used for sorting maps by key/value/etc, e.g. the sortmf DSL function. -func (mlrmap *Mlrmap) ToPairsArray() []MlrmapPair { - pairsArray := make([]MlrmapPair, mlrmap.FieldCount) - i := 0 - for pe := mlrmap.Head; pe != nil; pe = pe.Next { - pairsArray[i].Key = pe.Key - pairsArray[i].Value = pe.Value.Copy() - i++ - } - - return pairsArray -} - -// MlrmapFromPairsArray is used for sorting maps by key/value/etc, e.g. the sortmf DSL function. -func MlrmapFromPairsArray(pairsArray []MlrmapPair) *Mlrmap { - mlrmap := NewMlrmap() - for i := range pairsArray { - mlrmap.PutCopy(pairsArray[i].Key, pairsArray[i].Value) - } - - return mlrmap -} - -// ---------------------------------------------------------------- - -// GetFirstPair returns the first key-value pair as its own map. If the map is -// empty (i.e. there is no first pair) it returns nil. -func (mlrmap *Mlrmap) GetFirstPair() *Mlrmap { - if mlrmap.Head == nil { - return nil - } - pair := NewMlrmap() - pair.PutCopy(mlrmap.Head.Key, mlrmap.Head.Value) - return pair -} - -func (mlrmap *Mlrmap) IsSinglePair() bool { - return mlrmap.FieldCount == 1 -} +//// GetFirstPair returns the first key-value pair as its own map. If the map is +//// empty (i.e. there is no first pair) it returns nil. +//func (mlrmap *Mlrmap) GetFirstPair() *Mlrmap { +// if mlrmap.Head == nil { +// return nil +// } +// pair := NewMlrmap() +// pair.PutCopy(mlrmap.Head.Key, mlrmap.Head.Value) +// return pair +//} +// +//func (mlrmap *Mlrmap) IsSinglePair() bool { +// return mlrmap.FieldCount == 1 +//} diff --git a/internal/pkg/types/mlrval_accessors.go b/internal/pkg/types/mlrval_accessors.go index 09ec40a08..63cd6e3fd 100644 --- a/internal/pkg/types/mlrval_accessors.go +++ b/internal/pkg/types/mlrval_accessors.go @@ -22,203 +22,21 @@ func GetTypeName(mvtype MVType) string { } // ---------------------------------------------------------------- -func (mv *Mlrval) IsLegit() bool { - return mv.mvtype >= MT_VOID -} - -func (mv *Mlrval) IsErrorOrAbsent() bool { - return mv.mvtype == MT_ERROR || mv.mvtype == MT_ABSENT -} - -func (mv *Mlrval) IsError() bool { - return mv.mvtype == MT_ERROR -} - -func (mv *Mlrval) IsAbsent() bool { - return mv.mvtype == MT_ABSENT -} - -func (mv *Mlrval) IsNull() bool { - return mv.mvtype == MT_NULL -} - -func (mv *Mlrval) IsVoid() bool { - return mv.mvtype == MT_VOID -} - -func (mv *Mlrval) IsErrorOrVoid() bool { - return mv.mvtype == MT_ERROR || mv.mvtype == MT_VOID -} - -// * Error is non-empty -// * Absent is non-empty (shouldn't have been assigned in the first place; error -// should be surfaced) -// * Void is empty -// * Empty string is empty -// * Int/float/bool/array/map are all non-empty -func (mv *Mlrval) IsEmpty() bool { - if mv.mvtype == MT_VOID { - return true - } else if mv.mvtype == MT_STRING { - return mv.printrep == "" - } else { - return false - } -} - -func (mv *Mlrval) IsString() bool { - return mv.mvtype == MT_STRING -} - -func (mv *Mlrval) IsStringOrVoid() bool { - return mv.mvtype == MT_STRING || mv.mvtype == MT_VOID -} - -func (mv *Mlrval) IsInt() bool { - return mv.mvtype == MT_INT -} - -func (mv *Mlrval) IsFloat() bool { - return mv.mvtype == MT_FLOAT -} - -func (mv *Mlrval) IsNumeric() bool { - return mv.mvtype == MT_INT || mv.mvtype == MT_FLOAT -} - -func (mv *Mlrval) IsIntZero() bool { - return mv.mvtype == MT_INT && mv.intval == 0 -} - -func (mv *Mlrval) IsBool() bool { - return mv.mvtype == MT_BOOL -} - -func (mv *Mlrval) IsTrue() bool { - return mv.mvtype == MT_BOOL && mv.boolval == true -} -func (mv *Mlrval) IsFalse() bool { - return mv.mvtype == MT_BOOL && mv.boolval == false -} - -func (mv *Mlrval) IsArray() bool { - return mv.mvtype == MT_ARRAY -} -func (mv *Mlrval) IsMap() bool { - return mv.mvtype == MT_MAP -} -func (mv *Mlrval) IsArrayOrMap() bool { - return mv.mvtype == MT_ARRAY || mv.mvtype == MT_MAP -} - -func (mv *Mlrval) IsFunction() bool { - return mv.mvtype == MT_FUNC -} - -// ---------------------------------------------------------------- -func (mv *Mlrval) GetIntValue() (intValue int, isInt bool) { - if mv.mvtype == MT_INT { - return mv.intval, true - } else { - return -999, false - } -} - -func (mv *Mlrval) GetFloatValue() (floatValue float64, isFloat bool) { - if mv.mvtype == MT_FLOAT { - return mv.floatval, true - } else { - return -777.0, false - } -} - -func (mv *Mlrval) GetNumericToFloatValue() (floatValue float64, isFloat bool) { - if mv.mvtype == MT_FLOAT { - return mv.floatval, true - } else if mv.mvtype == MT_INT { - return float64(mv.intval), true - } else { - return -888.0, false - } -} - -func (mv *Mlrval) GetNumericToFloatValueOrDie() (floatValue float64) { - floatValue, ok := mv.GetNumericToFloatValue() - if !ok { - fmt.Fprintf( - os.Stderr, - "%s: couldn't parse \"%s\" as number.", - "mlr", mv.String(), - ) - os.Exit(1) - } - return floatValue -} - -func (mv *Mlrval) GetNumericNegativeorDie() bool { - floatValue, ok := mv.GetNumericToFloatValue() - lib.InternalCodingErrorIf(!ok) - return floatValue < 0.0 -} - -func (mv *Mlrval) AssertNumeric() { - _ = mv.GetNumericToFloatValueOrDie() -} - -func (mv *Mlrval) GetString() (stringValue string, isString bool) { - if mv.mvtype == MT_STRING || mv.mvtype == MT_VOID { - return mv.printrep, true - } else { - return "", false - } -} - -func (mv *Mlrval) GetBoolValue() (boolValue bool, isBool bool) { - if mv.mvtype == MT_BOOL { - return mv.boolval, true - } else { - return false, false - } -} - -func (mv *Mlrval) GetArray() []Mlrval { - if mv.mvtype == MT_ARRAY { - return mv.arrayval - } else { - return nil - } -} func (mv *Mlrval) GetArrayLength() (int, bool) { - if mv.mvtype == MT_ARRAY { + if mv.IsArray() { return len(mv.arrayval), true } else { return -999, false } } -func (mv *Mlrval) GetMap() *Mlrmap { - if mv.mvtype == MT_MAP { - return mv.mapval - } else { - return nil - } -} - -func (mv *Mlrval) GetFunction() interface{} { - if mv.mvtype == MT_FUNC { - return mv.funcval - } else { - return nil - } -} - // ---------------------------------------------------------------- func (mv *Mlrval) Copy() *Mlrval { other := *mv - if mv.mvtype == MT_MAP { + if mv.IsMap() { other.mapval = mv.mapval.Copy() - } else if mv.mvtype == MT_ARRAY { + } else if mv.IsArray() { other.arrayval = CopyMlrvalArray(mv.arrayval) } return &other @@ -246,7 +64,7 @@ func CopyMlrvalPointerArray(input []*Mlrval) []*Mlrval { func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { retval := NewMlrmap() - if mv.mvtype == MT_MAP { + if mv.IsMap() { // Without this, the for-loop below is zero-pass and fields with "{}" // values would disappear entirely in a JSON-to-CSV conversion. if mv.mapval.IsEmpty() { @@ -260,7 +78,7 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { if prefix != "" { nextPrefix = prefix + delimiter + nextPrefix } - if pe.Value.mvtype == MT_MAP || pe.Value.mvtype == MT_ARRAY { + if pe.Value.IsMap() || pe.Value.IsArray() { nextResult := pe.Value.FlattenToMap(nextPrefix, delimiter) lib.InternalCodingErrorIf(nextResult.mvtype != MT_MAP) for pf := nextResult.mapval.Head; pf != nil; pf = pf.Next { @@ -271,7 +89,7 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { } } - } else if mv.mvtype == MT_ARRAY { + } else if mv.IsArray() { // Without this, the for-loop below is zero-pass and fields with "[]" // values would disappear entirely in a JSON-to-CSV conversion. if len(mv.arrayval) == 0 { @@ -285,7 +103,7 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { if prefix != "" { nextPrefix = prefix + delimiter + nextPrefix } - if value.mvtype == MT_MAP || value.mvtype == MT_ARRAY { + if value.IsMap() || value.IsArray() { nextResult := value.FlattenToMap(nextPrefix, delimiter) lib.InternalCodingErrorIf(nextResult.mvtype != MT_MAP) for pf := nextResult.mapval.Head; pf != nil; pf = pf.Next { diff --git a/internal/pkg/types/mlrval_collections.go b/internal/pkg/types/mlrval_collections.go index 75a36741e..d12f5b017 100644 --- a/internal/pkg/types/mlrval_collections.go +++ b/internal/pkg/types/mlrval_collections.go @@ -79,10 +79,10 @@ import ( // ================================================================ // TODO: copy-reduction refactor func (mv *Mlrval) ArrayGet(mindex *Mlrval) Mlrval { - if mv.mvtype != MT_ARRAY { + if !mv.IsArray() { return *MLRVAL_ERROR } - if mindex.mvtype != MT_INT { + if !mindex.IsInt() { return *MLRVAL_ERROR } value := arrayGetAliased(&mv.arrayval, mindex.intval) @@ -96,7 +96,7 @@ func (mv *Mlrval) ArrayGet(mindex *Mlrval) Mlrval { // ---------------------------------------------------------------- // TODO: make this return error so caller can do 'if err == nil { ... }' func (mv *Mlrval) ArrayPut(mindex *Mlrval, value *Mlrval) { - if mv.mvtype != MT_ARRAY { + if !mv.IsArray() { fmt.Fprintf( os.Stderr, "mlr: expected array as indexed item in ArrayPut; got %s\n", @@ -104,7 +104,7 @@ func (mv *Mlrval) ArrayPut(mindex *Mlrval, value *Mlrval) { ) os.Exit(1) } - if mindex.mvtype != MT_INT { + if !mindex.IsInt() { // TODO: need to be careful about semantics here. // Silent no-ops are not good UX ... fmt.Fprintf( @@ -208,7 +208,7 @@ func UnaliasArrayLengthIndex(n int, mindex int) (int, bool) { // ---------------------------------------------------------------- // TODO: thinking about capacity-resizing func (mv *Mlrval) ArrayAppend(value *Mlrval) { - if mv.mvtype != MT_ARRAY { + if !mv.IsArray() { // TODO: need to be careful about semantics here. // Silent no-ops are not good UX ... return @@ -218,7 +218,7 @@ func (mv *Mlrval) ArrayAppend(value *Mlrval) { // ================================================================ func (mv *Mlrval) MapGet(key *Mlrval) Mlrval { - if mv.mvtype != MT_MAP { + if !mv.IsMap() { return *MLRVAL_ERROR } @@ -236,15 +236,15 @@ func (mv *Mlrval) MapGet(key *Mlrval) Mlrval { // ---------------------------------------------------------------- func (mv *Mlrval) MapPut(key *Mlrval, value *Mlrval) { - if mv.mvtype != MT_MAP { + if !mv.IsMap() { // TODO: need to be careful about semantics here. // Silent no-ops are not good UX ... return } - if key.mvtype == MT_STRING { + if key.IsString() { mv.mapval.PutCopy(key.printrep, value) - } else if key.mvtype == MT_INT { + } else if key.IsInt() { mv.mapval.PutCopy(key.String(), value) } // TODO: need to be careful about semantics here. @@ -290,18 +290,18 @@ func (mv *Mlrval) MapPut(key *Mlrval, value *Mlrval) { func (mv *Mlrval) PutIndexed(indices []*Mlrval, rvalue *Mlrval) error { lib.InternalCodingErrorIf(len(indices) < 1) - if mv.mvtype == MT_MAP { + if mv.IsMap() { return putIndexedOnMap(mv.mapval, indices, rvalue) - } else if mv.mvtype == MT_ARRAY { + } else if mv.IsArray() { return putIndexedOnArray(&mv.arrayval, indices, rvalue) } else { baseIndex := indices[0] - if baseIndex.mvtype == MT_STRING { + if baseIndex.IsString() { *mv = *MlrvalFromEmptyMap() return putIndexedOnMap(mv.mapval, indices, rvalue) - } else if baseIndex.mvtype == MT_INT { + } else if baseIndex.IsInt() { *mv = MlrvalEmptyArray() return putIndexedOnArray(&mv.arrayval, indices, rvalue) } else { @@ -340,7 +340,7 @@ func putIndexedOnMap(baseMap *Mlrmap, indices []*Mlrval, rvalue *Mlrval) error { } // If not last index, then recurse. - if baseIndex.mvtype != MT_STRING && baseIndex.mvtype != MT_INT { + if !baseIndex.IsString() && !baseIndex.IsInt() { // Base is map, index is invalid type return errors.New( "mlr: map indices must be string, int, or array thereof; got " + baseIndex.GetTypeName(), @@ -374,7 +374,7 @@ func putIndexedOnArray( lib.InternalCodingErrorIf(numIndices < 1) mindex := indices[0] - if mindex.mvtype != MT_INT { + if !mindex.IsInt() { return errors.New( "Array index must be int, but was " + mindex.GetTypeName() + @@ -406,11 +406,11 @@ func putIndexedOnArray( nextIndex := indices[1] // Overwrite what's in this slot if it's the wrong type - if nextIndex.mvtype == MT_STRING { + if nextIndex.IsString() { if (*baseArray)[zindex].mvtype != MT_MAP { (*baseArray)[zindex] = *MlrvalFromEmptyMap() } - } else if nextIndex.mvtype == MT_INT { + } else if nextIndex.IsInt() { if (*baseArray)[zindex].mvtype != MT_ARRAY { (*baseArray)[zindex] = MlrvalEmptyArray() } @@ -442,10 +442,10 @@ func putIndexedOnArray( func (mv *Mlrval) RemoveIndexed(indices []*Mlrval) error { lib.InternalCodingErrorIf(len(indices) < 1) - if mv.mvtype == MT_MAP { + if mv.IsMap() { return removeIndexedOnMap(mv.mapval, indices) - } else if mv.mvtype == MT_ARRAY { + } else if mv.IsArray() { return removeIndexedOnArray(&mv.arrayval, indices) } else { @@ -464,7 +464,7 @@ func removeIndexedOnMap(baseMap *Mlrmap, indices []*Mlrval) error { // If last index, then unset. if numIndices == 1 { - if baseIndex.mvtype == MT_STRING || baseIndex.mvtype == MT_INT { + if baseIndex.IsString() || baseIndex.IsInt() { baseMap.Remove(baseIndex.String()) return nil } else { @@ -476,7 +476,7 @@ func removeIndexedOnMap(baseMap *Mlrmap, indices []*Mlrval) error { } // If not last index, then recurse. - if baseIndex.mvtype == MT_STRING || baseIndex.mvtype == MT_INT { + if baseIndex.IsString() || baseIndex.IsInt() { // Base is map, index is string baseValue := baseMap.Get(baseIndex.String()) if baseValue != nil { @@ -503,7 +503,7 @@ func removeIndexedOnArray( lib.InternalCodingErrorIf(numIndices < 1) mindex := indices[0] - if mindex.mvtype != MT_INT { + if !mindex.IsInt() { return errors.New( "Array index must be int, but was " + mindex.GetTypeName() + diff --git a/internal/pkg/types/mlrval_new.go b/internal/pkg/types/mlrval_new.go deleted file mode 100644 index 0ae2dc7ae..000000000 --- a/internal/pkg/types/mlrval_new.go +++ /dev/null @@ -1,199 +0,0 @@ -// ================================================================ -// Constructors -// ================================================================ - -package types - -import ( - "errors" - - "github.com/johnkerl/miller/internal/pkg/lib" -) - -func MlrvalFromString(input string) *Mlrval { - if input == "" { - return MLRVAL_VOID - } - return &Mlrval{ - mvtype: MT_STRING, - printrep: input, - printrepValid: true, - } -} - -func MlrvalFromInt(input int) *Mlrval { - return &Mlrval{ - mvtype: MT_INT, - printrepValid: false, - intval: input, - } -} - -func MlrvalFromFloat64(input float64) *Mlrval { - return &Mlrval{ - mvtype: MT_FLOAT, - printrepValid: false, - floatval: input, - } -} - -func MlrvalFromBool(input bool) *Mlrval { - if input == true { - return MLRVAL_TRUE - } else { - return MLRVAL_FALSE - } -} - -func MlrvalFromBoolString(input string) *Mlrval { - if input == "true" { - return MLRVAL_TRUE - } else if input == "false" { - return MLRVAL_FALSE - } else { - lib.InternalCodingErrorIf(true) - return MLRVAL_ERROR // not reached - } -} - -// The user-defined function is of type 'interface{}' here to avoid what would -// otherwise be a package-dependency cycle between this package and -// github.com/johnkerl/miller/internal/pkg/dsl/cst. -// -// Nominally the name argument is the user-specified name if `func f(a, b) { -// ... }`, or some autogenerated UUID like `fl0052` if `func (a, b) { ... }`. -func MlrvalFromFunction(funcval interface{}, name string) *Mlrval { - return &Mlrval{ - mvtype: MT_FUNC, - printrep: name, - printrepValid: true, - funcval: funcval, - } -} - -func MlrvalFromEmptyMap() *Mlrval { - return &Mlrval{ - mvtype: MT_MAP, - printrep: "(bug-if-you-see-this-map-type)", - printrepValid: false, - mapval: NewMlrmap(), - } -} - -func MlrvalFromMap(mlrmap *Mlrmap) *Mlrval { - mv := MlrvalFromEmptyMap() - if mlrmap == nil { - // TODO maybe return 2nd-arg error in the API - return MLRVAL_ERROR - } - - for pe := mlrmap.Head; pe != nil; pe = pe.Next { - mv.mapval.PutCopy(pe.Key, pe.Value) - } - return mv -} - -// Like previous but doesn't copy. Only safe when the argument's sole purpose -// is to be passed into here. -func MlrvalFromMapReferenced(mlrmap *Mlrmap) *Mlrval { - mv := MlrvalFromEmptyMap() - if mlrmap == nil { - // xxx maybe return 2nd-arg error in the API - return MLRVAL_ERROR - } - - for pe := mlrmap.Head; pe != nil; pe = pe.Next { - mv.mapval.PutReference(pe.Key, pe.Value) - } - return mv -} - -// Does not copy the data. We can make a MlrvalFromArrayLiteralCopy if needed, -// using values.CopyMlrvalArray(). -func MlrvalEmptyArray() Mlrval { - return Mlrval{ - mvtype: MT_ARRAY, - printrep: "(bug-if-you-see-this-array-type)", - printrepValid: false, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: make([]Mlrval, 0, 10), - mapval: nil, - } -} - -// Users can do things like '$new[1][2][3] = 4' even if '$new' isn't already -// allocated. This function supports that. -func NewSizedMlrvalArray(length int) *Mlrval { - arrayval := make([]Mlrval, length, 2*length) - - for i := 0; i < int(length); i++ { - arrayval[i] = *MLRVAL_VOID - } - - return &Mlrval{ - mvtype: MT_ARRAY, - printrep: "(bug-if-you-see-this-array-type)", - printrepValid: false, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: arrayval, - mapval: nil, - } -} - -// Does not copy the data. We can make a SetFromArrayLiteralCopy if needed -// using values.CopyMlrvalArray(). -func MlrvalFromArrayReference(input []Mlrval) *Mlrval { - return &Mlrval{ - mvtype: MT_ARRAY, - printrepValid: false, - arrayval: input, - } -} - -func LengthenMlrvalArray(array *[]Mlrval, newLength64 int) { - newLength := int(newLength64) - lib.InternalCodingErrorIf(newLength <= len(*array)) - - if newLength <= cap(*array) { - newArray := (*array)[:newLength] - for zindex := len(*array); zindex < newLength; zindex++ { - // TODO: comment why not MT_ABSENT or MT_VOID - newArray[zindex] = *MLRVAL_NULL - } - *array = newArray - } else { - newArray := make([]Mlrval, newLength, 2*newLength) - zindex := 0 - for zindex = 0; zindex < len(*array); zindex++ { - newArray[zindex] = (*array)[zindex] - } - for zindex = len(*array); zindex < newLength; zindex++ { - // TODO: comment why not MT_ABSENT or MT_VOID - newArray[zindex] = *MLRVAL_NULL - } - *array = newArray - } -} - -// NewMlrvalForAutoDeepen is for auto-deepen of nested maps in things like -// -// $foo[1]["a"][2]["b"] = 3 -// -// Autocreated levels are maps. Array levels can be explicitly created e.g. -// -// $foo[1]["a"] ??= [] -// $foo[1]["a"][2]["b"] = 3 -func NewMlrvalForAutoDeepen(mvtype MVType) (*Mlrval, error) { - if mvtype == MT_STRING || mvtype == MT_INT { - empty := MlrvalFromEmptyMap() - return empty, nil - } else { - return nil, errors.New( - "mlr: indices must be string, int, or array thereof; got " + GetTypeName(mvtype), - ) - } -} diff --git a/internal/pkg/types/mlrval_output.go b/internal/pkg/types/mlrval_output.go index 022196758..7583d432c 100644 --- a/internal/pkg/types/mlrval_output.go +++ b/internal/pkg/types/mlrval_output.go @@ -90,7 +90,7 @@ func (mv *Mlrval) setPrintRep() { // Must have non-pointer receiver in order to implement the fmt.Stringer // interface to make this printable via fmt.Println et al. func (mv Mlrval) String() string { - if mv.mvtype == MT_FLOAT && mlrvalFloatOutputFormatter != nil { + if mv.IsFloat() && mlrvalFloatOutputFormatter != nil { // Use the format string from global --ofmt, if supplied return mlrvalFloatOutputFormatter.FormatFloat(mv.floatval) } else { diff --git a/todo.txt b/todo.txt index 7f4449726..6dff14a27 100644 --- a/todo.txt +++ b/todo.txt @@ -95,6 +95,55 @@ PUNCHDOWN LIST > try unconditional batching and hiding *first* to see how much best-case perf to be had ? lazy type-infer?? needs careful use of accessor-mutators in place of mv.type etc +---------------------------------------------------------------- +* JIT refactor + o x 'tens/dozens of UT cases' + ! re-check hashed/unhashed and do so periodically + > env-var back door + m internal/pkg/mlrval + - UT-driven mlrval defer/infer + - pkg/mlrval doc.go & README.md + o internal/pkg/types + - UT-driven mlrmap constructor refactor + - UT-driven JSON-parser refactor + o internal/pkg/input + - basic + mlrval.FromDeferredType + mlrmap.PutReference(key, value) + + - UT-driven DKVP-from-line + - UT-driven X-from-line: nidx, xtab, csvlite, csv + - UT-driven JSON-from-line + - defer channelizers to a separate PR + o internal/pkg/output + - basic + mlrval.String() + - UT-driven to-line + o internal/pkg/bifs + - more UT ... final authority is RT though + + ? stick w/ regtest ? + - internal/pkg/colorizer + - internal/pkg/platform + - internal/pkg/runtime + + - internal/pkg/stream + - internal/pkg/cli + - internal/pkg/climain + + - internal/pkg/transformers/utils + - internal/pkg/transformers + + - internal/pkg/parsing + - internal/pkg/dsl + - internal/pkg/dsl/cst + + - internal/pkg/auxents/repl + - internal/pkg/auxents/regtest + - internal/pkg/auxents + - internal/pkg/auxents/help +---------------------------------------------------------------- + * blockers: - keep checking issues - verslink old relnotes From 1f191cb7cba0434d1dfccbe5a73bf306e7196999 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Tue, 30 Nov 2021 23:17:00 -0500 Subject: [PATCH 02/19] mlrmap refactor --- .vimrc | 3 +- Makefile | 6 + internal/pkg/auxents/help/entry.go | 7 +- internal/pkg/auxents/repl/dsl.go | 3 +- internal/pkg/auxents/repl/session.go | 3 +- internal/pkg/cli/option_parse.go | 8 +- internal/pkg/dsl/cst/builtin_functions.go | 51 ++-- internal/pkg/dsl/cst/collections.go | 41 +-- internal/pkg/dsl/cst/emit_emitp.go | 43 ++-- internal/pkg/dsl/cst/env.go | 5 +- internal/pkg/dsl/cst/evaluable.go | 4 +- internal/pkg/dsl/cst/for.go | 19 +- internal/pkg/dsl/cst/hofs.go | 239 +++++++++--------- internal/pkg/dsl/cst/leaves.go | 94 +++---- internal/pkg/dsl/cst/lvalues.go | 127 +++++----- internal/pkg/dsl/cst/types.go | 11 +- internal/pkg/dsl/cst/udf.go | 17 +- internal/pkg/dsl/cst/uds.go | 3 +- internal/pkg/input/pseudo_reader_gen.go | 5 +- internal/pkg/input/record_reader_csv.go | 8 +- internal/pkg/input/record_reader_csvlite.go | 16 +- internal/pkg/input/record_reader_dkvp.go | 149 +++++++++++ internal/pkg/input/record_reader_dkvp_test.go | 59 +++++ internal/pkg/input/record_reader_json.go | 3 +- internal/pkg/input/record_reader_nidx.go | 138 ++++++++++ internal/pkg/input/record_reader_xtab.go | 4 +- internal/pkg/mlrval/constants.go | 20 +- internal/pkg/mlrval/is.go | 1 - internal/pkg/mlrval/is_test.go | 54 ++-- internal/pkg/mlrval/new.go | 12 +- internal/pkg/mlrval/tbf.go | 11 +- internal/pkg/mlrval/type.go | 1 - internal/pkg/runtime/stack.go | 43 ++-- internal/pkg/runtime/state.go | 3 + internal/pkg/transformers/bar.go | 5 +- internal/pkg/transformers/cat.go | 5 +- internal/pkg/transformers/clean-whitespace.go | 5 +- internal/pkg/transformers/count-similar.go | 3 +- internal/pkg/transformers/count.go | 9 +- internal/pkg/transformers/fill-down.go | 5 +- internal/pkg/transformers/fill-empty.go | 7 +- internal/pkg/transformers/fraction.go | 37 +-- internal/pkg/transformers/histogram.go | 13 +- .../transformers/most-or-least-frequent.go | 7 +- internal/pkg/transformers/nest.go | 17 +- internal/pkg/transformers/put-or-filter.go | 3 +- internal/pkg/transformers/reshape.go | 5 +- internal/pkg/transformers/sec2gmt.go | 3 +- internal/pkg/transformers/seqgen.go | 15 +- internal/pkg/transformers/sort.go | 5 +- internal/pkg/transformers/stats1.go | 15 +- internal/pkg/transformers/stats2.go | 3 +- internal/pkg/transformers/step.go | 63 ++--- internal/pkg/transformers/template.go | 5 +- internal/pkg/transformers/top.go | 9 +- internal/pkg/transformers/uniq.go | 17 +- internal/pkg/transformers/unsparsify.go | 5 +- .../transformers/utils/join-bucket-keeper.go | 11 +- .../pkg/transformers/utils/join-bucket.go | 5 +- .../transformers/utils/percentile-keeper.go | 23 +- .../transformers/utils/stats1-accumulators.go | 183 +++++++------- .../transformers/utils/stats2-accumulators.go | 51 ++-- internal/pkg/transformers/utils/top-keeper.go | 7 +- .../mlrmap_flatten_unflatten.go | 0 .../{types => types-parked}/mlrmap_json.go | 0 .../{types => types-parked}/mlrmap_print.go | 0 .../mlrval_accessors.go | 0 .../mlrval_collections.go | 0 .../{types => types-parked}/mlrval_format.go | 0 .../{types => types-parked}/mlrval_json.go | 0 .../{types => types-parked}/mlrval_output.go | 0 .../{types => types-parked}/mlrval_typing.go | 0 internal/pkg/types/context.go | 29 ++- internal/pkg/types/mlrmap.go | 2 +- internal/pkg/types/mlrmap_accessors.go | 4 +- internal/pkg/types/mlrmap_accessors_test.go | 51 ++++ internal/pkg/types/mlrmap_new_test.go | 17 ++ todo.txt | 24 +- 78 files changed, 1180 insertions(+), 699 deletions(-) create mode 100644 internal/pkg/input/record_reader_dkvp.go create mode 100644 internal/pkg/input/record_reader_dkvp_test.go create mode 100644 internal/pkg/input/record_reader_nidx.go rename internal/pkg/{types => types-parked}/mlrmap_flatten_unflatten.go (100%) rename internal/pkg/{types => types-parked}/mlrmap_json.go (100%) rename internal/pkg/{types => types-parked}/mlrmap_print.go (100%) rename internal/pkg/{types => types-parked}/mlrval_accessors.go (100%) rename internal/pkg/{types => types-parked}/mlrval_collections.go (100%) rename internal/pkg/{types => types-parked}/mlrval_format.go (100%) rename internal/pkg/{types => types-parked}/mlrval_json.go (100%) rename internal/pkg/{types => types-parked}/mlrval_output.go (100%) rename internal/pkg/{types => types-parked}/mlrval_typing.go (100%) create mode 100644 internal/pkg/types/mlrmap_accessors_test.go create mode 100644 internal/pkg/types/mlrmap_new_test.go diff --git a/.vimrc b/.vimrc index a2083dc8e..9d261becd 100644 --- a/.vimrc +++ b/.vimrc @@ -1,3 +1,4 @@ map \d :w:!clear;echo Building ...; echo; make mlr "map \f :w:!clear;echo Building ...; echo; make mlrval-tests -map \f :w:!clear;echo Building ...; echo; make mlrmap-tests +"map \f :w:!clear;echo Building ...; echo; make mlrmap-tests +map \f :w:!clear;echo Building ...; echo; make input-tests diff --git a/Makefile b/Makefile index 9873d3aed..52b1e8488 100644 --- a/Makefile +++ b/Makefile @@ -69,6 +69,12 @@ mlrmap-accessors-test: internal/pkg/types/mlrmap_accessors.go mlrmap-tests: mlrmap-new-test mlrmap-accessors-test +input-dkvp-test: + go test internal/pkg/input/record_reader_dkvp_test.go \ + internal/pkg/input/record_reader.go \ + internal/pkg/input/record_reader_dkvp.go +input-tests: input-dkvp-test + #mlrval_functions_test: # go test internal/pkg/types/mlrval_functions_test.go $(ls internal/pkg/types/*.go | grep -v test) #mlrval_format_test: diff --git a/internal/pkg/auxents/help/entry.go b/internal/pkg/auxents/help/entry.go index 7e8c14597..72dc42db0 100644 --- a/internal/pkg/auxents/help/entry.go +++ b/internal/pkg/auxents/help/entry.go @@ -13,6 +13,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/dsl/cst" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/transformers" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -447,9 +448,9 @@ func helpOutputColorization() { // ---------------------------------------------------------------- func helpTypeArithmeticInfo() { - mlrvals := []*types.Mlrval{ - types.MlrvalFromInt(1), - types.MlrvalFromFloat64(2.5), + mlrvals := []*mlrval.Mlrval{ + mlrval.MlrvalFromInt(1), + mlrval.MlrvalFromFloat64(2.5), types.MLRVAL_ABSENT, types.MLRVAL_ERROR, } diff --git a/internal/pkg/auxents/repl/dsl.go b/internal/pkg/auxents/repl/dsl.go index ae98538d8..222ce1553 100644 --- a/internal/pkg/auxents/repl/dsl.go +++ b/internal/pkg/auxents/repl/dsl.go @@ -25,6 +25,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/dsl/cst" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -83,7 +84,7 @@ func (repl *Repl) handleDSLStringAux( // The filter expression for the main Miller DSL is any non-assignment // statment like 'true' or '$x > 0.5' etc. For the REPL, we re-use this for // interactive expressions to be printed to the terminal. For the main DSL, - // the default is types.MlrvalFromTrue(); for the REPL, the default is + // the default is mlrval.MlrvalFromTrue(); for the REPL, the default is // types.MLRVAL_VOID. filterExpression := repl.runtimeState.FilterExpression if filterExpression.IsVoid() { diff --git a/internal/pkg/auxents/repl/session.go b/internal/pkg/auxents/repl/session.go index 04dfd3015..ee5043745 100644 --- a/internal/pkg/auxents/repl/session.go +++ b/internal/pkg/auxents/repl/session.go @@ -29,6 +29,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl/cst" "github.com/johnkerl/miller/internal/pkg/input" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/output" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" @@ -67,7 +68,7 @@ func NewRepl( // The filter expression for the main Miller DSL is any non-assignment // statment like 'true' or '$x > 0.5' etc. For the REPL, we re-use this for // interactive expressions to be printed to the terminal. For the main DSL, - // the default is types.MlrvalFromTrue(); for the REPL, the default is + // the default is mlrval.MlrvalFromTrue(); for the REPL, the default is // types.MLRVAL_VOID. runtimeState.FilterExpression = types.MLRVAL_VOID diff --git a/internal/pkg/cli/option_parse.go b/internal/pkg/cli/option_parse.go index 839e0282a..978b9c7fb 100644 --- a/internal/pkg/cli/option_parse.go +++ b/internal/pkg/cli/option_parse.go @@ -15,7 +15,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/colorizer" "github.com/johnkerl/miller/internal/pkg/lib" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // FinalizeReaderOptions does a few things. @@ -2603,7 +2603,7 @@ data having many fields, and many of them being processed during a given process altNames: []string{"-S"}, help: `Don't treat values like 123 or 456.7 in data files as int/float; leave them as strings.`, parser: func(args []string, argc int, pargi *int, options *TOptions) { - types.SetInferrerStringOnly() + mlrval.SetInferrerStringOnly() *pargi += 1 }, }, @@ -2613,7 +2613,7 @@ data having many fields, and many of them being processed during a given process altNames: []string{"-A"}, help: `Cast all integers in data files to floats.`, parser: func(args []string, argc int, pargi *int, options *TOptions) { - types.SetInferrerIntAsFloat() + mlrval.SetInferrerIntAsFloat() *pargi += 1 }, }, @@ -2623,7 +2623,7 @@ data having many fields, and many of them being processed during a given process altNames: []string{"-O"}, help: `Treat numbers like 0123 in data files as string "0123", not octal for decimal 83 etc.`, parser: func(args []string, argc int, pargi *int, options *TOptions) { - types.SetInferrerNoOctal() + mlrval.SetInferrerNoOctal() *pargi += 1 }, }, diff --git a/internal/pkg/dsl/cst/builtin_functions.go b/internal/pkg/dsl/cst/builtin_functions.go index 0a36f70ed..760303019 100644 --- a/internal/pkg/dsl/cst/builtin_functions.go +++ b/internal/pkg/dsl/cst/builtin_functions.go @@ -10,6 +10,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -116,7 +117,7 @@ func (root *RootNode) BuildZaryFunctionCallsiteNode( func (node *ZaryFunctionCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.zaryFunc() } @@ -157,7 +158,7 @@ func (root *RootNode) BuildUnaryFunctionCallsiteNode( func (node *UnaryFunctionCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.unaryFunc(node.evaluable1.Evaluate(state)) } @@ -198,7 +199,7 @@ func (root *RootNode) BuildUnaryFunctionWithContextCallsiteNode( func (node *UnaryFunctionWithContextCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.unaryFuncWithContext(node.evaluable1.Evaluate(state), state.Context) } @@ -271,7 +272,7 @@ func (root *RootNode) BuildBinaryFunctionCallsiteNode( func (node *BinaryFunctionCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.binaryFunc( node.evaluable1.Evaluate(state), node.evaluable2.Evaluate(state), @@ -321,7 +322,7 @@ func (root *RootNode) BuildBinaryFunctionWithStateCallsiteNode( func (node *BinaryFunctionWithStateCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.binaryFuncWithState( node.evaluable1.Evaluate(state), node.evaluable2.Evaluate(state), @@ -378,7 +379,7 @@ func (root *RootNode) BuildTernaryFunctionWithStateCallsiteNode( func (node *TernaryFunctionWithStateCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.ternaryFuncWithState( node.evaluable1.Evaluate(state), node.evaluable2.Evaluate(state), @@ -391,9 +392,9 @@ func (node *TernaryFunctionWithStateCallsiteNode) Evaluate( // RegexCaptureBinaryFunctionCallsiteNode special-cases the =~ and !=~ // operators which set the CST State object's captures array for "\1".."\9". // This is identical to BinaryFunctionCallsite except that -// BinaryFunctionCallsite's impl function takes two *types.Mlrval arguments and -// returns a *types.Mlrval, whereas RegexCaptureBinaryFunctionCallsiteNode's -// impl function takes two *types.Mlrval arguments but returns *types.Mlrval +// BinaryFunctionCallsite's impl function takes two *mlrval.Mlrval arguments and +// returns a *mlrval.Mlrval, whereas RegexCaptureBinaryFunctionCallsiteNode's +// impl function takes two *mlrval.Mlrval arguments but returns *mlrval.Mlrval // along with a []string captures array. The captures are stored in the State // object for use in subsequent statements. // @@ -461,7 +462,7 @@ func (root *RootNode) BuildRegexCaptureBinaryFunctionCallsiteNode( func (node *RegexCaptureBinaryFunctionCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { output, captures := node.regexCaptureBinaryFunc( node.evaluable1.Evaluate(state), node.evaluable2.Evaluate(state), @@ -515,7 +516,7 @@ func (root *RootNode) BuildDotCallsiteNode( func (node *DotCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { value1 := node.evaluable1.Evaluate(state) mapvalue1 := value1.GetMap() @@ -596,7 +597,7 @@ func (root *RootNode) BuildTernaryFunctionCallsiteNode( func (node *TernaryFunctionCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.ternaryFunc( node.evaluable1.Evaluate(state), node.evaluable2.Evaluate(state), @@ -658,8 +659,8 @@ func (root *RootNode) BuildVariadicFunctionCallsiteNode( func (node *VariadicFunctionCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { - args := make([]*types.Mlrval, len(node.evaluables)) +) *mlrval.Mlrval { + args := make([]*mlrval.Mlrval, len(node.evaluables)) for i := range node.evaluables { args[i] = node.evaluables[i].Evaluate(state) } @@ -720,8 +721,8 @@ func (root *RootNode) BuildVariadicFunctionWithStateCallsiteNode( func (node *VariadicFunctionWithStateCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { - args := make([]*types.Mlrval, len(node.evaluables)) +) *mlrval.Mlrval { + args := make([]*mlrval.Mlrval, len(node.evaluables)) for i := range node.evaluables { args[i] = node.evaluables[i].Evaluate(state) } @@ -778,7 +779,7 @@ func (root *RootNode) BuildLogicalANDOperatorNode(a, b IEvaluable) *LogicalANDOp func (node *LogicalANDOperatorNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { aout := node.a.Evaluate(state) atype := aout.GetType() if !(atype == types.MT_ABSENT || atype == types.MT_BOOL) { @@ -804,7 +805,7 @@ func (node *LogicalANDOperatorNode) Evaluate( return types.MLRVAL_ABSENT } - return types.MlrvalLogicalAND(aout, bout) + return mlrval.MlrvalLogicalAND(aout, bout) } // ================================================================ @@ -826,7 +827,7 @@ func (root *RootNode) BuildLogicalOROperatorNode(a, b IEvaluable) *LogicalOROper // See the disposition-matrix discussion for LogicalANDOperator. func (node *LogicalOROperatorNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { aout := node.a.Evaluate(state) atype := aout.GetType() if !(atype == types.MT_ABSENT || atype == types.MT_BOOL) { @@ -851,7 +852,7 @@ func (node *LogicalOROperatorNode) Evaluate( if btype == types.MT_ABSENT { return types.MLRVAL_ABSENT } - return types.MlrvalLogicalOR(aout, bout) + return mlrval.MlrvalLogicalOR(aout, bout) } // ================================================================ @@ -868,7 +869,7 @@ func (root *RootNode) BuildAbsentCoalesceOperatorNode(a, b IEvaluable) *AbsentCo // argument is not absent. func (node *AbsentCoalesceOperatorNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { aout := node.a.Evaluate(state) if aout.GetType() != types.MT_ABSENT { return aout @@ -891,7 +892,7 @@ func (root *RootNode) BuildEmptyCoalesceOperatorNode(a, b IEvaluable) *EmptyCoal // argument is not absent. func (node *EmptyCoalesceOperatorNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { aout := node.a.Evaluate(state) atype := aout.GetType() if atype == types.MT_ABSENT || atype == types.MT_VOID || (atype == types.MT_STRING && aout.String() == "") { @@ -909,7 +910,7 @@ func (root *RootNode) BuildStandardTernaryOperatorNode(a, b, c IEvaluable) *Stan } func (node *StandardTernaryOperatorNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { aout := node.a.Evaluate(state) boolValue, isBool := aout.GetBoolValue() @@ -936,12 +937,12 @@ func (node *StandardTernaryOperatorNode) Evaluate( // for the function-manager lookup table to indicate the arity of the function, // even though at runtime these functions should not get invoked. -func BinaryShortCircuitPlaceholder(input1, input2 *types.Mlrval) *types.Mlrval { +func BinaryShortCircuitPlaceholder(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { lib.InternalCodingErrorPanic("Short-circuting was not correctly implemented") return types.MLRVAL_ERROR // not reached } -func TernaryShortCircuitPlaceholder(input1, input2, input3 *types.Mlrval) *types.Mlrval { +func TernaryShortCircuitPlaceholder(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { lib.InternalCodingErrorPanic("Short-circuting was not correctly implemented") return types.MLRVAL_ERROR // not reached } diff --git a/internal/pkg/dsl/cst/collections.go b/internal/pkg/dsl/cst/collections.go index 6dd0c32a1..12cf11b15 100644 --- a/internal/pkg/dsl/cst/collections.go +++ b/internal/pkg/dsl/cst/collections.go @@ -8,6 +8,7 @@ package cst import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -42,12 +43,12 @@ func (node *RootNode) BuildArrayLiteralNode( func (node *ArrayLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { - mlrvals := make([]types.Mlrval, len(node.evaluables)) +) *mlrval.Mlrval { + mlrvals := make([]mlrval.Mlrval, len(node.evaluables)) for i := range node.evaluables { mlrvals[i] = *node.evaluables[i].Evaluate(state) } - return types.MlrvalFromArrayReference(mlrvals) + return mlrval.MlrvalFromArrayReference(mlrvals) } // ---------------------------------------------------------------- @@ -82,7 +83,7 @@ func (node *RootNode) BuildArrayOrMapIndexAccessNode( func (node *CollectionIndexAccessNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { baseMlrval := node.baseEvaluable.Evaluate(state) indexMlrval := node.indexEvaluable.Evaluate(state) @@ -105,7 +106,7 @@ func (node *CollectionIndexAccessNode) Evaluate( if !inBounds { return types.MLRVAL_ERROR } - return types.MlrvalFromString(string(runes[zindex])) + return mlrval.MlrvalFromString(string(runes[zindex])) } else if baseMlrval.IsAbsent() { return types.MLRVAL_ABSENT @@ -155,7 +156,7 @@ func (node *RootNode) BuildArraySliceAccessNode( func (node *ArraySliceAccessNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { baseMlrval := node.baseEvaluable.Evaluate(state) lowerIndexMlrval := node.lowerIndexEvaluable.Evaluate(state) upperIndexMlrval := node.upperIndexEvaluable.Evaluate(state) @@ -203,7 +204,7 @@ func (node *ArraySliceAccessNode) Evaluate( upperZindex, _ := types.UnaliasArrayIndex(&array, upperIndex) if lowerZindex > upperZindex { - return types.MlrvalFromArrayReference(make([]types.Mlrval, 0)) + return mlrval.MlrvalFromArrayReference(make([]mlrval.Mlrval, 0)) } // Semantics: say x=[1,2,3,4,5]. Then x[3:10] is [3,4,5]. @@ -220,13 +221,13 @@ func (node *ArraySliceAccessNode) Evaluate( if lowerZindex < 0 { lowerZindex = 0 if lowerZindex > upperZindex { - return types.MlrvalFromArrayReference(make([]types.Mlrval, 0)) + return mlrval.MlrvalFromArrayReference(make([]mlrval.Mlrval, 0)) } } if upperZindex > n-1 { upperZindex = n - 1 if lowerZindex > upperZindex { - return types.MlrvalFromArrayReference(make([]types.Mlrval, 0)) + return mlrval.MlrvalFromArrayReference(make([]mlrval.Mlrval, 0)) } } @@ -234,7 +235,7 @@ func (node *ArraySliceAccessNode) Evaluate( // Miller slices have inclusive lower bound, inclusive upper bound. var m = upperZindex - lowerZindex + 1 - retval := make([]types.Mlrval, m) + retval := make([]mlrval.Mlrval, m) di := 0 for si := lowerZindex; si <= upperZindex; si++ { @@ -242,7 +243,7 @@ func (node *ArraySliceAccessNode) Evaluate( di++ } - return types.MlrvalFromArrayReference(retval) + return mlrval.MlrvalFromArrayReference(retval) } // ================================================================ @@ -273,7 +274,7 @@ func (node *RootNode) BuildPositionalFieldNameNode( // TODO: code-dedupe these next four Evaluate methods func (node *PositionalFieldNameNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { indexMlrval := node.indexEvaluable.Evaluate(state) if indexMlrval.IsAbsent() { return types.MLRVAL_ABSENT @@ -289,7 +290,7 @@ func (node *PositionalFieldNameNode) Evaluate( return types.MLRVAL_ABSENT } - return types.MlrvalFromString(name) + return mlrval.MlrvalFromString(name) } // ================================================================ @@ -319,7 +320,7 @@ func (node *RootNode) BuildPositionalFieldValueNode( func (node *PositionalFieldValueNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { indexMlrval := node.indexEvaluable.Evaluate(state) if indexMlrval.IsAbsent() { return types.MLRVAL_ABSENT @@ -372,7 +373,7 @@ func (node *RootNode) BuildArrayOrMapPositionalNameAccessNode( func (node *ArrayOrMapPositionalNameAccessNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { baseMlrval := node.baseEvaluable.Evaluate(state) indexMlrval := node.indexEvaluable.Evaluate(state) @@ -389,7 +390,7 @@ func (node *ArrayOrMapPositionalNameAccessNode) Evaluate( n, _ := baseMlrval.GetArrayLength() zindex, ok := types.UnaliasArrayLengthIndex(int(n), index) if ok { - return types.MlrvalFromInt(zindex + 1) // Miller user-space indices are 1-up + return mlrval.MlrvalFromInt(zindex + 1) // Miller user-space indices are 1-up } else { return types.MLRVAL_ABSENT } @@ -399,7 +400,7 @@ func (node *ArrayOrMapPositionalNameAccessNode) Evaluate( if !ok { return types.MLRVAL_ABSENT } else { - return types.MlrvalFromString(name) + return mlrval.MlrvalFromString(name) } } else if baseMlrval.IsAbsent() { @@ -444,7 +445,7 @@ func (node *RootNode) BuildArrayOrMapPositionalValueAccessNode( func (node *ArrayOrMapPositionalValueAccessNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { baseMlrval := node.baseEvaluable.Evaluate(state) indexMlrval := node.indexEvaluable.Evaluate(state) @@ -536,8 +537,8 @@ func (node *RootNode) BuildMapLiteralNode( func (node *MapLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { - output := types.MlrvalFromEmptyMap() +) *mlrval.Mlrval { + output := mlrval.MlrvalFromEmptyMap() for i := range node.evaluablePairs { key := node.evaluablePairs[i].Key.Evaluate(state) diff --git a/internal/pkg/dsl/cst/emit_emitp.go b/internal/pkg/dsl/cst/emit_emitp.go index dd92ad2d5..e3b8ec743 100644 --- a/internal/pkg/dsl/cst/emit_emitp.go +++ b/internal/pkg/dsl/cst/emit_emitp.go @@ -45,6 +45,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/output" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" @@ -60,7 +61,7 @@ type tEmitToRedirectFunc func( type tEmitExecutorFunc func( names []string, - values []*types.Mlrval, + values []*mlrval.Mlrval, state *runtime.State, ) error @@ -299,7 +300,7 @@ func (node *EmitXStatementNode) Execute(state *runtime.State) (*BlockExitPayload if node.topLevelEvaluableMap == nil { // 'emit @a', 'emit (@a, @b)', etc. names := node.topLevelNameList - values := make([]*types.Mlrval, len(names)) + values := make([]*mlrval.Mlrval, len(names)) for i, evaluable := range node.topLevelEvaluableList { values[i] = evaluable.Evaluate(state) } @@ -317,7 +318,7 @@ func (node *EmitXStatementNode) Execute(state *runtime.State) (*BlockExitPayload return nil, nil } names := make([]string, parentMapValue.FieldCount) - values := make([]*types.Mlrval, parentMapValue.FieldCount) + values := make([]*mlrval.Mlrval, parentMapValue.FieldCount) i := 0 for pe := parentMapValue.Head; pe != nil; pe = pe.Next { @@ -362,7 +363,7 @@ func (node *EmitXStatementNode) Execute(state *runtime.State) (*BlockExitPayload func (node *EmitXStatementNode) executeNonIndexedNonLashedEmit( names []string, - values []*types.Mlrval, + values []*mlrval.Mlrval, state *runtime.State, ) error { for i, value := range values { @@ -394,7 +395,7 @@ func (node *EmitXStatementNode) executeNonIndexedNonLashedEmit( } else { // recurse nextLevelNames := make([]string, 0) - nextLevelValues := make([]*types.Mlrval, 0) + nextLevelValues := make([]*mlrval.Mlrval, 0) for pe := value.GetMap().Head; pe != nil; pe = pe.Next { nextLevelNames = append(nextLevelNames, pe.Key) nextLevelValues = append(nextLevelValues, pe.Value.Copy()) @@ -409,7 +410,7 @@ func (node *EmitXStatementNode) executeNonIndexedNonLashedEmit( func (node *EmitXStatementNode) executeNonIndexedNonLashedEmitP( names []string, - values []*types.Mlrval, + values []*mlrval.Mlrval, state *runtime.State, ) error { for i, value := range values { @@ -433,7 +434,7 @@ func (node *EmitXStatementNode) executeNonIndexedNonLashedEmitP( func (node *EmitXStatementNode) executeNonIndexedLashedEmit( names []string, - values []*types.Mlrval, + values []*mlrval.Mlrval, state *runtime.State, ) error { lib.InternalCodingErrorIf(len(values) < 1) @@ -483,7 +484,7 @@ func (node *EmitXStatementNode) executeNonIndexedLashedEmit( } else { // recurse nextLevelNames := make([]string, 0) - nextLevelValues := make([]*types.Mlrval, 0) + nextLevelValues := make([]*mlrval.Mlrval, 0) for pe := value.GetMap().Head; pe != nil; pe = pe.Next { nextLevelNames = append(nextLevelNames, pe.Key) nextLevelValues = append(nextLevelValues, pe.Value.Copy()) @@ -499,7 +500,7 @@ func (node *EmitXStatementNode) executeNonIndexedLashedEmit( func (node *EmitXStatementNode) executeNonIndexedLashedEmitP( names []string, - values []*types.Mlrval, + values []*mlrval.Mlrval, state *runtime.State, ) error { newrec := types.NewMlrmapAsRecord() @@ -518,7 +519,7 @@ func (node *EmitXStatementNode) executeNonIndexedLashedEmitP( // ---------------------------------------------------------------- func (node *EmitXStatementNode) executeIndexed( names []string, - values []*types.Mlrval, + values []*mlrval.Mlrval, state *runtime.State, ) error { @@ -535,7 +536,7 @@ func (node *EmitXStatementNode) executeIndexed( } // TODO: libify this - indices := make([]*types.Mlrval, len(node.indexEvaluables)) + indices := make([]*mlrval.Mlrval, len(node.indexEvaluables)) for i := range node.indexEvaluables { indices[i] = node.indexEvaluables[i].Evaluate(state) if indices[i].IsAbsent() { @@ -653,7 +654,7 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitAux( templateRecord *types.Mlrmap, names []string, emittableMaps []*types.Mlrmap, - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { lib.InternalCodingErrorIf(len(indices) < 1) @@ -663,7 +664,7 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitAux( for i, emittableMap := range emittableMaps { for pe := emittableMap.Head; pe != nil; pe = pe.Next { newrec := templateRecord.Copy() - newrec.PutCopy(indexString, types.MlrvalFromString(pe.Key)) + newrec.PutCopy(indexString, mlrval.MlrvalFromString(pe.Key)) if len(indices) == 1 { valueAsMap := pe.Value.GetMap() @@ -799,7 +800,7 @@ func (node *EmitXStatementNode) executeIndexedLashedEmitAux( templateRecord *types.Mlrmap, names []string, emittableMaps []*types.Mlrmap, - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { lib.InternalCodingErrorIf(len(indices) < 1) @@ -810,10 +811,10 @@ func (node *EmitXStatementNode) executeIndexedLashedEmitAux( for pe := leadingMap.Head; pe != nil; pe = pe.Next { newrec := templateRecord.Copy() - indexValue := types.MlrvalFromString(pe.Key) + indexValue := mlrval.MlrvalFromString(pe.Key) newrec.PutCopy(indexString, indexValue) - nextLevelValues := make([]*types.Mlrval, len(emittableMaps)) + nextLevelValues := make([]*mlrval.Mlrval, len(emittableMaps)) nextLevelMaps := make([]*types.Mlrmap, len(emittableMaps)) for i, emittableMap := range emittableMaps { if emittableMap != nil { @@ -868,7 +869,7 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitPAux( templateRecord *types.Mlrmap, names []string, emittableMaps []*types.Mlrmap, - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { lib.InternalCodingErrorIf(len(indices) < 1) @@ -878,7 +879,7 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitPAux( for i, emittableMap := range emittableMaps { for pe := emittableMap.Head; pe != nil; pe = pe.Next { newrec := templateRecord.Copy() - newrec.PutCopy(indexString, types.MlrvalFromString(pe.Key)) + newrec.PutCopy(indexString, mlrval.MlrvalFromString(pe.Key)) if len(indices) == 1 { newrec.PutCopy(names[i], pe.Value) @@ -914,7 +915,7 @@ func (node *EmitXStatementNode) executeIndexedLashedEmitPAux( templateRecord *types.Mlrmap, names []string, emittableMaps []*types.Mlrmap, - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { lib.InternalCodingErrorIf(len(indices) < 1) @@ -926,11 +927,11 @@ func (node *EmitXStatementNode) executeIndexedLashedEmitPAux( for pe := leadingMap.Head; pe != nil; pe = pe.Next { newrec := templateRecord.Copy() - indexValue := types.MlrvalFromString(pe.Key) + indexValue := mlrval.MlrvalFromString(pe.Key) newrec.PutCopy(indexString, indexValue) indexValueString := indexValue.String() - nextLevels := make([]*types.Mlrval, len(emittableMaps)) + nextLevels := make([]*mlrval.Mlrval, len(emittableMaps)) nextLevelMaps := make([]*types.Mlrmap, len(emittableMaps)) for i, emittableMap := range emittableMaps { if emittableMap != nil { diff --git a/internal/pkg/dsl/cst/env.go b/internal/pkg/dsl/cst/env.go index 371467235..0017c6dc7 100644 --- a/internal/pkg/dsl/cst/env.go +++ b/internal/pkg/dsl/cst/env.go @@ -12,6 +12,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -35,7 +36,7 @@ func (root *RootNode) BuildEnvironmentVariableNode(astNode *dsl.ASTNode) (*Envir func (node *EnvironmentVariableNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { name := node.nameEvaluable.Evaluate(state) if name.IsAbsent() { return types.MLRVAL_ABSENT @@ -44,5 +45,5 @@ func (node *EnvironmentVariableNode) Evaluate( return types.MLRVAL_ERROR } - return types.MlrvalFromString(os.Getenv(name.String())) + return mlrval.MlrvalFromString(os.Getenv(name.String())) } diff --git a/internal/pkg/dsl/cst/evaluable.go b/internal/pkg/dsl/cst/evaluable.go index 03722d4d2..e21aadf70 100644 --- a/internal/pkg/dsl/cst/evaluable.go +++ b/internal/pkg/dsl/cst/evaluable.go @@ -107,7 +107,7 @@ func (root *RootNode) BuildIndirectFieldValueNode( func (node *IndirectFieldValueNode) Evaluate( state *runtime.State, -) *types.Mlrval { // TODO: err +) *mlrval.Mlrval { // TODO: err fieldName := node.fieldNameEvaluable.Evaluate(state) if fieldName.IsAbsent() { return types.MLRVAL_ABSENT @@ -157,7 +157,7 @@ func (root *RootNode) BuildIndirectOosvarValueNode( func (node *IndirectOosvarValueNode) Evaluate( state *runtime.State, -) *types.Mlrval { // TODO: err +) *mlrval.Mlrval { // TODO: err oosvarName := node.oosvarNameEvaluable.Evaluate(state) if oosvarName.IsAbsent() { return types.MLRVAL_ABSENT diff --git a/internal/pkg/dsl/cst/for.go b/internal/pkg/dsl/cst/for.go index f312e092a..0a9ffb042 100644 --- a/internal/pkg/dsl/cst/for.go +++ b/internal/pkg/dsl/cst/for.go @@ -9,6 +9,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -126,7 +127,7 @@ func (node *ForLoopOneVariableNode) Execute(state *runtime.State) (*BlockExitPay state.Stack.PushStackFrame() defer state.Stack.PopStackFrame() for pe := mapval.Head; pe != nil; pe = pe.Next { - mapkey := types.MlrvalFromString(pe.Key) + mapkey := mlrval.MlrvalFromString(pe.Key) err := state.Stack.SetAtScope(node.indexVariable, mapkey) if err != nil { @@ -311,7 +312,7 @@ func (node *ForLoopTwoVariableNode) Execute(state *runtime.State) (*BlockExitPay state.Stack.PushStackFrame() defer state.Stack.PopStackFrame() for pe := mapval.Head; pe != nil; pe = pe.Next { - mapkey := types.MlrvalFromString(pe.Key) + mapkey := mlrval.MlrvalFromString(pe.Key) err := state.Stack.SetAtScope(node.keyIndexVariable, mapkey) if err != nil { @@ -352,7 +353,7 @@ func (node *ForLoopTwoVariableNode) Execute(state *runtime.State) (*BlockExitPay state.Stack.PushStackFrame() defer state.Stack.PopStackFrame() for zindex, element := range arrayval { - mindex := types.MlrvalFromInt(int(zindex + 1)) + mindex := mlrval.MlrvalFromInt(int(zindex + 1)) err := state.Stack.SetAtScope(node.keyIndexVariable, mindex) if err != nil { @@ -524,7 +525,7 @@ func (node *ForLoopMultivariableNode) Execute(state *runtime.State) (*BlockExitP // ---------------------------------------------------------------- func (node *ForLoopMultivariableNode) executeOuter( - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, keyIndexVariables []*runtime.StackVariable, state *runtime.State, ) (*BlockExitPayload, error) { @@ -537,7 +538,7 @@ func (node *ForLoopMultivariableNode) executeOuter( mapval := mlrval.GetMap() for pe := mapval.Head; pe != nil; pe = pe.Next { - mapkey := types.MlrvalFromString(pe.Key) + mapkey := mlrval.MlrvalFromString(pe.Key) err := state.Stack.SetAtScope(keyIndexVariables[0], mapkey) if err != nil { @@ -570,7 +571,7 @@ func (node *ForLoopMultivariableNode) executeOuter( // Go storage ("zindex") is 0-up. for zindex, element := range arrayval { - mindex := types.MlrvalFromInt(int(zindex + 1)) + mindex := mlrval.MlrvalFromInt(int(zindex + 1)) err := state.Stack.SetAtScope(keyIndexVariables[0], mindex) if err != nil { @@ -617,7 +618,7 @@ func (node *ForLoopMultivariableNode) executeOuter( // ---------------------------------------------------------------- func (node *ForLoopMultivariableNode) executeInner( - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, keyIndexVariable *runtime.StackVariable, state *runtime.State, ) (*BlockExitPayload, error) { @@ -625,7 +626,7 @@ func (node *ForLoopMultivariableNode) executeInner( mapval := mlrval.GetMap() for pe := mapval.Head; pe != nil; pe = pe.Next { - mapkey := types.MlrvalFromString(pe.Key) + mapkey := mlrval.MlrvalFromString(pe.Key) err := state.Stack.SetAtScope(keyIndexVariable, mapkey) if err != nil { @@ -663,7 +664,7 @@ func (node *ForLoopMultivariableNode) executeInner( // Go storage ("zindex") is 0-up. for zindex, element := range arrayval { - mindex := types.MlrvalFromInt(int(zindex + 1)) + mindex := mlrval.MlrvalFromInt(int(zindex + 1)) err := state.Stack.SetAtScope(keyIndexVariable, mindex) if err != nil { diff --git a/internal/pkg/dsl/cst/hofs.go b/internal/pkg/dsl/cst/hofs.go index 4e067685a..12cd187a4 100644 --- a/internal/pkg/dsl/cst/hofs.go +++ b/internal/pkg/dsl/cst/hofs.go @@ -13,6 +13,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -24,29 +25,29 @@ import ( // BinaryFuncWithState is for select, apply, and reduce. type BinaryFuncWithState func( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval +) *mlrval.Mlrval // TernaryFuncWithState is for fold. type TernaryFuncWithState func( - input1 *types.Mlrval, - input2 *types.Mlrval, - input3 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, + input3 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval +) *mlrval.Mlrval // VariadicFuncWithState is for sort. type VariadicFuncWithState func( - inputs []*types.Mlrval, + inputs []*mlrval.Mlrval, state *runtime.State, -) *types.Mlrval +) *mlrval.Mlrval // tHOFSpace is the datatype for the getHOFSpace cache-manager. type tHOFSpace struct { udfCallsite *UDFCallsite - argsArray []*types.Mlrval + argsArray []*mlrval.Mlrval } // hofCache is persistent data for the getHOFSpace cache-manager. @@ -56,7 +57,7 @@ var hofCache map[string]*tHOFSpace = make(map[string]*tHOFSpace) // Those functions may be invoked on every record of a big data file, so we try // to cache data they need for UDF-callsite setup. func getHOFSpace( - funcVal *types.Mlrval, + funcVal *mlrval.Mlrval, arity int, hofName string, arrayOrMap string, @@ -120,7 +121,7 @@ func getHOFSpace( } udfCallsite := NewUDFCallsiteForHigherOrderFunction(udf, arity) - argsArray := make([]*types.Mlrval, arity) + argsArray := make([]*mlrval.Mlrval, arity) entry = &tHOFSpace{ udfCallsite: udfCallsite, argsArray: argsArray, @@ -131,7 +132,7 @@ func getHOFSpace( } // mustBeNonAbsent checks that a UDF for array reduce/fold/apply returned a value. -func isNonAbsentOrDie(mlrval *types.Mlrval, hofName string) *types.Mlrval { +func isNonAbsentOrDie(mlrval *mlrval.Mlrval, hofName string) *mlrval.Mlrval { if mlrval.IsAbsent() { hofCheckDie(mlrval, hofName, "second-argument function must return a value") } @@ -140,7 +141,7 @@ func isNonAbsentOrDie(mlrval *types.Mlrval, hofName string) *types.Mlrval { // getKVPairForAccumulatorOrDie checks that a user-supplied accumulator value // for a map fold is indeed a single-element map. -func getKVPairForAccumulatorOrDie(mlrval *types.Mlrval, hofName string) *types.Mlrmap { +func getKVPairForAccumulatorOrDie(mlrval *mlrval.Mlrval, hofName string) *types.Mlrmap { kvPair := getKVPair(mlrval) if kvPair == nil { hofCheckDie(mlrval, hofName, "accumulator value must be a single-element map") @@ -150,7 +151,7 @@ func getKVPairForAccumulatorOrDie(mlrval *types.Mlrval, hofName string) *types.M // getKVPairForCallbackOrDie checks that a return value from a UDF for map // reduce/fold/apply is indeed a single-element map. -func getKVPairForCallbackOrDie(mlrval *types.Mlrval, hofName string) *types.Mlrmap { +func getKVPairForCallbackOrDie(mlrval *mlrval.Mlrval, hofName string) *types.Mlrmap { kvPair := getKVPair(mlrval) if kvPair == nil { hofCheckDie(mlrval, hofName, "second-argument function must return single-element map") @@ -160,7 +161,7 @@ func getKVPairForCallbackOrDie(mlrval *types.Mlrval, hofName string) *types.Mlrm // hofCheckDie is a helper function for HOFs on maps, to check that the // user-supplied UDF returned a single-entry map. -func hofCheckDie(mlrval *types.Mlrval, hofName string, message string) { +func hofCheckDie(mlrval *mlrval.Mlrval, hofName string, message string) { fmt.Fprintf( os.Stderr, "mlr: %s: %s; got \"%s\".\n", @@ -172,7 +173,7 @@ func hofCheckDie(mlrval *types.Mlrval, hofName string, message string) { } // getKVPair is a helper function getKVPairOrDie. -func getKVPair(mlrval *types.Mlrval) *types.Mlrmap { +func getKVPair(mlrval *mlrval.Mlrval) *types.Mlrmap { mapval := mlrval.GetMap() if mapval == nil { return nil @@ -183,7 +184,7 @@ func getKVPair(mlrval *types.Mlrval) *types.Mlrmap { return mapval } -func isFunctionOrDie(mlrval *types.Mlrval, hofName string) { +func isFunctionOrDie(mlrval *mlrval.Mlrval, hofName string) { if !mlrval.IsFunction() { fmt.Fprintf(os.Stderr, "mlr: %s: second argument must be a function; got %s.\n", hofName, mlrval.GetTypeName(), @@ -196,10 +197,10 @@ func isFunctionOrDie(mlrval *types.Mlrval, hofName string) { // SELECT HOF func SelectHOF( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { if input1.IsArray() { return selectArray(input1, input2, state) } else if input1.IsMap() { @@ -210,10 +211,10 @@ func SelectHOF( } func selectArray( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array return types.MLRVAL_ERROR @@ -224,7 +225,7 @@ func selectArray( udfCallsite := hofSpace.udfCallsite argsArray := hofSpace.argsArray - outputArray := make([]types.Mlrval, 0, len(inputArray)) + outputArray := make([]mlrval.Mlrval, 0, len(inputArray)) for i := range inputArray { argsArray[0] = &inputArray[i] @@ -242,14 +243,14 @@ func selectArray( outputArray = append(outputArray, *inputArray[i].Copy()) } } - return types.MlrvalFromArrayReference(outputArray) + return mlrval.MlrvalFromArrayReference(outputArray) } func selectMap( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map return types.MLRVAL_ERROR @@ -263,7 +264,7 @@ func selectMap( outputMap := types.NewMlrmap() for pe := inputMap.Head; pe != nil; pe = pe.Next { - argsArray[0] = types.MlrvalFromString(pe.Key) + argsArray[0] = mlrval.MlrvalFromString(pe.Key) argsArray[1] = pe.Value mret := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) bret, ok := mret.GetBoolValue() @@ -280,17 +281,17 @@ func selectMap( } } - return types.MlrvalFromMap(outputMap) + return mlrval.MlrvalFromMap(outputMap) } // ================================================================ // APPLY HOF func ApplyHOF( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { if input1.IsArray() { return applyArray(input1, input2, state) } else if input1.IsMap() { @@ -301,10 +302,10 @@ func ApplyHOF( } func applyArray( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array return types.MLRVAL_ERROR @@ -315,7 +316,7 @@ func applyArray( udfCallsite := hofSpace.udfCallsite argsArray := hofSpace.argsArray - outputArray := make([]types.Mlrval, len(inputArray)) + outputArray := make([]mlrval.Mlrval, len(inputArray)) for i := range inputArray { argsArray[0] = &inputArray[i] @@ -323,14 +324,14 @@ func applyArray( isNonAbsentOrDie(&retval, "apply") outputArray[i] = retval } - return types.MlrvalFromArrayReference(outputArray) + return mlrval.MlrvalFromArrayReference(outputArray) } func applyMap( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map return types.MLRVAL_ERROR @@ -344,23 +345,23 @@ func applyMap( outputMap := types.NewMlrmap() for pe := inputMap.Head; pe != nil; pe = pe.Next { - argsArray[0] = types.MlrvalFromString(pe.Key) + argsArray[0] = mlrval.MlrvalFromString(pe.Key) argsArray[1] = pe.Value retval := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) kvPair := getKVPairForCallbackOrDie(retval, "apply") outputMap.PutReference(kvPair.Head.Key, kvPair.Head.Value) } - return types.MlrvalFromMap(outputMap) + return mlrval.MlrvalFromMap(outputMap) } // ================================================================ // REDUCE HOF func ReduceHOF( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { if input1.IsArray() { return reduceArray(input1, input2, state) } else if input1.IsMap() { @@ -371,10 +372,10 @@ func ReduceHOF( } func reduceArray( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array return types.MLRVAL_ERROR @@ -401,10 +402,10 @@ func reduceArray( } func reduceMap( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map return types.MLRVAL_ERROR @@ -421,26 +422,26 @@ func reduceMap( } for pe := inputMap.Head.Next; pe != nil; pe = pe.Next { - argsArray[0] = types.MlrvalFromString(accumulator.Head.Key) + argsArray[0] = mlrval.MlrvalFromString(accumulator.Head.Key) argsArray[1] = accumulator.Head.Value - argsArray[2] = types.MlrvalFromString(pe.Key) + argsArray[2] = mlrval.MlrvalFromString(pe.Key) argsArray[3] = pe.Value.Copy() retval := (udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray)) kvPair := getKVPairForCallbackOrDie(retval, "reduce") accumulator = kvPair } - return types.MlrvalFromMap(accumulator) + return mlrval.MlrvalFromMap(accumulator) } // ================================================================ // FOLD HOF func FoldHOF( - input1 *types.Mlrval, - input2 *types.Mlrval, - input3 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, + input3 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { if input1.IsArray() { return foldArray(input1, input2, input3, state) } else if input1.IsMap() { @@ -451,11 +452,11 @@ func FoldHOF( } func foldArray( - input1 *types.Mlrval, - input2 *types.Mlrval, - input3 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, + input3 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array return types.MLRVAL_ERROR @@ -478,11 +479,11 @@ func foldArray( } func foldMap( - input1 *types.Mlrval, - input2 *types.Mlrval, - input3 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, + input3 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map return types.MLRVAL_ERROR @@ -500,24 +501,24 @@ func foldMap( accumulator := getKVPairForAccumulatorOrDie(input3, "reduce").Copy() for pe := inputMap.Head; pe != nil; pe = pe.Next { - argsArray[0] = types.MlrvalFromString(accumulator.Head.Key) + argsArray[0] = mlrval.MlrvalFromString(accumulator.Head.Key) argsArray[1] = accumulator.Head.Value - argsArray[2] = types.MlrvalFromString(pe.Key) + argsArray[2] = mlrval.MlrvalFromString(pe.Key) argsArray[3] = pe.Value.Copy() retval := (udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray)) kvPair := getKVPairForCallbackOrDie(retval, "reduce") accumulator = kvPair } - return types.MlrvalFromMap(accumulator) + return mlrval.MlrvalFromMap(accumulator) } // ================================================================ // SORT HOF func SortHOF( - inputs []*types.Mlrval, + inputs []*mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { if len(inputs) == 1 { if inputs[0].IsArray() { @@ -588,9 +589,9 @@ func decodeSortFlags(flags string) (tSortType, bool) { // sortA implements sort on array, with string flags rather than callback UDF. func sortA( - input1 *types.Mlrval, + input1 *mlrval.Mlrval, flags string, -) *types.Mlrval { +) *mlrval.Mlrval { if input1.GetArray() == nil { // not an array return types.MLRVAL_ERROR } @@ -612,19 +613,19 @@ func sortA( return output } -func sortANumerical(array []types.Mlrval, reverse bool) { +func sortANumerical(array []mlrval.Mlrval, reverse bool) { if !reverse { sort.Slice(array, func(i, j int) bool { - return types.MlrvalLessThanAsBool(&array[i], &array[j]) + return mlrval.MlrvalLessThanAsBool(&array[i], &array[j]) }) } else { sort.Slice(array, func(i, j int) bool { - return types.MlrvalGreaterThanAsBool(&array[i], &array[j]) + return mlrval.MlrvalGreaterThanAsBool(&array[i], &array[j]) }) } } -func sortALexical(array []types.Mlrval, reverse bool) { +func sortALexical(array []mlrval.Mlrval, reverse bool) { if !reverse { sort.Slice(array, func(i, j int) bool { return array[i].String() < array[j].String() @@ -636,7 +637,7 @@ func sortALexical(array []types.Mlrval, reverse bool) { } } -func sortACaseFold(array []types.Mlrval, reverse bool) { +func sortACaseFold(array []mlrval.Mlrval, reverse bool) { if !reverse { sort.Slice(array, func(i, j int) bool { return strings.ToLower(array[i].String()) < strings.ToLower(array[j].String()) @@ -650,9 +651,9 @@ func sortACaseFold(array []types.Mlrval, reverse bool) { // sortA implements sort on map, with string flags rather than callback UDF. func sortMK( - input1 *types.Mlrval, + input1 *mlrval.Mlrval, flags string, -) *types.Mlrval { +) *mlrval.Mlrval { inmap := input1.GetMap() if inmap == nil { // not a map return types.MLRVAL_ERROR @@ -688,7 +689,7 @@ func sortMK( outmap.PutCopy(key, inmap.Get(key)) } - return types.MlrvalFromMapReferenced(outmap) + return mlrval.MlrvalFromMapReferenced(outmap) } func sortMKNumerical(array []string, reverse bool) { @@ -742,10 +743,10 @@ func sortMKCaseFold(array []string, reverse bool) { // sortAF implements sort on arrays with callback UDF. func sortAF( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array return types.MLRVAL_ERROR @@ -765,7 +766,7 @@ func sortAF( argsArray[1] = &outputArray[j] // Call the user's comparator function. mret := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) - // Unpack the types.Mlrval return value into a number. + // Unpack the mlrval.Mlrval return value into a number. nret, ok := mret.GetNumericToFloatValue() if !ok { fmt.Fprintf( @@ -780,15 +781,15 @@ func sortAF( // Go sort-callback conventions: true if a < b, false otherwise. return nret < 0 }) - return types.MlrvalFromArrayReference(outputArray) + return mlrval.MlrvalFromArrayReference(outputArray) } // sortAF implements sort on arrays with callback UDF. func sortMF( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map return types.MLRVAL_ERROR @@ -804,14 +805,14 @@ func sortMF( argsArray := hofSpace.argsArray sort.Slice(pairsArray, func(i, j int) bool { - argsArray[0] = types.MlrvalFromString(pairsArray[i].Key) + argsArray[0] = mlrval.MlrvalFromString(pairsArray[i].Key) argsArray[1] = pairsArray[i].Value - argsArray[2] = types.MlrvalFromString(pairsArray[j].Key) + argsArray[2] = mlrval.MlrvalFromString(pairsArray[j].Key) argsArray[3] = pairsArray[j].Value // Call the user's comparator function. mret := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) - // Unpack the types.Mlrval return value into a number. + // Unpack the mlrval.Mlrval return value into a number. nret, ok := mret.GetNumericToFloatValue() if !ok { fmt.Fprintf( @@ -828,17 +829,17 @@ func sortMF( }) sortedMap := types.MlrmapFromPairsArray(pairsArray) - return types.MlrvalFromMapReferenced(sortedMap) + return mlrval.MlrvalFromMapReferenced(sortedMap) } // ================================================================ // ANY HOF func AnyHOF( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { if input1.IsArray() { return anyArray(input1, input2, state) } else if input1.IsMap() { @@ -849,10 +850,10 @@ func AnyHOF( } func anyArray( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array return types.MLRVAL_ERROR @@ -881,14 +882,14 @@ func anyArray( break } } - return types.MlrvalFromBool(boolAny) + return mlrval.MlrvalFromBool(boolAny) } func anyMap( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map return types.MLRVAL_ERROR @@ -902,7 +903,7 @@ func anyMap( boolAny := false for pe := inputMap.Head; pe != nil; pe = pe.Next { - argsArray[0] = types.MlrvalFromString(pe.Key) + argsArray[0] = mlrval.MlrvalFromString(pe.Key) argsArray[1] = pe.Value mret := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) bret, ok := mret.GetBoolValue() @@ -920,17 +921,17 @@ func anyMap( } } - return types.MlrvalFromBool(boolAny) + return mlrval.MlrvalFromBool(boolAny) } // ================================================================ // EVERY HOF func EveryHOF( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { if input1.IsArray() { return everyArray(input1, input2, state) } else if input1.IsMap() { @@ -941,10 +942,10 @@ func EveryHOF( } func everyArray( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array return types.MLRVAL_ERROR @@ -973,14 +974,14 @@ func everyArray( break } } - return types.MlrvalFromBool(boolEvery) + return mlrval.MlrvalFromBool(boolEvery) } func everyMap( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map return types.MLRVAL_ERROR @@ -994,7 +995,7 @@ func everyMap( boolEvery := true for pe := inputMap.Head; pe != nil; pe = pe.Next { - argsArray[0] = types.MlrvalFromString(pe.Key) + argsArray[0] = mlrval.MlrvalFromString(pe.Key) argsArray[1] = pe.Value mret := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) bret, ok := mret.GetBoolValue() @@ -1012,5 +1013,5 @@ func everyMap( } } - return types.MlrvalFromBool(boolEvery) + return mlrval.MlrvalFromBool(boolEvery) } diff --git a/internal/pkg/dsl/cst/leaves.go b/internal/pkg/dsl/cst/leaves.go index b39b0b16f..2c5e965b9 100644 --- a/internal/pkg/dsl/cst/leaves.go +++ b/internal/pkg/dsl/cst/leaves.go @@ -10,6 +10,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -114,7 +115,7 @@ func (root *RootNode) BuildDirectFieldRvalueNode(fieldName string) *DirectFieldR } func (node *DirectFieldRvalueNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { // For normal DSL use the CST validator will prohibit this from being // called in places the current record is undefined (begin and end blocks). // However in the REPL people can read past end of stream and still try to @@ -140,7 +141,7 @@ func (root *RootNode) BuildFullSrecRvalueNode() *FullSrecRvalueNode { } func (node *FullSrecRvalueNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { // For normal DSL use the CST validator will prohibit this from being // called in places the current record is undefined (begin and end blocks). // However in the REPL people can read past end of stream and still try to @@ -149,7 +150,7 @@ func (node *FullSrecRvalueNode) Evaluate( if state.Inrec == nil { return types.MLRVAL_ABSENT } else { - return types.MlrvalFromMap(state.Inrec) + return mlrval.MlrvalFromMap(state.Inrec) } } @@ -165,7 +166,7 @@ func (root *RootNode) BuildDirectOosvarRvalueNode(variableName string) *DirectOo } func (node *DirectOosvarRvalueNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { value := state.Oosvars.Get(node.variableName) if value == nil { return types.MLRVAL_ABSENT @@ -183,8 +184,8 @@ func (root *RootNode) BuildFullOosvarRvalueNode() *FullOosvarRvalueNode { } func (node *FullOosvarRvalueNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromMap(state.Oosvars) +) *mlrval.Mlrval { + return mlrval.MlrvalFromMap(state.Oosvars) } // ---------------------------------------------------------------- @@ -208,7 +209,7 @@ func (root *RootNode) BuildLocalVariableNode(variableName string) *LocalVariable } func (node *LocalVariableNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { value := state.Stack.Get(node.stackVariable) if value != nil { return value @@ -218,7 +219,7 @@ func (node *LocalVariableNode) Evaluate( udf := node.udfManager.LookUpDisregardingArity(functionName) if udf != nil { - return types.MlrvalFromFunction(udf, functionName) + return mlrval.MlrvalFromFunction(udf, functionName) } // TODO: allow built-in functions as well. Needs some API-merging as a @@ -237,18 +238,18 @@ func (node *LocalVariableNode) Evaluate( // regex literals differently from those for non-regex string literals. type RegexLiteralNode struct { - literal *types.Mlrval + literal *mlrval.Mlrval } func (root *RootNode) BuildRegexLiteralNode(literal string) IEvaluable { return &RegexLiteralNode{ - literal: types.MlrvalFromString(literal), + literal: mlrval.MlrvalFromString(literal), } } func (node *RegexLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.literal } @@ -256,7 +257,7 @@ func (node *RegexLiteralNode) Evaluate( // StringLiteralNode is for any string literal that doesn't have any "\0" .. // "\9" in it. type StringLiteralNode struct { - literal *types.Mlrval + literal *mlrval.Mlrval } // RegexCaptureReplacementNode is for any string literal that has any "\0" .. @@ -288,7 +289,7 @@ func (root *RootNode) BuildStringLiteralNode(literal string) IEvaluable { hasCaptures, replacementCaptureMatrix := lib.RegexReplacementHasCaptures(literal) if !hasCaptures { return &StringLiteralNode{ - literal: types.MlrvalFromString(literal), + literal: mlrval.MlrvalFromString(literal), } } else { return &RegexCaptureReplacementNode{ @@ -300,7 +301,7 @@ func (root *RootNode) BuildStringLiteralNode(literal string) IEvaluable { func (node *StringLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.literal } @@ -316,8 +317,8 @@ func (node *StringLiteralNode) Evaluate( // set on some previous invocation of =~ or !=~. func (node *RegexCaptureReplacementNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString( +) *mlrval.Mlrval { + return mlrval.MlrvalFromString( lib.InterpolateCaptures( node.replacementString, node.replacementCaptureMatrix, @@ -328,55 +329,55 @@ func (node *RegexCaptureReplacementNode) Evaluate( // ---------------------------------------------------------------- type IntLiteralNode struct { - literal *types.Mlrval + literal *mlrval.Mlrval } func (root *RootNode) BuildIntLiteralNode(literal string) *IntLiteralNode { return &IntLiteralNode{ - literal: types.MlrvalFromPrevalidatedIntString(literal), + literal: mlrval.MlrvalFromPrevalidatedIntString(literal), } } func (node *IntLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.literal } // ---------------------------------------------------------------- type FloatLiteralNode struct { - literal *types.Mlrval + literal *mlrval.Mlrval } func (root *RootNode) BuildFloatLiteralNode(literal string) *FloatLiteralNode { return &FloatLiteralNode{ - literal: types.MlrvalFromPrevalidatedFloat64String(literal), + literal: mlrval.MlrvalFromPrevalidatedFloat64String(literal), } } func (node *FloatLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.literal } // ---------------------------------------------------------------- type BoolLiteralNode struct { - literal *types.Mlrval + literal *mlrval.Mlrval } func (root *RootNode) BuildBoolLiteralNode(literal string) *BoolLiteralNode { return &BoolLiteralNode{ - literal: types.MlrvalFromBoolString(literal), + literal: mlrval.MlrvalFromBoolString(literal), } } func (node *BoolLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.literal } // ---------------------------------------------------------------- type NullLiteralNode struct { - literal *types.Mlrval + literal *mlrval.Mlrval } func (root *RootNode) BuildNullLiteralNode() *NullLiteralNode { @@ -386,7 +387,7 @@ func (root *RootNode) BuildNullLiteralNode() *NullLiteralNode { } func (node *NullLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.literal } @@ -453,8 +454,8 @@ func (root *RootNode) BuildFILENAMENode() *FILENAMENode { } func (node *FILENAMENode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Context.FILENAME) +) *mlrval.Mlrval { + return mlrval.MlrvalFromString(state.Context.FILENAME) } // ---------------------------------------------------------------- @@ -466,8 +467,8 @@ func (root *RootNode) BuildFILENUMNode() *FILENUMNode { } func (node *FILENUMNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromInt(state.Context.FILENUM) +) *mlrval.Mlrval { + return mlrval.MlrvalFromInt(state.Context.FILENUM) } // ---------------------------------------------------------------- @@ -479,8 +480,8 @@ func (root *RootNode) BuildNFNode() *NFNode { } func (node *NFNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromInt(state.Inrec.FieldCount) +) *mlrval.Mlrval { + return mlrval.MlrvalFromInt(state.Inrec.FieldCount) } // ---------------------------------------------------------------- @@ -492,8 +493,8 @@ func (root *RootNode) BuildNRNode() *NRNode { } func (node *NRNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromInt(state.Context.NR) +) *mlrval.Mlrval { + return mlrval.MlrvalFromInt(state.Context.NR) } // ---------------------------------------------------------------- @@ -505,8 +506,8 @@ func (root *RootNode) BuildFNRNode() *FNRNode { } func (node *FNRNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromInt(state.Context.FNR) +) *mlrval.Mlrval { + return mlrval.MlrvalFromInt(state.Context.FNR) } // ---------------------------------------------------------------- @@ -520,6 +521,7 @@ func (node *IRSNode) Evaluate( state *runtime.State, ) *types.Mlrval { return types.MlrvalFromString(state.Options.ReaderOptions.IRS) +>>>>>>> 8f0ac5233 (mlrmap refactor) } // ---------------------------------------------------------------- @@ -630,8 +632,8 @@ func (root *RootNode) BuildMathPINode() *MathPINode { } func (node *MathPINode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromFloat64(math.Pi) +) *mlrval.Mlrval { + return mlrval.MlrvalFromFloat64(math.Pi) } // ---------------------------------------------------------------- @@ -643,8 +645,8 @@ func (root *RootNode) BuildMathENode() *MathENode { } func (node *MathENode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromFloat64(math.E) +) *mlrval.Mlrval { + return mlrval.MlrvalFromFloat64(math.E) } // ================================================================ @@ -661,8 +663,8 @@ func (root *RootNode) BuildArraySliceEmptyLowerIndexNode( } func (node *LiteralOneNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromInt(1) +) *mlrval.Mlrval { + return mlrval.MlrvalFromInt(1) } // ================================================================ @@ -681,8 +683,8 @@ func (root *RootNode) BuildArraySliceEmptyUpperIndexNode( } func (node *LiteralEmptyStringNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString("") +) *mlrval.Mlrval { + return mlrval.MlrvalFromString("") } // ---------------------------------------------------------------- @@ -698,7 +700,7 @@ func (root *RootNode) BuildPanicNode(astNode *dsl.ASTNode) (*PanicNode, error) { } func (node *PanicNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { lib.InternalCodingErrorPanic("Panic token was evaluated, not short-circuited.") return nil // not reached } diff --git a/internal/pkg/dsl/cst/lvalues.go b/internal/pkg/dsl/cst/lvalues.go index 0e3572c54..9e531e4f1 100644 --- a/internal/pkg/dsl/cst/lvalues.go +++ b/internal/pkg/dsl/cst/lvalues.go @@ -12,6 +12,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -84,7 +85,7 @@ func (root *RootNode) BuildAssignableNode( // ---------------------------------------------------------------- type DirectFieldValueLvalueNode struct { - lhsFieldName *types.Mlrval + lhsFieldName *mlrval.Mlrval } func (root *RootNode) BuildDirectFieldValueLvalueNode( @@ -92,26 +93,26 @@ func (root *RootNode) BuildDirectFieldValueLvalueNode( ) (IAssignable, error) { lib.InternalCodingErrorIf(astNode.Type != dsl.NodeTypeDirectFieldValue) - lhsFieldName := types.MlrvalFromString(string(astNode.Token.Lit)) + lhsFieldName := mlrval.MlrvalFromString(string(astNode.Token.Lit)) return NewDirectFieldValueLvalueNode(lhsFieldName), nil } -func NewDirectFieldValueLvalueNode(lhsFieldName *types.Mlrval) *DirectFieldValueLvalueNode { +func NewDirectFieldValueLvalueNode(lhsFieldName *mlrval.Mlrval) *DirectFieldValueLvalueNode { return &DirectFieldValueLvalueNode{ lhsFieldName: lhsFieldName, } } func (node *DirectFieldValueLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *DirectFieldValueLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { @@ -134,7 +135,7 @@ func (node *DirectFieldValueLvalueNode) AssignIndexed( return nil } else { return state.Inrec.PutIndexed( - append([]*types.Mlrval{node.lhsFieldName}, indices...), + append([]*mlrval.Mlrval{node.lhsFieldName}, indices...), rvalue, ) } @@ -147,7 +148,7 @@ func (node *DirectFieldValueLvalueNode) Unassign( } func (node *DirectFieldValueLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { @@ -166,7 +167,7 @@ func (node *DirectFieldValueLvalueNode) UnassignIndexed( state.Inrec.Remove(name) } else { state.Inrec.RemoveIndexed( - append([]*types.Mlrval{node.lhsFieldName}, indices...), + append([]*mlrval.Mlrval{node.lhsFieldName}, indices...), ) } } @@ -199,15 +200,15 @@ func NewIndirectFieldValueLvalueNode( } func (node *IndirectFieldValueLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *IndirectFieldValueLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absentness of the rvalue, so we just assign @@ -233,7 +234,7 @@ func (node *IndirectFieldValueLvalueNode) AssignIndexed( return nil } else { return state.Inrec.PutIndexed( - append([]*types.Mlrval{lhsFieldName.Copy()}, indices...), + append([]*mlrval.Mlrval{lhsFieldName.Copy()}, indices...), rvalue, ) } @@ -246,7 +247,7 @@ func (node *IndirectFieldValueLvalueNode) Unassign( } func (node *IndirectFieldValueLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { // For normal DSL use the CST validator will prohibit this from being @@ -264,7 +265,7 @@ func (node *IndirectFieldValueLvalueNode) UnassignIndexed( state.Inrec.Remove(name) } else { state.Inrec.RemoveIndexed( - append([]*types.Mlrval{lhsFieldName.Copy()}, indices...), + append([]*mlrval.Mlrval{lhsFieldName.Copy()}, indices...), ) } } @@ -300,7 +301,7 @@ func NewPositionalFieldNameLvalueNode( } func (node *PositionalFieldNameLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absentness of the rvalue, so we just assign @@ -334,8 +335,8 @@ func (node *PositionalFieldNameLvalueNode) Assign( } func (node *PositionalFieldNameLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // TODO: reconsider this if /when we decide to allow string-slice @@ -352,7 +353,7 @@ func (node *PositionalFieldNameLvalueNode) Unassign( } func (node *PositionalFieldNameLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { lhsFieldIndex := node.lhsFieldIndexExpression.Evaluate(state) @@ -376,7 +377,7 @@ func (node *PositionalFieldNameLvalueNode) UnassignIndexed( } else { // xxx positional state.Inrec.RemoveIndexed( - append([]*types.Mlrval{lhsFieldIndex}, indices...), + append([]*mlrval.Mlrval{lhsFieldIndex}, indices...), ) } } @@ -412,15 +413,15 @@ func NewPositionalFieldValueLvalueNode( } func (node *PositionalFieldValueLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *PositionalFieldValueLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absentness of the rvalue, so we just assign @@ -460,7 +461,7 @@ func (node *PositionalFieldValueLvalueNode) AssignIndexed( } else { // xxx positional return state.Inrec.PutIndexed( - append([]*types.Mlrval{lhsFieldIndex}, indices...), + append([]*mlrval.Mlrval{lhsFieldIndex}, indices...), rvalue, ) } @@ -475,7 +476,7 @@ func (node *PositionalFieldValueLvalueNode) Unassign( } func (node *PositionalFieldValueLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { // For normal DSL use the CST validator will prohibit this from being @@ -498,7 +499,7 @@ func (node *PositionalFieldValueLvalueNode) UnassignIndexed( } else { // xxx positional state.Inrec.RemoveIndexed( - append([]*types.Mlrval{lhsFieldIndex}, indices...), + append([]*mlrval.Mlrval{lhsFieldIndex}, indices...), ) } } @@ -519,15 +520,15 @@ func NewFullSrecLvalueNode() *FullSrecLvalueNode { } func (node *FullSrecLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *FullSrecLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // For normal DSL use the CST validator will prohibit this from being @@ -558,7 +559,7 @@ func (node *FullSrecLvalueNode) Unassign( } func (node *FullSrecLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { // For normal DSL use the CST validator will prohibit this from being @@ -579,32 +580,32 @@ func (node *FullSrecLvalueNode) UnassignIndexed( // ---------------------------------------------------------------- type DirectOosvarValueLvalueNode struct { - lhsOosvarName *types.Mlrval + lhsOosvarName *mlrval.Mlrval } func (root *RootNode) BuildDirectOosvarValueLvalueNode(astNode *dsl.ASTNode) (IAssignable, error) { lib.InternalCodingErrorIf(astNode.Type != dsl.NodeTypeDirectOosvarValue) - lhsOosvarName := types.MlrvalFromString(string(astNode.Token.Lit)) + lhsOosvarName := mlrval.MlrvalFromString(string(astNode.Token.Lit)) return NewDirectOosvarValueLvalueNode(lhsOosvarName), nil } -func NewDirectOosvarValueLvalueNode(lhsOosvarName *types.Mlrval) *DirectOosvarValueLvalueNode { +func NewDirectOosvarValueLvalueNode(lhsOosvarName *mlrval.Mlrval) *DirectOosvarValueLvalueNode { return &DirectOosvarValueLvalueNode{ lhsOosvarName: lhsOosvarName, } } func (node *DirectOosvarValueLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *DirectOosvarValueLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absent, so we just assign whatever we get @@ -617,7 +618,7 @@ func (node *DirectOosvarValueLvalueNode) AssignIndexed( return nil } else { return state.Oosvars.PutIndexed( - append([]*types.Mlrval{node.lhsOosvarName}, indices...), + append([]*mlrval.Mlrval{node.lhsOosvarName}, indices...), rvalue, ) } @@ -630,7 +631,7 @@ func (node *DirectOosvarValueLvalueNode) Unassign( } func (node *DirectOosvarValueLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { if indices == nil { @@ -638,7 +639,7 @@ func (node *DirectOosvarValueLvalueNode) UnassignIndexed( state.Oosvars.Remove(name) } else { state.Oosvars.RemoveIndexed( - append([]*types.Mlrval{node.lhsOosvarName}, indices...), + append([]*mlrval.Mlrval{node.lhsOosvarName}, indices...), ) } } @@ -672,15 +673,15 @@ func NewIndirectOosvarValueLvalueNode( } func (node *IndirectOosvarValueLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *IndirectOosvarValueLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absentness of the rvalue, so we just assign @@ -697,7 +698,7 @@ func (node *IndirectOosvarValueLvalueNode) AssignIndexed( return nil } else { return state.Oosvars.PutIndexed( - append([]*types.Mlrval{lhsOosvarName.Copy()}, indices...), + append([]*mlrval.Mlrval{lhsOosvarName.Copy()}, indices...), rvalue, ) } @@ -710,7 +711,7 @@ func (node *IndirectOosvarValueLvalueNode) Unassign( } func (node *IndirectOosvarValueLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { lhsOosvarName := node.lhsOosvarNameExpression.Evaluate(state) @@ -720,7 +721,7 @@ func (node *IndirectOosvarValueLvalueNode) UnassignIndexed( state.Oosvars.Remove(sname) } else { state.Oosvars.RemoveIndexed( - append([]*types.Mlrval{lhsOosvarName}, indices...), + append([]*mlrval.Mlrval{lhsOosvarName}, indices...), ) } } @@ -741,15 +742,15 @@ func NewFullOosvarLvalueNode() *FullOosvarLvalueNode { } func (node *FullOosvarLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *FullOosvarLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absentness of the rvalue, so we just assign @@ -771,7 +772,7 @@ func (node *FullOosvarLvalueNode) Unassign( } func (node *FullOosvarLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { if indices == nil { @@ -827,15 +828,15 @@ func NewLocalVariableLvalueNode( } func (node *LocalVariableLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *LocalVariableLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absent, so we just assign whatever we get @@ -863,7 +864,7 @@ func (node *LocalVariableLvalueNode) Unassign( } func (node *LocalVariableLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { if indices == nil { @@ -1006,10 +1007,10 @@ func NewIndexedLvalueNode( // '$x[1][2] = 3' or '@x[1][2] = 3', the indices are [1,2], and the baseLvalue // is '$x' or '@x' respectively. func (node *IndexedLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { - indices := make([]*types.Mlrval, len(node.indexEvaluables)) + indices := make([]*mlrval.Mlrval, len(node.indexEvaluables)) for i := range node.indexEvaluables { indices[i] = node.indexEvaluables[i].Evaluate(state) @@ -1028,8 +1029,8 @@ func (node *IndexedLvalueNode) Assign( } func (node *IndexedLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // We are the delegator, not the delegatee @@ -1040,7 +1041,7 @@ func (node *IndexedLvalueNode) AssignIndexed( func (node *IndexedLvalueNode) Unassign( state *runtime.State, ) { - indices := make([]*types.Mlrval, len(node.indexEvaluables)) + indices := make([]*mlrval.Mlrval, len(node.indexEvaluables)) for i := range node.indexEvaluables { indices[i] = node.indexEvaluables[i].Evaluate(state) } @@ -1049,7 +1050,7 @@ func (node *IndexedLvalueNode) Unassign( } func (node *IndexedLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { // We are the delegator, not the delegatee @@ -1082,7 +1083,7 @@ func NewEnvironmentVariableLvalueNode( } func (node *EnvironmentVariableLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absentness of the rvalue, so we just assign @@ -1114,8 +1115,8 @@ func (node *EnvironmentVariableLvalueNode) Assign( } func (node *EnvironmentVariableLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { return errors.New("mlr: ENV[...] cannot be indexed.") @@ -1138,7 +1139,7 @@ func (node *EnvironmentVariableLvalueNode) Unassign( } func (node *EnvironmentVariableLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { // TODO: needs error return diff --git a/internal/pkg/dsl/cst/types.go b/internal/pkg/dsl/cst/types.go index f5ad98530..b28cb1c19 100644 --- a/internal/pkg/dsl/cst/types.go +++ b/internal/pkg/dsl/cst/types.go @@ -9,6 +9,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/dsl" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -67,16 +68,16 @@ type Executor func(state *runtime.State) (*BlockExitPayload, error) // ================================================================ // This is for any left-hand side (LHS or Lvalue) of an assignment statement. type IAssignable interface { - Assign(rvalue *types.Mlrval, state *runtime.State) error + Assign(rvalue *mlrval.Mlrval, state *runtime.State) error // 'foo = "bar"' or 'foo[3]["abc"] = "bar"' // For non-indexed assignment, which is the normal case, indices can be // zero-length or nil. - AssignIndexed(rvalue *types.Mlrval, indices []*types.Mlrval, state *runtime.State) error + AssignIndexed(rvalue *mlrval.Mlrval, indices []*mlrval.Mlrval, state *runtime.State) error Unassign(state *runtime.State) - UnassignIndexed(indices []*types.Mlrval, state *runtime.State) + UnassignIndexed(indices []*mlrval.Mlrval, state *runtime.State) } // ================================================================ @@ -84,7 +85,7 @@ type IAssignable interface { // Also, for computed field names on the left-hand side, like '$a . $b' in mlr // put '$[$a . $b]' = $x + $y'. type IEvaluable interface { - Evaluate(state *runtime.State) *types.Mlrval + Evaluate(state *runtime.State) *mlrval.Mlrval } // ================================================================ @@ -119,5 +120,5 @@ type BlockExitPayload struct { blockExitStatus BlockExitStatus // No multiple return yet in the Miller DSL -- if there were, this would be // an array. - blockReturnValue *types.Mlrval // TODO: TypeGatedMlrval + blockReturnValue *mlrval.Mlrval // TODO: TypeGatedMlrval } diff --git a/internal/pkg/dsl/cst/udf.go b/internal/pkg/dsl/cst/udf.go index b7f069665..d466e4929 100644 --- a/internal/pkg/dsl/cst/udf.go +++ b/internal/pkg/dsl/cst/udf.go @@ -11,6 +11,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -83,7 +84,7 @@ func NewUDFCallsite( // NewUDFCallsiteForHigherOrderFunction is for UDF callsites such as // sortaf/sortmf. Here, the array/map to be sorted has already been evaluated -// and is an array of *types.Mlrval. The UDF needs to be invoked on pairs of +// and is an array of *mlrval.Mlrval. The UDF needs to be invoked on pairs of // array elements. func NewUDFCallsiteForHigherOrderFunction( udf *UDF, @@ -123,7 +124,7 @@ func (site *UDFCallsite) findUDF(state *runtime.State) *UDF { // See comments above NewUDFCallsite. func (site *UDFCallsite) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { udf := site.findUDF(state) if udf == nil { @@ -188,7 +189,7 @@ func (site *UDFCallsite) Evaluate( os.Exit(1) } - arguments := make([]*types.Mlrval, numArguments) + arguments := make([]*mlrval.Mlrval, numArguments) for i := range udf.signature.typeGatedParameterNames { arguments[i] = site.argumentNodes[i].Evaluate(state) @@ -210,8 +211,8 @@ func (site *UDFCallsite) Evaluate( func (site *UDFCallsite) EvaluateWithArguments( state *runtime.State, udf *UDF, - arguments []*types.Mlrval, -) *types.Mlrval { + arguments []*mlrval.Mlrval, +) *mlrval.Mlrval { // Bind the arguments to the parameters. Function literals can access // locals in their enclosing scope; named functions cannot. Hence stack @@ -434,7 +435,7 @@ func genFunctionLiteralName() string { // UnnamedUDFNode holds function literals like 'func (a, b) { return b - a }'. type UnnamedUDFNode struct { - udfAsMlrval *types.Mlrval + udfAsMlrval *mlrval.Mlrval } func (root *RootNode) BuildUnnamedUDFNode(astNode *dsl.ASTNode) (IEvaluable, error) { @@ -447,14 +448,14 @@ func (root *RootNode) BuildUnnamedUDFNode(astNode *dsl.ASTNode) (IEvaluable, err return nil, err } - udfAsMlrval := types.MlrvalFromFunction(udf, name) + udfAsMlrval := mlrval.MlrvalFromFunction(udf, name) return &UnnamedUDFNode{ udfAsMlrval: udfAsMlrval, }, nil } -func (node *UnnamedUDFNode) Evaluate(state *runtime.State) *types.Mlrval { +func (node *UnnamedUDFNode) Evaluate(state *runtime.State) *mlrval.Mlrval { return node.udfAsMlrval } diff --git a/internal/pkg/dsl/cst/uds.go b/internal/pkg/dsl/cst/uds.go index 394c3fd88..7bc8ec91a 100644 --- a/internal/pkg/dsl/cst/uds.go +++ b/internal/pkg/dsl/cst/uds.go @@ -10,6 +10,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -106,7 +107,7 @@ func (site *UDSCallsite) Execute(state *runtime.State) (*BlockExitPayload, error // we push a new frameset and DefineTypedAtScope using the callee's frameset. // Evaluate the arguments - arguments := make([]*types.Mlrval, len(site.uds.signature.typeGatedParameterNames)) + arguments := make([]*mlrval.Mlrval, len(site.uds.signature.typeGatedParameterNames)) for i, typeGatedParameterName := range site.uds.signature.typeGatedParameterNames { arguments[i] = site.argumentNodes[i].Evaluate(state) diff --git a/internal/pkg/input/pseudo_reader_gen.go b/internal/pkg/input/pseudo_reader_gen.go index 143b2f5e9..7b3d1d5e4 100644 --- a/internal/pkg/input/pseudo_reader_gen.go +++ b/internal/pkg/input/pseudo_reader_gen.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -121,8 +122,8 @@ func (reader *PseudoReaderGen) process( func (reader *PseudoReaderGen) tryParse( name string, svalue string, -) (*types.Mlrval, error) { - mvalue := types.MlrvalFromInferredType(svalue) +) (*mlrval.Mlrval, error) { + mvalue := mlrval.MlrvalFromInferredType(svalue) if mvalue == nil || !mvalue.IsNumeric() { return nil, errors.New( fmt.Sprintf("mlr: gen: %s \"%s\" is not parseable as number", name, svalue), diff --git a/internal/pkg/input/record_reader_csv.go b/internal/pkg/input/record_reader_csv.go index a961e057c..da2199c7d 100644 --- a/internal/pkg/input/record_reader_csv.go +++ b/internal/pkg/input/record_reader_csv.go @@ -226,7 +226,7 @@ func (reader *RecordReaderCSV) getRecordBatch( if nh == nd { for i := 0; i < nh; i++ { key := reader.header[i] - value := types.MlrvalFromInferredTypeForDataFiles(csvRecord[i]) + value := mlrval.FromDeferredType(csvRecord[i]) record.PutReference(key, value) } @@ -246,19 +246,19 @@ func (reader *RecordReaderCSV) getRecordBatch( n := lib.IntMin2(nh, nd) for i = 0; i < n; i++ { key := reader.header[i] - value := types.MlrvalFromInferredTypeForDataFiles(csvRecord[i]) + value := mlrval.FromDeferredType(csvRecord[i]) record.PutReference(key, value) } if nh < nd { // if header shorter than data: use 1-up itoa keys key := strconv.Itoa(i + 1) - value := types.MlrvalFromInferredTypeForDataFiles(csvRecord[i]) + value := mlrval.FromDeferredType(csvRecord[i]) record.PutCopy(key, value) } if nh > nd { // if header longer than data: use "" values for i = nd; i < nh; i++ { - record.PutCopy(reader.header[i], types.MLRVAL_VOID) + record.PutCopy(reader.header[i], mlrval.VOID) } } } diff --git a/internal/pkg/input/record_reader_csvlite.go b/internal/pkg/input/record_reader_csvlite.go index b9f1c65d2..7fc1f4f01 100644 --- a/internal/pkg/input/record_reader_csvlite.go +++ b/internal/pkg/input/record_reader_csvlite.go @@ -245,7 +245,7 @@ func getRecordBatchExplicitCSVHeader( record := types.NewMlrmapAsRecord() if !reader.readerOptions.AllowRaggedCSVInput { for i, field := range fields { - value := types.MlrvalFromInferredTypeForDataFiles(field) + value := mlrval.FromDeferredType(field) record.PutCopy(reader.headerStrings[i], value) } } else { @@ -254,21 +254,21 @@ func getRecordBatchExplicitCSVHeader( n := lib.IntMin2(nh, nd) var i int for i = 0; i < n; i++ { - value := types.MlrvalFromInferredTypeForDataFiles(fields[i]) + value := mlrval.FromDeferredType(fields[i]) record.PutCopy(reader.headerStrings[i], value) } if nh < nd { // if header shorter than data: use 1-up itoa keys for i = nh; i < nd; i++ { key := strconv.Itoa(i + 1) - value := types.MlrvalFromInferredTypeForDataFiles(fields[i]) + value := mlrval.FromDeferredType(fields[i]) record.PutCopy(key, value) } } if nh > nd { // if header longer than data: use "" values for i = nd; i < nh; i++ { - record.PutCopy(reader.headerStrings[i], types.MLRVAL_VOID) + record.PutCopy(reader.headerStrings[i], mlrval.VOID) } } } @@ -362,7 +362,7 @@ func getRecordBatchImplicitCSVHeader( record := types.NewMlrmapAsRecord() if !reader.readerOptions.AllowRaggedCSVInput { for i, field := range fields { - value := types.MlrvalFromInferredTypeForDataFiles(field) + value := mlrval.FromDeferredType(field) record.PutCopy(reader.headerStrings[i], value) } } else { @@ -371,19 +371,19 @@ func getRecordBatchImplicitCSVHeader( n := lib.IntMin2(nh, nd) var i int for i = 0; i < n; i++ { - value := types.MlrvalFromInferredTypeForDataFiles(fields[i]) + value := mlrval.FromDeferredType(fields[i]) record.PutCopy(reader.headerStrings[i], value) } if nh < nd { // if header shorter than data: use 1-up itoa keys key := strconv.Itoa(i + 1) - value := types.MlrvalFromInferredTypeForDataFiles(fields[i]) + value := mlrval.FromDeferredType(fields[i]) record.PutCopy(key, value) } if nh > nd { // if header longer than data: use "" values for i = nd; i < nh; i++ { - record.PutCopy(reader.headerStrings[i], types.MLRVAL_VOID) + record.PutCopy(reader.headerStrings[i], mlrval.VOID) } } } diff --git a/internal/pkg/input/record_reader_dkvp.go b/internal/pkg/input/record_reader_dkvp.go new file mode 100644 index 000000000..47a46bcf9 --- /dev/null +++ b/internal/pkg/input/record_reader_dkvp.go @@ -0,0 +1,149 @@ +package input + +import ( + "io" + "strconv" + "strings" + + "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" + "github.com/johnkerl/miller/internal/pkg/types" +) + +type RecordReaderDKVP struct { + readerOptions *cli.TReaderOptions +} + +func NewRecordReaderDKVP(readerOptions *cli.TReaderOptions) (*RecordReaderDKVP, error) { + return &RecordReaderDKVP{ + readerOptions: readerOptions, + }, nil +} + +func (reader *RecordReaderDKVP) Read( + filenames []string, + context types.Context, + readerChannel chan<- *types.RecordAndContext, + errorChannel chan error, + downstreamDoneChannel <-chan bool, // for mlr head +) { + if filenames != nil { // nil for mlr -n + if len(filenames) == 0 { // read from stdin + handle, err := lib.OpenStdin( + reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, + reader.readerOptions.FileInputEncoding, + ) + if err != nil { + errorChannel <- err + } + reader.processHandle(handle, "(stdin)", &context, readerChannel, errorChannel, downstreamDoneChannel) + } else { + for _, filename := range filenames { + handle, err := lib.OpenFileForRead( + filename, + reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, + reader.readerOptions.FileInputEncoding, + ) + if err != nil { + errorChannel <- err + } else { + reader.processHandle(handle, filename, &context, readerChannel, errorChannel, downstreamDoneChannel) + handle.Close() + } + } + } + } + readerChannel <- types.NewEndOfStreamMarker(&context) +} + +func (reader *RecordReaderDKVP) processHandle( + handle io.Reader, + filename string, + context *types.Context, + readerChannel chan<- *types.RecordAndContext, + errorChannel chan error, + downstreamDoneChannel <-chan bool, // for mlr head +) { + context.UpdateForStartOfFile(filename) + + scanner := NewLineScanner(handle, reader.readerOptions.IRS) + for scanner.Scan() { + + // See if downstream processors will be ignoring further data (e.g. mlr + // head). If so, stop reading. This makes 'mlr head hugefile' exit + // quickly, as it should. + eof := false + select { + case _ = <-downstreamDoneChannel: + eof = true + break + default: + break + } + if eof { + break + } + + line := scanner.Text() + + // Check for comments-in-data feature + if strings.HasPrefix(line, reader.readerOptions.CommentString) { + if reader.readerOptions.CommentHandling == cli.PassComments { + readerChannel <- types.NewOutputString(line+"\n", context) + continue + } else if reader.readerOptions.CommentHandling == cli.SkipComments { + continue + } + // else comments are data + } + + record := reader.recordFromDKVPLine(line) + context.UpdateForInputRecord() + readerChannel <- types.NewRecordAndContext( + record, + context, + ) + } +} + +// ---------------------------------------------------------------- +func (reader *RecordReaderDKVP) recordFromDKVPLine( + line string, +) *types.Mlrmap { + record := types.NewMlrmap() + + var pairs []string + if reader.readerOptions.IFSRegex == nil { // e.g. --no-ifs-regex + pairs = lib.SplitString(line, reader.readerOptions.IFS) + } else { + pairs = lib.RegexSplitString(reader.readerOptions.IFSRegex, line, -1) + } + + for i, pair := range pairs { + var kv []string + if reader.readerOptions.IPSRegex == nil { // e.g. --no-ips-regex + kv = strings.SplitN(pair, reader.readerOptions.IPS, 2) + } else { + kv = lib.RegexSplitString(reader.readerOptions.IPSRegex, pair, 2) + } + + if len(kv) == 0 || (len(kv) == 1 && kv[0] == "") { + // Ignore. This is expected when splitting with repeated IFS. + } else if len(kv) == 1 { + // E.g the pair has no equals sign: "a" rather than "a=1" or + // "a=". Here we use the positional index as the key. This way + // DKVP is a generalization of NIDX. + key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up + value := mlrval.FromInferredType(kv[0]) + record.PutReference(key, value) + } else { + key := kv[0] + value := mlrval.FromInferredType(kv[1]) + record.PutReference(key, value) + } + } + return record +} diff --git a/internal/pkg/input/record_reader_dkvp_test.go b/internal/pkg/input/record_reader_dkvp_test.go new file mode 100644 index 000000000..00b89c73d --- /dev/null +++ b/internal/pkg/input/record_reader_dkvp_test.go @@ -0,0 +1,59 @@ +package input + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/johnkerl/miller/internal/pkg/cli" +) + +func TestRecordFromDKVPLine(t *testing.T) { + readerOptions := cli.DefaultReaderOptions() + cli.FinalizeReaderOptions(&readerOptions) // compute IPS, IFS -> IPSRegex, IFSRegex + reader, err := NewRecordReaderDKVP(&readerOptions) + assert.NotNil(t, reader) + assert.Nil(t, err) + + line := "" + record := reader.recordFromDKVPLine(line) + assert.NotNil(t, record) + assert.Equal(t, 0, record.FieldCount) + + line = "a=1,b=2,c=3" + record = reader.recordFromDKVPLine(line) + assert.NotNil(t, record) + assert.Equal(t, 3, record.FieldCount) + + assert.NotNil(t, record.Head) + assert.NotNil(t, record.Head.Next) + assert.NotNil(t, record.Head.Next.Next) + assert.Nil(t, record.Head.Next.Next.Next) + assert.Equal(t, record.Head.Key, "a") + assert.Equal(t, record.Head.Next.Key, "b") + assert.Equal(t, record.Head.Next.Next.Key, "c") + + line = "a=1,b=2,b=3" + record = reader.recordFromDKVPLine(line) + assert.NotNil(t, record) + assert.Equal(t, 2, record.FieldCount) + + assert.NotNil(t, record.Head) + assert.NotNil(t, record.Head.Next) + assert.Nil(t, record.Head.Next.Next) + assert.Equal(t, record.Head.Key, "a") + assert.Equal(t, record.Head.Next.Key, "b") + + line = "a,b,c" + record = reader.recordFromDKVPLine(line) + assert.NotNil(t, record) + assert.Equal(t, 3, record.FieldCount) + + assert.NotNil(t, record.Head) + assert.NotNil(t, record.Head.Next) + assert.NotNil(t, record.Head.Next.Next) + assert.Nil(t, record.Head.Next.Next.Next) + assert.Equal(t, record.Head.Key, "1") + assert.Equal(t, record.Head.Next.Key, "2") + assert.Equal(t, record.Head.Next.Next.Key, "3") +} diff --git a/internal/pkg/input/record_reader_json.go b/internal/pkg/input/record_reader_json.go index 13f76222e..7084e95c8 100644 --- a/internal/pkg/input/record_reader_json.go +++ b/internal/pkg/input/record_reader_json.go @@ -12,6 +12,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -107,7 +108,7 @@ func (reader *RecordReaderJSON) processHandle( } } - mlrval, eof, err := types.MlrvalDecodeFromJSON(decoder) + mlrval, eof, err := mlrval.MlrvalDecodeFromJSON(decoder) if eof { break } diff --git a/internal/pkg/input/record_reader_nidx.go b/internal/pkg/input/record_reader_nidx.go new file mode 100644 index 000000000..5ae729d26 --- /dev/null +++ b/internal/pkg/input/record_reader_nidx.go @@ -0,0 +1,138 @@ +package input + +import ( + "io" + "strconv" + "strings" + + "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/types" +) + +type RecordReaderNIDX struct { + readerOptions *cli.TReaderOptions +} + +func NewRecordReaderNIDX(readerOptions *cli.TReaderOptions) (*RecordReaderNIDX, error) { + return &RecordReaderNIDX{ + readerOptions: readerOptions, + }, nil +} + +func (reader *RecordReaderNIDX) Read( + filenames []string, + context types.Context, + readerChannel chan<- *types.RecordAndContext, + errorChannel chan error, + downstreamDoneChannel <-chan bool, // for mlr head +) { + if filenames != nil { // nil for mlr -n + if len(filenames) == 0 { // read from stdin + handle, err := lib.OpenStdin( + reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, + reader.readerOptions.FileInputEncoding, + ) + if err != nil { + errorChannel <- err + } + reader.processHandle(handle, "(stdin)", &context, readerChannel, errorChannel, downstreamDoneChannel) + } else { + for _, filename := range filenames { + handle, err := lib.OpenFileForRead( + filename, + reader.readerOptions.Prepipe, + reader.readerOptions.PrepipeIsRaw, + reader.readerOptions.FileInputEncoding, + ) + if err != nil { + errorChannel <- err + } else { + reader.processHandle(handle, filename, &context, readerChannel, errorChannel, downstreamDoneChannel) + handle.Close() + } + } + } + } + readerChannel <- types.NewEndOfStreamMarker(&context) +} + +func (reader *RecordReaderNIDX) processHandle( + handle io.Reader, + filename string, + context *types.Context, + readerChannel chan<- *types.RecordAndContext, + errorChannel chan error, + downstreamDoneChannel <-chan bool, // for mlr head +) { + context.UpdateForStartOfFile(filename) + + scanner := NewLineScanner(handle, reader.readerOptions.IRS) + + for scanner.Scan() { + + // See if downstream processors will be ignoring further data (e.g. mlr + // head). If so, stop reading. This makes 'mlr head hugefile' exit + // quickly, as it should. + eof := false + select { + case _ = <-downstreamDoneChannel: + eof = true + break + default: + break + } + if eof { + break + } + + // TODO: IRS + line := scanner.Text() + + // Check for comments-in-data feature + if strings.HasPrefix(line, reader.readerOptions.CommentString) { + if reader.readerOptions.CommentHandling == cli.PassComments { + readerChannel <- types.NewOutputString(line+"\n", context) + continue + } else if reader.readerOptions.CommentHandling == cli.SkipComments { + continue + } + // else comments are data + } + + record := reader.recordFromNIDXLine(line) + + context.UpdateForInputRecord() + readerChannel <- types.NewRecordAndContext( + record, + context, + ) + } +} + +// ---------------------------------------------------------------- +func (reader *RecordReaderNIDX) recordFromNIDXLine( + line string, +) *types.Mlrmap { + record := types.NewMlrmap() + + var values []string + if reader.readerOptions.IFSRegex == nil { // e.g. --no-ifs-regex + values = lib.SplitString(line, reader.readerOptions.IFS) + } else { + values = lib.RegexSplitString(reader.readerOptions.IFSRegex, line, -1) + } + if reader.readerOptions.AllowRepeatIFS { + values = lib.StripEmpties(values) // left/right trim + } + + var i int = 0 + for _, value := range values { + i++ + key := strconv.Itoa(i) + mval := mlrval.FromDeferredType(value) + record.PutReference(key, mval) + } + return record +} diff --git a/internal/pkg/input/record_reader_xtab.go b/internal/pkg/input/record_reader_xtab.go index 8213916ae..dfd6c5eca 100644 --- a/internal/pkg/input/record_reader_xtab.go +++ b/internal/pkg/input/record_reader_xtab.go @@ -272,10 +272,10 @@ func (reader *RecordReaderXTAB) recordFromXTABLines( key := kv[0] if len(kv) == 1 { - value := types.MLRVAL_VOID + value := mlrval.VOID record.PutReference(key, value) } else { - value := types.MlrvalFromInferredTypeForDataFiles(kv[1]) + value := mlrval.FromDeferredType(kv[1]) record.PutReference(key, value) } } diff --git a/internal/pkg/mlrval/constants.go b/internal/pkg/mlrval/constants.go index 1fe36ebc1..bbc6dbe1d 100644 --- a/internal/pkg/mlrval/constants.go +++ b/internal/pkg/mlrval/constants.go @@ -6,13 +6,13 @@ package mlrval // MlrvalFromPending is designed solely for the JSON API, for something // intended to be mutated after construction once its type is (later) known. -// Whereas MLRVAL_ERROR, MLRVAL_ABSENT, etc are all singletons, this one +// Whereas ERROR, ABSENT, etc are all singletons, this one // must be mutable and therefor non-singleton. func MlrvalFromPending() Mlrval { return Mlrval{ - mvtype: MT_PENDING, - printrep: INVALID_PRINTREP, + mvtype: MT_PENDING, + printrep: INVALID_PRINTREP, } } @@ -21,38 +21,38 @@ func MlrvalFromPending() Mlrval { // just using `const` isn't enough) but the gentelpersons' agreement is that // the caller should never modify these. -var MLRVAL_ERROR = &Mlrval{ +var ERROR = &Mlrval{ mvtype: MT_ERROR, printrep: ERROR_PRINTREP, printrepValid: true, } -var MLRVAL_ABSENT = &Mlrval{ +var ABSENT = &Mlrval{ mvtype: MT_ABSENT, - printrep: ABSENT_PRINTREP, + printrep: ABSENT_PRINTREP, printrepValid: true, } -var MLRVAL_NULL = &Mlrval{ +var NULL = &Mlrval{ mvtype: MT_NULL, printrep: "null", printrepValid: true, } -var MLRVAL_VOID = &Mlrval{ +var VOID = &Mlrval{ mvtype: MT_VOID, printrep: "", printrepValid: true, } -var MLRVAL_TRUE = &Mlrval{ +var TRUE = &Mlrval{ mvtype: MT_BOOL, printrep: "true", printrepValid: true, boolval: true, } -var MLRVAL_FALSE = &Mlrval{ +var FALSE = &Mlrval{ mvtype: MT_BOOL, printrep: "false", printrepValid: true, diff --git a/internal/pkg/mlrval/is.go b/internal/pkg/mlrval/is.go index e40529246..b154ec957 100644 --- a/internal/pkg/mlrval/is.go +++ b/internal/pkg/mlrval/is.go @@ -101,4 +101,3 @@ func (mv *Mlrval) IsArrayOrMap() bool { func (mv *Mlrval) IsFunction() bool { return mv.Type() == MT_FUNC } - diff --git a/internal/pkg/mlrval/is_test.go b/internal/pkg/mlrval/is_test.go index 4e4eaec42..6b4d9232e 100644 --- a/internal/pkg/mlrval/is_test.go +++ b/internal/pkg/mlrval/is_test.go @@ -11,9 +11,9 @@ import ( ) func TestIsLegit(t *testing.T) { - assert.False(t, MLRVAL_ERROR.IsLegit()) - assert.False(t, MLRVAL_ABSENT.IsLegit()) - assert.False(t, MLRVAL_NULL.IsLegit()) + assert.False(t, ERROR.IsLegit()) + assert.False(t, ABSENT.IsLegit()) + assert.False(t, NULL.IsLegit()) assert.True(t, FromString("").IsLegit()) assert.True(t, FromString("abc").IsLegit()) assert.True(t, FromInt(123).IsLegit()) @@ -24,37 +24,37 @@ func TestIsLegit(t *testing.T) { } func TestIsErrorOrAbsent(t *testing.T) { - assert.True(t, MLRVAL_ERROR.IsErrorOrAbsent()) - assert.True(t, MLRVAL_ABSENT.IsErrorOrAbsent()) - assert.False(t, MLRVAL_NULL.IsErrorOrAbsent()) + assert.True(t, ERROR.IsErrorOrAbsent()) + assert.True(t, ABSENT.IsErrorOrAbsent()) + assert.False(t, NULL.IsErrorOrAbsent()) assert.False(t, FromString("").IsErrorOrAbsent()) } func TestIsError(t *testing.T) { - assert.True(t, MLRVAL_ERROR.IsError()) - assert.False(t, MLRVAL_ABSENT.IsError()) - assert.False(t, MLRVAL_NULL.IsError()) + assert.True(t, ERROR.IsError()) + assert.False(t, ABSENT.IsError()) + assert.False(t, NULL.IsError()) assert.False(t, FromString("").IsError()) } func TestIsAbsent(t *testing.T) { - assert.False(t, MLRVAL_ERROR.IsAbsent()) - assert.True(t, MLRVAL_ABSENT.IsAbsent()) - assert.False(t, MLRVAL_NULL.IsAbsent()) + assert.False(t, ERROR.IsAbsent()) + assert.True(t, ABSENT.IsAbsent()) + assert.False(t, NULL.IsAbsent()) assert.False(t, FromString("").IsAbsent()) } func TestIsNull(t *testing.T) { - assert.False(t, MLRVAL_ERROR.IsNull()) - assert.False(t, MLRVAL_ABSENT.IsNull()) - assert.True(t, MLRVAL_NULL.IsNull()) + assert.False(t, ERROR.IsNull()) + assert.False(t, ABSENT.IsNull()) + assert.True(t, NULL.IsNull()) assert.False(t, FromString("").IsNull()) } func TestIsVoid(t *testing.T) { - assert.False(t, MLRVAL_ERROR.IsVoid()) - assert.False(t, MLRVAL_ABSENT.IsVoid()) - assert.False(t, MLRVAL_NULL.IsVoid()) + assert.False(t, ERROR.IsVoid()) + assert.False(t, ABSENT.IsVoid()) + assert.False(t, NULL.IsVoid()) assert.True(t, FromString("").IsVoid()) assert.True(t, FromDeferredType("").IsVoid()) assert.True(t, FromInferredType("").IsVoid()) @@ -63,9 +63,9 @@ func TestIsVoid(t *testing.T) { } func TestIsEmptyString(t *testing.T) { - assert.False(t, MLRVAL_ERROR.IsEmptyString()) - assert.False(t, MLRVAL_ABSENT.IsEmptyString()) - assert.False(t, MLRVAL_NULL.IsEmptyString()) + assert.False(t, ERROR.IsEmptyString()) + assert.False(t, ABSENT.IsEmptyString()) + assert.False(t, NULL.IsEmptyString()) assert.True(t, FromString("").IsEmptyString()) assert.True(t, FromDeferredType("").IsEmptyString()) assert.True(t, FromInferredType("").IsEmptyString()) @@ -74,9 +74,9 @@ func TestIsEmptyString(t *testing.T) { } func TestIsString(t *testing.T) { - assert.False(t, MLRVAL_ERROR.IsString()) - assert.False(t, MLRVAL_ABSENT.IsString()) - assert.False(t, MLRVAL_NULL.IsString()) + assert.False(t, ERROR.IsString()) + assert.False(t, ABSENT.IsString()) + assert.False(t, NULL.IsString()) assert.False(t, FromString("").IsString()) assert.False(t, FromDeferredType("").IsString()) assert.False(t, FromInferredType("").IsString()) @@ -85,9 +85,9 @@ func TestIsString(t *testing.T) { } func TestIsStringOrVoid(t *testing.T) { - assert.False(t, MLRVAL_ERROR.IsStringOrVoid()) - assert.False(t, MLRVAL_ABSENT.IsStringOrVoid()) - assert.False(t, MLRVAL_NULL.IsStringOrVoid()) + assert.False(t, ERROR.IsStringOrVoid()) + assert.False(t, ABSENT.IsStringOrVoid()) + assert.False(t, NULL.IsStringOrVoid()) assert.True(t, FromString("").IsStringOrVoid()) assert.True(t, FromDeferredType("").IsStringOrVoid()) assert.True(t, FromInferredType("").IsStringOrVoid()) diff --git a/internal/pkg/mlrval/new.go b/internal/pkg/mlrval/new.go index 585effc22..27d9f0089 100644 --- a/internal/pkg/mlrval/new.go +++ b/internal/pkg/mlrval/new.go @@ -31,13 +31,13 @@ func FromInferredType(input string) *Mlrval { printrepValid: true, } // TODO: comment re inferBool arg - packageLevelInferrer(mv, mv.printrep, true) + packageLevelInferrer(mv, mv.printrep, true) return mv } func FromString(input string) *Mlrval { if input == "" { - return MLRVAL_VOID + return VOID } return &Mlrval{ mvtype: MT_STRING, @@ -100,17 +100,17 @@ func (mv *Mlrval) SetFromPrevalidatedFloatString(input string, floatval float64) func FromBool(input bool) *Mlrval { if input == true { - return MLRVAL_TRUE + return TRUE } else { - return MLRVAL_FALSE + return FALSE } } func FromBoolString(input string) *Mlrval { if input == "true" { - return MLRVAL_TRUE + return TRUE } else if input == "false" { - return MLRVAL_FALSE + return FALSE } else { lib.InternalCodingErrorIf(true) return nil // not reached diff --git a/internal/pkg/mlrval/tbf.go b/internal/pkg/mlrval/tbf.go index 290cc306f..3ee116d72 100644 --- a/internal/pkg/mlrval/tbf.go +++ b/internal/pkg/mlrval/tbf.go @@ -25,7 +25,6 @@ package mlrval // } //} - //func MlrvalFromEmptyMap() *Mlrval { // return &Mlrval{ // mvtype: MT_MAP, @@ -39,7 +38,7 @@ package mlrval // mv := MlrvalFromEmptyMap() // if mlrmap == nil { // // TODO maybe return 2nd-arg error in the API -// return MLRVAL_ERROR +// return ERROR // } // // for pe := mlrmap.Head; pe != nil; pe = pe.Next { @@ -54,7 +53,7 @@ package mlrval // mv := MlrvalFromEmptyMap() // if mlrmap == nil { // // xxx maybe return 2nd-arg error in the API -// return MLRVAL_ERROR +// return ERROR // } // // for pe := mlrmap.Head; pe != nil; pe = pe.Next { @@ -84,7 +83,7 @@ package mlrval // arrayval := make([]Mlrval, length, 2*length) // // for i := 0; i < int(length); i++ { -// arrayval[i] = *MLRVAL_VOID +// arrayval[i] = *VOID // } // // return &Mlrval{ @@ -117,7 +116,7 @@ package mlrval // newArray := (*array)[:newLength] // for zindex := len(*array); zindex < newLength; zindex++ { // // TODO: comment why not MT_ABSENT or MT_VOID -// newArray[zindex] = *MLRVAL_NULL +// newArray[zindex] = *NULL // } // *array = newArray // } else { @@ -128,7 +127,7 @@ package mlrval // } // for zindex = len(*array); zindex < newLength; zindex++ { // // TODO: comment why not MT_ABSENT or MT_VOID -// newArray[zindex] = *MLRVAL_NULL +// newArray[zindex] = *NULL // } // *array = newArray // } diff --git a/internal/pkg/mlrval/type.go b/internal/pkg/mlrval/type.go index 8430c15fc..a43e5a010 100644 --- a/internal/pkg/mlrval/type.go +++ b/internal/pkg/mlrval/type.go @@ -182,4 +182,3 @@ var typeNameToMaskMap = map[string]int{ "any": MT_TYPE_MASK_ANY, "funct": MT_TYPE_MASK_FUNC, } - diff --git a/internal/pkg/runtime/stack.go b/internal/pkg/runtime/stack.go index 6ad9347be..038c107e2 100644 --- a/internal/pkg/runtime/stack.go +++ b/internal/pkg/runtime/stack.go @@ -31,6 +31,7 @@ import ( "fmt" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -114,7 +115,7 @@ func (stack *Stack) PopStackFrame() { // Returns nil on no-such func (stack *Stack) Get( stackVariable *StackVariable, -) *types.Mlrval { +) *mlrval.Mlrval { return stack.head.get(stackVariable) } @@ -124,7 +125,7 @@ func (stack *Stack) Get( func (stack *Stack) DefineTypedAtScope( stackVariable *StackVariable, typeName string, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { return stack.head.defineTypedAtScope(stackVariable, typeName, mlrval) } @@ -136,7 +137,7 @@ func (stack *Stack) DefineTypedAtScope( // E.g. 'for (i = 0; i < 10; i += 1)' uses Set. func (stack *Stack) SetAtScope( stackVariable *StackVariable, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { return stack.head.setAtScope(stackVariable, mlrval) } @@ -150,7 +151,7 @@ func (stack *Stack) SetAtScope( // assignment is OK. func (stack *Stack) Set( stackVariable *StackVariable, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { return stack.head.set(stackVariable, mlrval) } @@ -158,8 +159,8 @@ func (stack *Stack) Set( // E.g. 'x[1] = 2' where the variable x may or may not have been already set. func (stack *Stack) SetIndexed( stackVariable *StackVariable, - indices []*types.Mlrval, - mlrval *types.Mlrval, + indices []*mlrval.Mlrval, + mlrval *mlrval.Mlrval, ) error { return stack.head.setIndexed(stackVariable, indices, mlrval) } @@ -174,7 +175,7 @@ func (stack *Stack) Unset( // E.g. 'unset x[1]' func (stack *Stack) UnsetIndexed( stackVariable *StackVariable, - indices []*types.Mlrval, + indices []*mlrval.Mlrval, ) { stack.head.unsetIndexed(stackVariable, indices) } @@ -225,7 +226,7 @@ func (frameset *StackFrameSet) dump() { // Returns nil on no-such func (frameset *StackFrameSet) get( stackVariable *StackVariable, -) *types.Mlrval { +) *mlrval.Mlrval { // Scope-walk numStackFrames := len(frameset.stackFrames) for offset := numStackFrames - 1; offset >= 0; offset-- { @@ -242,7 +243,7 @@ func (frameset *StackFrameSet) get( func (frameset *StackFrameSet) defineTypedAtScope( stackVariable *StackVariable, typeName string, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { offset := len(frameset.stackFrames) - 1 // TODO: comment @@ -254,7 +255,7 @@ func (frameset *StackFrameSet) defineTypedAtScope( // See Stack.SetAtScope comments above func (frameset *StackFrameSet) setAtScope( stackVariable *StackVariable, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { offset := len(frameset.stackFrames) - 1 return frameset.stackFrames[offset].set(stackVariable, mlrval) @@ -263,7 +264,7 @@ func (frameset *StackFrameSet) setAtScope( // See Stack.Set comments above func (frameset *StackFrameSet) set( stackVariable *StackVariable, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { // Scope-walk numStackFrames := len(frameset.stackFrames) @@ -279,8 +280,8 @@ func (frameset *StackFrameSet) set( // See Stack.SetIndexed comments above func (frameset *StackFrameSet) setIndexed( stackVariable *StackVariable, - indices []*types.Mlrval, - mlrval *types.Mlrval, + indices []*mlrval.Mlrval, + mlrval *mlrval.Mlrval, ) error { // Scope-walk numStackFrames := len(frameset.stackFrames) @@ -312,7 +313,7 @@ func (frameset *StackFrameSet) unset( // See Stack.UnsetIndexed comments above func (frameset *StackFrameSet) unsetIndexed( stackVariable *StackVariable, - indices []*types.Mlrval, + indices []*mlrval.Mlrval, ) { // Scope-walk numStackFrames := len(frameset.stackFrames) @@ -353,7 +354,7 @@ func newStackFrame() *StackFrame { // Returns nil on no such func (frame *StackFrame) get( stackVariable *StackVariable, -) *types.Mlrval { +) *mlrval.Mlrval { offset, ok := frame.namesToOffsets[stackVariable.name] if ok { return frame.vars[offset].GetValue() @@ -372,7 +373,7 @@ func (frame *StackFrame) has( // TODO: audit for honor of error-return at callsites func (frame *StackFrame) set( stackVariable *StackVariable, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { offset, ok := frame.namesToOffsets[stackVariable.name] if !ok { @@ -393,7 +394,7 @@ func (frame *StackFrame) set( func (frame *StackFrame) defineTyped( stackVariable *StackVariable, typeName string, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { _, ok := frame.namesToOffsets[stackVariable.name] if !ok { @@ -418,15 +419,15 @@ func (frame *StackFrame) defineTyped( // TODO: audit for honor of error-return at callsites func (frame *StackFrame) setIndexed( stackVariable *StackVariable, - indices []*types.Mlrval, - mlrval *types.Mlrval, + indices []*mlrval.Mlrval, + mlrval *mlrval.Mlrval, ) error { value := frame.get(stackVariable) if value == nil { lib.InternalCodingErrorIf(len(indices) < 1) leadingIndex := indices[0] if leadingIndex.IsString() || leadingIndex.IsInt() { - newval := types.MlrvalFromEmptyMap() + newval := mlrval.MlrvalFromEmptyMap() newval.PutIndexed(indices, mlrval) return frame.set(stackVariable, newval) } else { @@ -455,7 +456,7 @@ func (frame *StackFrame) unset( func (frame *StackFrame) unsetIndexed( stackVariable *StackVariable, - indices []*types.Mlrval, + indices []*mlrval.Mlrval, ) { value := frame.get(stackVariable) if value == nil { diff --git a/internal/pkg/runtime/state.go b/internal/pkg/runtime/state.go index 19b11414c..e7f725412 100644 --- a/internal/pkg/runtime/state.go +++ b/internal/pkg/runtime/state.go @@ -11,16 +11,19 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) type State struct { + Inrec *types.Mlrmap Context *types.Context Oosvars *types.Mlrmap FilterExpression *types.Mlrval Stack *Stack OutputRecordsAndContexts *list.List // list of *types.RecordAndContext + // For holding "\0".."\9" between where they are set via things like // '$x =~ "(..)_(...)"', and interpolated via things like '$y = "\2:\1"'. RegexCaptures []string diff --git a/internal/pkg/transformers/bar.go b/internal/pkg/transformers/bar.go index 993895b22..70a15440e 100644 --- a/internal/pkg/transformers/bar.go +++ b/internal/pkg/transformers/bar.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -251,7 +252,7 @@ func (tr *TransformerBar) processNoAuto( if idx > tr.width { idx = tr.width } - inrec.PutReference(fieldName, types.MlrvalFromString(tr.bars[idx])) + inrec.PutReference(fieldName, mlrval.MlrvalFromString(tr.bars[idx])) } outputRecordsAndContexts.PushBack(inrecAndContext) @@ -339,7 +340,7 @@ func (tr *TransformerBar) processAuto( buffer.WriteString("[") buffer.WriteString(shi) buffer.WriteString("]") - record.PutReference(fieldName, types.MlrvalFromString(buffer.String())) + record.PutReference(fieldName, mlrval.MlrvalFromString(buffer.String())) } } diff --git a/internal/pkg/transformers/cat.go b/internal/pkg/transformers/cat.go index 8a0a5919a..3865a578c 100644 --- a/internal/pkg/transformers/cat.go +++ b/internal/pkg/transformers/cat.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -178,7 +179,7 @@ func (tr *TransformerCat) countersUngrouped( inrec := inrecAndContext.Record tr.counter++ key := tr.counterFieldName - inrec.PrependCopy(key, types.MlrvalFromInt(tr.counter)) + inrec.PrependCopy(key, mlrval.MlrvalFromInt(tr.counter)) } outputRecordsAndContexts.PushBack(inrecAndContext) } @@ -210,7 +211,7 @@ func (tr *TransformerCat) countersGrouped( } key := tr.counterFieldName - inrec.PrependCopy(key, types.MlrvalFromInt(counter)) + inrec.PrependCopy(key, mlrval.MlrvalFromInt(counter)) } outputRecordsAndContexts.PushBack(inrecAndContext) } diff --git a/internal/pkg/transformers/clean-whitespace.go b/internal/pkg/transformers/clean-whitespace.go index 85f349759..4e6b236db 100644 --- a/internal/pkg/transformers/clean-whitespace.go +++ b/internal/pkg/transformers/clean-whitespace.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -152,7 +153,7 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeysAndValues( newrec := types.NewMlrmapAsRecord() for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { - oldKey := types.MlrvalFromString(pe.Key) + oldKey := mlrval.MlrvalFromString(pe.Key) // xxx temp newKey := types.BIF_clean_whitespace(oldKey) newValue := types.BIF_clean_whitespace(pe.Value) @@ -177,7 +178,7 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeys( newrec := types.NewMlrmapAsRecord() for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { - oldKey := types.MlrvalFromString(pe.Key) + oldKey := mlrval.MlrvalFromString(pe.Key) newKey := types.BIF_clean_whitespace(oldKey) // Transferring ownership from old record to new record; no copy needed newrec.PutReference(newKey.String(), pe.Value) diff --git a/internal/pkg/transformers/count-similar.go b/internal/pkg/transformers/count-similar.go index 39d381c64..8e94fc629 100644 --- a/internal/pkg/transformers/count-similar.go +++ b/internal/pkg/transformers/count-similar.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -154,7 +155,7 @@ func (tr *TransformerCountSimilar) Transform( recordListForGroup := outer.Value.(*list.List) // TODO: make 64-bit friendly groupSize := recordListForGroup.Len() - mgroupSize := types.MlrvalFromInt(int(groupSize)) + mgroupSize := mlrval.MlrvalFromInt(int(groupSize)) for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() { recordAndContext := inner.Value.(*types.RecordAndContext) recordAndContext.Record.PutCopy(tr.counterFieldName, mgroupSize) diff --git a/internal/pkg/transformers/count.go b/internal/pkg/transformers/count.go index 02ddd80fe..83a79ad87 100644 --- a/internal/pkg/transformers/count.go +++ b/internal/pkg/transformers/count.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -172,7 +173,7 @@ func (tr *TransformerCount) countUngrouped( tr.ungroupedCount++ } else { newrec := types.NewMlrmapAsRecord() - newrec.PutCopy(tr.outputFieldName, types.MlrvalFromInt(tr.ungroupedCount)) + newrec.PutCopy(tr.outputFieldName, mlrval.MlrvalFromInt(tr.ungroupedCount)) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker @@ -210,7 +211,7 @@ func (tr *TransformerCount) countGrouped( } else { if tr.showCountsOnly { newrec := types.NewMlrmapAsRecord() - newrec.PutCopy(tr.outputFieldName, types.MlrvalFromInt(tr.groupedCounts.FieldCount)) + newrec.PutCopy(tr.outputFieldName, mlrval.MlrvalFromInt(tr.groupedCounts.FieldCount)) outrecAndContext := types.NewRecordAndContext(newrec, &inrecAndContext.Context) outputRecordsAndContexts.PushBack(outrecAndContext) @@ -227,7 +228,7 @@ func (tr *TransformerCount) countGrouped( // * Grouping values for key is ["foo", "bar"] // Here we populate a record with "a=foo,b=bar". - groupingValuesForKey := tr.groupingValues.Get(groupingKey).([]*types.Mlrval) + groupingValuesForKey := tr.groupingValues.Get(groupingKey).([]*mlrval.Mlrval) i := 0 for _, groupingValueForKey := range groupingValuesForKey { newrec.PutCopy(tr.groupByFieldNames[i], groupingValueForKey) @@ -235,7 +236,7 @@ func (tr *TransformerCount) countGrouped( } countForGroup := outer.Value.(int) - newrec.PutCopy(tr.outputFieldName, types.MlrvalFromInt(countForGroup)) + newrec.PutCopy(tr.outputFieldName, mlrval.MlrvalFromInt(countForGroup)) outrecAndContext := types.NewRecordAndContext(newrec, &inrecAndContext.Context) outputRecordsAndContexts.PushBack(outrecAndContext) diff --git a/internal/pkg/transformers/fill-down.go b/internal/pkg/transformers/fill-down.go index f96429a07..fc3b16789 100644 --- a/internal/pkg/transformers/fill-down.go +++ b/internal/pkg/transformers/fill-down.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -122,7 +123,7 @@ type TransformerFillDown struct { onlyIfAbsent bool // state - lastNonNullValues map[string]*types.Mlrval + lastNonNullValues map[string]*mlrval.Mlrval recordTransformerFunc RecordTransformerFunc } @@ -135,7 +136,7 @@ func NewTransformerFillDown( tr := &TransformerFillDown{ fillDownFieldNames: fillDownFieldNames, onlyIfAbsent: onlyIfAbsent, - lastNonNullValues: make(map[string]*types.Mlrval), + lastNonNullValues: make(map[string]*mlrval.Mlrval), } if doAll { diff --git a/internal/pkg/transformers/fill-empty.go b/internal/pkg/transformers/fill-empty.go index 420656bd1..2e619c7be 100644 --- a/internal/pkg/transformers/fill-empty.go +++ b/internal/pkg/transformers/fill-empty.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -93,7 +94,7 @@ func transformerFillEmptyParseCLI( // ---------------------------------------------------------------- type TransformerFillEmpty struct { - fillValue *types.Mlrval + fillValue *mlrval.Mlrval } func NewTransformerFillEmpty( @@ -102,9 +103,9 @@ func NewTransformerFillEmpty( ) (*TransformerFillEmpty, error) { tr := &TransformerFillEmpty{} if inferType { - tr.fillValue = types.MlrvalFromInferredType(fillString) + tr.fillValue = mlrval.MlrvalFromInferredType(fillString) } else { - tr.fillValue = types.MlrvalFromString(fillString) + tr.fillValue = mlrval.MlrvalFromString(fillString) } return tr, nil } diff --git a/internal/pkg/transformers/fraction.go b/internal/pkg/transformers/fraction.go index 75a330ba4..7d474257e 100644 --- a/internal/pkg/transformers/fraction.go +++ b/internal/pkg/transformers/fraction.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -135,12 +136,12 @@ type TransformerFraction struct { recordsAndContexts *list.List // Two-level map: Group-by field names are the first keyset; // fraction field names are keys into the second. - sums map[string]map[string]*types.Mlrval - cumus map[string]map[string]*types.Mlrval + sums map[string]map[string]*mlrval.Mlrval + cumus map[string]map[string]*mlrval.Mlrval - outputFieldNameSuffix string // "_fraction" or "_percent" - multiplier *types.Mlrval // 1.0 for fraction or 100.0 for percent - zero *types.Mlrval + outputFieldNameSuffix string // "_fraction" or "_percent" + multiplier *mlrval.Mlrval // 1.0 for fraction or 100.0 for percent + zero *mlrval.Mlrval } // ---------------------------------------------------------------- @@ -152,20 +153,20 @@ func NewTransformerFraction( ) (*TransformerFraction, error) { recordsAndContexts := list.New() - sums := make(map[string]map[string]*types.Mlrval) - cumus := make(map[string]map[string]*types.Mlrval) + sums := make(map[string]map[string]*mlrval.Mlrval) + cumus := make(map[string]map[string]*mlrval.Mlrval) - var multiplier *types.Mlrval + var multiplier *mlrval.Mlrval var outputFieldNameSuffix string if doPercents { - multiplier = types.MlrvalFromInt(100) + multiplier = mlrval.MlrvalFromInt(100) if doCumu { outputFieldNameSuffix = "_cumulative_percent" } else { outputFieldNameSuffix = "_percent" } } else { - multiplier = types.MlrvalFromInt(1) + multiplier = mlrval.MlrvalFromInt(1) if doCumu { outputFieldNameSuffix = "_cumulative_fraction" } else { @@ -173,7 +174,7 @@ func NewTransformerFraction( } } - zero := types.MlrvalFromInt(0) + zero := mlrval.MlrvalFromInt(0) return &TransformerFraction{ fractionFieldNames: fractionFieldNames, @@ -208,11 +209,11 @@ func (tr *TransformerFraction) Transform( if hasAll { sumsForGroup := tr.sums[groupingKey] - var cumusForGroup map[string]*types.Mlrval = nil + var cumusForGroup map[string]*mlrval.Mlrval = nil if sumsForGroup == nil { - sumsForGroup = make(map[string]*types.Mlrval) + sumsForGroup = make(map[string]*mlrval.Mlrval) tr.sums[groupingKey] = sumsForGroup - cumusForGroup = make(map[string]*types.Mlrval) + cumusForGroup = make(map[string]*mlrval.Mlrval) tr.cumus[groupingKey] = cumusForGroup } for _, fractionFieldName := range tr.fractionFieldNames { @@ -254,9 +255,9 @@ func (tr *TransformerFraction) Transform( if value != nil { value.AssertNumeric() // may fatal the process - var numerator *types.Mlrval = nil - var cumu *types.Mlrval = nil - var outputValue *types.Mlrval = nil + var numerator *mlrval.Mlrval = nil + var cumu *mlrval.Mlrval = nil + var outputValue *mlrval.Mlrval = nil if tr.doCumu { cumu = cumusForGroup[fractionFieldName] @@ -266,7 +267,7 @@ func (tr *TransformerFraction) Transform( } denominator := sumsForGroup[fractionFieldName] - if !types.MlrvalEqualsAsBool(value, tr.zero) { + if !mlrval.MlrvalEqualsAsBool(value, tr.zero) { outputValue = types.BIF_divide(numerator, denominator) outputValue = types.BIF_times(outputValue, tr.multiplier) } else { diff --git a/internal/pkg/transformers/histogram.go b/internal/pkg/transformers/histogram.go index 47f625b5f..83ba99d98 100644 --- a/internal/pkg/transformers/histogram.go +++ b/internal/pkg/transformers/histogram.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -261,17 +262,17 @@ func (tr *TransformerHistogram) emitNonAuto( outrec.PutReference( tr.outputPrefix+"bin_lo", - types.MlrvalFromFloat64((tr.lo+float64(i))/tr.mul), + mlrval.MlrvalFromFloat64((tr.lo+float64(i))/tr.mul), ) outrec.PutReference( tr.outputPrefix+"bin_hi", - types.MlrvalFromFloat64((tr.lo+float64(i+1))/tr.mul), + mlrval.MlrvalFromFloat64((tr.lo+float64(i+1))/tr.mul), ) for _, valueFieldName := range tr.valueFieldNames { outrec.PutReference( countFieldNames[valueFieldName], - types.MlrvalFromInt(tr.countsByField[valueFieldName][i]), + mlrval.MlrvalFromInt(tr.countsByField[valueFieldName][i]), ) } @@ -367,17 +368,17 @@ func (tr *TransformerHistogram) emitAuto( outrec.PutReference( tr.outputPrefix+"bin_lo", - types.MlrvalFromFloat64((lo+float64(i))/mul), + mlrval.MlrvalFromFloat64((lo+float64(i))/mul), ) outrec.PutReference( tr.outputPrefix+"bin_hi", - types.MlrvalFromFloat64((lo+float64(i+1))/mul), + mlrval.MlrvalFromFloat64((lo+float64(i+1))/mul), ) for _, valueFieldName := range tr.valueFieldNames { outrec.PutReference( countFieldNames[valueFieldName], - types.MlrvalFromInt(tr.countsByField[valueFieldName][i]), + mlrval.MlrvalFromInt(tr.countsByField[valueFieldName][i]), ) } diff --git a/internal/pkg/transformers/most-or-least-frequent.go b/internal/pkg/transformers/most-or-least-frequent.go index fb7a6c7de..c0e6bda07 100644 --- a/internal/pkg/transformers/most-or-least-frequent.go +++ b/internal/pkg/transformers/most-or-least-frequent.go @@ -9,6 +9,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -178,7 +179,7 @@ type TransformerMostOrLeastFrequent struct { outputFieldName string descending bool countsByGroup *lib.OrderedMap // map[string]int - valuesForGroup map[string][]*types.Mlrval + valuesForGroup map[string][]*mlrval.Mlrval } type tMostOrLeastFrequentSortPair struct { @@ -201,7 +202,7 @@ func NewTransformerMostOrLeastFrequent( outputFieldName: outputFieldName, descending: descending, countsByGroup: lib.NewOrderedMap(), - valuesForGroup: make(map[string][]*types.Mlrval), + valuesForGroup: make(map[string][]*mlrval.Mlrval), } return tr, nil @@ -284,7 +285,7 @@ func (tr *TransformerMostOrLeastFrequent) Transform( } if tr.showCounts { - outrec.PutReference(tr.outputFieldName, types.MlrvalFromInt(sortPairs[i].count)) + outrec.PutReference(tr.outputFieldName, mlrval.MlrvalFromInt(sortPairs[i].count)) } outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } diff --git a/internal/pkg/transformers/nest.go b/internal/pkg/transformers/nest.go index 71e7e7fa6..f1a81bfec 100644 --- a/internal/pkg/transformers/nest.go +++ b/internal/pkg/transformers/nest.go @@ -11,6 +11,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -335,7 +336,7 @@ func (tr *TransformerNest) explodeValuesAcrossFields( i := 1 for _, piece := range pieces { key := tr.fieldName + "_" + strconv.Itoa(i) - value := types.MlrvalFromString(piece) + value := mlrval.MlrvalFromString(piece) recordEntry = inrec.PutReferenceAfter(recordEntry, key, value) i++ } @@ -368,7 +369,7 @@ func (tr *TransformerNest) explodeValuesAcrossRecords( pieces := strings.Split(svalue, tr.nestedFS) for _, piece := range pieces { outrec := inrec.Copy() - outrec.PutReference(tr.fieldName, types.MlrvalFromString(piece)) + outrec.PutReference(tr.fieldName, mlrval.MlrvalFromString(piece)) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } @@ -404,13 +405,13 @@ func (tr *TransformerNest) explodePairsAcrossFields( recordEntry = inrec.PutReferenceAfter( recordEntry, pair[0], - types.MlrvalFromString(pair[1]), + mlrval.MlrvalFromString(pair[1]), ) } else { // there is not a pair recordEntry = inrec.PutReferenceAfter( recordEntry, tr.fieldName, - types.MlrvalFromString(piece), + mlrval.MlrvalFromString(piece), ) } } @@ -449,9 +450,9 @@ func (tr *TransformerNest) explodePairsAcrossRecords( // name, in which case replace its value. pair := strings.SplitN(piece, tr.nestedPS, 2) if len(pair) == 2 { // there is a pair - outrec.PutReferenceAfter(originalEntry, pair[0], types.MlrvalFromString(pair[1])) + outrec.PutReferenceAfter(originalEntry, pair[0], mlrval.MlrvalFromString(pair[1])) } else { // there is not a pair - outrec.PutReferenceAfter(originalEntry, tr.fieldName, types.MlrvalFromString(piece)) + outrec.PutReferenceAfter(originalEntry, tr.fieldName, mlrval.MlrvalFromString(piece)) } outrec.Unlink(originalEntry) @@ -497,7 +498,7 @@ func (tr *TransformerNest) implodeValuesAcrossFields( } if fieldCount > 0 { - newValue := types.MlrvalFromString(buffer.String()) + newValue := mlrval.MlrvalFromString(buffer.String()) if previousEntry == nil { // No record before the unlinked one, i.e. list-head. inrec.PrependReference(tr.fieldName, newValue) } else { @@ -576,7 +577,7 @@ func (tr *TransformerNest) implodeValueAcrossRecords( } // tr.fieldName was already present so we'll overwrite it in-place here. - outrec.PutReference(tr.fieldName, types.MlrvalFromString(buffer.String())) + outrec.PutReference(tr.fieldName, mlrval.MlrvalFromString(buffer.String())) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } } diff --git a/internal/pkg/transformers/put-or-filter.go b/internal/pkg/transformers/put-or-filter.go index 11d5755b4..17f09937f 100644 --- a/internal/pkg/transformers/put-or-filter.go +++ b/internal/pkg/transformers/put-or-filter.go @@ -11,6 +11,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/dsl/cst" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/parsing/lexer" "github.com/johnkerl/miller/internal/pkg/parsing/parser" "github.com/johnkerl/miller/internal/pkg/runtime" @@ -455,7 +456,7 @@ func NewTransformerPut( } key := pair[0] svalue := pair[1] - mvalue := types.MlrvalFromInferredType(svalue) + mvalue := mlrval.MlrvalFromInferredType(svalue) runtimeState.Oosvars.PutCopy(key, mvalue) } } diff --git a/internal/pkg/transformers/reshape.go b/internal/pkg/transformers/reshape.go index 45de4430f..8323a2a51 100644 --- a/internal/pkg/transformers/reshape.go +++ b/internal/pkg/transformers/reshape.go @@ -36,6 +36,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -320,7 +321,7 @@ func (tr *TransformerReshape) wideToLongNoRegex( } else { for pf := pairs.Head; pf != nil; pf = pf.Next { outrec := inrec.Copy() - outrec.PutReference(tr.outputKeyFieldName, types.MlrvalFromString(pf.Key)) + outrec.PutReference(tr.outputKeyFieldName, mlrval.MlrvalFromString(pf.Key)) outrec.PutReference(tr.outputValueFieldName, pf.Value) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } @@ -362,7 +363,7 @@ func (tr *TransformerReshape) wideToLongRegex( } else { for pf := pairs.Head; pf != nil; pf = pf.Next { outrec := inrec.Copy() - outrec.PutReference(tr.outputKeyFieldName, types.MlrvalFromString(pf.Key)) + outrec.PutReference(tr.outputKeyFieldName, mlrval.MlrvalFromString(pf.Key)) outrec.PutReference(tr.outputValueFieldName, pf.Value) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } diff --git a/internal/pkg/transformers/sec2gmt.go b/internal/pkg/transformers/sec2gmt.go index 9d99e9129..4b9c8f640 100644 --- a/internal/pkg/transformers/sec2gmt.go +++ b/internal/pkg/transformers/sec2gmt.go @@ -7,6 +7,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -163,7 +164,7 @@ func (tr *TransformerSec2GMT) Transform( if value != nil { floatval, ok := value.GetNumericToFloatValue() if ok { - newValue := types.MlrvalFromString(lib.Sec2GMT( + newValue := mlrval.MlrvalFromString(lib.Sec2GMT( floatval/tr.preDivide, tr.numDecimalPlaces, )) diff --git a/internal/pkg/transformers/seqgen.go b/internal/pkg/transformers/seqgen.go index 563501682..ab73725dd 100644 --- a/internal/pkg/transformers/seqgen.go +++ b/internal/pkg/transformers/seqgen.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -117,11 +118,11 @@ func transformerSeqgenParseCLI( // ---------------------------------------------------------------- type TransformerSeqgen struct { fieldName string - start *types.Mlrval - stop *types.Mlrval - step *types.Mlrval + start *mlrval.Mlrval + stop *mlrval.Mlrval + step *mlrval.Mlrval doneComparator types.BinaryFunc - mdone *types.Mlrval + mdone *mlrval.Mlrval } // ---------------------------------------------------------------- @@ -131,9 +132,9 @@ func NewTransformerSeqgen( stopString string, stepString string, ) (*TransformerSeqgen, error) { - start := types.MlrvalFromInferredType(startString) - stop := types.MlrvalFromInferredType(stopString) - step := types.MlrvalFromInferredType(stepString) + start := mlrval.MlrvalFromInferredType(startString) + stop := mlrval.MlrvalFromInferredType(stopString) + step := mlrval.MlrvalFromInferredType(stepString) var doneComparator types.BinaryFunc = nil fstart, startIsNumeric := start.GetNumericToFloatValue() diff --git a/internal/pkg/transformers/sort.go b/internal/pkg/transformers/sort.go index a2b90223a..90d8d5767 100644 --- a/internal/pkg/transformers/sort.go +++ b/internal/pkg/transformers/sort.go @@ -50,6 +50,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -298,7 +299,7 @@ func NewTransformerSort( // ---------------------------------------------------------------- type GroupingKeysAndMlrvals struct { groupingKey string - mlrvals []*types.Mlrval + mlrvals []*mlrval.Mlrval } func (tr *TransformerSort) Transform( @@ -386,7 +387,7 @@ func groupHeadsToArray(groupHeads *lib.OrderedMap) []GroupingKeysAndMlrvals { for entry := groupHeads.Head; entry != nil; entry = entry.Next { retval[i] = GroupingKeysAndMlrvals{ groupingKey: entry.Key, - mlrvals: entry.Value.([]*types.Mlrval), + mlrvals: entry.Value.([]*mlrval.Mlrval), } i++ } diff --git a/internal/pkg/transformers/stats1.go b/internal/pkg/transformers/stats1.go index 602b2b357..03b875b78 100644 --- a/internal/pkg/transformers/stats1.go +++ b/internal/pkg/transformers/stats1.go @@ -11,6 +11,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/transformers/utils" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -253,7 +254,7 @@ type TransformerStats1 struct { // except we need maps that preserve insertion order. namedAccumulators *lib.OrderedMap - // map[string]OrderedMap[string]*types.Mlrval + // map[string]OrderedMap[string]*mlrval.Mlrval groupingKeysToGroupByFieldValues map[string]*lib.OrderedMap } @@ -382,7 +383,7 @@ func (tr *TransformerStats1) handleInputRecord( // E.g. if grouping by "a" and "b", and the current record has a=circle, b=blue, // then groupingKey is the string "circle,blue". var groupingKey string - var groupByFieldValues *lib.OrderedMap // OrderedMap[string]*types.Mlrval + var groupByFieldValues *lib.OrderedMap // OrderedMap[string]*mlrval.Mlrval var ok bool if tr.doRegexGroupByFieldNames { groupingKey, groupByFieldValues, ok = tr.getGroupByFieldNamesWithRegexes(inrec) @@ -428,10 +429,10 @@ func (tr *TransformerStats1) getGroupByFieldNamesWithoutRegexes( inrec *types.Mlrmap, ) ( groupingKey string, - groupByFieldValues *lib.OrderedMap, // OrderedMap[string]*types.Mlrval, + groupByFieldValues *lib.OrderedMap, // OrderedMap[string]*mlrval.Mlrval, ok bool, ) { - var groupByFieldValuesArray []*types.Mlrval + var groupByFieldValuesArray []*mlrval.Mlrval groupingKey, groupByFieldValuesArray, ok = inrec.GetSelectedValuesAndJoined(tr.groupByFieldNameList) if !ok { return groupingKey, nil, false @@ -451,7 +452,7 @@ func (tr *TransformerStats1) getGroupByFieldNamesWithRegexes( inrec *types.Mlrmap, ) ( groupingKey string, - groupByFieldValues *lib.OrderedMap, // OrderedMap[string]*types.Mlrval, + groupByFieldValues *lib.OrderedMap, // OrderedMap[string]*mlrval.Mlrval, ok bool, ) { @@ -616,7 +617,7 @@ func (tr *TransformerStats1) handleEndOfRecordStream( func (tr *TransformerStats1) emitIntoOutputRecord( inrec *types.Mlrmap, - groupByFieldValues *lib.OrderedMap, // OrderedMap[string]*types.Mlrval, + groupByFieldValues *lib.OrderedMap, // OrderedMap[string]*mlrval.Mlrval, level2accumulators *lib.OrderedMap, outrec *types.Mlrmap, ) { @@ -625,7 +626,7 @@ func (tr *TransformerStats1) emitIntoOutputRecord( groupByFieldName := pa.Key iValue := groupByFieldValues.Get(groupByFieldName) if iValue != nil { - outrec.PutCopy(groupByFieldName, iValue.(*types.Mlrval)) + outrec.PutCopy(groupByFieldName, iValue.(*mlrval.Mlrval)) } } diff --git a/internal/pkg/transformers/stats2.go b/internal/pkg/transformers/stats2.go index 59d771abf..067268296 100644 --- a/internal/pkg/transformers/stats2.go +++ b/internal/pkg/transformers/stats2.go @@ -9,6 +9,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/transformers/utils" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -397,7 +398,7 @@ func (tr *TransformerStats2) emit( // Add in a=s,b=t fields: groupingKey := pa.Key - groupByFieldValues := tr.groupingKeysToGroupByFieldValues.Get(groupingKey).([]*types.Mlrval) + groupByFieldValues := tr.groupingKeysToGroupByFieldValues.Get(groupingKey).([]*mlrval.Mlrval) for i, groupByFieldName := range tr.groupByFieldNameList { outrec.PutReference(groupByFieldName, groupByFieldValues[i].Copy()) } diff --git a/internal/pkg/transformers/step.go b/internal/pkg/transformers/step.go index b71b5bcf8..d22c163ba 100644 --- a/internal/pkg/transformers/step.go +++ b/internal/pkg/transformers/step.go @@ -9,6 +9,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -163,7 +164,7 @@ type TransformerStep struct { // STATE // Scratch space used per-record - valueFieldValues []types.Mlrval + valueFieldValues []mlrval.Mlrval // Map from group-by field names to value-field names to array of // stepper objects. See the Transform method below for more details. groups map[string]map[string]map[string]tStepper @@ -316,7 +317,7 @@ type tStepperAllocator func( ) tStepper type tStepper interface { - process(valueFieldValue *types.Mlrval, inputRecord *types.Mlrmap) + process(valueFieldValue *mlrval.Mlrval, inputRecord *types.Mlrmap) } type tStepperLookup struct { @@ -358,7 +359,7 @@ func allocateStepper( // ================================================================ type tStepperDelta struct { - previous *types.Mlrval + previous *mlrval.Mlrval outputFieldName string } @@ -374,7 +375,7 @@ func stepperDeltaAlloc( } func (stepper *tStepperDelta) process( - valueFieldValue *types.Mlrval, + valueFieldValue *mlrval.Mlrval, inrec *types.Mlrmap, ) { if valueFieldValue.IsEmpty() { @@ -382,7 +383,7 @@ func (stepper *tStepperDelta) process( return } - delta := types.MlrvalFromInt(0) + delta := mlrval.MlrvalFromInt(0) if stepper.previous != nil { delta = types.BIF_minus_binary(valueFieldValue, stepper.previous) } @@ -393,7 +394,7 @@ func (stepper *tStepperDelta) process( // ================================================================ type tStepperShift struct { - previous *types.Mlrval + previous *mlrval.Mlrval outputFieldName string } @@ -409,7 +410,7 @@ func stepperShiftAlloc( } func (stepper *tStepperShift) process( - valueFieldValue *types.Mlrval, + valueFieldValue *mlrval.Mlrval, inrec *types.Mlrmap, ) { if stepper.previous == nil { @@ -424,7 +425,7 @@ func (stepper *tStepperShift) process( // ================================================================ type tStepperFromFirst struct { - first *types.Mlrval + first *mlrval.Mlrval outputFieldName string } @@ -440,10 +441,10 @@ func stepperFromFirstAlloc( } func (stepper *tStepperFromFirst) process( - valueFieldValue *types.Mlrval, + valueFieldValue *mlrval.Mlrval, inrec *types.Mlrmap, ) { - fromFirst := types.MlrvalFromInt(0) + fromFirst := mlrval.MlrvalFromInt(0) if stepper.first == nil { stepper.first = valueFieldValue.Copy() } else { @@ -454,7 +455,7 @@ func (stepper *tStepperFromFirst) process( // ================================================================ type tStepperRatio struct { - previous *types.Mlrval + previous *mlrval.Mlrval outputFieldName string } @@ -470,7 +471,7 @@ func stepperRatioAlloc( } func (stepper *tStepperRatio) process( - valueFieldValue *types.Mlrval, + valueFieldValue *mlrval.Mlrval, inrec *types.Mlrmap, ) { if valueFieldValue.IsEmpty() { @@ -478,7 +479,7 @@ func (stepper *tStepperRatio) process( return } - ratio := types.MlrvalFromInt(1) + ratio := mlrval.MlrvalFromInt(1) if stepper.previous != nil { ratio = types.BIF_divide(valueFieldValue, stepper.previous) } @@ -489,7 +490,7 @@ func (stepper *tStepperRatio) process( // ================================================================ type tStepperRsum struct { - rsum *types.Mlrval + rsum *mlrval.Mlrval outputFieldName string } @@ -499,13 +500,13 @@ func stepperRsumAlloc( _unused2 []string, ) tStepper { return &tStepperRsum{ - rsum: types.MlrvalFromInt(0), + rsum: mlrval.MlrvalFromInt(0), outputFieldName: inputFieldName + "_rsum", } } func (stepper *tStepperRsum) process( - valueFieldValue *types.Mlrval, + valueFieldValue *mlrval.Mlrval, inrec *types.Mlrmap, ) { if valueFieldValue.IsEmpty() { @@ -518,8 +519,8 @@ func (stepper *tStepperRsum) process( // ================================================================ type tStepperCounter struct { - counter *types.Mlrval - one *types.Mlrval + counter *mlrval.Mlrval + one *mlrval.Mlrval outputFieldName string } @@ -529,14 +530,14 @@ func stepperCounterAlloc( _unused2 []string, ) tStepper { return &tStepperCounter{ - counter: types.MlrvalFromInt(0), - one: types.MlrvalFromInt(1), + counter: mlrval.MlrvalFromInt(0), + one: mlrval.MlrvalFromInt(1), outputFieldName: inputFieldName + "_counter", } } func (stepper *tStepperCounter) process( - valueFieldValue *types.Mlrval, + valueFieldValue *mlrval.Mlrval, inrec *types.Mlrmap, ) { if valueFieldValue.IsEmpty() { @@ -552,9 +553,9 @@ func (stepper *tStepperCounter) process( // ================================================================ type tStepperEWMA struct { - alphas []*types.Mlrval - oneMinusAlphas []*types.Mlrval - prevs []*types.Mlrval + alphas []*mlrval.Mlrval + oneMinusAlphas []*mlrval.Mlrval + prevs []*mlrval.Mlrval outputFieldNames []string havePrevs bool } @@ -569,9 +570,9 @@ func stepperEWMAAlloc( // len(ewmaSuffixes) in the CLI parser. n := len(stringAlphas) - alphas := make([]*types.Mlrval, n) - oneMinusAlphas := make([]*types.Mlrval, n) - prevs := make([]*types.Mlrval, n) + alphas := make([]*mlrval.Mlrval, n) + oneMinusAlphas := make([]*mlrval.Mlrval, n) + prevs := make([]*mlrval.Mlrval, n) outputFieldNames := make([]string, n) suffixes := stringAlphas @@ -591,9 +592,9 @@ func stepperEWMAAlloc( ) os.Exit(1) } - alphas[i] = types.MlrvalFromFloat64(dalpha) - oneMinusAlphas[i] = types.MlrvalFromFloat64(1.0 - dalpha) - prevs[i] = types.MlrvalFromFloat64(0.0) + alphas[i] = mlrval.MlrvalFromFloat64(dalpha) + oneMinusAlphas[i] = mlrval.MlrvalFromFloat64(1.0 - dalpha) + prevs[i] = mlrval.MlrvalFromFloat64(0.0) outputFieldNames[i] = inputFieldName + "_ewma_" + suffix } @@ -607,7 +608,7 @@ func stepperEWMAAlloc( } func (stepper *tStepperEWMA) process( - valueFieldValue *types.Mlrval, + valueFieldValue *mlrval.Mlrval, inrec *types.Mlrmap, ) { if !stepper.havePrevs { diff --git a/internal/pkg/transformers/template.go b/internal/pkg/transformers/template.go index 21a65c088..ac1716052 100644 --- a/internal/pkg/transformers/template.go +++ b/internal/pkg/transformers/template.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -119,7 +120,7 @@ func transformerTemplateParseCLI( type TransformerTemplate struct { fieldNameList []string fieldNameSet map[string]bool - fillWith *types.Mlrval + fillWith *mlrval.Mlrval } func NewTransformerTemplate( @@ -129,7 +130,7 @@ func NewTransformerTemplate( return &TransformerTemplate{ fieldNameList: fieldNames, fieldNameSet: lib.StringListToSet(fieldNames), - fillWith: types.MlrvalFromString(fillWith), + fillWith: mlrval.MlrvalFromString(fillWith), }, nil } diff --git a/internal/pkg/transformers/top.go b/internal/pkg/transformers/top.go index 647fab2c9..c2012dd24 100644 --- a/internal/pkg/transformers/top.go +++ b/internal/pkg/transformers/top.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/transformers/utils" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -141,7 +142,7 @@ type TransformerTop struct { // Two-level map from grouping key (string of joined-together group-by field values), // to string value-field name, to *utils.TopKeeper groups *lib.OrderedMap - groupingKeysToGroupByFieldValues map[string][]*types.Mlrval + groupingKeysToGroupByFieldValues map[string][]*mlrval.Mlrval } // ---------------------------------------------------------------- @@ -163,7 +164,7 @@ func NewTransformerTop( outputFieldName: outputFieldName, groups: lib.NewOrderedMap(), - groupingKeysToGroupByFieldValues: make(map[string][]*types.Mlrval), + groupingKeysToGroupByFieldValues: make(map[string][]*mlrval.Mlrval), } return tr, nil @@ -273,10 +274,10 @@ func (tr *TransformerTop) emit( topKeeper := pb.Value.(*utils.TopKeeper) key := valueFieldName + "_top" if i < topKeeper.GetSize() { - newrec.PutReference(tr.outputFieldName, types.MlrvalFromInt(i+1)) + newrec.PutReference(tr.outputFieldName, mlrval.MlrvalFromInt(i+1)) newrec.PutReference(key, topKeeper.TopValues[i].Copy()) } else { - newrec.PutReference(tr.outputFieldName, types.MlrvalFromInt(i+1)) + newrec.PutReference(tr.outputFieldName, mlrval.MlrvalFromInt(i+1)) newrec.PutCopy(key, types.MLRVAL_VOID) } } diff --git a/internal/pkg/transformers/uniq.go b/internal/pkg/transformers/uniq.go index d4f6ddd07..5fa100424 100644 --- a/internal/pkg/transformers/uniq.go +++ b/internal/pkg/transformers/uniq.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -372,7 +373,7 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowCounts( for pe := tr.uniqifiedRecords.Head; pe != nil; pe = pe.Next { outrecAndContext := pe.Value.(*types.RecordAndContext) icount := tr.uniqifiedRecordCounts.Get(pe.Key) - mcount := types.MlrvalFromInt(icount.(int)) + mcount := mlrval.MlrvalFromInt(icount.(int)) outrecAndContext.Record.PrependReference(tr.outputFieldName, mcount) outputRecordsAndContexts.PushBack(outrecAndContext) } @@ -402,7 +403,7 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowNumDistinctOnly( outrec := types.NewMlrmapAsRecord() outrec.PutReference( tr.outputFieldName, - types.MlrvalFromInt(tr.uniqifiedRecordCounts.FieldCount), + mlrval.MlrvalFromInt(tr.uniqifiedRecordCounts.FieldCount), ) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) @@ -474,12 +475,12 @@ func (tr *TransformerUniq) transformUnlashed( for pf := countsForFieldName.Head; pf != nil; pf = pf.Next { fieldValueString := pf.Key outrec := types.NewMlrmapAsRecord() - outrec.PutReference("field", types.MlrvalFromString(fieldName)) + outrec.PutReference("field", mlrval.MlrvalFromString(fieldName)) outrec.PutCopy( "value", - tr.unlashedCountValues.Get(fieldName).(*lib.OrderedMap).Get(fieldValueString).(*types.Mlrval), + tr.unlashedCountValues.Get(fieldName).(*lib.OrderedMap).Get(fieldValueString).(*mlrval.Mlrval), ) - outrec.PutReference("count", types.MlrvalFromInt(pf.Value.(int))) + outrec.PutReference("count", mlrval.MlrvalFromInt(pf.Value.(int))) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } } @@ -512,7 +513,7 @@ func (tr *TransformerUniq) transformNumDistinctOnly( outrec := types.NewMlrmapAsRecord() outrec.PutReference( "count", - types.MlrvalFromInt(tr.countsByGroup.FieldCount), + mlrval.MlrvalFromInt(tr.countsByGroup.FieldCount), ) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) @@ -545,7 +546,7 @@ func (tr *TransformerUniq) transformWithCounts( for pa := tr.countsByGroup.Head; pa != nil; pa = pa.Next { outrec := types.NewMlrmapAsRecord() - valuesForGroup := tr.valuesByGroup.Get(pa.Key).([]*types.Mlrval) + valuesForGroup := tr.valuesByGroup.Get(pa.Key).([]*mlrval.Mlrval) for i, fieldName := range tr.fieldNames { outrec.PutCopy( fieldName, @@ -555,7 +556,7 @@ func (tr *TransformerUniq) transformWithCounts( if tr.showCounts { outrec.PutReference( tr.outputFieldName, - types.MlrvalFromInt(pa.Value.(int)), + mlrval.MlrvalFromInt(pa.Value.(int)), ) } outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) diff --git a/internal/pkg/transformers/unsparsify.go b/internal/pkg/transformers/unsparsify.go index caaa3301a..33970f192 100644 --- a/internal/pkg/transformers/unsparsify.go +++ b/internal/pkg/transformers/unsparsify.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -110,7 +111,7 @@ func transformerUnsparsifyParseCLI( // ---------------------------------------------------------------- type TransformerUnsparsify struct { - fillerMlrval *types.Mlrval + fillerMlrval *mlrval.Mlrval recordsAndContexts *list.List fieldNamesSeen *lib.OrderedMap recordTransformerFunc RecordTransformerFunc @@ -127,7 +128,7 @@ func NewTransformerUnsparsify( } tr := &TransformerUnsparsify{ - fillerMlrval: types.MlrvalFromString(fillerString), + fillerMlrval: mlrval.MlrvalFromString(fillerString), recordsAndContexts: list.New(), fieldNamesSeen: fieldNamesSeen, } diff --git a/internal/pkg/transformers/utils/join-bucket-keeper.go b/internal/pkg/transformers/utils/join-bucket-keeper.go index c5be1253b..6f14eebd8 100644 --- a/internal/pkg/transformers/utils/join-bucket-keeper.go +++ b/internal/pkg/transformers/utils/join-bucket-keeper.go @@ -116,6 +116,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/input" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -251,7 +252,7 @@ func (keeper *JoinBucketKeeper) computeState() tJoinBucketKeeperState { // will also be moved to keeper.leftUnpaired. func (keeper *JoinBucketKeeper) FindJoinBucket( - rightFieldValues []*types.Mlrval, // nil means right-file EOF + rightFieldValues []*mlrval.Mlrval, // nil means right-file EOF ) bool { // TODO: comment me isPaired := false @@ -367,7 +368,7 @@ func (keeper *JoinBucketKeeper) prepareForFirstJoinBucket() { // leftvals < rightvals && !eof. func (keeper *JoinBucketKeeper) prepareForNewJoinBucket( - rightFieldValues []*types.Mlrval, + rightFieldValues []*mlrval.Mlrval, ) { if !keeper.JoinBucket.WasPaired { moveRecordsAndContexts(keeper.leftUnpaireds, keeper.JoinBucket.RecordsAndContexts) @@ -607,8 +608,8 @@ func moveRecordsAndContexts( // for numerical values). func compareLexically( - leftFieldValues []*types.Mlrval, - rightFieldValues []*types.Mlrval, + leftFieldValues []*mlrval.Mlrval, + rightFieldValues []*mlrval.Mlrval, ) int { lib.InternalCodingErrorIf(len(leftFieldValues) != len(rightFieldValues)) n := len(leftFieldValues) @@ -659,7 +660,7 @@ func (keeper *JoinBucketKeeper) dump(prefix string) { fmt.Printf("------------------------------------------------------\n") } -func dumpFieldValues(name string, values []*types.Mlrval) { +func dumpFieldValues(name string, values []*mlrval.Mlrval) { for i, value := range values { fmt.Printf("-- %s[%d] = %s\n", name, i, value.String()) } diff --git a/internal/pkg/transformers/utils/join-bucket.go b/internal/pkg/transformers/utils/join-bucket.go index ef9336cfb..8cf9ef6e8 100644 --- a/internal/pkg/transformers/utils/join-bucket.go +++ b/internal/pkg/transformers/utils/join-bucket.go @@ -7,18 +7,19 @@ package utils import ( "container/list" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- type JoinBucket struct { - leftFieldValues []*types.Mlrval + leftFieldValues []*mlrval.Mlrval RecordsAndContexts *list.List WasPaired bool } func NewJoinBucket( - leftFieldValues []*types.Mlrval, + leftFieldValues []*mlrval.Mlrval, ) *JoinBucket { return &JoinBucket{ leftFieldValues: leftFieldValues, diff --git a/internal/pkg/transformers/utils/percentile-keeper.go b/internal/pkg/transformers/utils/percentile-keeper.go index 30e20cbd2..3219bccd7 100644 --- a/internal/pkg/transformers/utils/percentile-keeper.go +++ b/internal/pkg/transformers/utils/percentile-keeper.go @@ -9,11 +9,12 @@ import ( "math" "sort" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) type PercentileKeeper struct { - data []*types.Mlrval + data []*mlrval.Mlrval sorted bool doInterpolatedPercentiles bool } @@ -23,21 +24,21 @@ func NewPercentileKeeper( doInterpolatedPercentiles bool, ) *PercentileKeeper { return &PercentileKeeper{ - data: make([]*types.Mlrval, 0, 1000), + data: make([]*mlrval.Mlrval, 0, 1000), sorted: false, doInterpolatedPercentiles: doInterpolatedPercentiles, } } func (keeper *PercentileKeeper) Reset() { - keeper.data = make([]*types.Mlrval, 0, 1000) + keeper.data = make([]*mlrval.Mlrval, 0, 1000) keeper.sorted = false } // ---------------------------------------------------------------- -func (keeper *PercentileKeeper) Ingest(value *types.Mlrval) { +func (keeper *PercentileKeeper) Ingest(value *mlrval.Mlrval) { if len(keeper.data) >= cap(keeper.data) { - newData := make([]*types.Mlrval, len(keeper.data), 2*cap(keeper.data)) + newData := make([]*mlrval.Mlrval, len(keeper.data), 2*cap(keeper.data)) copy(newData, keeper.data) keeper.data = newData } @@ -235,7 +236,7 @@ func computeIndexNoninterpolated(n int, p float64) int { } // xxx pending pointer-output refactor -func getPercentileLinearlyInterpolated(array []*types.Mlrval, n int, p float64) types.Mlrval { +func getPercentileLinearlyInterpolated(array []*mlrval.Mlrval, n int, p float64) mlrval.Mlrval { findex := (p / 100.0) * (float64(n) - 1) if findex < 0.0 { findex = 0.0 @@ -246,7 +247,7 @@ func getPercentileLinearlyInterpolated(array []*types.Mlrval, n int, p float64) } else { // array[iindex] + frac * (array[iindex+1] - array[iindex]) // TODO: just do this in float64. - frac := types.MlrvalFromFloat64(findex - float64(iindex)) + frac := mlrval.MlrvalFromFloat64(findex - float64(iindex)) diff := types.BIF_minus_binary(array[iindex+1], array[iindex]) prod := types.BIF_times(frac, diff) return *types.BIF_plus_binary(array[iindex], prod) @@ -257,14 +258,14 @@ func getPercentileLinearlyInterpolated(array []*types.Mlrval, n int, p float64) func (keeper *PercentileKeeper) sortIfNecessary() { if !keeper.sorted { sort.Slice(keeper.data, func(i, j int) bool { - return types.MlrvalLessThanAsBool(keeper.data[i], keeper.data[j]) + return mlrval.MlrvalLessThanAsBool(keeper.data[i], keeper.data[j]) }) keeper.sorted = true } } // ---------------------------------------------------------------- -func (keeper *PercentileKeeper) Emit(percentile float64) *types.Mlrval { +func (keeper *PercentileKeeper) Emit(percentile float64) *mlrval.Mlrval { if keeper.doInterpolatedPercentiles { return keeper.EmitLinearlyInterpolated(percentile) } else { @@ -272,7 +273,7 @@ func (keeper *PercentileKeeper) Emit(percentile float64) *types.Mlrval { } } -func (keeper *PercentileKeeper) EmitNonInterpolated(percentile float64) *types.Mlrval { +func (keeper *PercentileKeeper) EmitNonInterpolated(percentile float64) *mlrval.Mlrval { if len(keeper.data) == 0 { return types.MLRVAL_VOID } @@ -280,7 +281,7 @@ func (keeper *PercentileKeeper) EmitNonInterpolated(percentile float64) *types.M return keeper.data[computeIndexNoninterpolated(int(len(keeper.data)), percentile)].Copy() } -func (keeper *PercentileKeeper) EmitLinearlyInterpolated(percentile float64) *types.Mlrval { +func (keeper *PercentileKeeper) EmitLinearlyInterpolated(percentile float64) *mlrval.Mlrval { if len(keeper.data) == 0 { return types.MLRVAL_VOID } diff --git a/internal/pkg/transformers/utils/stats1-accumulators.go b/internal/pkg/transformers/utils/stats1-accumulators.go index 4bd3bb565..c0fee9e84 100644 --- a/internal/pkg/transformers/utils/stats1-accumulators.go +++ b/internal/pkg/transformers/utils/stats1-accumulators.go @@ -10,13 +10,14 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- type IStats1Accumulator interface { - Ingest(value *types.Mlrval) - Emit() *types.Mlrval + Ingest(value *mlrval.Mlrval) + Emit() *mlrval.Mlrval Reset() // for merge-fields where we reset after each record instead of replace/recreate } @@ -121,11 +122,11 @@ func NewStats1NamedAccumulator( } } -func (nacc *Stats1NamedAccumulator) Ingest(value *types.Mlrval) { +func (nacc *Stats1NamedAccumulator) Ingest(value *mlrval.Mlrval) { nacc.accumulator.Ingest(value) } -func (nacc *Stats1NamedAccumulator) Emit() (key string, value *types.Mlrval) { +func (nacc *Stats1NamedAccumulator) Emit() (key string, value *mlrval.Mlrval) { return nacc.outputFieldName, nacc.accumulator.Emit() } @@ -288,11 +289,11 @@ func NewStats1CountAccumulator() IStats1Accumulator { count: 0, } } -func (acc *Stats1CountAccumulator) Ingest(value *types.Mlrval) { +func (acc *Stats1CountAccumulator) Ingest(value *mlrval.Mlrval) { acc.count++ } -func (acc *Stats1CountAccumulator) Emit() *types.Mlrval { - return types.MlrvalFromInt(acc.count) +func (acc *Stats1CountAccumulator) Emit() *mlrval.Mlrval { + return mlrval.MlrvalFromInt(acc.count) } func (acc *Stats1CountAccumulator) Reset() { acc.count = 0 @@ -310,7 +311,7 @@ func NewStats1ModeAccumulator() IStats1Accumulator { countsByValue: lib.NewOrderedMap(), } } -func (acc *Stats1ModeAccumulator) Ingest(value *types.Mlrval) { +func (acc *Stats1ModeAccumulator) Ingest(value *mlrval.Mlrval) { key := value.String() // 1, 1.0, and 1.000 are distinct iPrevious, ok := acc.countsByValue.GetWithCheck(key) if !ok { @@ -319,7 +320,7 @@ func (acc *Stats1ModeAccumulator) Ingest(value *types.Mlrval) { acc.countsByValue.Put(key, iPrevious.(int)+1) } } -func (acc *Stats1ModeAccumulator) Emit() *types.Mlrval { +func (acc *Stats1ModeAccumulator) Emit() *mlrval.Mlrval { if acc.countsByValue.IsEmpty() { return types.MLRVAL_VOID } @@ -333,7 +334,7 @@ func (acc *Stats1ModeAccumulator) Emit() *types.Mlrval { maxCount = count } } - return types.MlrvalFromString(maxValue) + return mlrval.MlrvalFromString(maxValue) } func (acc *Stats1ModeAccumulator) Reset() { acc.countsByValue = lib.NewOrderedMap() @@ -351,7 +352,7 @@ func NewStats1AntimodeAccumulator() IStats1Accumulator { countsByValue: lib.NewOrderedMap(), } } -func (acc *Stats1AntimodeAccumulator) Ingest(value *types.Mlrval) { +func (acc *Stats1AntimodeAccumulator) Ingest(value *mlrval.Mlrval) { key := value.String() // 1, 1.0, and 1.000 are distinct iPrevious, ok := acc.countsByValue.GetWithCheck(key) if !ok { @@ -360,7 +361,7 @@ func (acc *Stats1AntimodeAccumulator) Ingest(value *types.Mlrval) { acc.countsByValue.Put(key, iPrevious.(int)+1) } } -func (acc *Stats1AntimodeAccumulator) Emit() *types.Mlrval { +func (acc *Stats1AntimodeAccumulator) Emit() *mlrval.Mlrval { if acc.countsByValue.IsEmpty() { return types.MLRVAL_VOID } @@ -374,7 +375,7 @@ func (acc *Stats1AntimodeAccumulator) Emit() *types.Mlrval { minCount = count } } - return types.MlrvalFromString(minValue) + return mlrval.MlrvalFromString(minValue) } func (acc *Stats1AntimodeAccumulator) Reset() { acc.countsByValue = lib.NewOrderedMap() @@ -382,55 +383,55 @@ func (acc *Stats1AntimodeAccumulator) Reset() { // ---------------------------------------------------------------- type Stats1SumAccumulator struct { - sum *types.Mlrval + sum *mlrval.Mlrval } func NewStats1SumAccumulator() IStats1Accumulator { return &Stats1SumAccumulator{ - sum: types.MlrvalFromInt(0), + sum: mlrval.MlrvalFromInt(0), } } -func (acc *Stats1SumAccumulator) Ingest(value *types.Mlrval) { +func (acc *Stats1SumAccumulator) Ingest(value *mlrval.Mlrval) { acc.sum = types.BIF_plus_binary(acc.sum, value) } -func (acc *Stats1SumAccumulator) Emit() *types.Mlrval { +func (acc *Stats1SumAccumulator) Emit() *mlrval.Mlrval { return acc.sum.Copy() } func (acc *Stats1SumAccumulator) Reset() { - acc.sum = types.MlrvalFromInt(0) + acc.sum = mlrval.MlrvalFromInt(0) } // ---------------------------------------------------------------- type Stats1MeanAccumulator struct { - sum *types.Mlrval + sum *mlrval.Mlrval count int } func NewStats1MeanAccumulator() IStats1Accumulator { return &Stats1MeanAccumulator{ - sum: types.MlrvalFromInt(0), + sum: mlrval.MlrvalFromInt(0), count: 0, } } -func (acc *Stats1MeanAccumulator) Ingest(value *types.Mlrval) { +func (acc *Stats1MeanAccumulator) Ingest(value *mlrval.Mlrval) { acc.sum = types.BIF_plus_binary(acc.sum, value) acc.count++ } -func (acc *Stats1MeanAccumulator) Emit() *types.Mlrval { +func (acc *Stats1MeanAccumulator) Emit() *mlrval.Mlrval { if acc.count == 0 { return types.MLRVAL_VOID } else { - return types.BIF_divide(acc.sum, types.MlrvalFromInt(acc.count)) + return types.BIF_divide(acc.sum, mlrval.MlrvalFromInt(acc.count)) } } func (acc *Stats1MeanAccumulator) Reset() { - acc.sum = types.MlrvalFromInt(0) + acc.sum = mlrval.MlrvalFromInt(0) acc.count = 0 } // ---------------------------------------------------------------- type Stats1MinAccumulator struct { - min *types.Mlrval + min *mlrval.Mlrval } func NewStats1MinAccumulator() IStats1Accumulator { @@ -438,10 +439,10 @@ func NewStats1MinAccumulator() IStats1Accumulator { min: types.MLRVAL_ABSENT, } } -func (acc *Stats1MinAccumulator) Ingest(value *types.Mlrval) { - acc.min = types.MlrvalBinaryMin(acc.min, value) +func (acc *Stats1MinAccumulator) Ingest(value *mlrval.Mlrval) { + acc.min = mlrval.MlrvalBinaryMin(acc.min, value) } -func (acc *Stats1MinAccumulator) Emit() *types.Mlrval { +func (acc *Stats1MinAccumulator) Emit() *mlrval.Mlrval { if acc.min.IsAbsent() { return types.MLRVAL_VOID } else { @@ -454,7 +455,7 @@ func (acc *Stats1MinAccumulator) Reset() { // ---------------------------------------------------------------- type Stats1MaxAccumulator struct { - max *types.Mlrval + max *mlrval.Mlrval } func NewStats1MaxAccumulator() IStats1Accumulator { @@ -462,10 +463,10 @@ func NewStats1MaxAccumulator() IStats1Accumulator { max: types.MLRVAL_ABSENT, } } -func (acc *Stats1MaxAccumulator) Ingest(value *types.Mlrval) { - acc.max = types.MlrvalBinaryMax(acc.max, value) +func (acc *Stats1MaxAccumulator) Ingest(value *mlrval.Mlrval) { + acc.max = mlrval.MlrvalBinaryMax(acc.max, value) } -func (acc *Stats1MaxAccumulator) Emit() *types.Mlrval { +func (acc *Stats1MaxAccumulator) Emit() *mlrval.Mlrval { if acc.max.IsAbsent() { return types.MLRVAL_VOID } else { @@ -479,108 +480,108 @@ func (acc *Stats1MaxAccumulator) Reset() { // ---------------------------------------------------------------- type Stats1VarAccumulator struct { count int - sum *types.Mlrval - sum2 *types.Mlrval + sum *mlrval.Mlrval + sum2 *mlrval.Mlrval } func NewStats1VarAccumulator() IStats1Accumulator { return &Stats1VarAccumulator{ count: 0, - sum: types.MlrvalFromInt(0), - sum2: types.MlrvalFromInt(0), + sum: mlrval.MlrvalFromInt(0), + sum2: mlrval.MlrvalFromInt(0), } } -func (acc *Stats1VarAccumulator) Ingest(value *types.Mlrval) { +func (acc *Stats1VarAccumulator) Ingest(value *mlrval.Mlrval) { value2 := types.BIF_times(value, value) acc.count++ acc.sum = types.BIF_plus_binary(acc.sum, value) acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) } -func (acc *Stats1VarAccumulator) Emit() *types.Mlrval { - return types.MlrvalGetVar(types.MlrvalFromInt(acc.count), acc.sum, acc.sum2) +func (acc *Stats1VarAccumulator) Emit() *mlrval.Mlrval { + return mlrval.MlrvalGetVar(mlrval.MlrvalFromInt(acc.count), acc.sum, acc.sum2) } func (acc *Stats1VarAccumulator) Reset() { acc.count = 0 - acc.sum = types.MlrvalFromInt(0) - acc.sum2 = types.MlrvalFromInt(0) + acc.sum = mlrval.MlrvalFromInt(0) + acc.sum2 = mlrval.MlrvalFromInt(0) } // ---------------------------------------------------------------- type Stats1StddevAccumulator struct { count int - sum *types.Mlrval - sum2 *types.Mlrval + sum *mlrval.Mlrval + sum2 *mlrval.Mlrval } func NewStats1StddevAccumulator() IStats1Accumulator { return &Stats1StddevAccumulator{ count: 0, - sum: types.MlrvalFromInt(0), - sum2: types.MlrvalFromInt(0), + sum: mlrval.MlrvalFromInt(0), + sum2: mlrval.MlrvalFromInt(0), } } -func (acc *Stats1StddevAccumulator) Ingest(value *types.Mlrval) { +func (acc *Stats1StddevAccumulator) Ingest(value *mlrval.Mlrval) { value2 := types.BIF_times(value, value) acc.count++ acc.sum = types.BIF_plus_binary(acc.sum, value) acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) } -func (acc *Stats1StddevAccumulator) Emit() *types.Mlrval { - return types.MlrvalGetStddev(types.MlrvalFromInt(acc.count), acc.sum, acc.sum2) +func (acc *Stats1StddevAccumulator) Emit() *mlrval.Mlrval { + return mlrval.MlrvalGetStddev(mlrval.MlrvalFromInt(acc.count), acc.sum, acc.sum2) } func (acc *Stats1StddevAccumulator) Reset() { acc.count = 0 - acc.sum = types.MlrvalFromInt(0) - acc.sum2 = types.MlrvalFromInt(0) + acc.sum = mlrval.MlrvalFromInt(0) + acc.sum2 = mlrval.MlrvalFromInt(0) } // ---------------------------------------------------------------- type Stats1MeanEBAccumulator struct { count int - sum *types.Mlrval - sum2 *types.Mlrval + sum *mlrval.Mlrval + sum2 *mlrval.Mlrval } func NewStats1MeanEBAccumulator() IStats1Accumulator { return &Stats1MeanEBAccumulator{ count: 0, - sum: types.MlrvalFromInt(0), - sum2: types.MlrvalFromInt(0), + sum: mlrval.MlrvalFromInt(0), + sum2: mlrval.MlrvalFromInt(0), } } -func (acc *Stats1MeanEBAccumulator) Ingest(value *types.Mlrval) { +func (acc *Stats1MeanEBAccumulator) Ingest(value *mlrval.Mlrval) { value2 := types.BIF_times(value, value) acc.count++ acc.sum = types.BIF_plus_binary(acc.sum, value) acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) } -func (acc *Stats1MeanEBAccumulator) Emit() *types.Mlrval { - mcount := types.MlrvalFromInt(acc.count) - return types.MlrvalGetMeanEB(mcount, acc.sum, acc.sum2) +func (acc *Stats1MeanEBAccumulator) Emit() *mlrval.Mlrval { + mcount := mlrval.MlrvalFromInt(acc.count) + return mlrval.MlrvalGetMeanEB(mcount, acc.sum, acc.sum2) } func (acc *Stats1MeanEBAccumulator) Reset() { acc.count = 0 - acc.sum = types.MlrvalFromInt(0) - acc.sum2 = types.MlrvalFromInt(0) + acc.sum = mlrval.MlrvalFromInt(0) + acc.sum2 = mlrval.MlrvalFromInt(0) } // ---------------------------------------------------------------- type Stats1SkewnessAccumulator struct { count int - sum *types.Mlrval - sum2 *types.Mlrval - sum3 *types.Mlrval + sum *mlrval.Mlrval + sum2 *mlrval.Mlrval + sum3 *mlrval.Mlrval } func NewStats1SkewnessAccumulator() IStats1Accumulator { return &Stats1SkewnessAccumulator{ count: 0, - sum: types.MlrvalFromInt(0), - sum2: types.MlrvalFromInt(0), - sum3: types.MlrvalFromInt(0), + sum: mlrval.MlrvalFromInt(0), + sum2: mlrval.MlrvalFromInt(0), + sum3: mlrval.MlrvalFromInt(0), } } -func (acc *Stats1SkewnessAccumulator) Ingest(value *types.Mlrval) { +func (acc *Stats1SkewnessAccumulator) Ingest(value *mlrval.Mlrval) { value2 := types.BIF_times(value, value) value3 := types.BIF_times(value, value2) acc.count++ @@ -588,36 +589,36 @@ func (acc *Stats1SkewnessAccumulator) Ingest(value *types.Mlrval) { acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) acc.sum3 = types.BIF_plus_binary(acc.sum3, value3) } -func (acc *Stats1SkewnessAccumulator) Emit() *types.Mlrval { - mcount := types.MlrvalFromInt(acc.count) - return types.MlrvalGetSkewness(mcount, acc.sum, acc.sum2, acc.sum3) +func (acc *Stats1SkewnessAccumulator) Emit() *mlrval.Mlrval { + mcount := mlrval.MlrvalFromInt(acc.count) + return mlrval.MlrvalGetSkewness(mcount, acc.sum, acc.sum2, acc.sum3) } func (acc *Stats1SkewnessAccumulator) Reset() { acc.count = 0 - acc.sum = types.MlrvalFromInt(0) - acc.sum2 = types.MlrvalFromInt(0) - acc.sum3 = types.MlrvalFromInt(0) + acc.sum = mlrval.MlrvalFromInt(0) + acc.sum2 = mlrval.MlrvalFromInt(0) + acc.sum3 = mlrval.MlrvalFromInt(0) } // ---------------------------------------------------------------- type Stats1KurtosisAccumulator struct { count int - sum *types.Mlrval - sum2 *types.Mlrval - sum3 *types.Mlrval - sum4 *types.Mlrval + sum *mlrval.Mlrval + sum2 *mlrval.Mlrval + sum3 *mlrval.Mlrval + sum4 *mlrval.Mlrval } func NewStats1KurtosisAccumulator() IStats1Accumulator { return &Stats1KurtosisAccumulator{ count: 0, - sum: types.MlrvalFromInt(0), - sum2: types.MlrvalFromInt(0), - sum3: types.MlrvalFromInt(0), - sum4: types.MlrvalFromInt(0), + sum: mlrval.MlrvalFromInt(0), + sum2: mlrval.MlrvalFromInt(0), + sum3: mlrval.MlrvalFromInt(0), + sum4: mlrval.MlrvalFromInt(0), } } -func (acc *Stats1KurtosisAccumulator) Ingest(value *types.Mlrval) { +func (acc *Stats1KurtosisAccumulator) Ingest(value *mlrval.Mlrval) { value2 := types.BIF_times(value, value) value3 := types.BIF_times(value, value2) value4 := types.BIF_times(value, value3) @@ -627,16 +628,16 @@ func (acc *Stats1KurtosisAccumulator) Ingest(value *types.Mlrval) { acc.sum3 = types.BIF_plus_binary(acc.sum3, value3) acc.sum4 = types.BIF_plus_binary(acc.sum4, value4) } -func (acc *Stats1KurtosisAccumulator) Emit() *types.Mlrval { - mcount := types.MlrvalFromInt(acc.count) - return types.MlrvalGetKurtosis(mcount, acc.sum, acc.sum2, acc.sum3, acc.sum4) +func (acc *Stats1KurtosisAccumulator) Emit() *mlrval.Mlrval { + mcount := mlrval.MlrvalFromInt(acc.count) + return mlrval.MlrvalGetKurtosis(mcount, acc.sum, acc.sum2, acc.sum3, acc.sum4) } func (acc *Stats1KurtosisAccumulator) Reset() { acc.count = 0 - acc.sum = types.MlrvalFromInt(0) - acc.sum2 = types.MlrvalFromInt(0) - acc.sum3 = types.MlrvalFromInt(0) - acc.sum4 = types.MlrvalFromInt(0) + acc.sum = mlrval.MlrvalFromInt(0) + acc.sum2 = mlrval.MlrvalFromInt(0) + acc.sum3 = mlrval.MlrvalFromInt(0) + acc.sum4 = mlrval.MlrvalFromInt(0) } // ---------------------------------------------------------------- @@ -663,13 +664,13 @@ func NewStats1PercentileAccumulator( } } -func (acc *Stats1PercentileAccumulator) Ingest(value *types.Mlrval) { +func (acc *Stats1PercentileAccumulator) Ingest(value *mlrval.Mlrval) { if acc.isPrimary { acc.percentileKeeper.Ingest(value) } } -func (acc *Stats1PercentileAccumulator) Emit() *types.Mlrval { +func (acc *Stats1PercentileAccumulator) Emit() *mlrval.Mlrval { return acc.percentileKeeper.Emit(acc.percentile) } diff --git a/internal/pkg/transformers/utils/stats2-accumulators.go b/internal/pkg/transformers/utils/stats2-accumulators.go index 13d86949a..3859c0680 100644 --- a/internal/pkg/transformers/utils/stats2-accumulators.go +++ b/internal/pkg/transformers/utils/stats2-accumulators.go @@ -10,6 +10,7 @@ import ( "os" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -186,10 +187,10 @@ func (acc *Stats2LinRegOLSAccumulator) Populate( m, b := lib.GetLinearRegressionOLS(acc.count, acc.sumx, acc.sumx2, acc.sumxy, acc.sumy) - outrec.PutReference(acc.mOutputFieldName, types.MlrvalFromFloat64(m)) - outrec.PutReference(acc.bOutputFieldName, types.MlrvalFromFloat64(b)) + outrec.PutReference(acc.mOutputFieldName, mlrval.MlrvalFromFloat64(m)) + outrec.PutReference(acc.bOutputFieldName, mlrval.MlrvalFromFloat64(b)) } - outrec.PutReference(acc.nOutputFieldName, types.MlrvalFromInt(acc.count)) + outrec.PutReference(acc.nOutputFieldName, mlrval.MlrvalFromInt(acc.count)) } func (acc *Stats2LinRegOLSAccumulator) Fit( @@ -212,7 +213,7 @@ func (acc *Stats2LinRegOLSAccumulator) Fit( outrec.PutCopy(acc.fitOutputFieldName, types.MLRVAL_VOID) } else { yfit := acc.m*x + acc.b - outrec.PutReference(acc.fitOutputFieldName, types.MlrvalFromFloat64(yfit)) + outrec.PutReference(acc.fitOutputFieldName, mlrval.MlrvalFromFloat64(yfit)) } } @@ -270,10 +271,10 @@ func (acc *Stats2LogiRegAccumulator) Populate( outrec.PutCopy(acc.bOutputFieldName, types.MLRVAL_VOID) } else { m, b := lib.LogisticRegression(acc.xs, acc.ys) - outrec.PutCopy(acc.mOutputFieldName, types.MlrvalFromFloat64(m)) - outrec.PutCopy(acc.bOutputFieldName, types.MlrvalFromFloat64(b)) + outrec.PutCopy(acc.mOutputFieldName, mlrval.MlrvalFromFloat64(m)) + outrec.PutCopy(acc.bOutputFieldName, mlrval.MlrvalFromFloat64(b)) } - outrec.PutReference(acc.nOutputFieldName, types.MlrvalFromInt(len(acc.xs))) + outrec.PutReference(acc.nOutputFieldName, mlrval.MlrvalFromInt(len(acc.xs))) } func (acc *Stats2LogiRegAccumulator) Fit( @@ -296,7 +297,7 @@ func (acc *Stats2LogiRegAccumulator) Fit( outrec.PutCopy(acc.fitOutputFieldName, types.MLRVAL_VOID) } else { yfit := 1.0 / (1.0 + math.Exp(-acc.m*x-acc.b)) - outrec.PutReference(acc.fitOutputFieldName, types.MlrvalFromFloat64(yfit)) + outrec.PutReference(acc.fitOutputFieldName, mlrval.MlrvalFromFloat64(yfit)) } } @@ -363,7 +364,7 @@ func (acc *Stats2R2Accumulator) Populate( numerator = numerator * numerator denominator := (n*sumx2 - sumx*sumx) * (n*sumy2 - sumy*sumy) output := numerator / denominator - outrec.PutReference(acc.r2OutputFieldName, types.MlrvalFromFloat64(output)) + outrec.PutReference(acc.r2OutputFieldName, mlrval.MlrvalFromFloat64(output)) } } @@ -510,10 +511,10 @@ func (acc *Stats2CorrCovAccumulator) Populate( acc.sumy2, acc.sumxy, ) - outrec.PutReference(key00, types.MlrvalFromFloat64(Q[0][0])) - outrec.PutReference(key01, types.MlrvalFromFloat64(Q[0][1])) - outrec.PutReference(key10, types.MlrvalFromFloat64(Q[1][0])) - outrec.PutReference(key11, types.MlrvalFromFloat64(Q[1][1])) + outrec.PutReference(key00, mlrval.MlrvalFromFloat64(Q[0][0])) + outrec.PutReference(key01, mlrval.MlrvalFromFloat64(Q[0][1])) + outrec.PutReference(key10, mlrval.MlrvalFromFloat64(Q[1][0])) + outrec.PutReference(key11, mlrval.MlrvalFromFloat64(Q[1][1])) } } else if acc.doWhich == DO_LINREG_PCA { @@ -560,18 +561,18 @@ func (acc *Stats2CorrCovAccumulator) Populate( y_mean := acc.sumy / float64(acc.count) m, b, q := lib.GetLinearRegressionPCA(l1, l2, v1, v2, x_mean, y_mean) - outrec.PutReference(keym, types.MlrvalFromFloat64(m)) - outrec.PutReference(keyb, types.MlrvalFromFloat64(b)) - outrec.PutReference(keyn, types.MlrvalFromInt(acc.count)) - outrec.PutReference(keyq, types.MlrvalFromFloat64(q)) + outrec.PutReference(keym, mlrval.MlrvalFromFloat64(m)) + outrec.PutReference(keyb, mlrval.MlrvalFromFloat64(b)) + outrec.PutReference(keyn, mlrval.MlrvalFromInt(acc.count)) + outrec.PutReference(keyq, mlrval.MlrvalFromFloat64(q)) if acc.doVerbose { - outrec.PutReference(keyl1, types.MlrvalFromFloat64(l1)) - outrec.PutReference(keyl2, types.MlrvalFromFloat64(l2)) - outrec.PutReference(keyv11, types.MlrvalFromFloat64(v1[0])) - outrec.PutReference(keyv12, types.MlrvalFromFloat64(v1[1])) - outrec.PutReference(keyv21, types.MlrvalFromFloat64(v2[0])) - outrec.PutReference(keyv22, types.MlrvalFromFloat64(v2[1])) + outrec.PutReference(keyl1, mlrval.MlrvalFromFloat64(l1)) + outrec.PutReference(keyl2, mlrval.MlrvalFromFloat64(l2)) + outrec.PutReference(keyv11, mlrval.MlrvalFromFloat64(v1[0])) + outrec.PutReference(keyv12, mlrval.MlrvalFromFloat64(v1[1])) + outrec.PutReference(keyv21, mlrval.MlrvalFromFloat64(v2[0])) + outrec.PutReference(keyv22, mlrval.MlrvalFromFloat64(v2[1])) } } } else { @@ -588,7 +589,7 @@ func (acc *Stats2CorrCovAccumulator) Populate( sigmay := math.Sqrt(lib.GetVar(acc.count, acc.sumy, acc.sumy2)) output = output / sigmax / sigmay } - outrec.PutReference(key, types.MlrvalFromFloat64(output)) + outrec.PutReference(key, mlrval.MlrvalFromFloat64(output)) } } } @@ -622,7 +623,7 @@ func (acc *Stats2CorrCovAccumulator) Fit( outrec.PutCopy(acc.pca_fitOutputFieldName, types.MLRVAL_VOID) } else { yfit := acc.m*x + acc.b - outrec.PutCopy(acc.pca_fitOutputFieldName, types.MlrvalFromFloat64(yfit)) + outrec.PutCopy(acc.pca_fitOutputFieldName, mlrval.MlrvalFromFloat64(yfit)) } } diff --git a/internal/pkg/transformers/utils/top-keeper.go b/internal/pkg/transformers/utils/top-keeper.go index d36700ad1..073e99c47 100644 --- a/internal/pkg/transformers/utils/top-keeper.go +++ b/internal/pkg/transformers/utils/top-keeper.go @@ -5,12 +5,13 @@ package utils import ( + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- type TopKeeper struct { - TopValues []*types.Mlrval + TopValues []*mlrval.Mlrval TopRecordsAndContexts []*types.RecordAndContext size int capacity int @@ -20,7 +21,7 @@ type TopKeeper struct { // ---------------------------------------------------------------- func NewTopKeeper(capacity int, doMax bool) *TopKeeper { keeper := &TopKeeper{ - TopValues: make([]*types.Mlrval, capacity), + TopValues: make([]*mlrval.Mlrval, capacity), TopRecordsAndContexts: make([]*types.RecordAndContext, capacity), size: 0, capacity: capacity, @@ -69,7 +70,7 @@ func (keeper *TopKeeper) GetSize() int { // Our caller, the 'top' verb, feeds us records. We keep them or not; in the // latter case, the Go runtime GCs them. -func (keeper *TopKeeper) Add(value *types.Mlrval, recordAndContext *types.RecordAndContext) { +func (keeper *TopKeeper) Add(value *mlrval.Mlrval, recordAndContext *types.RecordAndContext) { destidx := keeper.bsearchFunc(&keeper.TopValues, keeper.size, value) if keeper.size < keeper.capacity { diff --git a/internal/pkg/types/mlrmap_flatten_unflatten.go b/internal/pkg/types-parked/mlrmap_flatten_unflatten.go similarity index 100% rename from internal/pkg/types/mlrmap_flatten_unflatten.go rename to internal/pkg/types-parked/mlrmap_flatten_unflatten.go diff --git a/internal/pkg/types/mlrmap_json.go b/internal/pkg/types-parked/mlrmap_json.go similarity index 100% rename from internal/pkg/types/mlrmap_json.go rename to internal/pkg/types-parked/mlrmap_json.go diff --git a/internal/pkg/types/mlrmap_print.go b/internal/pkg/types-parked/mlrmap_print.go similarity index 100% rename from internal/pkg/types/mlrmap_print.go rename to internal/pkg/types-parked/mlrmap_print.go diff --git a/internal/pkg/types/mlrval_accessors.go b/internal/pkg/types-parked/mlrval_accessors.go similarity index 100% rename from internal/pkg/types/mlrval_accessors.go rename to internal/pkg/types-parked/mlrval_accessors.go diff --git a/internal/pkg/types/mlrval_collections.go b/internal/pkg/types-parked/mlrval_collections.go similarity index 100% rename from internal/pkg/types/mlrval_collections.go rename to internal/pkg/types-parked/mlrval_collections.go diff --git a/internal/pkg/types/mlrval_format.go b/internal/pkg/types-parked/mlrval_format.go similarity index 100% rename from internal/pkg/types/mlrval_format.go rename to internal/pkg/types-parked/mlrval_format.go diff --git a/internal/pkg/types/mlrval_json.go b/internal/pkg/types-parked/mlrval_json.go similarity index 100% rename from internal/pkg/types/mlrval_json.go rename to internal/pkg/types-parked/mlrval_json.go diff --git a/internal/pkg/types/mlrval_output.go b/internal/pkg/types-parked/mlrval_output.go similarity index 100% rename from internal/pkg/types/mlrval_output.go rename to internal/pkg/types-parked/mlrval_output.go diff --git a/internal/pkg/types/mlrval_typing.go b/internal/pkg/types-parked/mlrval_typing.go similarity index 100% rename from internal/pkg/types/mlrval_typing.go rename to internal/pkg/types-parked/mlrval_typing.go diff --git a/internal/pkg/types/context.go b/internal/pkg/types/context.go index 1aff93811..311edb520 100644 --- a/internal/pkg/types/context.go +++ b/internal/pkg/types/context.go @@ -38,20 +38,21 @@ func NewRecordAndContext( } } -// For the record-readers to update their initial context as each new record is read. -func (rac *RecordAndContext) Copy() *RecordAndContext { - if rac == nil { - return nil - } - recordCopy := rac.Record.Copy() - contextCopy := rac.Context - return &RecordAndContext{ - Record: recordCopy, - Context: contextCopy, - OutputString: "", - EndOfStream: false, - } -} +// TODO: uncomment and integrate +//// For the record-readers to update their initial context as each new record is read. +//func (rac *RecordAndContext) Copy() *RecordAndContext { +// if rac == nil { +// return nil +// } +// recordCopy := rac.Record.Copy() +// contextCopy := rac.Context +// return &RecordAndContext{ +// Record: recordCopy, +// Context: contextCopy, +// OutputString: "", +// EndOfStream: false, +// } +//} // For print/dump/etc to insert strings sequenced into the record-output // stream. This avoids race conditions between different goroutines printing diff --git a/internal/pkg/types/mlrmap.go b/internal/pkg/types/mlrmap.go index c91827b74..821e85f07 100644 --- a/internal/pkg/types/mlrmap.go +++ b/internal/pkg/types/mlrmap.go @@ -53,7 +53,7 @@ package types -import( +import ( "github.com/johnkerl/miller/internal/pkg/mlrval" ) diff --git a/internal/pkg/types/mlrmap_accessors.go b/internal/pkg/types/mlrmap_accessors.go index 7bac89e99..d183097bd 100644 --- a/internal/pkg/types/mlrmap_accessors.go +++ b/internal/pkg/types/mlrmap_accessors.go @@ -1,9 +1,6 @@ package types import ( - //"bytes" - //"errors" - "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" ) @@ -77,6 +74,7 @@ func (mlrmap *Mlrmap) PutReference(key string, value *mlrval.Mlrval) { // } //} +// PrependReference is the same as PutReference, but puts a new entry first, not last. func (mlrmap *Mlrmap) PrependReference(key string, value *mlrval.Mlrval) { pe := mlrmap.findEntry(key) if pe == nil { diff --git a/internal/pkg/types/mlrmap_accessors_test.go b/internal/pkg/types/mlrmap_accessors_test.go new file mode 100644 index 000000000..ae4678d7f --- /dev/null +++ b/internal/pkg/types/mlrmap_accessors_test.go @@ -0,0 +1,51 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + +func TestIsEmpty(t *testing.T) { + mlrmap := NewMlrmap() + assert.Equal(t, true, mlrmap.IsEmpty()) +} + +func TestPutReference(t *testing.T) { + mlrmap := NewMlrmap() + key1 := "a" + val1 := mlrval.FromInt(1) + mlrmap.PutReference(key1, val1) + + assert.False(t, mlrmap.IsEmpty()) + + assert.True(t, mlrmap.Has("a")) + assert.False(t, mlrmap.Has("b")) + assert.Equal(t, 1, mlrmap.FieldCount) + + read := mlrmap.Get("b") + assert.Nil(t, read) + + read = mlrmap.Get("a") + assert.NotNil(t, read) + intval, ok := read.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 1, intval) + + key2 := "b" + val2 := mlrval.FromBool(true) + mlrmap.PutReference(key2, val2) + + assert.True(t, mlrmap.Has("a")) + assert.True(t, mlrmap.Has("b")) + assert.Equal(t, 2, mlrmap.FieldCount) + + read = mlrmap.Get("a") + assert.NotNil(t, read) + read = mlrmap.Get("b") + assert.NotNil(t, read) +} + +// TODO: TestPrependReference diff --git a/internal/pkg/types/mlrmap_new_test.go b/internal/pkg/types/mlrmap_new_test.go new file mode 100644 index 000000000..5671bb836 --- /dev/null +++ b/internal/pkg/types/mlrmap_new_test.go @@ -0,0 +1,17 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewMlrmapAsRecord(t *testing.T) { + mlrmap := newMlrmapUnhashed() + assert.Equal(t, false, mlrmap.isHashed()) +} + +func TestNewMlrmap(t *testing.T) { + mlrmap := NewMlrmap() + assert.Equal(t, true, mlrmap.isHashed()) +} diff --git a/todo.txt b/todo.txt index 6dff14a27..146d559ce 100644 --- a/todo.txt +++ b/todo.txt @@ -98,20 +98,18 @@ PUNCHDOWN LIST ---------------------------------------------------------------- * JIT refactor o x 'tens/dozens of UT cases' - ! re-check hashed/unhashed and do so periodically - > env-var back door m internal/pkg/mlrval - UT-driven mlrval defer/infer - pkg/mlrval doc.go & README.md o internal/pkg/types - - UT-driven mlrmap constructor refactor + k UT-driven mlrmap constructor refactor - UT-driven JSON-parser refactor o internal/pkg/input - - basic + k basic mlrval.FromDeferredType mlrmap.PutReference(key, value) - - UT-driven DKVP-from-line + k UT-driven DKVP-from-line - UT-driven X-from-line: nidx, xtab, csvlite, csv - UT-driven JSON-from-line - defer channelizers to a separate PR @@ -144,6 +142,22 @@ PUNCHDOWN LIST - internal/pkg/auxents/help ---------------------------------------------------------------- +* perf more + o see other branch + o nim6 note on perf: baseline/cat; processor/pipelining; + o update https://miller.readthedocs.io/en/latest/cpu/, and/or + https://miller.readthedocs.io/en/latest/streaming-and-memory/, and/or + make a new page w/ info on per + o JIT: maasure + o pipelined reader + o batched reader separately + o buffered output + o re-check hashed/unhashed and do so periodically + - env-var back door + - check with more than just abixy + +---------------------------------------------------------------- + * blockers: - keep checking issues - verslink old relnotes From 6773f6770e65558d21fca56076db0ee1cf407ca9 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 13 Dec 2021 08:01:40 -0500 Subject: [PATCH 03/19] complete merge from #779 --- Makefile | 2 +- internal/pkg/cli/option_parse.go | 1 + internal/pkg/input/pseudo_reader_gen.go | 2 +- internal/pkg/input/record_reader_csv.go | 3 +- internal/pkg/input/record_reader_csvlite.go | 1 + internal/pkg/input/record_reader_dkvp.go | 149 ------------- internal/pkg/input/record_reader_dkvp_nidx.go | 7 +- internal/pkg/input/record_reader_dkvp_test.go | 10 +- internal/pkg/input/record_reader_nidx.go | 138 ------------ internal/pkg/types/mlrmap_accessors.go | 87 ++++---- internal/pkg/types/mlrval_infer.go | 198 ------------------ todo.txt | 3 + 12 files changed, 59 insertions(+), 542 deletions(-) delete mode 100644 internal/pkg/input/record_reader_dkvp.go delete mode 100644 internal/pkg/input/record_reader_nidx.go delete mode 100644 internal/pkg/types/mlrval_infer.go diff --git a/Makefile b/Makefile index 52b1e8488..a310c599f 100644 --- a/Makefile +++ b/Makefile @@ -72,7 +72,7 @@ mlrmap-tests: mlrmap-new-test mlrmap-accessors-test input-dkvp-test: go test internal/pkg/input/record_reader_dkvp_test.go \ internal/pkg/input/record_reader.go \ - internal/pkg/input/record_reader_dkvp.go + internal/pkg/input/record_reader_dkvp_nidx.go input-tests: input-dkvp-test #mlrval_functions_test: diff --git a/internal/pkg/cli/option_parse.go b/internal/pkg/cli/option_parse.go index 978b9c7fb..665803e18 100644 --- a/internal/pkg/cli/option_parse.go +++ b/internal/pkg/cli/option_parse.go @@ -16,6 +16,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/colorizer" "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" + "github.com/johnkerl/miller/internal/pkg/types" ) // FinalizeReaderOptions does a few things. diff --git a/internal/pkg/input/pseudo_reader_gen.go b/internal/pkg/input/pseudo_reader_gen.go index 7b3d1d5e4..cb1a1d3f1 100644 --- a/internal/pkg/input/pseudo_reader_gen.go +++ b/internal/pkg/input/pseudo_reader_gen.go @@ -123,7 +123,7 @@ func (reader *PseudoReaderGen) tryParse( name string, svalue string, ) (*mlrval.Mlrval, error) { - mvalue := mlrval.MlrvalFromInferredType(svalue) + mvalue := mlrval.FromInferredType(svalue) if mvalue == nil || !mvalue.IsNumeric() { return nil, errors.New( fmt.Sprintf("mlr: gen: %s \"%s\" is not parseable as number", name, svalue), diff --git a/internal/pkg/input/record_reader_csv.go b/internal/pkg/input/record_reader_csv.go index da2199c7d..a03f96bf0 100644 --- a/internal/pkg/input/record_reader_csv.go +++ b/internal/pkg/input/record_reader_csv.go @@ -12,6 +12,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -27,7 +28,6 @@ type RecordReaderCSV struct { header []string } -// ---------------------------------------------------------------- func NewRecordReaderCSV( readerOptions *cli.TReaderOptions, recordsPerBatch int, @@ -45,7 +45,6 @@ func NewRecordReaderCSV( }, nil } -// ---------------------------------------------------------------- func (reader *RecordReaderCSV) Read( filenames []string, context types.Context, diff --git a/internal/pkg/input/record_reader_csvlite.go b/internal/pkg/input/record_reader_csvlite.go index 7fc1f4f01..6ab3dcbe4 100644 --- a/internal/pkg/input/record_reader_csvlite.go +++ b/internal/pkg/input/record_reader_csvlite.go @@ -28,6 +28,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) diff --git a/internal/pkg/input/record_reader_dkvp.go b/internal/pkg/input/record_reader_dkvp.go deleted file mode 100644 index 47a46bcf9..000000000 --- a/internal/pkg/input/record_reader_dkvp.go +++ /dev/null @@ -1,149 +0,0 @@ -package input - -import ( - "io" - "strconv" - "strings" - - "github.com/johnkerl/miller/internal/pkg/cli" - "github.com/johnkerl/miller/internal/pkg/lib" - "github.com/johnkerl/miller/internal/pkg/mlrval" - "github.com/johnkerl/miller/internal/pkg/types" -) - -type RecordReaderDKVP struct { - readerOptions *cli.TReaderOptions -} - -func NewRecordReaderDKVP(readerOptions *cli.TReaderOptions) (*RecordReaderDKVP, error) { - return &RecordReaderDKVP{ - readerOptions: readerOptions, - }, nil -} - -func (reader *RecordReaderDKVP) Read( - filenames []string, - context types.Context, - readerChannel chan<- *types.RecordAndContext, - errorChannel chan error, - downstreamDoneChannel <-chan bool, // for mlr head -) { - if filenames != nil { // nil for mlr -n - if len(filenames) == 0 { // read from stdin - handle, err := lib.OpenStdin( - reader.readerOptions.Prepipe, - reader.readerOptions.PrepipeIsRaw, - reader.readerOptions.FileInputEncoding, - ) - if err != nil { - errorChannel <- err - } - reader.processHandle(handle, "(stdin)", &context, readerChannel, errorChannel, downstreamDoneChannel) - } else { - for _, filename := range filenames { - handle, err := lib.OpenFileForRead( - filename, - reader.readerOptions.Prepipe, - reader.readerOptions.PrepipeIsRaw, - reader.readerOptions.FileInputEncoding, - ) - if err != nil { - errorChannel <- err - } else { - reader.processHandle(handle, filename, &context, readerChannel, errorChannel, downstreamDoneChannel) - handle.Close() - } - } - } - } - readerChannel <- types.NewEndOfStreamMarker(&context) -} - -func (reader *RecordReaderDKVP) processHandle( - handle io.Reader, - filename string, - context *types.Context, - readerChannel chan<- *types.RecordAndContext, - errorChannel chan error, - downstreamDoneChannel <-chan bool, // for mlr head -) { - context.UpdateForStartOfFile(filename) - - scanner := NewLineScanner(handle, reader.readerOptions.IRS) - for scanner.Scan() { - - // See if downstream processors will be ignoring further data (e.g. mlr - // head). If so, stop reading. This makes 'mlr head hugefile' exit - // quickly, as it should. - eof := false - select { - case _ = <-downstreamDoneChannel: - eof = true - break - default: - break - } - if eof { - break - } - - line := scanner.Text() - - // Check for comments-in-data feature - if strings.HasPrefix(line, reader.readerOptions.CommentString) { - if reader.readerOptions.CommentHandling == cli.PassComments { - readerChannel <- types.NewOutputString(line+"\n", context) - continue - } else if reader.readerOptions.CommentHandling == cli.SkipComments { - continue - } - // else comments are data - } - - record := reader.recordFromDKVPLine(line) - context.UpdateForInputRecord() - readerChannel <- types.NewRecordAndContext( - record, - context, - ) - } -} - -// ---------------------------------------------------------------- -func (reader *RecordReaderDKVP) recordFromDKVPLine( - line string, -) *types.Mlrmap { - record := types.NewMlrmap() - - var pairs []string - if reader.readerOptions.IFSRegex == nil { // e.g. --no-ifs-regex - pairs = lib.SplitString(line, reader.readerOptions.IFS) - } else { - pairs = lib.RegexSplitString(reader.readerOptions.IFSRegex, line, -1) - } - - for i, pair := range pairs { - var kv []string - if reader.readerOptions.IPSRegex == nil { // e.g. --no-ips-regex - kv = strings.SplitN(pair, reader.readerOptions.IPS, 2) - } else { - kv = lib.RegexSplitString(reader.readerOptions.IPSRegex, pair, 2) - } - - if len(kv) == 0 || (len(kv) == 1 && kv[0] == "") { - // Ignore. This is expected when splitting with repeated IFS. - } else if len(kv) == 1 { - // E.g the pair has no equals sign: "a" rather than "a=1" or - // "a=". Here we use the positional index as the key. This way - // DKVP is a generalization of NIDX. - key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := mlrval.FromInferredType(kv[0]) - record.PutReference(key, value) - } else { - key := kv[0] - value := mlrval.FromInferredType(kv[1]) - record.PutReference(key, value) - } - } - return record -} diff --git a/internal/pkg/input/record_reader_dkvp_nidx.go b/internal/pkg/input/record_reader_dkvp_nidx.go index c2e3140ed..7283c6100 100644 --- a/internal/pkg/input/record_reader_dkvp_nidx.go +++ b/internal/pkg/input/record_reader_dkvp_nidx.go @@ -10,6 +10,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -179,11 +180,11 @@ func recordFromDKVPLine(reader *RecordReaderDKVPNIDX, line string) *types.Mlrmap // "a=". Here we use the positional index as the key. This way // DKVP is a generalization of NIDX. key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := types.MlrvalFromInferredTypeForDataFiles(kv[0]) + value := mlrval.FromInferredType(kv[0]) record.PutReference(key, value) } else { key := kv[0] - value := types.MlrvalFromInferredTypeForDataFiles(kv[1]) + value := mlrval.FromInferredType(kv[1]) record.PutReference(key, value) } } @@ -208,7 +209,7 @@ func recordFromNIDXLine(reader *RecordReaderDKVPNIDX, line string) *types.Mlrmap for _, value := range values { i++ key := strconv.Itoa(i) - mval := types.MlrvalFromInferredTypeForDataFiles(value) + mval := mlrval.FromInferredType(value) record.PutReference(key, mval) } return record diff --git a/internal/pkg/input/record_reader_dkvp_test.go b/internal/pkg/input/record_reader_dkvp_test.go index 00b89c73d..33e9aa194 100644 --- a/internal/pkg/input/record_reader_dkvp_test.go +++ b/internal/pkg/input/record_reader_dkvp_test.go @@ -11,17 +11,17 @@ import ( func TestRecordFromDKVPLine(t *testing.T) { readerOptions := cli.DefaultReaderOptions() cli.FinalizeReaderOptions(&readerOptions) // compute IPS, IFS -> IPSRegex, IFSRegex - reader, err := NewRecordReaderDKVP(&readerOptions) + reader, err := NewRecordReaderDKVP(&readerOptions, 1) assert.NotNil(t, reader) assert.Nil(t, err) line := "" - record := reader.recordFromDKVPLine(line) + record := recordFromDKVPLine(reader, line) assert.NotNil(t, record) assert.Equal(t, 0, record.FieldCount) line = "a=1,b=2,c=3" - record = reader.recordFromDKVPLine(line) + record = recordFromDKVPLine(reader, line) assert.NotNil(t, record) assert.Equal(t, 3, record.FieldCount) @@ -34,7 +34,7 @@ func TestRecordFromDKVPLine(t *testing.T) { assert.Equal(t, record.Head.Next.Next.Key, "c") line = "a=1,b=2,b=3" - record = reader.recordFromDKVPLine(line) + record = recordFromDKVPLine(reader, line) assert.NotNil(t, record) assert.Equal(t, 2, record.FieldCount) @@ -45,7 +45,7 @@ func TestRecordFromDKVPLine(t *testing.T) { assert.Equal(t, record.Head.Next.Key, "b") line = "a,b,c" - record = reader.recordFromDKVPLine(line) + record = recordFromDKVPLine(reader, line) assert.NotNil(t, record) assert.Equal(t, 3, record.FieldCount) diff --git a/internal/pkg/input/record_reader_nidx.go b/internal/pkg/input/record_reader_nidx.go deleted file mode 100644 index 5ae729d26..000000000 --- a/internal/pkg/input/record_reader_nidx.go +++ /dev/null @@ -1,138 +0,0 @@ -package input - -import ( - "io" - "strconv" - "strings" - - "github.com/johnkerl/miller/internal/pkg/cli" - "github.com/johnkerl/miller/internal/pkg/lib" - "github.com/johnkerl/miller/internal/pkg/types" -) - -type RecordReaderNIDX struct { - readerOptions *cli.TReaderOptions -} - -func NewRecordReaderNIDX(readerOptions *cli.TReaderOptions) (*RecordReaderNIDX, error) { - return &RecordReaderNIDX{ - readerOptions: readerOptions, - }, nil -} - -func (reader *RecordReaderNIDX) Read( - filenames []string, - context types.Context, - readerChannel chan<- *types.RecordAndContext, - errorChannel chan error, - downstreamDoneChannel <-chan bool, // for mlr head -) { - if filenames != nil { // nil for mlr -n - if len(filenames) == 0 { // read from stdin - handle, err := lib.OpenStdin( - reader.readerOptions.Prepipe, - reader.readerOptions.PrepipeIsRaw, - reader.readerOptions.FileInputEncoding, - ) - if err != nil { - errorChannel <- err - } - reader.processHandle(handle, "(stdin)", &context, readerChannel, errorChannel, downstreamDoneChannel) - } else { - for _, filename := range filenames { - handle, err := lib.OpenFileForRead( - filename, - reader.readerOptions.Prepipe, - reader.readerOptions.PrepipeIsRaw, - reader.readerOptions.FileInputEncoding, - ) - if err != nil { - errorChannel <- err - } else { - reader.processHandle(handle, filename, &context, readerChannel, errorChannel, downstreamDoneChannel) - handle.Close() - } - } - } - } - readerChannel <- types.NewEndOfStreamMarker(&context) -} - -func (reader *RecordReaderNIDX) processHandle( - handle io.Reader, - filename string, - context *types.Context, - readerChannel chan<- *types.RecordAndContext, - errorChannel chan error, - downstreamDoneChannel <-chan bool, // for mlr head -) { - context.UpdateForStartOfFile(filename) - - scanner := NewLineScanner(handle, reader.readerOptions.IRS) - - for scanner.Scan() { - - // See if downstream processors will be ignoring further data (e.g. mlr - // head). If so, stop reading. This makes 'mlr head hugefile' exit - // quickly, as it should. - eof := false - select { - case _ = <-downstreamDoneChannel: - eof = true - break - default: - break - } - if eof { - break - } - - // TODO: IRS - line := scanner.Text() - - // Check for comments-in-data feature - if strings.HasPrefix(line, reader.readerOptions.CommentString) { - if reader.readerOptions.CommentHandling == cli.PassComments { - readerChannel <- types.NewOutputString(line+"\n", context) - continue - } else if reader.readerOptions.CommentHandling == cli.SkipComments { - continue - } - // else comments are data - } - - record := reader.recordFromNIDXLine(line) - - context.UpdateForInputRecord() - readerChannel <- types.NewRecordAndContext( - record, - context, - ) - } -} - -// ---------------------------------------------------------------- -func (reader *RecordReaderNIDX) recordFromNIDXLine( - line string, -) *types.Mlrmap { - record := types.NewMlrmap() - - var values []string - if reader.readerOptions.IFSRegex == nil { // e.g. --no-ifs-regex - values = lib.SplitString(line, reader.readerOptions.IFS) - } else { - values = lib.RegexSplitString(reader.readerOptions.IFSRegex, line, -1) - } - if reader.readerOptions.AllowRepeatIFS { - values = lib.StripEmpties(values) // left/right trim - } - - var i int = 0 - for _, value := range values { - i++ - key := strconv.Itoa(i) - mval := mlrval.FromDeferredType(value) - record.PutReference(key, mval) - } - return record -} diff --git a/internal/pkg/types/mlrmap_accessors.go b/internal/pkg/types/mlrmap_accessors.go index d183097bd..ff21cc6a9 100644 --- a/internal/pkg/types/mlrmap_accessors.go +++ b/internal/pkg/types/mlrmap_accessors.go @@ -239,16 +239,15 @@ func (mlrmap *Mlrmap) findEntryByPositionalIndex(position int) *MlrmapEntry { // return mlrmap.findEntry(key) //} -//// ---------------------------------------------------------------- -//func (mlrmap *Mlrmap) GetKeys() []string { -// keys := make([]string, mlrmap.FieldCount) -// i := 0 -// for pe := mlrmap.Head; pe != nil; pe = pe.Next { -// keys[i] = pe.Key -// i++ -// } -// return keys -//} +func (mlrmap *Mlrmap) GetKeys() []string { + keys := make([]string, mlrmap.FieldCount) + i := 0 + for pe := mlrmap.Head; pe != nil; pe = pe.Next { + keys[i] = pe.Key + i++ + } + return keys +} // ---------------------------------------------------------------- // For '$[[[1]]]' etc. in the DSL. @@ -271,41 +270,39 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { // ---------------------------------------------------------------- // TODO: put error-return into this API -func (mlrmap *Mlrmap) PutNameWithPositionalIndex(position int, name *Mlrval) { - positionalEntry := mlrmap.findEntryByPositionalIndex(position) - - if positionalEntry == nil { - // TODO: handle out-of-bounds accesses - return - } - - s := "" - if name.mvtype == MT_STRING { - s = name.printrep - } else if name.mvtype == MT_INT { - s = name.String() - } else { - // TODO: return MlrvalFromError() - return - } - - // E.g. there are fields named 'a' and 'b', as positions 1 and 2, - // and the user does '$[[1]] = $[[2]]'. Then there would be two b's. - mapEntry := mlrmap.findEntry(s) - if mapEntry != nil && mapEntry != positionalEntry { - if mlrmap.keysToEntries != nil { - delete(mlrmap.keysToEntries, positionalEntry.Key) - } - mlrmap.Unlink(mapEntry) - } - - lib.InternalCodingErrorIf(s == "") - positionalEntry.Key = s - - if mlrmap.keysToEntries != nil { - mlrmap.keysToEntries[s] = positionalEntry - } -} +//func (mlrmap *Mlrmap) PutNameWithPositionalIndex(position int, name *mlrval.Mlrval) { +// positionalEntry := mlrmap.findEntryByPositionalIndex(position) +// +// if positionalEntry == nil { +// // TODO: handle out-of-bounds accesses +// return +// } +// +// s := "" +// if name.IsString() || name.IsInt() { +// s = name.String() +// } else { +// // TODO: return MlrvalFromError() +// return +// } +// +// // E.g. there are fields named 'a' and 'b', as positions 1 and 2, +// // and the user does '$[[1]] = $[[2]]'. Then there would be two b's. +// mapEntry := mlrmap.findEntry(s) +// if mapEntry != nil && mapEntry != positionalEntry { +// if mlrmap.keysToEntries != nil { +// delete(mlrmap.keysToEntries, positionalEntry.Key) +// } +// mlrmap.Unlink(mapEntry) +// } +// +// lib.InternalCodingErrorIf(s == "") +// positionalEntry.Key = s +// +// if mlrmap.keysToEntries != nil { +// mlrmap.keysToEntries[s] = positionalEntry +// } +//} //func (mlrmap *Mlrmap) GetWithPositionalIndex(position int) *mlrval.Mlrval { // mapEntry := mlrmap.findEntryByPositionalIndex(position) diff --git a/internal/pkg/types/mlrval_infer.go b/internal/pkg/types/mlrval_infer.go deleted file mode 100644 index 171a6bf06..000000000 --- a/internal/pkg/types/mlrval_infer.go +++ /dev/null @@ -1,198 +0,0 @@ -// ================================================================ -// Constructors -// ================================================================ - -package types - -import ( - "regexp" - "strings" - - "github.com/johnkerl/miller/internal/pkg/lib" -) - -// ---------------------------------------------------------------- - -// MlrvalFromPrevalidatedIntString is for situations where the string has -// already been determined to be parseable as int. For example, int literals in -// the Miller DSL fall into this category, as the parser has already matched -// them. -func MlrvalFromPrevalidatedIntString(input string) *Mlrval { - ival, ok := lib.TryIntFromString(input) - lib.InternalCodingErrorIf(!ok) - return &Mlrval{ - mvtype: MT_INT, - printrep: input, - printrepValid: true, - intval: ival, - } -} - -// MlrvalFromPrevalidatedFloat64String is for situations where the string has -// already been determined to be parseable as int. For example, int literals in -// the Miller DSL fall into this category, as the parser has already matched -// them. -func MlrvalFromPrevalidatedFloat64String(input string) *Mlrval { - fval, ok := lib.TryFloat64FromString(input) - lib.InternalCodingErrorIf(!ok) - return &Mlrval{ - mvtype: MT_FLOAT, - printrep: input, - printrepValid: true, - floatval: fval, - } -} - -// ---------------------------------------------------------------- - -// MlrvalTryPointerFromFloatString is used by MlrvalFormatter (fmtnum DSL -// function, format-values verb, etc). Each mlrval has printrep and a -// printrepValid for its original string, then a type-code like MT_INT or -// MT_FLOAT, and type-specific storage like intval or floatval. -// -// If the user has taken a mlrval with original string "3.14" and formatted it -// with "%.4f" then its printrep will be "3.1400" but its type should still be -// MT_FLOAT. -// -// If on the other hand the user has formatted the same mlrval with "[[%.4f]]" -// then its printrep will be "[[3.1400]]" and it will be MT_STRING. -// This function supports that. -func MlrvalTryPointerFromFloatString(input string) *Mlrval { - _, fok := lib.TryFloat64FromString(input) - if fok { - return MlrvalFromPrevalidatedFloat64String(input) - } else { - return MlrvalFromString(input) - } -} - -// MlrvalTryPointerFromIntString is used by MlrvalFormatter (fmtnum DSL -// function, format-values verb, etc). Each mlrval has printrep and a -// printrepValid for its original string, then a type-code like MT_INT or -// MT_FLOAT, and type-specific storage like intval or floatval. -// -// If the user has taken a mlrval with original string "314" and formatted it -// with "0x%04x" then its printrep will be "0x013a" but its type should still be -// MT_INT. -// -// If on the other hand the user has formatted the same mlrval with -// "[[%0x04x]]" then its printrep will be "[[0x013a]]" and it will be -// MT_STRING. This function supports that. -func MlrvalTryPointerFromIntString(input string) *Mlrval { - _, iok := lib.TryIntFromString(input) - if iok { - return MlrvalFromPrevalidatedIntString(input) - } else { - return MlrvalFromString(input) - } -} - -// ---------------------------------------------------------------- - -// When loading data files, don't scan these words into floats -- even though -// the Go library is willing to do so. -var downcasedFloatNamesToNotInfer = map[string]bool{ - "inf": true, - "+inf": true, - "-inf": true, - "infinity": true, - "+infinity": true, - "-infinity": true, - "nan": true, -} - -// ---------------------------------------------------------------- -type tInferrer func(input string, inferBool bool) *Mlrval - -var inferrer tInferrer = inferNormally - -func SetInferrerNoOctal() { - inferrer = inferWithOctalSuppress -} -func SetInferrerIntAsFloat() { - inferrer = inferWithIntAsFloat -} -func SetInferrerStringOnly() { - inferrer = inferStringOnly -} - -// MlrvalFromInferredTypeForDataFiles is for parsing field values directly from -// data files (except JSON, which is typed -- "true" and true are distinct). -// Mostly the same as MlrvalFromInferredType, except it doesn't auto-infer -// true/false to bool; don't auto-infer NaN/Inf to float; etc. -func MlrvalFromInferredTypeForDataFiles(input string) *Mlrval { - return inferrer(input, false) -} - -// MlrvalFromInferredType is for parsing field values not directly from data -// files. Mostly the same as MlrvalFromInferredTypeForDataFiles, except it -// auto-infers true/false to bool; don't auto-infer NaN/Inf to float; etc. -func MlrvalFromInferredType(input string) *Mlrval { - return inferrer(input, true) -} - -func inferNormally(input string, inferBool bool) *Mlrval { - if input == "" { - return MLRVAL_VOID - } - - _, iok := lib.TryIntFromString(input) - if iok { - return MlrvalFromPrevalidatedIntString(input) - } - - if downcasedFloatNamesToNotInfer[strings.ToLower(input)] == false { - _, fok := lib.TryFloat64FromString(input) - if fok { - return MlrvalFromPrevalidatedFloat64String(input) - } - } - - if inferBool { - _, bok := lib.TryBoolFromBoolString(input) - if bok { - return MlrvalFromBoolString(input) - } - } - - return MlrvalFromString(input) -} - -var octalDetector = regexp.MustCompile("^-?0[0-9]+") - -func inferWithOctalSuppress(input string, inferBool bool) *Mlrval { - output := inferNormally(input, inferBool) - if output.mvtype != MT_INT && output.mvtype != MT_FLOAT { - return output - } - - if octalDetector.MatchString(input) { - return MlrvalFromString(input) - } else { - return output - } - - return output -} - -func inferWithIntAsFloat(input string, inferBool bool) *Mlrval { - output := inferNormally(input, inferBool) - if output.mvtype == MT_INT { - return &Mlrval{ - mvtype: MT_FLOAT, - printrepValid: true, - printrep: input, - floatval: float64(output.intval), - } - } else { - return output - } -} - -func inferStringOnly(input string, inferBool bool) *Mlrval { - if input == "" { - return MLRVAL_VOID - } - - return MlrvalFromString(input) -} diff --git a/todo.txt b/todo.txt index 146d559ce..315566e53 100644 --- a/todo.txt +++ b/todo.txt @@ -2,6 +2,9 @@ PUNCHDOWN LIST * --ifs-regex & --ips-regex -- guessing is not safe as evidence by '.' and '|' +* JIT next: + o mlrval copy + o mlrmap copy * big-picture item @ Rmd; also webdoc intro page From f8fd0da3477f38c3ad049abea51a0c1d1de6a287 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 13 Dec 2021 09:56:04 -0500 Subject: [PATCH 04/19] iterating --- .vimrc | 1 + Makefile | 18 +++++-- internal/pkg/mlrval/get.go | 4 ++ internal/pkg/mlrval/get_test.go | 26 +++++++++ internal/pkg/types-parked/mlrval_accessors.go | 12 ----- internal/pkg/types/mlrval_stringify.go | 7 +++ internal/pkg/types/mlrval_stringify_test.go | 53 +++++++++++++++++++ todo.txt | 2 + 8 files changed, 107 insertions(+), 16 deletions(-) create mode 100644 internal/pkg/types/mlrval_stringify.go create mode 100644 internal/pkg/types/mlrval_stringify_test.go diff --git a/.vimrc b/.vimrc index 9d261becd..860112cd2 100644 --- a/.vimrc +++ b/.vimrc @@ -2,3 +2,4 @@ map \d :w:!clear;echo Building ...; echo; make mlr "map \f :w:!clear;echo Building ...; echo; make mlrval-tests "map \f :w:!clear;echo Building ...; echo; make mlrmap-tests map \f :w:!clear;echo Building ...; echo; make input-tests +map \r :w:!clear;echo Building ...; echo; make tests-in-order diff --git a/Makefile b/Makefile index a310c599f..bddc90815 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ PREFIX=/usr/local INSTALLDIR=$(PREFIX)/bin -# ---------------------------------------------------------------- +# ================================================================ # General-use targets # This must remain the first target in this file, which is what 'make' with no @@ -23,9 +23,10 @@ install: build cp mlr $(DESTDIR)/$(INSTALLDIR) make -C man install -# ---------------------------------------------------------------- +# ================================================================ # Dev targets +# ---------------------------------------------------------------- # Unit tests (small number) unit-test ut: go test github.com/johnkerl/miller/internal/pkg/... @@ -58,7 +59,10 @@ mlrval-get-test: internal/pkg/mlrval/infer.go \ internal/pkg/mlrval/is.go \ internal/pkg/mlrval/get.go -mlrval-tests: mlrval-new-test mlrval-is-test mlrval-get-test +mlrval-stringify-test: + go test internal/pkg/types/mlrval_stringify_test.go \ + internal/pkg/types/mlrval_stringify.go +mlrval-tests: mlrval-new-test mlrval-is-test mlrval-get-test mlrval-stringify-test mlrmap-new-test: go test internal/pkg/types/mlrmap_new_test.go \ @@ -80,6 +84,9 @@ input-tests: input-dkvp-test #mlrval_format_test: # go test internal/pkg/types/mlrval_format_test.go $(ls internal/pkg/types/*.go|grep -v test) +tests-in-order: mlrval-tests mlrmap-tests input-tests + +# ---------------------------------------------------------------- # Regression tests (large number) # # See ./regression_test.go for information on how to get more details @@ -92,6 +99,7 @@ regression-test: fmt: -go fmt ./... +# ---------------------------------------------------------------- # For developers before pushing to GitHub. # # These steps are done in a particular order: @@ -113,6 +121,7 @@ dev: make -C docs @echo DONE +# ---------------------------------------------------------------- # Keystroke-savers it: build check so: install @@ -131,11 +140,12 @@ mprof5: go build github.com/johnkerl/miller/cmd/mprof5 mall: mprof5 mprof4 mprof3 mprof2 mprof mlr +# ---------------------------------------------------------------- # Please see comments in ./create-release-tarball as well as # https://miller.readthedocs.io/en/latest/build/#creating-a-new-release-for-developers release_tarball: build check ./create-release-tarball -# ---------------------------------------------------------------- +# ================================================================ # Go does its own dependency management, outside of make. .PHONY: build mlr mprof mprof2 mprof3 mprof4 mprof5 check unit_test regression_test fmt dev diff --git a/internal/pkg/mlrval/get.go b/internal/pkg/mlrval/get.go index 57759c288..b76a61e0e 100644 --- a/internal/pkg/mlrval/get.go +++ b/internal/pkg/mlrval/get.go @@ -79,3 +79,7 @@ func (mv *Mlrval) GetFunction() interface{} { return nil } } + +func (mv *Mlrval) GetTypeName() string { + return TYPE_NAMES[mv.Type()] +} diff --git a/internal/pkg/mlrval/get_test.go b/internal/pkg/mlrval/get_test.go index 7fdbf8287..c633b4491 100644 --- a/internal/pkg/mlrval/get_test.go +++ b/internal/pkg/mlrval/get_test.go @@ -161,3 +161,29 @@ func TestGetBoolValue(t *testing.T) { boolval, ok = mv.GetBoolValue() assert.False(t, ok, "from-data-file \"false\" should infer to string") } + +func TestGetTypeName(t *testing.T) { + mv := FromInferredType("234") + assert.Equal(t, "int", mv.GetTypeName()) + + mv = FromDeferredType("234") + assert.Equal(t, "int", mv.GetTypeName()) + + mv = FromInferredType("234.5") + assert.Equal(t, "float", mv.GetTypeName()) + + mv = FromDeferredType("234.5") + assert.Equal(t, "float", mv.GetTypeName()) + + mv = FromInferredType("abc") + assert.Equal(t, "string", mv.GetTypeName()) + + mv = FromDeferredType("abc") + assert.Equal(t, "string", mv.GetTypeName()) + + mv = FromInferredType("") + assert.Equal(t, "empty", mv.GetTypeName()) + + mv = FromDeferredType("") + assert.Equal(t, "empty", mv.GetTypeName()) +} diff --git a/internal/pkg/types-parked/mlrval_accessors.go b/internal/pkg/types-parked/mlrval_accessors.go index 63cd6e3fd..86d80a313 100644 --- a/internal/pkg/types-parked/mlrval_accessors.go +++ b/internal/pkg/types-parked/mlrval_accessors.go @@ -8,21 +8,10 @@ import ( "github.com/johnkerl/miller/internal/pkg/lib" ) -// ---------------------------------------------------------------- -func (mv *Mlrval) GetType() MVType { - return mv.mvtype -} - -func (mv *Mlrval) GetTypeName() string { - return TYPE_NAMES[mv.mvtype] -} - func GetTypeName(mvtype MVType) string { return TYPE_NAMES[mvtype] } -// ---------------------------------------------------------------- - func (mv *Mlrval) GetArrayLength() (int, bool) { if mv.IsArray() { return len(mv.arrayval), true @@ -31,7 +20,6 @@ func (mv *Mlrval) GetArrayLength() (int, bool) { } } -// ---------------------------------------------------------------- func (mv *Mlrval) Copy() *Mlrval { other := *mv if mv.IsMap() { diff --git a/internal/pkg/types/mlrval_stringify.go b/internal/pkg/types/mlrval_stringify.go new file mode 100644 index 000000000..ac99ed37a --- /dev/null +++ b/internal/pkg/types/mlrval_stringify.go @@ -0,0 +1,7 @@ +package types + +import ( + //"github.com/johnkerl/miller/internal/pkg/mlrval" +) + +// ... diff --git a/internal/pkg/types/mlrval_stringify_test.go b/internal/pkg/types/mlrval_stringify_test.go new file mode 100644 index 000000000..5bb53fe94 --- /dev/null +++ b/internal/pkg/types/mlrval_stringify_test.go @@ -0,0 +1,53 @@ +// ================================================================ +// Tests mlrval-to-string logic +// ================================================================ + +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFoo(t *testing.T) { + assert.Equal(t, false, true) +} + +//func TestGetString(t *testing.T) { +// mv := FromInferredType("234") +// stringval, ok := mv.GetString() +// assert.False(t, ok) +// +// mv = FromDeferredType("234") +// stringval, ok = mv.GetString() +// assert.False(t, ok) +// +// mv = FromInferredType("234.5") +// stringval, ok = mv.GetString() +// assert.False(t, ok) +// +// mv = FromDeferredType("234.5") +// stringval, ok = mv.GetString() +// assert.False(t, ok) +// +// mv = FromInferredType("abc") +// stringval, ok = mv.GetString() +// assert.Equal(t, "abc", stringval) +// assert.True(t, ok) +// +// mv = FromDeferredType("abc") +// stringval, ok = mv.GetString() +// assert.Equal(t, "abc", stringval) +// assert.True(t, ok) +// +// mv = FromInferredType("") +// stringval, ok = mv.GetString() +// assert.Equal(t, "", stringval) +// assert.True(t, ok) +// +// mv = FromDeferredType("") +// stringval, ok = mv.GetString() +// assert.Equal(t, "", stringval) +// assert.True(t, ok) +//} diff --git a/todo.txt b/todo.txt index 315566e53..9894fc2d1 100644 --- a/todo.txt +++ b/todo.txt @@ -5,6 +5,8 @@ PUNCHDOWN LIST * JIT next: o mlrval copy o mlrmap copy + o s, ok := mlrval.ScalarString -- ? + o MlrvalString * big-picture item @ Rmd; also webdoc intro page From 1d3a918f7afc7ff37371b9dd78149ae50ef62429 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 13 Dec 2021 23:20:51 -0500 Subject: [PATCH 05/19] mlrval/format.go --- .vimrc | 5 +- Makefile | 25 +++- .../mlrval_format.go => mlrval/format.go} | 117 +++++++++-------- .../format_test.go} | 122 ++++++++---------- internal/pkg/mlrval/get_test.go | 1 - internal/pkg/mlrval/new.go | 56 ++++++++ internal/pkg/mlrval/new_test.go | 20 +++ .../mlrval_output.go => mlrval/output.go} | 65 ++++------ internal/pkg/mlrval/output_test.go | 18 +++ internal/pkg/mlrval/tbf.go | 44 ------- internal/pkg/types/mlrval_stringify.go | 7 - internal/pkg/types/mlrval_stringify_test.go | 53 -------- todo.txt | 4 +- 13 files changed, 267 insertions(+), 270 deletions(-) rename internal/pkg/{types-parked/mlrval_format.go => mlrval/format.go} (61%) rename internal/pkg/{types/mlrval_format_test.go => mlrval/format_test.go} (59%) rename internal/pkg/{types-parked/mlrval_output.go => mlrval/output.go} (64%) create mode 100644 internal/pkg/mlrval/output_test.go delete mode 100644 internal/pkg/types/mlrval_stringify.go delete mode 100644 internal/pkg/types/mlrval_stringify_test.go diff --git a/.vimrc b/.vimrc index 860112cd2..1a413f44a 100644 --- a/.vimrc +++ b/.vimrc @@ -1,5 +1,6 @@ map \d :w:!clear;echo Building ...; echo; make mlr +map \r :w:!clear;echo Building ...; echo; make tests-in-order "map \f :w:!clear;echo Building ...; echo; make mlrval-tests "map \f :w:!clear;echo Building ...; echo; make mlrmap-tests -map \f :w:!clear;echo Building ...; echo; make input-tests -map \r :w:!clear;echo Building ...; echo; make tests-in-order +"map \f :w:!clear;echo Building ...; echo; make input-tests +map \f :w:!clear;echo Building ...; echo; make mlrval-format-test diff --git a/Makefile b/Makefile index bddc90815..c89b2acd7 100644 --- a/Makefile +++ b/Makefile @@ -59,10 +59,27 @@ mlrval-get-test: internal/pkg/mlrval/infer.go \ internal/pkg/mlrval/is.go \ internal/pkg/mlrval/get.go -mlrval-stringify-test: - go test internal/pkg/types/mlrval_stringify_test.go \ - internal/pkg/types/mlrval_stringify.go -mlrval-tests: mlrval-new-test mlrval-is-test mlrval-get-test mlrval-stringify-test +mlrval-output-test: + go test internal/pkg/mlrval/output_test.go \ + internal/pkg/mlrval/type.go \ + internal/pkg/mlrval/constants.go \ + internal/pkg/mlrval/new.go \ + internal/pkg/mlrval/infer.go \ + internal/pkg/mlrval/is.go \ + internal/pkg/mlrval/get.go \ + internal/pkg/mlrval/output.go \ + internal/pkg/mlrval/format.go +mlrval-format-test: + go test internal/pkg/mlrval/format_test.go \ + internal/pkg/mlrval/type.go \ + internal/pkg/mlrval/constants.go \ + internal/pkg/mlrval/new.go \ + internal/pkg/mlrval/infer.go \ + internal/pkg/mlrval/is.go \ + internal/pkg/mlrval/get.go \ + internal/pkg/mlrval/output.go \ + internal/pkg/mlrval/format.go +mlrval-tests: mlrval-new-test mlrval-is-test mlrval-get-test mlrval-output-test mlrval-format-test mlrmap-new-test: go test internal/pkg/types/mlrmap_new_test.go \ diff --git a/internal/pkg/types-parked/mlrval_format.go b/internal/pkg/mlrval/format.go similarity index 61% rename from internal/pkg/types-parked/mlrval_format.go rename to internal/pkg/mlrval/format.go index 4c211441d..ae3bf2810 100644 --- a/internal/pkg/types-parked/mlrval_format.go +++ b/internal/pkg/mlrval/format.go @@ -1,4 +1,4 @@ -package types +package mlrval import ( "errors" @@ -38,40 +38,51 @@ import ( // - -i, -f, -s // ---------------------------------------------------------------- -// ---------------------------------------------------------------- -var mlrvalFormatterCache map[string]IMlrvalFormatter = make(map[string]IMlrvalFormatter) +// Nil means use default format. +// Set from the CLI parser using mlr --ofmt. +var floatOutputFormatter IFormatter = nil -func GetMlrvalFormatter( +func SetFloatOutputFormat(formatString string) error { + formatter, err := GetFormatter(formatString) + if err != nil { + return err + } + floatOutputFormatter = formatter + return nil +} + +var formatterCache map[string]IFormatter = make(map[string]IFormatter) + +type IFormatter interface { + Format(mlrval *Mlrval) *Mlrval + FormatFloat(floatValue float64) string // for --ofmt +} + +func GetFormatter( userLevelFormatString string, -) (IMlrvalFormatter, error) { +) (IFormatter, error) { // Cache hit - formatter, ok := mlrvalFormatterCache[userLevelFormatString] + formatter, ok := formatterCache[userLevelFormatString] if ok { return formatter, nil } // Cache miss - formatter, err := newMlrvalFormatter(userLevelFormatString) + formatter, err := newFormatter(userLevelFormatString) if err != nil { // TODO: temp exit fmt.Printf("mlr: %v\n", err) return nil, err } - mlrvalFormatterCache[userLevelFormatString] = formatter + formatterCache[userLevelFormatString] = formatter return formatter, nil } -// ---------------------------------------------------------------- -type IMlrvalFormatter interface { - Format(mlrval *Mlrval) *Mlrval - FormatFloat(floatValue float64) string // for --ofmt -} - // People can pass in things like "X%sX" unfortunately :( -func newMlrvalFormatter( +func newFormatter( userLevelFormatString string, -) (IMlrvalFormatter, error) { +) (IFormatter, error) { numPercents := strings.Count(userLevelFormatString, "%") if numPercents < 1 { return nil, errors.New( @@ -99,111 +110,113 @@ func newMlrvalFormatter( // and double-precision floats: e.g. "%08lld" and "%9.6lf". For Miller 6, // We must still accept these for backward compatibility. if strings.HasSuffix(goFormatString, "d") { - return newMlrvalFormatterToInt(goFormatString), nil + return newFormatterToInt(goFormatString), nil } if strings.HasSuffix(goFormatString, "x") { - return newMlrvalFormatterToInt(goFormatString), nil + return newFormatterToInt(goFormatString), nil } if strings.HasSuffix(goFormatString, "f") { - return newMlrvalFormatterToFloat(goFormatString), nil + return newFormatterToFloat(goFormatString), nil } if strings.HasSuffix(goFormatString, "e") { - return newMlrvalFormatterToFloat(goFormatString), nil + return newFormatterToFloat(goFormatString), nil } if strings.HasSuffix(goFormatString, "g") { - return newMlrvalFormatterToFloat(goFormatString), nil + return newFormatterToFloat(goFormatString), nil } if strings.HasSuffix(goFormatString, "s") { - return newMlrvalFormatterToString(goFormatString), nil + return newFormatterToString(goFormatString), nil } // TODO: // return nil, errors.New(fmt.Sprintf("unhandled format string \"%s\"", userLevelFormatString)) - return newMlrvalFormatterToString(goFormatString), nil + return newFormatterToString(goFormatString), nil } -//func regularizeFormat - // ---------------------------------------------------------------- -type mlrvalFormatterToFloat struct { + +type formatterToFloat struct { goFormatString string } -func newMlrvalFormatterToFloat(goFormatString string) IMlrvalFormatter { - return &mlrvalFormatterToFloat{ +func newFormatterToFloat(goFormatString string) IFormatter { + return &formatterToFloat{ goFormatString: goFormatString, } } -func (formatter *mlrvalFormatterToFloat) Format(mlrval *Mlrval) *Mlrval { - floatValue, isFloat := mlrval.GetFloatValue() +func (formatter *formatterToFloat) Format(mv *Mlrval) *Mlrval { + floatValue, isFloat := mv.GetFloatValue() if isFloat { formatted := fmt.Sprintf(formatter.goFormatString, floatValue) - return MlrvalTryPointerFromFloatString(formatted) + return TryFromFloatString(formatted) } - intValue, isInt := mlrval.GetIntValue() + intValue, isInt := mv.GetIntValue() if isInt { formatted := fmt.Sprintf(formatter.goFormatString, float64(intValue)) - return MlrvalTryPointerFromFloatString(formatted) + return TryFromFloatString(formatted) } - return mlrval + return mv } -func (formatter *mlrvalFormatterToFloat) FormatFloat(floatValue float64) string { +func (formatter *formatterToFloat) FormatFloat(floatValue float64) string { return fmt.Sprintf(formatter.goFormatString, floatValue) } // ---------------------------------------------------------------- -type mlrvalFormatterToInt struct { + +type formatterToInt struct { goFormatString string } -func newMlrvalFormatterToInt(goFormatString string) IMlrvalFormatter { - return &mlrvalFormatterToInt{ +func newFormatterToInt(goFormatString string) IFormatter { + return &formatterToInt{ goFormatString: goFormatString, } } -func (formatter *mlrvalFormatterToInt) Format(mlrval *Mlrval) *Mlrval { - intValue, isInt := mlrval.GetIntValue() +func (formatter *formatterToInt) Format(mv *Mlrval) *Mlrval { + intValue, isInt := mv.GetIntValue() if isInt { formatted := fmt.Sprintf(formatter.goFormatString, intValue) - return MlrvalTryPointerFromIntString(formatted) + return TryFromIntString(formatted) } - floatValue, isFloat := mlrval.GetFloatValue() + floatValue, isFloat := mv.GetFloatValue() if isFloat { formatted := fmt.Sprintf(formatter.goFormatString, int(floatValue)) - return MlrvalTryPointerFromIntString(formatted) + return TryFromIntString(formatted) } - return mlrval + return mv } -func (formatter *mlrvalFormatterToInt) FormatFloat(floatValue float64) string { +func (formatter *formatterToInt) FormatFloat(floatValue float64) string { return fmt.Sprintf(formatter.goFormatString, int(floatValue)) } + // ---------------------------------------------------------------- -type mlrvalFormatterToString struct { + +type formatterToString struct { goFormatString string } -func newMlrvalFormatterToString(goFormatString string) IMlrvalFormatter { - return &mlrvalFormatterToString{ +func newFormatterToString(goFormatString string) IFormatter { + return &formatterToString{ goFormatString: goFormatString, } } -func (formatter *mlrvalFormatterToString) Format(mlrval *Mlrval) *Mlrval { - return MlrvalFromString( +func (formatter *formatterToString) Format(mv *Mlrval) *Mlrval { + return FromString( fmt.Sprintf( formatter.goFormatString, - mlrval.String(), + mv.String(), ), ) } -func (formatter *mlrvalFormatterToString) FormatFloat(floatValue float64) string { +func (formatter *formatterToString) FormatFloat(floatValue float64) string { return strconv.FormatFloat(floatValue, 'g', -1, 64) } diff --git a/internal/pkg/types/mlrval_format_test.go b/internal/pkg/mlrval/format_test.go similarity index 59% rename from internal/pkg/types/mlrval_format_test.go rename to internal/pkg/mlrval/format_test.go index 330732917..c26bf1f6d 100644 --- a/internal/pkg/types/mlrval_format_test.go +++ b/internal/pkg/mlrval/format_test.go @@ -1,25 +1,18 @@ -// ================================================================ -// Most Miller tests (thousands of them) are command-line-driven via -// mlr regtest. Here are some cases needing special focus. -// ================================================================ - -// Invoke as: -// * cd internal/pkg/types -// * go test -// Or: -// * cd go -// * go test github.com/johnkerl/miller/internal/pkg/types/... - -package types +package mlrval import ( "testing" + + "github.com/stretchr/testify/assert" ) -func TestFormatterToString(t *testing.T) { +func TestFoo(t *testing.T) { + assert.Equal(t, true, true) +} - mv := MlrvalFromString("hello") - formatter := newMlrvalFormatterToString("%s") +func TestFormatterToString(t *testing.T) { + mv := FromString("hello") + formatter := newFormatterToString("%s") fmv := formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -28,8 +21,8 @@ func TestFormatterToString(t *testing.T) { t.Fatal() } - mv = MLRVAL_NULL - formatter = newMlrvalFormatterToString("%s") + mv = NULL + formatter = newFormatterToString("%s") fmv = formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -38,8 +31,8 @@ func TestFormatterToString(t *testing.T) { t.Fatal() } - mv = MLRVAL_TRUE - formatter = newMlrvalFormatterToString("%s") + mv = TRUE + formatter = newFormatterToString("%s") fmv = formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -48,8 +41,8 @@ func TestFormatterToString(t *testing.T) { t.Fatal() } - mv = MLRVAL_FALSE - formatter = newMlrvalFormatterToString("%s") + mv = FALSE + formatter = newFormatterToString("%s") fmv = formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -58,8 +51,8 @@ func TestFormatterToString(t *testing.T) { t.Fatal() } - mv = MlrvalFromInt(10) - formatter = newMlrvalFormatterToString("%s") + mv = FromInt(10) + formatter = newFormatterToString("%s") fmv = formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -68,8 +61,8 @@ func TestFormatterToString(t *testing.T) { t.Fatal() } - mv = MlrvalFromString("hello") - formatter = newMlrvalFormatterToString("[[%s]]") + mv = FromString("hello") + formatter = newFormatterToString("[[%s]]") fmv = formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -81,9 +74,8 @@ func TestFormatterToString(t *testing.T) { } func TestFormatterToInt(t *testing.T) { - - mv := MlrvalFromString("hello") - formatter := newMlrvalFormatterToInt("%d") + mv := FromString("hello") + formatter := newFormatterToInt("%d") fmv := formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -92,8 +84,8 @@ func TestFormatterToInt(t *testing.T) { t.Fatal() } - mv = MLRVAL_NULL - formatter = newMlrvalFormatterToInt("%d") + mv = NULL + formatter = newFormatterToInt("%d") fmv = formatter.Format(mv) if !fmv.IsNull() { t.Fatal() @@ -102,8 +94,8 @@ func TestFormatterToInt(t *testing.T) { t.Fatal() } - mv = MLRVAL_TRUE - formatter = newMlrvalFormatterToInt("%d") + mv = TRUE + formatter = newFormatterToInt("%d") fmv = formatter.Format(mv) if !fmv.IsBool() { t.Fatal() @@ -112,8 +104,8 @@ func TestFormatterToInt(t *testing.T) { t.Fatal() } - mv = MLRVAL_FALSE - formatter = newMlrvalFormatterToInt("%d") + mv = FALSE + formatter = newFormatterToInt("%d") fmv = formatter.Format(mv) if !fmv.IsBool() { t.Fatal() @@ -122,8 +114,8 @@ func TestFormatterToInt(t *testing.T) { t.Fatal() } - mv = MlrvalFromInt(10) - formatter = newMlrvalFormatterToInt("%d") + mv = FromInt(10) + formatter = newFormatterToInt("%d") fmv = formatter.Format(mv) if !fmv.IsInt() { t.Fatal() @@ -132,8 +124,8 @@ func TestFormatterToInt(t *testing.T) { t.Fatal() } - mv = MlrvalFromInt(10) - formatter = newMlrvalFormatterToInt("[[0x%x]]") + mv = FromInt(10) + formatter = newFormatterToInt("[[0x%x]]") fmv = formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -142,8 +134,8 @@ func TestFormatterToInt(t *testing.T) { t.Fatal() } - mv = MlrvalFromFloat64(10.1) - formatter = newMlrvalFormatterToInt("%d") + mv = FromFloat(10.1) + formatter = newFormatterToInt("%d") fmv = formatter.Format(mv) if !fmv.IsInt() { t.Fatal() @@ -156,8 +148,8 @@ func TestFormatterToInt(t *testing.T) { func TestFormatterToFloat(t *testing.T) { - mv := MlrvalFromString("hello") - formatter := newMlrvalFormatterToFloat("%.4f") + mv := FromString("hello") + formatter := newFormatterToFloat("%.4f") fmv := formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -166,8 +158,8 @@ func TestFormatterToFloat(t *testing.T) { t.Fatal() } - mv = MLRVAL_NULL - formatter = newMlrvalFormatterToFloat("%.4f") + mv = NULL + formatter = newFormatterToFloat("%.4f") fmv = formatter.Format(mv) if !fmv.IsNull() { t.Fatal() @@ -176,8 +168,8 @@ func TestFormatterToFloat(t *testing.T) { t.Fatal() } - mv = MLRVAL_TRUE - formatter = newMlrvalFormatterToFloat("%.4f") + mv = TRUE + formatter = newFormatterToFloat("%.4f") fmv = formatter.Format(mv) if !fmv.IsBool() { t.Fatal() @@ -186,8 +178,8 @@ func TestFormatterToFloat(t *testing.T) { t.Fatal() } - mv = MLRVAL_FALSE - formatter = newMlrvalFormatterToFloat("%.4f") + mv = FALSE + formatter = newFormatterToFloat("%.4f") fmv = formatter.Format(mv) if !fmv.IsBool() { t.Fatal() @@ -196,8 +188,8 @@ func TestFormatterToFloat(t *testing.T) { t.Fatal() } - mv = MlrvalFromFloat64(10) - formatter = newMlrvalFormatterToFloat("%.4f") + mv = FromFloat(10) + formatter = newFormatterToFloat("%.4f") fmv = formatter.Format(mv) if !fmv.IsFloat() { t.Fatal() @@ -206,8 +198,8 @@ func TestFormatterToFloat(t *testing.T) { t.Fatal() } - mv = MlrvalFromFloat64(10.1) - formatter = newMlrvalFormatterToFloat("%.4f") + mv = FromFloat(10.1) + formatter = newFormatterToFloat("%.4f") fmv = formatter.Format(mv) if !fmv.IsFloat() { t.Fatal() @@ -219,8 +211,8 @@ func TestFormatterToFloat(t *testing.T) { func TestFormatter(t *testing.T) { - mv := MlrvalFromString("hello") - formatter, err := GetMlrvalFormatter("%d") + mv := FromString("hello") + formatter, err := GetFormatter("%d") if err != nil { t.Fatal() } @@ -232,8 +224,8 @@ func TestFormatter(t *testing.T) { t.Fatal() } - mv = MLRVAL_NULL - formatter, err = GetMlrvalFormatter("%d") + mv = NULL + formatter, err = GetFormatter("%d") if err != nil { t.Fatal() } @@ -245,8 +237,8 @@ func TestFormatter(t *testing.T) { t.Fatal() } - mv = MLRVAL_TRUE - formatter, err = GetMlrvalFormatter("%d") + mv = TRUE + formatter, err = GetFormatter("%d") if err != nil { t.Fatal() } @@ -258,8 +250,8 @@ func TestFormatter(t *testing.T) { t.Fatal() } - mv = MLRVAL_FALSE - formatter, err = GetMlrvalFormatter("%d") + mv = FALSE + formatter, err = GetFormatter("%d") if err != nil { t.Fatal() } @@ -271,8 +263,8 @@ func TestFormatter(t *testing.T) { t.Fatal() } - mv = MlrvalFromFloat64(10.123) - formatter, err = GetMlrvalFormatter("%d") + mv = FromFloat(10.123) + formatter, err = GetFormatter("%d") if err != nil { t.Fatal() } @@ -284,8 +276,8 @@ func TestFormatter(t *testing.T) { t.Fatal() } - mv = MlrvalFromFloat64(10.1) - formatter, err = GetMlrvalFormatter("%d") + mv = FromFloat(10.1) + formatter, err = GetFormatter("%d") if err != nil { t.Fatal() } diff --git a/internal/pkg/mlrval/get_test.go b/internal/pkg/mlrval/get_test.go index c633b4491..43ca70c6a 100644 --- a/internal/pkg/mlrval/get_test.go +++ b/internal/pkg/mlrval/get_test.go @@ -8,7 +8,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - //"github.com/johnkerl/miller/internal/pkg/lib" ) func TestGetString(t *testing.T) { diff --git a/internal/pkg/mlrval/new.go b/internal/pkg/mlrval/new.go index 27d9f0089..1a4e9157b 100644 --- a/internal/pkg/mlrval/new.go +++ b/internal/pkg/mlrval/new.go @@ -72,6 +72,27 @@ func FromInt(input int) *Mlrval { } } +// TryFromIntString is used by the mlrval Formatter (fmtnum DSL function, +// format-values verb, etc). Each mlrval has printrep and a printrepValid for +// its original string, then a type-code like MT_INT or MT_FLOAT, and +// type-specific storage like intval or floatval. +// +// If the user has taken a mlrval with original string "314" and formatted it +// with "0x%04x" then its printrep will be "0x013a" but its type should still +// be MT_INT. +// +// If on the other hand the user has formatted the same mlrval with +// "[[%0x04x]]" then its printrep will be "[[0x013a]]" and it will be +// MT_STRING. This function supports that. +func TryFromIntString(input string) *Mlrval { + intval, ok := lib.TryIntFromString(input) + if ok { + return FromPrevalidatedIntString(input, intval) + } else { + return FromString(input) + } +} + // TODO: comment func (mv *Mlrval) SetFromPrevalidatedIntString(input string, intval int) *Mlrval { mv.printrep = input @@ -81,6 +102,13 @@ func (mv *Mlrval) SetFromPrevalidatedIntString(input string, intval int) *Mlrval return mv } +// TODO: comment +func FromPrevalidatedIntString(input string, intval int) *Mlrval { + mv := &Mlrval{} + mv.SetFromPrevalidatedIntString(input, intval) + return mv +} + func FromFloat(input float64) *Mlrval { return &Mlrval{ mvtype: MT_FLOAT, @@ -89,6 +117,27 @@ func FromFloat(input float64) *Mlrval { } } +// TryFromFloatString is used by the mlrval Formatter (fmtnum DSL function, +// format-values verb, etc). Each mlrval has printrep and a printrepValid for +// its original string, then a type-code like MT_INT or MT_FLOAT, and +// type-specific storage like intval or floatval. +// +// If the user has taken a mlrval with original string "3.14" and formatted it +// with "%.4f" then its printrep will be "3.1400" but its type should still be +// MT_FLOAT. +// +// If on the other hand the user has formatted the same mlrval with "[[%.4f]]" +// then its printrep will be "[[3.1400]]" and it will be MT_STRING. This +// function supports that. +func TryFromFloatString(input string) *Mlrval { + floatval, ok := lib.TryFloat64FromString(input) + if ok { + return FromPrevalidatedFloatString(input, floatval) + } else { + return FromString(input) + } +} + // TODO: comment func (mv *Mlrval) SetFromPrevalidatedFloatString(input string, floatval float64) *Mlrval { mv.printrep = input @@ -98,6 +147,13 @@ func (mv *Mlrval) SetFromPrevalidatedFloatString(input string, floatval float64) return mv } +// TODO: comment +func FromPrevalidatedFloatString(input string, floatval float64) *Mlrval { + mv := &Mlrval{} + mv.SetFromPrevalidatedFloatString(input, floatval) + return mv +} + func FromBool(input bool) *Mlrval { if input == true { return TRUE diff --git a/internal/pkg/mlrval/new_test.go b/internal/pkg/mlrval/new_test.go index 16b713b89..48214e9b1 100644 --- a/internal/pkg/mlrval/new_test.go +++ b/internal/pkg/mlrval/new_test.go @@ -74,12 +74,32 @@ func TestFromInt(t *testing.T) { assert.False(t, mv.printrepValid, "printrep should not be computed yet") } +func TestTryFromIntString(t *testing.T) { + mv := TryFromIntString("123") + assert.Equal(t, MVType(MT_INT), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid, "printrep should be computed") + + mv = TryFromIntString("[123]") + assert.Equal(t, MVType(MT_STRING), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid, "printrep should be computed") +} + func TestFromFloat(t *testing.T) { mv := FromFloat(123.4) assert.Equal(t, MVType(MT_FLOAT), MVType(mv.mvtype)) assert.False(t, mv.printrepValid, "printrep should not be computed yet") } +func TestTryFromFloatString(t *testing.T) { + mv := TryFromFloatString("123.4") + assert.Equal(t, MVType(MT_FLOAT), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid, "printrep should be computed") + + mv = TryFromIntString("[123.4]") + assert.Equal(t, MVType(MT_STRING), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid, "printrep should be computed") +} + func TestFromBool(t *testing.T) { mv := FromBool(true) assert.Equal(t, MVType(MT_BOOL), MVType(mv.mvtype)) diff --git a/internal/pkg/types-parked/mlrval_output.go b/internal/pkg/mlrval/output.go similarity index 64% rename from internal/pkg/types-parked/mlrval_output.go rename to internal/pkg/mlrval/output.go index 7583d432c..47afa9f22 100644 --- a/internal/pkg/types-parked/mlrval_output.go +++ b/internal/pkg/mlrval/output.go @@ -1,22 +1,19 @@ -package types +package mlrval import ( - "fmt" - "os" "strconv" ) -// Empty string means use default format. -// Set from the CLI parser using mlr --ofmt. -var mlrvalFloatOutputFormatter IMlrvalFormatter = nil - -func SetMlrvalFloatOutputFormat(formatString string) error { - formatter, err := GetMlrvalFormatter(formatString) - if err != nil { - return err +// Must have non-pointer receiver in order to implement the fmt.Stringer +// interface to make this printable via fmt.Println et al. +func (mv Mlrval) String() string { + if mv.IsFloat() && floatOutputFormatter != nil { + // Use the format string from global --ofmt, if supplied + return floatOutputFormatter.FormatFloat(mv.floatval) + } else { + mv.setPrintRep() + return mv.printrep } - mlrvalFloatOutputFormatter = formatter - return nil } // See mlrval.go for more about JIT-formatting of string backings @@ -62,39 +59,29 @@ func (mv *Mlrval) setPrintRep() { // TODO: handling indentation case MT_ARRAY: - bytes, err := mv.MarshalJSON(JSON_MULTILINE, false) - // maybe just InternalCodingErrorIf(err != nil) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - mv.printrep = string(bytes) + panic("mlrval array-to-string unimplemented") + //bytes, err := mv.MarshalJSON(JSON_MULTILINE, false) + //// maybe just InternalCodingErrorIf(err != nil) + //if err != nil { + // fmt.Fprintln(os.Stderr, err) + // os.Exit(1) + //} + //mv.printrep = string(bytes) break case MT_MAP: - bytes, err := mv.MarshalJSON(JSON_MULTILINE, false) - // maybe just InternalCodingErrorIf(err != nil) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - mv.printrep = string(bytes) + panic("mlrval map-to-string unimplemented") + //bytes, err := mv.MarshalJSON(JSON_MULTILINE, false) + //// maybe just InternalCodingErrorIf(err != nil) + //if err != nil { + // fmt.Fprintln(os.Stderr, err) + // os.Exit(1) + //} + //mv.printrep = string(bytes) break } mv.printrepValid = true } } - -// Must have non-pointer receiver in order to implement the fmt.Stringer -// interface to make this printable via fmt.Println et al. -func (mv Mlrval) String() string { - if mv.IsFloat() && mlrvalFloatOutputFormatter != nil { - // Use the format string from global --ofmt, if supplied - return mlrvalFloatOutputFormatter.FormatFloat(mv.floatval) - } else { - mv.setPrintRep() - return mv.printrep - } -} diff --git a/internal/pkg/mlrval/output_test.go b/internal/pkg/mlrval/output_test.go new file mode 100644 index 000000000..3bee43f52 --- /dev/null +++ b/internal/pkg/mlrval/output_test.go @@ -0,0 +1,18 @@ +package mlrval + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestString(t *testing.T) { + assert.Equal(t, "234", FromInferredType("234").String()) + assert.Equal(t, "234", FromDeferredType("234").String()) + assert.Equal(t, "234.5", FromInferredType("234.5").String()) + assert.Equal(t, "234.5", FromDeferredType("234.5").String()) + assert.Equal(t, "abc", FromInferredType("abc").String()) + assert.Equal(t, "abc", FromDeferredType("abc").String()) + assert.Equal(t, "", FromInferredType("").String()) + assert.Equal(t, "", FromDeferredType("").String()) +} diff --git a/internal/pkg/mlrval/tbf.go b/internal/pkg/mlrval/tbf.go index 3ee116d72..822052dbb 100644 --- a/internal/pkg/mlrval/tbf.go +++ b/internal/pkg/mlrval/tbf.go @@ -161,50 +161,6 @@ package mlrval // } //} -//// MlrvalTryPointerFromFloatString is used by MlrvalFormatter (fmtnum DSL -//// function, format-values verb, etc). Each mlrval has printrep and a -//// printrepValid for its original string, then a type-code like MT_INT or -//// MT_FLOAT, and type-specific storage like intval or floatval. -//// -//// If the user has taken a mlrval with original string "3.14" and formatted it -//// with "%.4f" then its printrep will be "3.1400" but its type should still be -//// MT_FLOAT. -//// -//// If on the other hand the user has formatted the same mlrval with "[[%.4f]]" -//// then its printrep will be "[[3.1400]]" and it will be MT_STRING. -//// This function supports that. -//func MlrvalTryPointerFromFloatString(input string) *Mlrval { -// _, fok := lib.TryFloat64FromString(input) -// if fok { -// return MlrvalFromPrevalidatedFloat64String(input) -// } else { -// return MlrvalFromString(input) -// } -//} -// -//// MlrvalTryPointerFromIntString is used by MlrvalFormatter (fmtnum DSL -//// function, format-values verb, etc). Each mlrval has printrep and a -//// printrepValid for its original string, then a type-code like MT_INT or -//// MT_FLOAT, and type-specific storage like intval or floatval. -//// -//// If the user has taken a mlrval with original string "314" and formatted it -//// with "0x%04x" then its printrep will be "0x013a" but its type should still be -//// MT_INT. -//// -//// If on the other hand the user has formatted the same mlrval with -//// "[[%0x04x]]" then its printrep will be "[[0x013a]]" and it will be -//// MT_STRING. This function supports that. -//func MlrvalTryPointerFromIntString(input string) *Mlrval { -// _, iok := lib.TryIntFromString(input) -// if iok { -// return MlrvalFromPrevalidatedIntString(input) -// } else { -// return MlrvalFromString(input) -// } -//} -// -//// ---------------------------------------------------------------- - // TODO: FILE // //// MlrvalFromInferredTypeForDataFiles is for parsing field values directly from diff --git a/internal/pkg/types/mlrval_stringify.go b/internal/pkg/types/mlrval_stringify.go deleted file mode 100644 index ac99ed37a..000000000 --- a/internal/pkg/types/mlrval_stringify.go +++ /dev/null @@ -1,7 +0,0 @@ -package types - -import ( - //"github.com/johnkerl/miller/internal/pkg/mlrval" -) - -// ... diff --git a/internal/pkg/types/mlrval_stringify_test.go b/internal/pkg/types/mlrval_stringify_test.go deleted file mode 100644 index 5bb53fe94..000000000 --- a/internal/pkg/types/mlrval_stringify_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// ================================================================ -// Tests mlrval-to-string logic -// ================================================================ - -package types - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestFoo(t *testing.T) { - assert.Equal(t, false, true) -} - -//func TestGetString(t *testing.T) { -// mv := FromInferredType("234") -// stringval, ok := mv.GetString() -// assert.False(t, ok) -// -// mv = FromDeferredType("234") -// stringval, ok = mv.GetString() -// assert.False(t, ok) -// -// mv = FromInferredType("234.5") -// stringval, ok = mv.GetString() -// assert.False(t, ok) -// -// mv = FromDeferredType("234.5") -// stringval, ok = mv.GetString() -// assert.False(t, ok) -// -// mv = FromInferredType("abc") -// stringval, ok = mv.GetString() -// assert.Equal(t, "abc", stringval) -// assert.True(t, ok) -// -// mv = FromDeferredType("abc") -// stringval, ok = mv.GetString() -// assert.Equal(t, "abc", stringval) -// assert.True(t, ok) -// -// mv = FromInferredType("") -// stringval, ok = mv.GetString() -// assert.Equal(t, "", stringval) -// assert.True(t, ok) -// -// mv = FromDeferredType("") -// stringval, ok = mv.GetString() -// assert.Equal(t, "", stringval) -// assert.True(t, ok) -//} diff --git a/todo.txt b/todo.txt index 9894fc2d1..364a94f46 100644 --- a/todo.txt +++ b/todo.txt @@ -5,10 +5,8 @@ PUNCHDOWN LIST * JIT next: o mlrval copy o mlrmap copy - o s, ok := mlrval.ScalarString -- ? - o MlrvalString -* big-picture item @ Rmd; also webdoc intro page +* big-picture item @ Rmd (csv memes; and beyond); also webdoc intro page * mr -S test/cases/dsl-array-map-indexing From 012e7079172e03c98ede930bd4b19e8b8245883d Mon Sep 17 00:00:00 2001 From: John Kerl Date: Tue, 14 Dec 2021 00:14:04 -0500 Subject: [PATCH 06/19] mlrval/copy.go --- internal/pkg/bifs/arithmetic.go | 452 ++++++++-------- internal/pkg/bifs/base.go | 97 ++-- internal/pkg/bifs/bifs_test.go | 16 +- internal/pkg/bifs/bits.go | 90 ++-- internal/pkg/bifs/booleans.go | 336 ++++++------ internal/pkg/bifs/collections.go | 484 +++++++++--------- internal/pkg/bifs/datetime.go | 165 +++--- internal/pkg/bifs/hashing.go | 34 +- internal/pkg/bifs/math.go | 20 +- internal/pkg/bifs/mathlib.go | 153 +++--- internal/pkg/bifs/random.go | 29 +- internal/pkg/bifs/regex.go | 83 +-- internal/pkg/bifs/relative_time.go | 170 +++--- internal/pkg/bifs/sort.go | 62 +-- internal/pkg/bifs/stats.go | 23 +- internal/pkg/bifs/strings.go | 155 +++--- internal/pkg/bifs/system.go | 25 +- internal/pkg/bifs/types.go | 212 ++++---- internal/pkg/input/pseudo_reader_gen.go | 7 +- internal/pkg/input/record_reader_xtab.go | 1 + internal/pkg/mlrval/copy.go | 13 + internal/pkg/mlrval/get.go | 38 +- internal/pkg/mlrval/get_test.go | 16 +- internal/pkg/mlrval/output.go | 20 +- internal/pkg/runtime/state.go | 4 +- internal/pkg/transformers/bar.go | 4 +- internal/pkg/transformers/clean-whitespace.go | 12 +- internal/pkg/transformers/fill-empty.go | 2 +- internal/pkg/transformers/fraction.go | 10 +- internal/pkg/transformers/nest.go | 16 +- internal/pkg/transformers/reshape.go | 4 +- internal/pkg/transformers/sec2gmt.go | 2 +- internal/pkg/transformers/sec2gmtdate.go | 2 +- internal/pkg/transformers/seqgen.go | 8 +- internal/pkg/transformers/step.go | 16 +- internal/pkg/transformers/template.go | 2 +- internal/pkg/transformers/uniq.go | 2 +- internal/pkg/transformers/unsparsify.go | 2 +- internal/pkg/types-parked/mlrmap_json.go | 2 +- internal/pkg/types-parked/mlrval_accessors.go | 4 +- internal/pkg/types-parked/mlrval_json.go | 2 +- internal/pkg/types/mlrmap_accessors.go | 54 +- todo.txt | 1 - 43 files changed, 1469 insertions(+), 1381 deletions(-) create mode 100644 internal/pkg/mlrval/copy.go diff --git a/internal/pkg/bifs/arithmetic.go b/internal/pkg/bifs/arithmetic.go index 21312d80e..ad830656d 100644 --- a/internal/pkg/bifs/arithmetic.go +++ b/internal/pkg/bifs/arithmetic.go @@ -2,12 +2,14 @@ package types import ( "math" + + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ================================================================ // Unary plus operator -var upos_dispositions = [MT_DIM]UnaryFunc{ +var upos_dispositions = [mlrval.MT_DIM]UnaryFunc{ /*ERROR */ _erro1, /*ABSENT */ _absn1, /*NULL */ _null1, @@ -21,22 +23,22 @@ var upos_dispositions = [MT_DIM]UnaryFunc{ /*FUNC */ _erro1, } -func BIF_plus_unary(input1 *Mlrval) *Mlrval { - return upos_dispositions[input1.mvtype](input1) +func BIF_plus_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return upos_dispositions[input1.Type()](input1) } // ================================================================ // Unary minus operator -func uneg_i_i(input1 *Mlrval) *Mlrval { - return MlrvalFromInt(-input1.intval) +func uneg_i_i(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(-input1.AcquireIntValue()) } -func uneg_f_f(input1 *Mlrval) *Mlrval { - return MlrvalFromFloat64(-input1.floatval) +func uneg_f_f(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(-input1.AcquireFloatValue()) } -var uneg_dispositions = [MT_DIM]UnaryFunc{ +var uneg_dispositions = [mlrval.MT_DIM]UnaryFunc{ /*ERROR */ _erro1, /*ABSENT */ _absn1, /*NULL */ _null1, @@ -50,18 +52,18 @@ var uneg_dispositions = [MT_DIM]UnaryFunc{ /*FUNC */ _erro1, } -func BIF_minus_unary(input1 *Mlrval) *Mlrval { - return uneg_dispositions[input1.mvtype](input1) +func BIF_minus_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return uneg_dispositions[input1.Type()](input1) } // ================================================================ // Logical NOT operator -func BIF_logicalnot(input1 *Mlrval) *Mlrval { +func BIF_logicalnot(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsBool() { - return MlrvalFromBool(!input1.boolval) + return mlrval.FromBool(!input1.AcquireBoolValue()) } else { - return MLRVAL_ERROR + return mlrval.ERROR } } @@ -71,9 +73,9 @@ func BIF_logicalnot(input1 *Mlrval) *Mlrval { // Auto-overflows up to float. Additions & subtractions overflow by at most // one bit so it suffices to check sign-changes. -func plus_n_ii(input1, input2 *Mlrval) *Mlrval { - a := input1.intval - b := input2.intval +func plus_n_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() c := a + b overflowed := false @@ -88,23 +90,23 @@ func plus_n_ii(input1, input2 *Mlrval) *Mlrval { } if overflowed { - return MlrvalFromFloat64(float64(a) + float64(b)) + return mlrval.FromFloat(float64(a) + float64(b)) } else { - return MlrvalFromInt(c) + return mlrval.FromInt(c) } } -func plus_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) + input2.floatval) +func plus_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) + input2.AcquireFloatValue()) } -func plus_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval + float64(input2.intval)) +func plus_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() + float64(input2.AcquireIntValue())) } -func plus_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval + input2.floatval) +func plus_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() + input2.AcquireFloatValue()) } -var plus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var plus_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _2___, _erro, _absn, _absn, _erro}, @@ -119,8 +121,8 @@ var plus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_plus_binary(input1, input2 *Mlrval) *Mlrval { - return plus_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_plus_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return plus_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ @@ -129,9 +131,9 @@ func BIF_plus_binary(input1, input2 *Mlrval) *Mlrval { // Adds & subtracts overflow by at most one bit so it suffices to check // sign-changes. -func minus_n_ii(input1, input2 *Mlrval) *Mlrval { - a := input1.intval - b := input2.intval +func minus_n_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() c := a - b overflowed := false @@ -146,23 +148,23 @@ func minus_n_ii(input1, input2 *Mlrval) *Mlrval { } if overflowed { - return MlrvalFromFloat64(float64(a) - float64(b)) + return mlrval.FromFloat(float64(a) - float64(b)) } else { - return MlrvalFromInt(c) + return mlrval.FromInt(c) } } -func minus_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) - input2.floatval) +func minus_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) - input2.AcquireFloatValue()) } -func minus_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval - float64(input2.intval)) +func minus_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() - float64(input2.AcquireIntValue())) } -func minus_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval - input2.floatval) +func minus_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() - input2.AcquireFloatValue()) } -var minus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var minus_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _2___, _erro, _absn, _absn, _erro}, @@ -177,8 +179,8 @@ var minus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_minus_binary(input1, input2 *Mlrval) *Mlrval { - return minus_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_minus_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return minus_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ @@ -214,29 +216,29 @@ func BIF_minus_binary(input1, input2 *Mlrval) *Mlrval { // double less than 2**63. (An alterative would be to do all integer multiplies // using handcrafted multi-word 128-bit arithmetic). -func times_n_ii(input1, input2 *Mlrval) *Mlrval { - a := input1.intval - b := input2.intval +func times_n_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() c := float64(a) * float64(b) if math.Abs(c) > 9223372036854774784.0 { - return MlrvalFromFloat64(c) + return mlrval.FromFloat(c) } else { - return MlrvalFromInt(a * b) + return mlrval.FromInt(a * b) } } -func times_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) * input2.floatval) +func times_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) * input2.AcquireFloatValue()) } -func times_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval * float64(input2.intval)) +func times_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() * float64(input2.AcquireIntValue())) } -func times_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval * input2.floatval) +func times_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() * input2.AcquireFloatValue()) } -var times_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var times_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _2___, _erro, _absn, _absn, _erro}, @@ -251,8 +253,8 @@ var times_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_times(input1, input2 *Mlrval) *Mlrval { - return times_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_times(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return times_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ @@ -272,42 +274,42 @@ func BIF_times(input1, input2 *Mlrval) *Mlrval { // $ echo 'x=1e-300,y=1e300' | mlr put '$z=$y/$x' // x=1e-300,y=1e300,z=+Inf -func divide_n_ii(input1, input2 *Mlrval) *Mlrval { - a := input1.intval - b := input2.intval +func divide_n_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() if b == 0 { // Compute inf/nan as with floats rather than fatal runtime FPE on integer divide by zero - return MlrvalFromFloat64(float64(a) / float64(b)) + return mlrval.FromFloat(float64(a) / float64(b)) } // Pythonic division, not C division. if a%b == 0 { - return MlrvalFromInt(a / b) + return mlrval.FromInt(a / b) } else { - return MlrvalFromFloat64(float64(a) / float64(b)) + return mlrval.FromFloat(float64(a) / float64(b)) } c := float64(a) * float64(b) if math.Abs(c) > 9223372036854774784.0 { - return MlrvalFromFloat64(c) + return mlrval.FromFloat(c) } else { - return MlrvalFromInt(a * b) + return mlrval.FromInt(a * b) } } -func divide_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) / input2.floatval) +func divide_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) / input2.AcquireFloatValue()) } -func divide_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval / float64(input2.intval)) +func divide_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() / float64(input2.AcquireIntValue())) } -func divide_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval / input2.floatval) +func divide_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() / input2.AcquireFloatValue()) } -var divide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var divide_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _i0__, _f0__, _erro, _absn, _absn, _erro}, @@ -322,21 +324,21 @@ var divide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_divide(input1, input2 *Mlrval) *Mlrval { - return divide_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return divide_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ // Integer division: DSL operator '//' as in Python. See also // https://johnkerl.org/miller6/reference-main-arithmetic.html -func int_divide_n_ii(input1, input2 *Mlrval) *Mlrval { - a := input1.intval - b := input2.intval +func int_divide_n_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() if b == 0 { // Compute inf/nan as with floats rather than fatal runtime FPE on integer divide by zero - return MlrvalFromFloat64(float64(a) / float64(b)) + return mlrval.FromFloat(float64(a) / float64(b)) } // Pythonic division, not C division. @@ -355,20 +357,20 @@ func int_divide_n_ii(input1, input2 *Mlrval) *Mlrval { } } } - return MlrvalFromInt(q) + return mlrval.FromInt(q) } -func int_divide_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Floor(float64(input1.intval) / input2.floatval)) +func int_divide_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Floor(float64(input1.AcquireIntValue()) / input2.AcquireFloatValue())) } -func int_divide_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Floor(input1.floatval / float64(input2.intval))) +func int_divide_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Floor(input1.AcquireFloatValue() / float64(input2.AcquireIntValue()))) } -func int_divide_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Floor(input1.floatval / input2.floatval)) +func int_divide_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Floor(input1.AcquireFloatValue() / input2.AcquireFloatValue())) } -var int_divide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var int_divide_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _i0__, _f0__, _erro, _absn, _absn, _erro}, @@ -383,28 +385,28 @@ var int_divide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_int_divide(input1, input2 *Mlrval) *Mlrval { - return int_divide_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_int_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return int_divide_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ // Non-auto-overflowing addition: DSL operator '.+'. See also // https://johnkerl.org/miller6/reference-main-arithmetic.html -func dotplus_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval + input2.intval) +func dotplus_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() + input2.AcquireIntValue()) } -func dotplus_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) + input2.floatval) +func dotplus_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) + input2.AcquireFloatValue()) } -func dotplus_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval + float64(input2.intval)) +func dotplus_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() + float64(input2.AcquireIntValue())) } -func dotplus_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval + input2.floatval) +func dotplus_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() + input2.AcquireFloatValue()) } -var dot_plus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var dot_plus_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _2___, _erro, _absn, _absn, _erro}, @@ -419,28 +421,28 @@ var dot_plus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_dot_plus(input1, input2 *Mlrval) *Mlrval { - return dot_plus_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_dot_plus(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return dot_plus_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ // Non-auto-overflowing subtraction: DSL operator '.-'. See also // https://johnkerl.org/miller6/reference-main-arithmetic.html -func dotminus_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval - input2.intval) +func dotminus_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() - input2.AcquireIntValue()) } -func dotminus_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) - input2.floatval) +func dotminus_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) - input2.AcquireFloatValue()) } -func dotminus_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval - float64(input2.intval)) +func dotminus_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() - float64(input2.AcquireIntValue())) } -func dotminus_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval - input2.floatval) +func dotminus_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() - input2.AcquireFloatValue()) } -var dotminus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var dotminus_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _n2__, _n2__, _erro, _absn, _absn, _erro}, @@ -455,28 +457,28 @@ var dotminus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_dot_minus(input1, input2 *Mlrval) *Mlrval { - return dotminus_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_dot_minus(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return dotminus_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // Non-auto-overflowing multiplication: DSL operator '.*'. See also // https://johnkerl.org/miller6/reference-main-arithmetic.html -func dottimes_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval * input2.intval) +func dottimes_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() * input2.AcquireIntValue()) } -func dottimes_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) * input2.floatval) +func dottimes_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) * input2.AcquireFloatValue()) } -func dottimes_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval * float64(input2.intval)) +func dottimes_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() * float64(input2.AcquireIntValue())) } -func dottimes_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval * input2.floatval) +func dottimes_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() * input2.AcquireFloatValue()) } -var dottimes_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var dottimes_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _2___, _erro, _absn, _absn, _erro}, @@ -491,28 +493,28 @@ var dottimes_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_dot_times(input1, input2 *Mlrval) *Mlrval { - return dottimes_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_dot_times(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return dottimes_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // 64-bit integer division: DSL operator './'. See also // https://johnkerl.org/miller6/reference-main-arithmetic.html -func dotdivide_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval / input2.intval) +func dotdivide_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() / input2.AcquireIntValue()) } -func dotdivide_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) / input2.floatval) +func dotdivide_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) / input2.AcquireFloatValue()) } -func dotdivide_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval / float64(input2.intval)) +func dotdivide_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() / float64(input2.AcquireIntValue())) } -func dotdivide_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval / input2.floatval) +func dotdivide_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() / input2.AcquireFloatValue()) } -var dotdivide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var dotdivide_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _2___, _erro, _absn, _absn, _erro}, @@ -527,21 +529,21 @@ var dotdivide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_dot_divide(input1, input2 *Mlrval) *Mlrval { - return dotdivide_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_dot_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return dotdivide_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // 64-bit integer division: DSL operator './/'. See also // https://johnkerl.org/miller6/reference-main-arithmetic.html -func dotidivide_i_ii(input1, input2 *Mlrval) *Mlrval { - a := input1.intval - b := input2.intval +func dotidivide_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() if b == 0 { // Compute inf/nan as with floats rather than fatal runtime FPE on integer divide by zero - return MlrvalFromFloat64(float64(a) / float64(b)) + return mlrval.FromFloat(float64(a) / float64(b)) } // Pythonic division, not C division. @@ -560,20 +562,20 @@ func dotidivide_i_ii(input1, input2 *Mlrval) *Mlrval { } } } - return MlrvalFromInt(q) + return mlrval.FromInt(q) } -func dotidivide_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Floor(float64(input1.intval) / input2.floatval)) +func dotidivide_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Floor(float64(input1.AcquireIntValue()) / input2.AcquireFloatValue())) } -func dotidivide_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Floor(input1.floatval / float64(input2.intval))) +func dotidivide_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Floor(input1.AcquireFloatValue() / float64(input2.AcquireIntValue()))) } -func dotidivide_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Floor(input1.floatval / input2.floatval)) +func dotidivide_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Floor(input1.AcquireFloatValue() / input2.AcquireFloatValue())) } -var dotidivide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var dotidivide_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _erro, _absn, _erro, _2___, _2___, _erro, _absn, _absn, _erro}, @@ -588,20 +590,20 @@ var dotidivide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func MlrvalDotIntDivide(input1, input2 *Mlrval) *Mlrval { - return dotidivide_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func MlrvalDotIntDivide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return dotidivide_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // Modulus -func modulus_i_ii(input1, input2 *Mlrval) *Mlrval { - a := input1.intval - b := input2.intval +func modulus_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() if b == 0 { // Compute inf/nan as with floats rather than fatal runtime FPE on integer divide by zero - return MlrvalFromFloat64(float64(a) / float64(b)) + return mlrval.FromFloat(float64(a) / float64(b)) } // Pythonic division, not C division. @@ -616,28 +618,28 @@ func modulus_i_ii(input1, input2 *Mlrval) *Mlrval { } } - return MlrvalFromInt(m) + return mlrval.FromInt(m) } -func modulus_f_fi(input1, input2 *Mlrval) *Mlrval { - a := input1.floatval - b := float64(input2.intval) - return MlrvalFromFloat64(a - b*math.Floor(a/b)) +func modulus_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireFloatValue() + b := float64(input2.AcquireIntValue()) + return mlrval.FromFloat(a - b*math.Floor(a/b)) } -func modulus_f_if(input1, input2 *Mlrval) *Mlrval { - a := float64(input1.intval) - b := input2.floatval - return MlrvalFromFloat64(a - b*math.Floor(a/b)) +func modulus_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := float64(input1.AcquireIntValue()) + b := input2.AcquireFloatValue() + return mlrval.FromFloat(a - b*math.Floor(a/b)) } -func modulus_f_ff(input1, input2 *Mlrval) *Mlrval { - a := input1.floatval - b := input2.floatval - return MlrvalFromFloat64(a - b*math.Floor(a/b)) +func modulus_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireFloatValue() + b := input2.AcquireFloatValue() + return mlrval.FromFloat(a - b*math.Floor(a/b)) } -var modulus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var modulus_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _i0__, _f0__, _erro, _absn, _absn, _erro}, @@ -652,8 +654,8 @@ var modulus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_modulus(input1, input2 *Mlrval) *Mlrval { - return modulus_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_modulus(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return modulus_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ @@ -701,7 +703,7 @@ func imodexp(a, e, m int) int { return c } -func imodop(input1, input2, input3 *Mlrval, iop i_iii_func) *Mlrval { +func imodop(input1, input2, input3 *mlrval.Mlrval, iop i_iii_func) *mlrval.Mlrval { if !input1.IsLegit() { return input1 } @@ -712,34 +714,34 @@ func imodop(input1, input2, input3 *Mlrval, iop i_iii_func) *Mlrval { return input3 } if !input1.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input3.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromInt(iop(input1.intval, input2.intval, input3.intval)) + return mlrval.FromInt(iop(input1.AcquireIntValue(), input2.AcquireIntValue(), input3.AcquireIntValue())) } -func BIF_mod_add(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_mod_add(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { return imodop(input1, input2, input3, imodadd) } -func BIF_mod_sub(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_mod_sub(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { return imodop(input1, input2, input3, imodsub) } -func BIF_mod_mul(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_mod_mul(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { return imodop(input1, input2, input3, imodmul) } -func BIF_mod_exp(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_mod_exp(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { // Pre-check for negative exponent - if input2.IsInt() && input2.intval < 0 { - return MLRVAL_ERROR + if input2.IsInt() && input2.AcquireIntValue() < 0 { + return mlrval.ERROR } return imodop(input1, input2, input3, imodexp) } @@ -759,27 +761,27 @@ func BIF_mod_exp(input1, input2, input3 *Mlrval) *Mlrval { // * empty-null always loses against numbers // ---------------------------------------------------------------- -func min_f_ff(input1, input2 *Mlrval) *Mlrval { - var a float64 = input1.floatval - var b float64 = input2.floatval - return MlrvalFromFloat64(math.Min(a, b)) +func min_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a float64 = input1.AcquireFloatValue() + var b float64 = input2.AcquireFloatValue() + return mlrval.FromFloat(math.Min(a, b)) } -func min_f_fi(input1, input2 *Mlrval) *Mlrval { - var a float64 = input1.floatval - var b float64 = float64(input2.intval) - return MlrvalFromFloat64(math.Min(a, b)) +func min_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a float64 = input1.AcquireFloatValue() + var b float64 = float64(input2.AcquireIntValue()) + return mlrval.FromFloat(math.Min(a, b)) } -func min_f_if(input1, input2 *Mlrval) *Mlrval { - var a float64 = float64(input1.intval) - var b float64 = input2.floatval - return MlrvalFromFloat64(math.Min(a, b)) +func min_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a float64 = float64(input1.AcquireIntValue()) + var b float64 = input2.AcquireFloatValue() + return mlrval.FromFloat(math.Min(a, b)) } -func min_i_ii(input1, input2 *Mlrval) *Mlrval { - var a int = input1.intval - var b int = input2.intval +func min_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a int = input1.AcquireIntValue() + var b int = input2.AcquireIntValue() if a < b { return input1 } else { @@ -791,17 +793,17 @@ func min_i_ii(input1, input2 *Mlrval) *Mlrval { // --- + ----- ----- // a=F | min=a min=a // a=T | min=b min=b -func min_b_bb(input1, input2 *Mlrval) *Mlrval { - if input1.boolval == false { +func min_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.AcquireBoolValue() == false { return input1 } else { return input2 } } -func min_s_ss(input1, input2 *Mlrval) *Mlrval { - var a string = input1.printrep - var b string = input2.printrep +func min_s_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a string = input1.AcquireStringValue() + var b string = input2.AcquireStringValue() if a < b { return input1 } else { @@ -809,7 +811,7 @@ func min_s_ss(input1, input2 *Mlrval) *Mlrval { } } -var min_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var min_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _null, _2___, _2___, _2___, _2___, _2___, _absn, _absn, _erro}, @@ -824,13 +826,13 @@ var min_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func MlrvalBinaryMin(input1, input2 *Mlrval) *Mlrval { - return (min_dispositions[input1.mvtype][input2.mvtype])(input1, input2) +func MlrvalBinaryMin(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return (min_dispositions[input1.Type()][input2.Type()])(input1, input2) } -func BIF_min_variadic(mlrvals []*Mlrval) *Mlrval { +func BIF_min_variadic(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { if len(mlrvals) == 0 { - return MLRVAL_VOID + return mlrval.VOID } else { retval := mlrvals[0] for i := range mlrvals { @@ -843,27 +845,27 @@ func BIF_min_variadic(mlrvals []*Mlrval) *Mlrval { } // ---------------------------------------------------------------- -func max_f_ff(input1, input2 *Mlrval) *Mlrval { - var a float64 = input1.floatval - var b float64 = input2.floatval - return MlrvalFromFloat64(math.Max(a, b)) +func max_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a float64 = input1.AcquireFloatValue() + var b float64 = input2.AcquireFloatValue() + return mlrval.FromFloat(math.Max(a, b)) } -func max_f_fi(input1, input2 *Mlrval) *Mlrval { - var a float64 = input1.floatval - var b float64 = float64(input2.intval) - return MlrvalFromFloat64(math.Max(a, b)) +func max_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a float64 = input1.AcquireFloatValue() + var b float64 = float64(input2.AcquireIntValue()) + return mlrval.FromFloat(math.Max(a, b)) } -func max_f_if(input1, input2 *Mlrval) *Mlrval { - var a float64 = float64(input1.intval) - var b float64 = input2.floatval - return MlrvalFromFloat64(math.Max(a, b)) +func max_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a float64 = float64(input1.AcquireIntValue()) + var b float64 = input2.AcquireFloatValue() + return mlrval.FromFloat(math.Max(a, b)) } -func max_i_ii(input1, input2 *Mlrval) *Mlrval { - var a int = input1.intval - var b int = input2.intval +func max_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a int = input1.AcquireIntValue() + var b int = input2.AcquireIntValue() if a > b { return input1 } else { @@ -875,17 +877,17 @@ func max_i_ii(input1, input2 *Mlrval) *Mlrval { // --- + ----- ----- // a=F | max=a max=b // a=T | max=a max=b -func max_b_bb(input1, input2 *Mlrval) *Mlrval { - if input2.boolval == false { +func max_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input2.AcquireBoolValue() == false { return input1 } else { return input2 } } -func max_s_ss(input1, input2 *Mlrval) *Mlrval { - var a string = input1.printrep - var b string = input2.printrep +func max_s_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a string = input1.AcquireStringValue() + var b string = input2.AcquireStringValue() if a > b { return input1 } else { @@ -893,7 +895,7 @@ func max_s_ss(input1, input2 *Mlrval) *Mlrval { } } -var max_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var max_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _null, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _2___, _2___, _2___, _2___, _2___, _absn, _absn, _erro}, @@ -908,13 +910,13 @@ var max_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func MlrvalBinaryMax(input1, input2 *Mlrval) *Mlrval { - return (max_dispositions[input1.mvtype][input2.mvtype])(input1, input2) +func MlrvalBinaryMax(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return (max_dispositions[input1.Type()][input2.Type()])(input1, input2) } -func BIF_max_variadic(mlrvals []*Mlrval) *Mlrval { +func BIF_max_variadic(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { if len(mlrvals) == 0 { - return MLRVAL_VOID + return mlrval.VOID } else { retval := mlrvals[0] for i := range mlrvals { diff --git a/internal/pkg/bifs/base.go b/internal/pkg/bifs/base.go index 68559ab6d..435117993 100644 --- a/internal/pkg/bifs/base.go +++ b/internal/pkg/bifs/base.go @@ -12,7 +12,7 @@ // example: adding two ints, or an int and a float, or int and boolean (the // latter being an error). // -// The next-past-highest mlrval type enum is called MT_DIM and that is the +// The next-past-highest mlrval type enum is called mlrval.MT_DIM and that is the // dimension of the binary-operator disposition matrices and unary-operator // disposition vectors. // @@ -47,36 +47,41 @@ package types +import ( + "github.com/johnkerl/miller/internal/pkg/mlrval" + "github.com/johnkerl/miller/internal/pkg/types" +) + // Function-pointer type for zary functions. -type ZaryFunc func() *Mlrval +type ZaryFunc func() *mlrval.Mlrval // Function-pointer type for unary-operator disposition vectors. -type UnaryFunc func(input1 *Mlrval) *Mlrval +type UnaryFunc func(input1 *mlrval.Mlrval) *mlrval.Mlrval // The asserting_{type} need access to the context to say things like 'Assertion ... failed // at filename {FILENAME} record number {NR}'. -type UnaryFuncWithContext func(input1 *Mlrval, context *Context) *Mlrval +type UnaryFuncWithContext func(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval // Returns nil, or one-up captures array as array slots 1..9 of 10-element // array for "\1".."\9". -type RegexCaptureBinaryFunc func(input *Mlrval, sregex *Mlrval) (*Mlrval, []string) +type RegexCaptureBinaryFunc func(input *mlrval.Mlrval, sregex *mlrval.Mlrval) (*mlrval.Mlrval, []string) // Helps keystroke-saving for wrapping Go math-library functions // Examples: cos, sin, etc. type mathLibUnaryFunc func(float64) float64 -type mathLibUnaryFuncWrapper func(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval +type mathLibUnaryFuncWrapper func(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval // Function-pointer type for binary-operator disposition matrices. -type BinaryFunc func(input1, input2 *Mlrval) *Mlrval +type BinaryFunc func(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval // Function-pointer type for ternary functions -type TernaryFunc func(input1, input2, input3 *Mlrval) *Mlrval +type TernaryFunc func(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval // Function-pointer type for variadic functions. -type VariadicFunc func(inputs []*Mlrval) *Mlrval +type VariadicFunc func(inputs []*mlrval.Mlrval) *mlrval.Mlrval // Function-pointer type for sorting. Returns < 0 if a < b, 0 if a == b, > 0 if a > b. -type ComparatorFunc func(*Mlrval, *Mlrval) int +type ComparatorFunc func(*mlrval.Mlrval, *mlrval.Mlrval) int // ================================================================ // The following are frequently used in disposition matrices for various @@ -86,102 +91,102 @@ type ComparatorFunc func(*Mlrval, *Mlrval) int // ---------------------------------------------------------------- // Return error (unary) -func _erro1(input1 *Mlrval) *Mlrval { - return MLRVAL_ERROR +func _erro1(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.ERROR } // Return absent (unary) -func _absn1(input1 *Mlrval) *Mlrval { - return MLRVAL_ABSENT +func _absn1(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.ABSENT } // Return zero (unary) -func _zero1(input1 *Mlrval) *Mlrval { - return MlrvalFromInt(0) +func _zero1(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(0) } // Return null (unary) -func _null1(input1 *Mlrval) *Mlrval { - return MLRVAL_NULL +func _null1(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.NULL } // Return void (unary) -func _void1(input1 *Mlrval) *Mlrval { - return MLRVAL_VOID +func _void1(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.VOID } // Return argument (unary) -func _1u___(input1 *Mlrval) *Mlrval { +func _1u___(input1 *mlrval.Mlrval) *mlrval.Mlrval { return input1 } // ---------------------------------------------------------------- // Return error (binary) -func _erro(input1, input2 *Mlrval) *Mlrval { - return MLRVAL_ERROR +func _erro(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.ERROR } // Return absent (binary) -func _absn(input1, input2 *Mlrval) *Mlrval { - return MLRVAL_ABSENT +func _absn(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.ABSENT } // Return null (binary) -func _null(input1, input2 *Mlrval) *Mlrval { - return MLRVAL_NULL +func _null(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.NULL } // Return void (binary) -func _void(input1, input2 *Mlrval) *Mlrval { - return MLRVAL_VOID +func _void(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.VOID } // Return 0 (binary) -func _zero2(input1 *Mlrval) *Mlrval { - return MlrvalFromInt(0) +func _zero2(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(0) } // Return first argument (binary) -func _1___(input1, input2 *Mlrval) *Mlrval { +func _1___(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return input1 } // Return second argument (binary) -func _2___(input1, input2 *Mlrval) *Mlrval { +func _2___(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return input2 } // Return negative second argument (binary) -func _n2__(input1, input2 *Mlrval) *Mlrval { +func _n2__(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return BIF_minus_unary(input2) } // Return first argument, as string (binary) -func _s1__(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromString(input1.String()) +func _s1__(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromString(input1.String()) } // Return second argument, as string (binary) -func _s2__(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromString(input2.String()) +func _s2__(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromString(input2.String()) } // Return integer zero (binary) -func _i0__(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(0) +func _i0__(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(0) } // Return float zero (binary) -func _f0__(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(0) +func _f0__(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(0) } // Return boolean true (binary) -func _true(input1, input2 *Mlrval) *Mlrval { - return MLRVAL_TRUE +func _true(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.TRUE } // Return boolean false (binary) -func _fals(input1, input2 *Mlrval) *Mlrval { - return MLRVAL_FALSE +func _fals(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FALSE } diff --git a/internal/pkg/bifs/bifs_test.go b/internal/pkg/bifs/bifs_test.go index 26b69a898..67ee073c2 100644 --- a/internal/pkg/bifs/bifs_test.go +++ b/internal/pkg/bifs/bifs_test.go @@ -25,18 +25,18 @@ import ( func TestComparators(t *testing.T) { - i10 := MlrvalFromInt(10) - i2 := MlrvalFromInt(2) + i10 := mlrval.FromInt(10) + i2 := mlrval.FromInt(2) - bfalse := MlrvalFromBool(false) - btrue := MlrvalFromBool(true) + bfalse := mlrval.FromBool(false) + btrue := mlrval.FromBool(true) - sabc := MlrvalFromString("abc") - sdef := MlrvalFromString("def") + sabc := mlrval.FromString("abc") + sdef := mlrval.FromString("def") - e := *MLRVAL_ERROR + e := *mlrval.ERROR - a := *MLRVAL_ABSENT + a := *mlrval.ABSENT // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // Within-type lexical comparisons diff --git a/internal/pkg/bifs/bits.go b/internal/pkg/bifs/bits.go index f722f8391..78bdbbbd3 100644 --- a/internal/pkg/bifs/bits.go +++ b/internal/pkg/bifs/bits.go @@ -1,13 +1,17 @@ package types +import ( + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + // ================================================================ // Bitwise NOT -func bitwise_not_i_i(input1 *Mlrval) *Mlrval { - return MlrvalFromInt(^input1.intval) +func bitwise_not_i_i(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(^input1.AcquireIntValue()) } -var bitwise_not_dispositions = [MT_DIM]UnaryFunc{ +var bitwise_not_dispositions = [mlrval.MT_DIM]UnaryFunc{ /*ERROR */ _erro1, /*ABSENT */ _absn1, /*NULL */ _null1, @@ -21,8 +25,8 @@ var bitwise_not_dispositions = [MT_DIM]UnaryFunc{ /*FUNC */ _erro1, } -func BIF_bitwise_not(input1 *Mlrval) *Mlrval { - return bitwise_not_dispositions[input1.mvtype](input1) +func BIF_bitwise_not(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return bitwise_not_dispositions[input1.Type()](input1) } // ================================================================ @@ -36,18 +40,18 @@ const _m08 uint64 = 0x00ff00ff00ff00ff const _m16 uint64 = 0x0000ffff0000ffff const _m32 uint64 = 0x00000000ffffffff -func bitcount_i_i(input1 *Mlrval) *Mlrval { - a := uint64(input1.intval) +func bitcount_i_i(input1 *mlrval.Mlrval) *mlrval.Mlrval { + a := uint64(input1.AcquireIntValue()) a = (a & _m01) + ((a >> 1) & _m01) a = (a & _m02) + ((a >> 2) & _m02) a = (a & _m04) + ((a >> 4) & _m04) a = (a & _m08) + ((a >> 8) & _m08) a = (a & _m16) + ((a >> 16) & _m16) a = (a & _m32) + ((a >> 32) & _m32) - return MlrvalFromInt(int(a)) + return mlrval.FromInt(int(a)) } -var bitcount_dispositions = [MT_DIM]UnaryFunc{ +var bitcount_dispositions = [mlrval.MT_DIM]UnaryFunc{ /*ERROR */ _erro1, /*ABSENT */ _absn1, /*NULL */ _zero1, @@ -61,18 +65,18 @@ var bitcount_dispositions = [MT_DIM]UnaryFunc{ /*FUNC */ _erro1, } -func BIF_bitcount(input1 *Mlrval) *Mlrval { - return bitcount_dispositions[input1.mvtype](input1) +func BIF_bitcount(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return bitcount_dispositions[input1.Type()](input1) } // ================================================================ // Bitwise AND -func bitwise_and_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval & input2.intval) +func bitwise_and_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() & input2.AcquireIntValue()) } -var bitwise_and_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var bitwise_and_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _erro, _erro, _absn, _absn, _erro}, @@ -87,18 +91,18 @@ var bitwise_and_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_bitwise_and(input1, input2 *Mlrval) *Mlrval { - return bitwise_and_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_bitwise_and(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return bitwise_and_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // Bitwise OR -func bitwise_or_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval | input2.intval) +func bitwise_or_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() | input2.AcquireIntValue()) } -var bitwise_or_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var bitwise_or_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _erro, _erro, _absn, _absn, _erro}, @@ -113,18 +117,18 @@ var bitwise_or_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_bitwise_or(input1, input2 *Mlrval) *Mlrval { - return bitwise_or_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_bitwise_or(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return bitwise_or_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // Bitwise XOR -func bitwise_xor_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval ^ input2.intval) +func bitwise_xor_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() ^ input2.AcquireIntValue()) } -var bitwise_xor_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var bitwise_xor_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _erro, _erro, _absn, _absn, _erro}, @@ -139,18 +143,18 @@ var bitwise_xor_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_bitwise_xor(input1, input2 *Mlrval) *Mlrval { - return bitwise_xor_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_bitwise_xor(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return bitwise_xor_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // Left shift -func lsh_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval << uint64(input2.intval)) +func lsh_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() << uint64(input2.AcquireIntValue())) } -var left_shift_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var left_shift_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _erro, _erro, _absn, _absn, _erro}, @@ -165,18 +169,18 @@ var left_shift_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_left_shift(input1, input2 *Mlrval) *Mlrval { - return left_shift_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_left_shift(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return left_shift_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // Signed right shift -func srsh_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval >> uint64(input2.intval)) +func srsh_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() >> uint64(input2.AcquireIntValue())) } -var signed_right_shift_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var signed_right_shift_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _erro, _erro, _absn, _absn, _erro}, @@ -191,21 +195,21 @@ var signed_right_shift_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_signed_right_shift(input1, input2 *Mlrval) *Mlrval { - return signed_right_shift_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_signed_right_shift(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return signed_right_shift_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // Unsigned right shift -func ursh_i_ii(input1, input2 *Mlrval) *Mlrval { - var ua uint64 = uint64(input1.intval) - var ub uint64 = uint64(input2.intval) +func ursh_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var ua uint64 = uint64(input1.AcquireIntValue()) + var ub uint64 = uint64(input2.AcquireIntValue()) var uc = ua >> ub - return MlrvalFromInt(int(uc)) + return mlrval.FromInt(int(uc)) } -var unsigned_right_shift_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var unsigned_right_shift_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _erro, _erro, _absn, _absn, _erro}, @@ -220,6 +224,6 @@ var unsigned_right_shift_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_unsigned_right_shift(input1, input2 *Mlrval) *Mlrval { - return unsigned_right_shift_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_unsigned_right_shift(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return unsigned_right_shift_dispositions[input1.Type()][input2.Type()](input1, input2) } diff --git a/internal/pkg/bifs/booleans.go b/internal/pkg/bifs/booleans.go index 1e704c6ce..89cd8b3fd 100644 --- a/internal/pkg/bifs/booleans.go +++ b/internal/pkg/bifs/booleans.go @@ -6,6 +6,8 @@ package types import ( "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" + "github.com/johnkerl/miller/internal/pkg/types" ) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -43,233 +45,233 @@ func float_cmp(a, b float64) int { } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_ss(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep == input2.printrep) +func eq_b_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() == input2.AcquireStringValue()) } -func ne_b_ss(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep != input2.printrep) +func ne_b_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() != input2.AcquireStringValue()) } -func gt_b_ss(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep > input2.printrep) +func gt_b_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() > input2.AcquireStringValue()) } -func ge_b_ss(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep >= input2.printrep) +func ge_b_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() >= input2.AcquireStringValue()) } -func lt_b_ss(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep < input2.printrep) +func lt_b_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() < input2.AcquireStringValue()) } -func le_b_ss(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep <= input2.printrep) +func le_b_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() <= input2.AcquireStringValue()) } -func cmp_b_ss(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(string_cmp(input1.printrep, input2.printrep)) +func cmp_b_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(string_cmp(input1.AcquireStringValue(), input2.AcquireStringValue())) } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_xs(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.String() == input2.printrep) +func eq_b_xs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.String() == input2.AcquireStringValue()) } -func ne_b_xs(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.String() != input2.printrep) +func ne_b_xs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.String() != input2.AcquireStringValue()) } -func gt_b_xs(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.String() > input2.printrep) +func gt_b_xs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.String() > input2.AcquireStringValue()) } -func ge_b_xs(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.String() >= input2.printrep) +func ge_b_xs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.String() >= input2.AcquireStringValue()) } -func lt_b_xs(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.String() < input2.printrep) +func lt_b_xs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.String() < input2.AcquireStringValue()) } -func le_b_xs(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.String() <= input2.printrep) +func le_b_xs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.String() <= input2.AcquireStringValue()) } -func cmp_b_xs(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(string_cmp(input1.String(), input2.printrep)) +func cmp_b_xs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(string_cmp(input1.String(), input2.AcquireStringValue())) } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_sx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep == input2.String()) +func eq_b_sx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() == input2.String()) } -func ne_b_sx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep != input2.String()) +func ne_b_sx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() != input2.String()) } -func gt_b_sx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep > input2.String()) +func gt_b_sx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() > input2.String()) } -func ge_b_sx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep >= input2.String()) +func ge_b_sx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() >= input2.String()) } -func lt_b_sx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep < input2.String()) +func lt_b_sx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() < input2.String()) } -func le_b_sx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep <= input2.String()) +func le_b_sx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() <= input2.String()) } -func cmp_b_sx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(string_cmp(input1.printrep, input2.String())) +func cmp_b_sx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(string_cmp(input1.AcquireStringValue(), input2.String())) } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.intval == input2.intval) +func eq_b_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireIntValue() == input2.AcquireIntValue()) } -func ne_b_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.intval != input2.intval) +func ne_b_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireIntValue() != input2.AcquireIntValue()) } -func gt_b_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.intval > input2.intval) +func gt_b_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireIntValue() > input2.AcquireIntValue()) } -func ge_b_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.intval >= input2.intval) +func ge_b_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireIntValue() >= input2.AcquireIntValue()) } -func lt_b_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.intval < input2.intval) +func lt_b_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireIntValue() < input2.AcquireIntValue()) } -func le_b_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.intval <= input2.intval) +func le_b_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireIntValue() <= input2.AcquireIntValue()) } -func cmp_b_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(int_cmp(input1.intval, input2.intval)) +func cmp_b_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(int_cmp(input1.AcquireIntValue(), input2.AcquireIntValue())) } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(float64(input1.intval) == input2.floatval) +func eq_b_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(float64(input1.AcquireIntValue()) == input2.AcquireFloatValue()) } -func ne_b_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(float64(input1.intval) != input2.floatval) +func ne_b_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(float64(input1.AcquireIntValue()) != input2.AcquireFloatValue()) } -func gt_b_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(float64(input1.intval) > input2.floatval) +func gt_b_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(float64(input1.AcquireIntValue()) > input2.AcquireFloatValue()) } -func ge_b_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(float64(input1.intval) >= input2.floatval) +func ge_b_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(float64(input1.AcquireIntValue()) >= input2.AcquireFloatValue()) } -func lt_b_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(float64(input1.intval) < input2.floatval) +func lt_b_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(float64(input1.AcquireIntValue()) < input2.AcquireFloatValue()) } -func le_b_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(float64(input1.intval) <= input2.floatval) +func le_b_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(float64(input1.AcquireIntValue()) <= input2.AcquireFloatValue()) } -func cmp_b_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(float_cmp(float64(input1.intval), input2.floatval)) +func cmp_b_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(float_cmp(float64(input1.AcquireIntValue()), input2.AcquireFloatValue())) } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval == float64(input2.intval)) +func eq_b_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() == float64(input2.AcquireIntValue())) } -func ne_b_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval != float64(input2.intval)) +func ne_b_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() != float64(input2.AcquireIntValue())) } -func gt_b_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval > float64(input2.intval)) +func gt_b_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() > float64(input2.AcquireIntValue())) } -func ge_b_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval >= float64(input2.intval)) +func ge_b_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() >= float64(input2.AcquireIntValue())) } -func lt_b_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval < float64(input2.intval)) +func lt_b_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() < float64(input2.AcquireIntValue())) } -func le_b_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval <= float64(input2.intval)) +func le_b_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() <= float64(input2.AcquireIntValue())) } -func cmp_b_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(float_cmp(input1.floatval, float64(input2.intval))) +func cmp_b_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(float_cmp(input1.AcquireFloatValue(), float64(input2.AcquireIntValue()))) } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval == input2.floatval) +func eq_b_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() == input2.AcquireFloatValue()) } -func ne_b_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval != input2.floatval) +func ne_b_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() != input2.AcquireFloatValue()) } -func gt_b_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval > input2.floatval) +func gt_b_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() > input2.AcquireFloatValue()) } -func ge_b_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval >= input2.floatval) +func ge_b_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() >= input2.AcquireFloatValue()) } -func lt_b_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval < input2.floatval) +func lt_b_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() < input2.AcquireFloatValue()) } -func le_b_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval <= input2.floatval) +func le_b_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() <= input2.AcquireFloatValue()) } -func cmp_b_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(float_cmp(input1.floatval, input2.floatval)) +func cmp_b_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(float_cmp(input1.AcquireFloatValue(), input2.AcquireFloatValue())) } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_bb(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.boolval == input2.boolval) +func eq_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireBoolValue() == input2.AcquireBoolValue()) } -func ne_b_bb(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.boolval != input2.boolval) +func ne_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireBoolValue() != input2.AcquireBoolValue()) } // We could say ordering on bool is error, but, Miller allows // sorting on bool so it should allow ordering on bool. -func gt_b_bb(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(lib.BoolToInt(input1.boolval) > lib.BoolToInt(input2.boolval)) +func gt_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(lib.BoolToInt(input1.AcquireBoolValue()) > lib.BoolToInt(input2.AcquireBoolValue())) } -func ge_b_bb(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(lib.BoolToInt(input1.boolval) >= lib.BoolToInt(input2.boolval)) +func ge_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(lib.BoolToInt(input1.AcquireBoolValue()) >= lib.BoolToInt(input2.AcquireBoolValue())) } -func lt_b_bb(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(lib.BoolToInt(input1.boolval) < lib.BoolToInt(input2.boolval)) +func lt_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(lib.BoolToInt(input1.AcquireBoolValue()) < lib.BoolToInt(input2.AcquireBoolValue())) } -func le_b_bb(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(lib.BoolToInt(input1.boolval) <= lib.BoolToInt(input2.boolval)) +func le_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(lib.BoolToInt(input1.AcquireBoolValue()) <= lib.BoolToInt(input2.AcquireBoolValue())) } -func cmp_b_bb(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(int_cmp(lib.BoolToInt(input1.boolval), lib.BoolToInt(input2.boolval))) +func cmp_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(int_cmp(lib.BoolToInt(input1.AcquireBoolValue()), lib.BoolToInt(input2.AcquireBoolValue()))) } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_aa(input1, input2 *Mlrval) *Mlrval { - a := input1.arrayval - b := input2.arrayval +func eq_b_aa(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireArrayValue().([]mlrval.Mlrval) + b := input2.AcquireArrayValue().([]mlrval.Mlrval) // Different-length arrays are not equal if len(a) != len(b) { - return MLRVAL_FALSE + return mlrval.FALSE } // Same-length arrays: return false if any slot is not equal, else true. for i := range a { eq := BIF_equals(&a[i], &b[i]) - lib.InternalCodingErrorIf(eq.mvtype != MT_BOOL) - if eq.boolval == false { - return MLRVAL_FALSE + lib.InternalCodingErrorIf(eq.Type() != mlrval.MT_BOOL) + if eq.AcquireBoolValue() == false { + return mlrval.FALSE } } - return MLRVAL_TRUE + return mlrval.TRUE } -func ne_b_aa(input1, input2 *Mlrval) *Mlrval { +func ne_b_aa(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { output := eq_b_aa(input1, input2) - return MlrvalFromBool(!output.boolval) + return mlrval.FromBool(!output.AcquireBoolValue()) } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_mm(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mapval.Equals(input2.mapval)) +func eq_b_mm(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireMapValue().(types.Mlrmap).Equals(input2.AcquireMapValue().(types.Mlrmap))) } -func ne_b_mm(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(!input1.mapval.Equals(input2.mapval)) +func ne_b_mm(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(!input1.AcquireArrayValue().(types.Mlrmap).Equals(input2.AcquireArrayValue().(types.Mlrmap))) } // We get a Golang "initialization loop" due to recursive depth computation // if this is defined statically. So, we use a "package init" function. -var eq_dispositions = [MT_DIM][MT_DIM]BinaryFunc{} +var eq_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{} func init() { - eq_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ + eq_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, @@ -285,7 +287,7 @@ func init() { } } -var ne_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var ne_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, @@ -300,7 +302,7 @@ var ne_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -var gt_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var gt_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _fals, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, /*ABSENT */ {_erro, _absn, _true, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, @@ -315,7 +317,7 @@ var gt_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -var ge_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var ge_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _fals, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, /*ABSENT */ {_erro, _absn, _true, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, @@ -330,7 +332,7 @@ var ge_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -var lt_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var lt_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _true, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, /*ABSENT */ {_erro, _absn, _fals, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, @@ -345,7 +347,7 @@ var lt_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -var le_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var le_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _true, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, /*ABSENT */ {_erro, _absn, _fals, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, @@ -360,7 +362,7 @@ var le_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -var cmp_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var cmp_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _true, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, /*ABSENT */ {_erro, _absn, _fals, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, @@ -375,97 +377,97 @@ var cmp_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_equals(input1, input2 *Mlrval) *Mlrval { - return eq_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_equals(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return eq_dispositions[input1.Type()][input2.Type()](input1, input2) } -func BIF_not_equals(input1, input2 *Mlrval) *Mlrval { - return ne_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_not_equals(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return ne_dispositions[input1.Type()][input2.Type()](input1, input2) } -func BIF_greater_than(input1, input2 *Mlrval) *Mlrval { - return gt_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_greater_than(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return gt_dispositions[input1.Type()][input2.Type()](input1, input2) } -func BIF_greater_than_or_equals(input1, input2 *Mlrval) *Mlrval { - return ge_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_greater_than_or_equals(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return ge_dispositions[input1.Type()][input2.Type()](input1, input2) } -func BIF_less_than(input1, input2 *Mlrval) *Mlrval { - return lt_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_less_than(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return lt_dispositions[input1.Type()][input2.Type()](input1, input2) } -func BIF_less_than_or_equals(input1, input2 *Mlrval) *Mlrval { - return le_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_less_than_or_equals(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return le_dispositions[input1.Type()][input2.Type()](input1, input2) } -func BIF_cmp(input1, input2 *Mlrval) *Mlrval { - return cmp_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_cmp(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return cmp_dispositions[input1.Type()][input2.Type()](input1, input2) } // For Go's sort.Slice. -func MlrvalLessThanAsBool(input1, input2 *Mlrval) bool { +func MlrvalLessThanAsBool(input1, input2 *mlrval.Mlrval) bool { // TODO refactor to avoid copy // This is a hot path for sort GC and is worth significant hand-optimization - mretval := lt_dispositions[input1.mvtype][input2.mvtype](input1, input2) + mretval := lt_dispositions[input1.Type()][input2.Type()](input1, input2) retval, ok := mretval.GetBoolValue() lib.InternalCodingErrorIf(!ok) return retval } // For Go's sort.Slice. -func MlrvalLessThanOrEqualsAsBool(input1, input2 *Mlrval) bool { +func MlrvalLessThanOrEqualsAsBool(input1, input2 *mlrval.Mlrval) bool { // TODO refactor to avoid copy // This is a hot path for sort GC and is worth significant hand-optimization - mretval := le_dispositions[input1.mvtype][input2.mvtype](input1, input2) + mretval := le_dispositions[input1.Type()][input2.Type()](input1, input2) retval, ok := mretval.GetBoolValue() lib.InternalCodingErrorIf(!ok) return retval } // For top-keeper -func MlrvalGreaterThanAsBool(input1, input2 *Mlrval) bool { +func MlrvalGreaterThanAsBool(input1, input2 *mlrval.Mlrval) bool { // TODO refactor to avoid copy // This is a hot path for sort GC and is worth significant hand-optimization - mretval := gt_dispositions[input1.mvtype][input2.mvtype](input1, input2) + mretval := gt_dispositions[input1.Type()][input2.Type()](input1, input2) retval, ok := mretval.GetBoolValue() lib.InternalCodingErrorIf(!ok) return retval } // For top-keeper -func MlrvalGreaterThanOrEqualsAsBool(input1, input2 *Mlrval) bool { +func MlrvalGreaterThanOrEqualsAsBool(input1, input2 *mlrval.Mlrval) bool { // TODO refactor to avoid copy // This is a hot path for sort GC and is worth significant hand-optimization - mretval := ge_dispositions[input1.mvtype][input2.mvtype](input1, input2) + mretval := ge_dispositions[input1.Type()][input2.Type()](input1, input2) retval, ok := mretval.GetBoolValue() lib.InternalCodingErrorIf(!ok) return retval } // Convenience wrapper for non-DSL callsites that just want a bool -func MlrvalEqualsAsBool(input1, input2 *Mlrval) bool { - mretval := eq_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func MlrvalEqualsAsBool(input1, input2 *mlrval.Mlrval) bool { + mretval := eq_dispositions[input1.Type()][input2.Type()](input1, input2) retval, ok := mretval.GetBoolValue() lib.InternalCodingErrorIf(!ok) return retval } // ---------------------------------------------------------------- -func MlrvalLogicalAND(input1, input2 *Mlrval) *Mlrval { +func MlrvalLogicalAND(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsBool() && input2.IsBool() { - return MlrvalFromBool(input1.boolval && input2.boolval) + return mlrval.FromBool(input1.AcquireBoolValue() && input2.AcquireBoolValue()) } else { - return MLRVAL_ERROR + return mlrval.ERROR } } -func MlrvalLogicalOR(input1, input2 *Mlrval) *Mlrval { +func MlrvalLogicalOR(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsBool() && input2.IsBool() { - return MlrvalFromBool(input1.boolval || input2.boolval) + return mlrval.FromBool(input1.AcquireBoolValue() || input2.AcquireBoolValue()) } else { - return MLRVAL_ERROR + return mlrval.ERROR } } -func BIF_logicalxor(input1, input2 *Mlrval) *Mlrval { +func BIF_logicalxor(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsBool() && input2.IsBool() { - return MlrvalFromBool(input1.boolval != input2.boolval) + return mlrval.FromBool(input1.AcquireBoolValue() != input2.AcquireBoolValue()) } else { - return MLRVAL_ERROR + return mlrval.ERROR } } diff --git a/internal/pkg/bifs/collections.go b/internal/pkg/bifs/collections.go index dca279999..9d003f5c6 100644 --- a/internal/pkg/bifs/collections.go +++ b/internal/pkg/bifs/collections.go @@ -6,66 +6,68 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" + "github.com/johnkerl/miller/internal/pkg/types" ) // ================================================================ // Map/array count. Scalars (including strings) have length 1. -func BIF_length(input1 *Mlrval) *Mlrval { - switch input1.mvtype { - case MT_ERROR: - return MlrvalFromInt(0) +func BIF_length(input1 *mlrval.Mlrval) *mlrval.Mlrval { + switch input1.Type() { + case mlrval.MT_ERROR: + return mlrval.FromInt(0) break - case MT_ABSENT: - return MlrvalFromInt(0) + case mlrval.MT_ABSENT: + return mlrval.FromInt(0) break - case MT_ARRAY: - return MlrvalFromInt(int(len(input1.arrayval))) + case mlrval.MT_ARRAY: + return mlrval.FromInt(int(len(input1.AcquireArrayValue().([]mlrval.Mlrval)))) break - case MT_MAP: - return MlrvalFromInt(int(input1.mapval.FieldCount)) + case mlrval.MT_MAP: + return mlrval.FromInt(int(input1.AcquireArrayValue().(types.Mlrmap).FieldCount)) break } - return MlrvalFromInt(1) + return mlrval.FromInt(1) } // ================================================================ -func depth_from_array(input1 *Mlrval) *Mlrval { +func depth_from_array(input1 *mlrval.Mlrval) *mlrval.Mlrval { maxChildDepth := 0 - for _, child := range input1.arrayval { + for _, child := range input1.AcquireArrayValue().([]mlrval.Mlrval) { childDepth := BIF_depth(&child) - lib.InternalCodingErrorIf(childDepth.mvtype != MT_INT) - iChildDepth := int(childDepth.intval) + lib.InternalCodingErrorIf(!childDepth.IsInt()) + iChildDepth := int(childDepth.AcquireIntValue()) if iChildDepth > maxChildDepth { maxChildDepth = iChildDepth } } - return MlrvalFromInt(int(1 + maxChildDepth)) + return mlrval.FromInt(int(1 + maxChildDepth)) } -func depth_from_map(input1 *Mlrval) *Mlrval { +func depth_from_map(input1 *mlrval.Mlrval) *mlrval.Mlrval { maxChildDepth := 0 - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { + for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { child := pe.Value childDepth := BIF_depth(child) - lib.InternalCodingErrorIf(childDepth.mvtype != MT_INT) - iChildDepth := int(childDepth.intval) + lib.InternalCodingErrorIf(!childDepth.IsInt()) + iChildDepth := int(childDepth.AcquireIntValue()) if iChildDepth > maxChildDepth { maxChildDepth = iChildDepth } } - return MlrvalFromInt(int(1 + maxChildDepth)) + return mlrval.FromInt(int(1 + maxChildDepth)) } -func depth_from_scalar(input1 *Mlrval) *Mlrval { - return MlrvalFromInt(0) +func depth_from_scalar(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(0) } // We get a Golang "initialization loop" due to recursive depth computation // if this is defined statically. So, we use a "package init" function. -var depth_dispositions = [MT_DIM]UnaryFunc{} +var depth_dispositions = [mlrval.MT_DIM]UnaryFunc{} func init() { - depth_dispositions = [MT_DIM]UnaryFunc{ + depth_dispositions = [mlrval.MT_DIM]UnaryFunc{ /*ERROR */ _erro1, /*ABSENT */ _absn1, /*NULL */ _zero1, @@ -80,58 +82,58 @@ func init() { } } -func BIF_depth(input1 *Mlrval) *Mlrval { - return depth_dispositions[input1.mvtype](input1) +func BIF_depth(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return depth_dispositions[input1.Type()](input1) } // ================================================================ -func leafcount_from_array(input1 *Mlrval) *Mlrval { +func leafcount_from_array(input1 *mlrval.Mlrval) *mlrval.Mlrval { sumChildLeafCount := 0 - for _, child := range input1.arrayval { + for _, child := range input1.AcquireArrayValue().([]mlrval.Mlrval) { // Golang initialization loop if we do this :( // childLeafCount := BIF_leafcount(&child) - childLeafCount := MlrvalFromInt(1) + childLeafCount := mlrval.FromInt(1) if child.IsArray() { childLeafCount = leafcount_from_array(&child) } else if child.IsMap() { childLeafCount = leafcount_from_map(&child) } - lib.InternalCodingErrorIf(childLeafCount.mvtype != MT_INT) - iChildLeafCount := int(childLeafCount.intval) + lib.InternalCodingErrorIf(!childLeafCount.IsInt()) + iChildLeafCount := int(childLeafCount.AcquireIntValue()) sumChildLeafCount += iChildLeafCount } - return MlrvalFromInt(int(sumChildLeafCount)) + return mlrval.FromInt(int(sumChildLeafCount)) } -func leafcount_from_map(input1 *Mlrval) *Mlrval { +func leafcount_from_map(input1 *mlrval.Mlrval) *mlrval.Mlrval { sumChildLeafCount := 0 - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { + for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { child := pe.Value // Golang initialization loop if we do this :( // childLeafCount := BIF_leafcount(child) - childLeafCount := MlrvalFromInt(1) + childLeafCount := mlrval.FromInt(1) if child.IsArray() { childLeafCount = leafcount_from_array(child) } else if child.IsMap() { childLeafCount = leafcount_from_map(child) } - lib.InternalCodingErrorIf(childLeafCount.mvtype != MT_INT) - iChildLeafCount := int(childLeafCount.intval) + lib.InternalCodingErrorIf(!childLeafCount.IsInt()) + iChildLeafCount := int(childLeafCount.AcquireIntValue()) sumChildLeafCount += iChildLeafCount } - return MlrvalFromInt(int(sumChildLeafCount)) + return mlrval.FromInt(int(sumChildLeafCount)) } -func leafcount_from_scalar(input1 *Mlrval) *Mlrval { - return MlrvalFromInt(1) +func leafcount_from_scalar(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(1) } -var leafcount_dispositions = [MT_DIM]UnaryFunc{ +var leafcount_dispositions = [mlrval.MT_DIM]UnaryFunc{ /*ERROR */ _erro1, /*ABSENT */ _absn1, /*NULL */ _zero1, @@ -145,67 +147,67 @@ var leafcount_dispositions = [MT_DIM]UnaryFunc{ /*FUNC */ _erro1, } -func BIF_leafcount(input1 *Mlrval) *Mlrval { - return leafcount_dispositions[input1.mvtype](input1) +func BIF_leafcount(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return leafcount_dispositions[input1.Type()](input1) } // ---------------------------------------------------------------- -func has_key_in_array(input1, input2 *Mlrval) *Mlrval { +func has_key_in_array(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if input2.IsString() { - return MLRVAL_FALSE + return mlrval.FALSE } - if input2.mvtype != MT_INT { - return MLRVAL_ERROR + if !input2.IsInt() { + return mlrval.ERROR } - _, ok := UnaliasArrayIndex(&input1.arrayval, input2.intval) - return MlrvalFromBool(ok) + _, ok := UnaliasArrayIndex(&input1.AcquireArrayValue().([]mlrval.Mlrval), input2.AcquireIntValue()) + return mlrval.FromBool(ok) } -func has_key_in_map(input1, input2 *Mlrval) *Mlrval { +func has_key_in_map(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if input2.IsString() || input2.IsInt() { - return MlrvalFromBool(input1.mapval.Has(input2.String())) + return mlrval.FromBool(input1.AcquireArrayValue().(types.Mlrmap).Has(input2.String())) } else { - return MLRVAL_ERROR + return mlrval.ERROR } } -func BIF_haskey(input1, input2 *Mlrval) *Mlrval { +func BIF_haskey(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsArray() { return has_key_in_array(input1, input2) } else if input1.IsMap() { return has_key_in_map(input1, input2) } else { - return MLRVAL_ERROR + return mlrval.ERROR } } // ================================================================ -func BIF_mapselect(mlrvals []*Mlrval) *Mlrval { +func BIF_mapselect(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { if len(mlrvals) < 1 { - return MLRVAL_ERROR + return mlrval.ERROR } - if mlrvals[0].mvtype != MT_MAP { - return MLRVAL_ERROR + if !mlrvals[0].IsMap() { + return mlrval.ERROR } - oldmap := mlrvals[0].mapval + oldmap := mlrvals[0].AcquireArrayValue().(types.Mlrmap) newMap := NewMlrmap() newKeys := make(map[string]bool) for _, selectArg := range mlrvals[1:] { if selectArg.IsString() { - newKeys[selectArg.printrep] = true + newKeys[selectArg.AcquireStringValue()] = true } else if selectArg.IsInt() { newKeys[selectArg.String()] = true } else if selectArg.IsArray() { - for _, element := range selectArg.arrayval { + for _, element := range selectArg.AcquireArrayValue().([]mlrval.Mlrval) { if element.IsString() { - newKeys[element.printrep] = true + newKeys[element.AcquireStringValue()] = true } else { - return MLRVAL_ERROR + return mlrval.ERROR } } } else { - return MLRVAL_ERROR + return mlrval.ERROR } } @@ -217,115 +219,115 @@ func BIF_mapselect(mlrvals []*Mlrval) *Mlrval { } } - return MlrvalFromMap(newMap) + return mlrval.FromMap(newMap) } // ---------------------------------------------------------------- -func BIF_mapexcept(mlrvals []*Mlrval) *Mlrval { +func BIF_mapexcept(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { if len(mlrvals) < 1 { - return MLRVAL_ERROR + return mlrval.ERROR } - if mlrvals[0].mvtype != MT_MAP { - return MLRVAL_ERROR + if !mlrvals[0].IsMap() { + return mlrval.ERROR } - newMap := mlrvals[0].mapval.Copy() + newMap := mlrvals[0].AcquireArrayValue().(types.Mlrmap).Copy() for _, exceptArg := range mlrvals[1:] { if exceptArg.IsString() { - newMap.Remove(exceptArg.printrep) + newMap.Remove(exceptArg.AcquireStringValue()) } else if exceptArg.IsInt() { newMap.Remove(exceptArg.String()) } else if exceptArg.IsArray() { - for _, element := range exceptArg.arrayval { + for _, element := range exceptArg.AcquireArrayValue().([]mlrval.Mlrval) { if element.IsString() { - newMap.Remove(element.printrep) + newMap.Remove(element.AcquireStringValue()) } else { - return MLRVAL_ERROR + return mlrval.ERROR } } } else { - return MLRVAL_ERROR + return mlrval.ERROR } } - return MlrvalFromMap(newMap) + return mlrval.FromMap(newMap) } // ---------------------------------------------------------------- -func BIF_mapsum(mlrvals []*Mlrval) *Mlrval { +func BIF_mapsum(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { if len(mlrvals) == 0 { - return MlrvalFromEmptyMap() + return mlrval.FromEmptyMap() } if len(mlrvals) == 1 { return mlrvals[0] } - if mlrvals[0].mvtype != MT_MAP { - return MLRVAL_ERROR + if mlrvals[0].Type() != MT_MAP { + return mlrval.ERROR } - newMap := mlrvals[0].mapval.Copy() + newMap := mlrvals[0].AcquireArrayValue().(types.Mlrmap).Copy() for _, otherMapArg := range mlrvals[1:] { - if otherMapArg.mvtype != MT_MAP { - return MLRVAL_ERROR + if otherMapArg.Type() != MT_MAP { + return mlrval.ERROR } - for pe := otherMapArg.mapval.Head; pe != nil; pe = pe.Next { + for pe := otherMapArg.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { newMap.PutCopy(pe.Key, pe.Value) } } - return MlrvalFromMap(newMap) + return mlrval.FromMap(newMap) } // ---------------------------------------------------------------- -func BIF_mapdiff(mlrvals []*Mlrval) *Mlrval { +func BIF_mapdiff(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { if len(mlrvals) == 0 { - return MlrvalFromEmptyMap() + return mlrval.FromEmptyMap() } if len(mlrvals) == 1 { return mlrvals[0] } - if mlrvals[0].mvtype != MT_MAP { - return MLRVAL_ERROR + if mlrvals[0].Type() != MT_MAP { + return mlrval.ERROR } - newMap := mlrvals[0].mapval.Copy() + newMap := mlrvals[0].AcquireArrayValue().(types.Mlrmap).Copy() for _, otherMapArg := range mlrvals[1:] { - if otherMapArg.mvtype != MT_MAP { - return MLRVAL_ERROR + if otherMapArg.Type() != MT_MAP { + return mlrval.ERROR } - for pe := otherMapArg.mapval.Head; pe != nil; pe = pe.Next { + for pe := otherMapArg.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { newMap.Remove(pe.Key) } } - return MlrvalFromMap(newMap) + return mlrval.FromMap(newMap) } // ================================================================ // joink([1,2,3], ",") -> "1,2,3" // joink({"a":3,"b":4,"c":5}, ",") -> "a,b,c" -func BIF_joink(input1, input2 *Mlrval) *Mlrval { - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR +func BIF_joink(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input2.Type() != MT_STRING { + return mlrval.ERROR } - fieldSeparator := input2.printrep + fieldSeparator := input2.AcquireStringValue() if input1.IsMap() { var buffer bytes.Buffer - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { + for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { buffer.WriteString(pe.Key) if pe.Next != nil { buffer.WriteString(fieldSeparator) } } - return MlrvalFromString(buffer.String()) + return mlrval.FromString(buffer.String()) } else if input1.IsArray() { var buffer bytes.Buffer - for i := range input1.arrayval { + for i := range input1.AcquireArrayValue().([]mlrval.Mlrval) { if i > 0 { buffer.WriteString(fieldSeparator) } @@ -333,65 +335,65 @@ func BIF_joink(input1, input2 *Mlrval) *Mlrval { buffer.WriteString(strconv.Itoa(i + 1)) } - return MlrvalFromString(buffer.String()) + return mlrval.FromString(buffer.String()) } else { - return MLRVAL_ERROR + return mlrval.ERROR } } // ---------------------------------------------------------------- // joinv([3,4,5], ",") -> "3,4,5" // joinv({"a":3,"b":4,"c":5}, ",") -> "3,4,5" -func BIF_joinv(input1, input2 *Mlrval) *Mlrval { - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR +func BIF_joinv(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input2.Type() != MT_STRING { + return mlrval.ERROR } - fieldSeparator := input2.printrep + fieldSeparator := input2.AcquireStringValue() if input1.IsMap() { var buffer bytes.Buffer - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { + for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { buffer.WriteString(pe.Value.String()) if pe.Next != nil { buffer.WriteString(fieldSeparator) } } - return MlrvalFromString(buffer.String()) + return mlrval.FromString(buffer.String()) } else if input1.IsArray() { var buffer bytes.Buffer - for i, element := range input1.arrayval { + for i, element := range input1.AcquireArrayValue().([]mlrval.Mlrval) { if i > 0 { buffer.WriteString(fieldSeparator) } buffer.WriteString(element.String()) } - return MlrvalFromString(buffer.String()) + return mlrval.FromString(buffer.String()) } else { - return MLRVAL_ERROR + return mlrval.ERROR } } // ---------------------------------------------------------------- // joinkv([3,4,5], "=", ",") -> "1=3,2=4,3=5" // joinkv({"a":3,"b":4,"c":5}, "=", ",") -> "a=3,b=4,c=5" -func BIF_joinkv(input1, input2, input3 *Mlrval) *Mlrval { - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR +func BIF_joinkv(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { + if input2.Type() != MT_STRING { + return mlrval.ERROR } - pairSeparator := input2.printrep - if input3.mvtype != MT_STRING { - return MLRVAL_ERROR + pairSeparator := input2.AcquireStringValue() + if input3.Type() != MT_STRING { + return mlrval.ERROR } - fieldSeparator := input3.printrep + fieldSeparator := input3.AcquireStringValue() if input1.IsMap() { var buffer bytes.Buffer - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { + for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { buffer.WriteString(pe.Key) buffer.WriteString(pairSeparator) buffer.WriteString(pe.Value.String()) @@ -400,11 +402,11 @@ func BIF_joinkv(input1, input2, input3 *Mlrval) *Mlrval { } } - return MlrvalFromString(buffer.String()) + return mlrval.FromString(buffer.String()) } else if input1.IsArray() { var buffer bytes.Buffer - for i, element := range input1.arrayval { + for i, element := range input1.AcquireArrayValue().([]mlrval.Mlrval) { if i > 0 { buffer.WriteString(fieldSeparator) } @@ -414,40 +416,40 @@ func BIF_joinkv(input1, input2, input3 *Mlrval) *Mlrval { buffer.WriteString(element.String()) } - return MlrvalFromString(buffer.String()) + return mlrval.FromString(buffer.String()) } else { - return MLRVAL_ERROR + return mlrval.ERROR } } // ================================================================ // splitkv("a=3,b=4,c=5", "=", ",") -> {"a":3,"b":4,"c":5} -func BIF_splitkv(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_splitkv(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR + if input2.Type() != MT_STRING { + return mlrval.ERROR } - pairSeparator := input2.printrep - if input3.mvtype != MT_STRING { - return MLRVAL_ERROR + pairSeparator := input2.AcquireStringValue() + if input3.Type() != MT_STRING { + return mlrval.ERROR } - fieldSeparator := input3.printrep + fieldSeparator := input3.AcquireStringValue() - output := MlrvalFromEmptyMap() + output := mlrval.FromEmptyMap() - fields := lib.SplitString(input1.printrep, fieldSeparator) + fields := lib.SplitString(input1.AcquireStringValue(), fieldSeparator) for i, field := range fields { pair := strings.SplitN(field, pairSeparator, 2) if len(pair) == 1 { key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up - value := MlrvalFromInferredType(pair[0]) - output.mapval.PutReference(key, value) + value := mlrval.FromInferredType(pair[0]) + output.AcquireArrayValue().(types.Mlrmap).PutReference(key, value) } else if len(pair) == 2 { key := pair[0] - value := MlrvalFromInferredType(pair[1]) - output.mapval.PutReference(key, value) + value := mlrval.FromInferredType(pair[1]) + output.AcquireArrayValue().(types.Mlrmap).PutReference(key, value) } else { lib.InternalCodingErrorIf(true) } @@ -457,32 +459,32 @@ func BIF_splitkv(input1, input2, input3 *Mlrval) *Mlrval { // ---------------------------------------------------------------- // splitkvx("a=3,b=4,c=5", "=", ",") -> {"a":"3","b":"4","c":"5"} -func BIF_splitkvx(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_splitkvx(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR + if input2.Type() != MT_STRING { + return mlrval.ERROR } - pairSeparator := input2.printrep - if input3.mvtype != MT_STRING { - return MLRVAL_ERROR + pairSeparator := input2.AcquireStringValue() + if input3.Type() != MT_STRING { + return mlrval.ERROR } - fieldSeparator := input3.printrep + fieldSeparator := input3.AcquireStringValue() - output := MlrvalFromEmptyMap() + output := mlrval.FromEmptyMap() - fields := lib.SplitString(input1.printrep, fieldSeparator) + fields := lib.SplitString(input1.AcquireStringValue(), fieldSeparator) for i, field := range fields { pair := strings.SplitN(field, pairSeparator, 2) if len(pair) == 1 { key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up - value := MlrvalFromString(pair[0]) - output.mapval.PutReference(key, value) + value := mlrval.FromString(pair[0]) + output.AcquireArrayValue().(types.Mlrmap).PutReference(key, value) } else if len(pair) == 2 { key := pair[0] - value := MlrvalFromString(pair[1]) - output.mapval.PutReference(key, value) + value := mlrval.FromString(pair[1]) + output.AcquireArrayValue().(types.Mlrmap).PutReference(key, value) } else { lib.InternalCodingErrorIf(true) } @@ -493,21 +495,21 @@ func BIF_splitkvx(input1, input2, input3 *Mlrval) *Mlrval { // ---------------------------------------------------------------- // splitnv("a,b,c", ",") -> {"1":"a","2":"b","3":"c"} -func BIF_splitnv(input1, input2 *Mlrval) *Mlrval { +func BIF_splitnv(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR + if input2.Type() != MT_STRING { + return mlrval.ERROR } - output := MlrvalFromEmptyMap() + output := mlrval.FromEmptyMap() - fields := lib.SplitString(input1.printrep, input2.printrep) + fields := lib.SplitString(input1.AcquireStringValue(), input2.AcquireStringValue()) for i, field := range fields { key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up - value := MlrvalFromInferredType(field) - output.mapval.PutReference(key, value) + value := mlrval.FromInferredType(field) + output.AcquireArrayValue().(types.Mlrmap).PutReference(key, value) } return output @@ -515,21 +517,21 @@ func BIF_splitnv(input1, input2 *Mlrval) *Mlrval { // ---------------------------------------------------------------- // splitnvx("3,4,5", ",") -> {"1":"3","2":"4","3":"5"} -func BIF_splitnvx(input1, input2 *Mlrval) *Mlrval { +func BIF_splitnvx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR + if input2.Type() != MT_STRING { + return mlrval.ERROR } - output := MlrvalFromEmptyMap() + output := mlrval.FromEmptyMap() - fields := lib.SplitString(input1.printrep, input2.printrep) + fields := lib.SplitString(input1.AcquireStringValue(), input2.AcquireStringValue()) for i, field := range fields { key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up - value := MlrvalFromString(field) - output.mapval.PutReference(key, value) + value := mlrval.FromString(field) + output.AcquireArrayValue().(types.Mlrmap).PutReference(key, value) } return output @@ -537,22 +539,22 @@ func BIF_splitnvx(input1, input2 *Mlrval) *Mlrval { // ---------------------------------------------------------------- // splita("3,4,5", ",") -> [3,4,5] -func BIF_splita(input1, input2 *Mlrval) *Mlrval { +func BIF_splita(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } - fieldSeparator := input2.printrep + fieldSeparator := input2.AcquireStringValue() - fields := lib.SplitString(input1.printrep, fieldSeparator) + fields := lib.SplitString(input1.AcquireStringValue(), fieldSeparator) output := NewSizedMlrvalArray(int(len(fields))) for i, field := range fields { - value := MlrvalFromInferredType(field) - output.arrayval[i] = *value + value := mlrval.FromInferredType(field) + output.AcquireArrayValue().([]mlrval.Mlrval)[i] = *value } return output @@ -561,85 +563,85 @@ func BIF_splita(input1, input2 *Mlrval) *Mlrval { // ---------------------------------------------------------------- // BIF_splitax splits a string to an array, without type-inference: // e.g. splitax("3,4,5", ",") -> ["3","4","5"] -func BIF_splitax(input1, input2 *Mlrval) *Mlrval { +func BIF_splitax(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR + if input2.Type() != MT_STRING { + return mlrval.ERROR } - input := input1.printrep - fieldSeparator := input2.printrep + input := input1.AcquireStringValue() + fieldSeparator := input2.AcquireStringValue() return mlrvalSplitAXHelper(input, fieldSeparator) } // mlrvalSplitAXHelper is Split out for the benefit of BIF_splitax and // BIF_unflatten. -func mlrvalSplitAXHelper(input string, separator string) *Mlrval { +func mlrvalSplitAXHelper(input string, separator string) *mlrval.Mlrval { fields := lib.SplitString(input, separator) output := NewSizedMlrvalArray(int(len(fields))) for i, field := range fields { - output.arrayval[i] = *MlrvalFromString(field) + output.AcquireArrayValue().([]mlrval.Mlrval)[i] = *mlrval.FromString(field) } return output } // ---------------------------------------------------------------- -func BIF_get_keys(input1 *Mlrval) *Mlrval { +func BIF_get_keys(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsMap() { // TODO: make a ReferenceFrom with comments - output := NewSizedMlrvalArray(input1.mapval.FieldCount) + output := NewSizedMlrvalArray(input1.AcquireArrayValue().(types.Mlrmap).FieldCount) i := 0 - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { - output.arrayval[i] = *MlrvalFromString(pe.Key) + for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { + output.AcquireArrayValue().([]mlrval.Mlrval)[i] = *mlrval.FromString(pe.Key) i++ } return output } else if input1.IsArray() { - output := NewSizedMlrvalArray(int(len(input1.arrayval))) - for i := range input1.arrayval { - output.arrayval[i] = *MlrvalFromInt(int(i + 1)) // Miller user-space indices are 1-up + output := NewSizedMlrvalArray(int(len(input1.AcquireArrayValue().([]mlrval.Mlrval)))) + for i := range input1.AcquireArrayValue().([]mlrval.Mlrval) { + output.AcquireArrayValue().([]mlrval.Mlrval)[i] = *mlrval.FromInt(int(i + 1)) // Miller user-space indices are 1-up } return output } else { - return MLRVAL_ERROR + return mlrval.ERROR } } // ---------------------------------------------------------------- -func BIF_get_values(input1 *Mlrval) *Mlrval { +func BIF_get_values(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsMap() { // TODO: make a ReferenceFrom with commenbs - output := NewSizedMlrvalArray(input1.mapval.FieldCount) + output := NewSizedMlrvalArray(input1.AcquireArrayValue().(types.Mlrmap).FieldCount) i := 0 - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { - output.arrayval[i] = *pe.Value.Copy() + for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { + output.AcquireArrayValue().([]mlrval.Mlrval)[i] = *pe.Value.Copy() i++ } return output } else if input1.IsArray() { - output := NewSizedMlrvalArray(int(len(input1.arrayval))) - for i, value := range input1.arrayval { - output.arrayval[i] = *value.Copy() + output := NewSizedMlrvalArray(int(len(input1.AcquireArrayValue().([]mlrval.Mlrval)))) + for i, value := range input1.AcquireArrayValue().([]mlrval.Mlrval) { + output.AcquireArrayValue().([]mlrval.Mlrval)[i] = *value.Copy() } return output } else { - return MLRVAL_ERROR + return mlrval.ERROR } } // ---------------------------------------------------------------- -func BIF_append(input1, input2 *Mlrval) *Mlrval { - if input1.mvtype != MT_ARRAY { - return MLRVAL_ERROR +func BIF_append(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.Type() != MT_ARRAY { + return mlrval.ERROR } output := input1.Copy() @@ -653,16 +655,16 @@ func BIF_append(input1, input2 *Mlrval) *Mlrval { // Third argument is map or array. // flatten("a", ".", {"b": { "c": 4 }}) is {"a.b.c" : 4}. // flatten("", ".", {"a": { "b": 3 }}) is {"a.b" : 3}. -func BIF_flatten(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_flatten(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if input3.IsMap() || input3.IsArray() { - if input1.mvtype != MT_STRING && input1.mvtype != MT_VOID { - return MLRVAL_ERROR + if input1.Type() != MT_STRING && input1.Type() != MT_VOID { + return mlrval.ERROR } - prefix := input1.printrep - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR + prefix := input1.AcquireStringValue() + if input2.Type() != MT_STRING { + return mlrval.ERROR } - delimiter := input2.printrep + delimiter := input2.AcquireStringValue() retval := input3.FlattenToMap(prefix, delimiter) return &retval @@ -672,38 +674,38 @@ func BIF_flatten(input1, input2, input3 *Mlrval) *Mlrval { } // flatten($*, ".") is the same as flatten("", ".", $*) -func BIF_flatten_binary(input1, input2 *Mlrval) *Mlrval { - return BIF_flatten(MLRVAL_VOID, input2, input1) +func BIF_flatten_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return BIF_flatten(mlrval.VOID, input2, input1) } // ---------------------------------------------------------------- // First argument is a map. // Second argument is a delimiter string. // unflatten({"a.b.c", ".") is {"a": { "b": { "c": 4}}}. -func BIF_unflatten(input1, input2 *Mlrval) *Mlrval { - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR +func BIF_unflatten(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input2.Type() != MT_STRING { + return mlrval.ERROR } - if input1.mvtype != MT_MAP { + if input1.Type() != MT_MAP { return input1 } - oldmap := input1.mapval - separator := input2.printrep + oldmap := input1.AcquireArrayValue().(types.Mlrmap) + separator := input2.AcquireStringValue() newmap := oldmap.CopyUnflattened(separator) - return MlrvalFromMapReferenced(newmap) + return mlrval.FromMapReferenced(newmap) } // ---------------------------------------------------------------- // Converts maps with "1", "2", ... keys into arrays. Recurses nested data structures. -func BIF_arrayify(input1 *Mlrval) *Mlrval { +func BIF_arrayify(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsMap() { - if input1.mapval.IsEmpty() { + if input1.AcquireArrayValue().(types.Mlrmap).IsEmpty() { return input1 } convertible := true i := 0 - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { + for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { sval := strconv.Itoa(i + 1) // Miller user-space indices are 1-up i++ if pe.Key != sval { @@ -713,13 +715,13 @@ func BIF_arrayify(input1 *Mlrval) *Mlrval { } if convertible { - arrayval := make([]Mlrval, input1.mapval.FieldCount) + AcquireArrayValue().([]mlrval.Mlrval) := make([]mlrval.Mlrval, input1.AcquireArrayValue().(types.Mlrmap).FieldCount) i := 0 - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { - arrayval[i] = *pe.Value.Copy() + for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { + AcquireArrayValue().([]mlrval.Mlrval)[i] = *pe.Value.Copy() i++ } - return MlrvalFromArrayReference(arrayval) + return mlrval.FromArrayReference(AcquireArrayValue().([]mlrval.Mlrval)) } else { return input1 @@ -728,8 +730,8 @@ func BIF_arrayify(input1 *Mlrval) *Mlrval { } else if input1.IsArray() { // TODO: comment (or rethink) that this modifies its inputs!! output := input1.Copy() - for i := range input1.arrayval { - output.arrayval[i] = *BIF_arrayify(&output.arrayval[i]) + for i := range input1.AcquireArrayValue().([]mlrval.Mlrval) { + output.AcquireArrayValue().([]mlrval.Mlrval)[i] = *BIF_arrayify(&output.AcquireArrayValue().([]mlrval.Mlrval)[i]) } return output @@ -739,35 +741,35 @@ func BIF_arrayify(input1 *Mlrval) *Mlrval { } // ---------------------------------------------------------------- -func BIF_json_parse(input1 *Mlrval) *Mlrval { +func BIF_json_parse(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsVoid() { return input1 - } else if input1.mvtype != MT_STRING { - return MLRVAL_ERROR + } else if input1.Type() != MT_STRING { + return mlrval.ERROR } else { - output := MlrvalFromPending() - err := output.UnmarshalJSON([]byte(input1.printrep)) + output := mlrval.FromPending() + err := output.UnmarshalJSON([]byte(input1.AcquireStringValue())) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } return &output } } -func BIF_json_stringify_unary(input1 *Mlrval) *Mlrval { +func BIF_json_stringify_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { outputBytes, err := input1.MarshalJSON(JSON_SINGLE_LINE, false) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } else { - return MlrvalFromString(string(outputBytes)) + return mlrval.FromString(string(outputBytes)) } } -func BIF_json_stringify_binary(input1, input2 *Mlrval) *Mlrval { +func BIF_json_stringify_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { var jsonFormatting TJSONFormatting = JSON_SINGLE_LINE useMultiline, ok := input2.GetBoolValue() if !ok { - return MLRVAL_ERROR + return mlrval.ERROR } if useMultiline { jsonFormatting = JSON_MULTILINE @@ -775,8 +777,8 @@ func BIF_json_stringify_binary(input1, input2 *Mlrval) *Mlrval { outputBytes, err := input1.MarshalJSON(jsonFormatting, false) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } else { - return MlrvalFromString(string(outputBytes)) + return mlrval.FromString(string(outputBytes)) } } diff --git a/internal/pkg/bifs/datetime.go b/internal/pkg/bifs/datetime.go index 65c946e6d..a35c10c3d 100644 --- a/internal/pkg/bifs/datetime.go +++ b/internal/pkg/bifs/datetime.go @@ -9,22 +9,23 @@ import ( "github.com/lestrrat-go/strftime" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) const ISO8601_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" -var ptr_ISO8601_TIME_FORMAT = MlrvalFromString("%Y-%m-%dT%H:%M:%SZ") -var ptr_ISO8601_LOCAL_TIME_FORMAT = MlrvalFromString("%Y-%m-%d %H:%M:%S") -var ptr_YMD_FORMAT = MlrvalFromString("%Y-%m-%d") +var ptr_ISO8601_TIME_FORMAT = mlrval.FromString("%Y-%m-%dT%H:%M:%SZ") +var ptr_ISO8601_LOCAL_TIME_FORMAT = mlrval.FromString("%Y-%m-%d %H:%M:%S") +var ptr_YMD_FORMAT = mlrval.FromString("%Y-%m-%d") // ================================================================ -func BIF_systime() *Mlrval { - return MlrvalFromFloat64( +func BIF_systime() *mlrval.Mlrval { + return mlrval.FromFloat( float64(time.Now().UnixNano()) / 1.0e9, ) } -func BIF_systimeint() *Mlrval { - return MlrvalFromInt(int(time.Now().Unix())) +func BIF_systimeint() *mlrval.Mlrval { + return mlrval.FromInt(int(time.Now().Unix())) } var startTime float64 @@ -32,15 +33,15 @@ var startTime float64 func init() { startTime = float64(time.Now().UnixNano()) / 1.0e9 } -func BIF_uptime() *Mlrval { - return MlrvalFromFloat64( +func BIF_uptime() *mlrval.Mlrval { + return mlrval.FromFloat( float64(time.Now().UnixNano())/1.0e9 - startTime, ) } // ================================================================ -func BIF_sec2gmt_unary(input1 *Mlrval) *Mlrval { +func BIF_sec2gmt_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { floatValue, isNumeric := input1.GetNumericToFloatValue() if !isNumeric { return input1 @@ -48,10 +49,10 @@ func BIF_sec2gmt_unary(input1 *Mlrval) *Mlrval { numDecimalPlaces := 0 - return MlrvalFromString(lib.Sec2GMT(floatValue, numDecimalPlaces)) + return mlrval.FromString(lib.Sec2GMT(floatValue, numDecimalPlaces)) } -func BIF_sec2gmt_binary(input1, input2 *Mlrval) *Mlrval { +func BIF_sec2gmt_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { floatValue, isNumeric := input1.GetNumericToFloatValue() if !isNumeric { return input1 @@ -59,13 +60,13 @@ func BIF_sec2gmt_binary(input1, input2 *Mlrval) *Mlrval { numDecimalPlaces, isInt := input2.GetIntValue() if !isInt { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromString(lib.Sec2GMT(floatValue, numDecimalPlaces)) + return mlrval.FromString(lib.Sec2GMT(floatValue, numDecimalPlaces)) } -func BIF_sec2localtime_unary(input1 *Mlrval) *Mlrval { +func BIF_sec2localtime_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { floatValue, isNumeric := input1.GetNumericToFloatValue() if !isNumeric { return input1 @@ -73,10 +74,10 @@ func BIF_sec2localtime_unary(input1 *Mlrval) *Mlrval { numDecimalPlaces := 0 - return MlrvalFromString(lib.Sec2LocalTime(floatValue, numDecimalPlaces)) + return mlrval.FromString(lib.Sec2LocalTime(floatValue, numDecimalPlaces)) } -func BIF_sec2localtime_binary(input1, input2 *Mlrval) *Mlrval { +func BIF_sec2localtime_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { floatValue, isNumeric := input1.GetNumericToFloatValue() if !isNumeric { return input1 @@ -84,13 +85,13 @@ func BIF_sec2localtime_binary(input1, input2 *Mlrval) *Mlrval { numDecimalPlaces, isInt := input2.GetIntValue() if !isInt { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromString(lib.Sec2LocalTime(floatValue, numDecimalPlaces)) + return mlrval.FromString(lib.Sec2LocalTime(floatValue, numDecimalPlaces)) } -func BIF_sec2localtime_ternary(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_sec2localtime_ternary(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { floatValue, isNumeric := input1.GetNumericToFloatValue() if !isNumeric { return input1 @@ -98,37 +99,37 @@ func BIF_sec2localtime_ternary(input1, input2, input3 *Mlrval) *Mlrval { numDecimalPlaces, isInt := input2.GetIntValue() if !isInt { - return MLRVAL_ERROR + return mlrval.ERROR } locationString, isString := input3.GetString() if !isString { - return MLRVAL_ERROR + return mlrval.ERROR } location, err := time.LoadLocation(locationString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromString(lib.Sec2LocationTime(floatValue, numDecimalPlaces, location)) + return mlrval.FromString(lib.Sec2LocationTime(floatValue, numDecimalPlaces, location)) } -func BIF_sec2gmtdate(input1 *Mlrval) *Mlrval { +func BIF_sec2gmtdate(input1 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsNumeric() { return input1 } return BIF_strftime(input1, ptr_YMD_FORMAT) } -func BIF_sec2localdate_unary(input1 *Mlrval) *Mlrval { +func BIF_sec2localdate_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsNumeric() { return input1 } return BIF_strftime_local_binary(input1, ptr_YMD_FORMAT) } -func BIF_sec2localdate_binary(input1, input2 *Mlrval) *Mlrval { +func BIF_sec2localdate_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsNumeric() { return input1 } @@ -137,32 +138,32 @@ func BIF_sec2localdate_binary(input1, input2 *Mlrval) *Mlrval { // ---------------------------------------------------------------- -func BIF_localtime2gmt_unary(input1 *Mlrval) *Mlrval { +func BIF_localtime2gmt_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } return BIF_sec2gmt_unary(BIF_localtime2sec_unary(input1)) } -func BIF_localtime2gmt_binary(input1, input2 *Mlrval) *Mlrval { +func BIF_localtime2gmt_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } return BIF_sec2gmt_unary(BIF_localtime2sec_binary(input1, input2)) } -func BIF_gmt2localtime_unary(input1 *Mlrval) *Mlrval { +func BIF_gmt2localtime_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } return BIF_sec2localtime_unary(BIF_gmt2sec(input1)) } -func BIF_gmt2localtime_binary(input1, input2 *Mlrval) *Mlrval { +func BIF_gmt2localtime_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } - return BIF_sec2localtime_ternary(BIF_gmt2sec(input1), MlrvalFromInt(0), input2) + return BIF_sec2localtime_ternary(BIF_gmt2sec(input1), mlrval.FromInt(0), input2) } // ================================================================ @@ -171,38 +172,38 @@ func BIF_gmt2localtime_binary(input1, input2 *Mlrval) *Mlrval { var extensionRegex = regexp.MustCompile("([1-9])S") -func BIF_strftime(input1, input2 *Mlrval) *Mlrval { +func BIF_strftime(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return strftimeHelper(input1, input2, false, nil) } -func BIF_strftime_local_binary(input1, input2 *Mlrval) *Mlrval { +func BIF_strftime_local_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return strftimeHelper(input1, input2, true, nil) } -func BIF_strftime_local_ternary(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_strftime_local_ternary(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { locationString, isString := input3.GetString() if !isString { - return MLRVAL_ERROR + return mlrval.ERROR } location, err := time.LoadLocation(locationString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } return strftimeHelper(input1, input2, true, location) } -func strftimeHelper(input1, input2 *Mlrval, doLocal bool, location *time.Location) *Mlrval { +func strftimeHelper(input1, input2 *mlrval.Mlrval, doLocal bool, location *time.Location) *mlrval.Mlrval { if input1.IsVoid() { return input1 } epochSeconds, ok := input1.GetNumericToFloatValue() if !ok { - return MLRVAL_ERROR + return mlrval.ERROR } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR + if input2.Type() != MT_STRING { + return mlrval.ERROR } // Convert argument1 from float seconds since the epoch to a Go time. @@ -223,16 +224,16 @@ func strftimeHelper(input1, input2 *Mlrval, doLocal bool, location *time.Locatio // implementation. However, in the strftime package we're using in the Go // port, extension-formats are only a single byte so we need to rewrite // them to "%6". - formatString := extensionRegex.ReplaceAllString(input2.printrep, "$1") + formatString := extensionRegex.ReplaceAllString(input2.AcquireStringValue(), "$1") formatter, err := strftime.New(formatString, strftimeExtensions) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } outputString := formatter.FormatString(inputTime) - return MlrvalFromString(outputString) + return mlrval.FromString(outputString) } // ---------------------------------------------------------------- @@ -299,82 +300,82 @@ func init() { // ================================================================ // Argument 1 is formatted date string like "2021-03-04 02:59:50". // Argument 2 is format string like "%Y-%m-%d %H:%M:%S". -func BIF_strptime(input1, input2 *Mlrval) *Mlrval { - if input1.mvtype != MT_STRING { - return MLRVAL_ERROR +func BIF_strptime(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.Type() != MT_STRING { + return mlrval.ERROR } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR + if input2.Type() != MT_STRING { + return mlrval.ERROR } - timeString := input1.printrep - formatString := input2.printrep + timeString := input1.AcquireStringValue() + formatString := input2.AcquireStringValue() t, err := strptime.Parse(timeString, formatString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromFloat64(float64(t.UnixNano()) / 1.0e9) + return mlrval.FromFloat(float64(t.UnixNano()) / 1.0e9) } -func BIF_strptime_local_binary(input1, input2 *Mlrval) *Mlrval { - if input1.mvtype != MT_STRING { - return MLRVAL_ERROR +func BIF_strptime_local_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.Type() != MT_STRING { + return mlrval.ERROR } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR + if input2.Type() != MT_STRING { + return mlrval.ERROR } - timeString := input1.printrep - formatString := input2.printrep + timeString := input1.AcquireStringValue() + formatString := input2.AcquireStringValue() t, err := strptime.ParseLocal(timeString, formatString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromFloat64(float64(t.UnixNano()) / 1.0e9) + return mlrval.FromFloat(float64(t.UnixNano()) / 1.0e9) } -func BIF_strptime_local_ternary(input1, input2, input3 *Mlrval) *Mlrval { - if input1.mvtype != MT_STRING { - return MLRVAL_ERROR +func BIF_strptime_local_ternary(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.Type() != MT_STRING { + return mlrval.ERROR } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR + if input2.Type() != MT_STRING { + return mlrval.ERROR } - if input3.mvtype != MT_STRING { - return MLRVAL_ERROR + if input3.Type() != MT_STRING { + return mlrval.ERROR } - timeString := input1.printrep - formatString := input2.printrep - locationString := input3.printrep + timeString := input1.AcquireStringValue() + formatString := input2.AcquireStringValue() + locationString := input3.AcquireStringValue() location, err := time.LoadLocation(locationString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } // TODO: use location t, err := strptime.ParseLocation(timeString, formatString, location) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromFloat64(float64(t.UnixNano()) / 1.0e9) + return mlrval.FromFloat(float64(t.UnixNano()) / 1.0e9) } // ================================================================ // Argument 1 is formatted date string like "2021-03-04T02:59:50Z". -func BIF_gmt2sec(input1 *Mlrval) *Mlrval { +func BIF_gmt2sec(input1 *mlrval.Mlrval) *mlrval.Mlrval { return BIF_strptime(input1, ptr_ISO8601_TIME_FORMAT) } -func BIF_localtime2sec_unary(input1 *Mlrval) *Mlrval { +func BIF_localtime2sec_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { return BIF_strptime_local_binary(input1, ptr_ISO8601_LOCAL_TIME_FORMAT) } -func BIF_localtime2sec_binary(input1, input2 *Mlrval) *Mlrval { +func BIF_localtime2sec_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return BIF_strptime_local_ternary(input1, ptr_ISO8601_LOCAL_TIME_FORMAT, input2) } diff --git a/internal/pkg/bifs/hashing.go b/internal/pkg/bifs/hashing.go index 3a3adc7d0..976202bb9 100644 --- a/internal/pkg/bifs/hashing.go +++ b/internal/pkg/bifs/hashing.go @@ -6,55 +6,57 @@ import ( "crypto/sha256" "crypto/sha512" "fmt" + + "github.com/johnkerl/miller/internal/pkg/mlrval" ) -func BIF_md5(input1 *Mlrval) *Mlrval { +func BIF_md5(input1 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } else { - return MlrvalFromString( + return mlrval.FromString( fmt.Sprintf( "%x", - md5.Sum([]byte(input1.printrep)), + md5.Sum([]byte(input1.AcquireStringValue())), ), ) } } -func BIF_sha1(input1 *Mlrval) *Mlrval { +func BIF_sha1(input1 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } else { - return MlrvalFromString( + return mlrval.FromString( fmt.Sprintf( "%x", - sha1.Sum([]byte(input1.printrep)), + sha1.Sum([]byte(input1.AcquireStringValue())), ), ) } } -func BIF_sha256(input1 *Mlrval) *Mlrval { +func BIF_sha256(input1 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } else { - return MlrvalFromString( + return mlrval.FromString( fmt.Sprintf( "%x", - sha256.Sum256([]byte(input1.printrep)), + sha256.Sum256([]byte(input1.AcquireStringValue())), ), ) } } -func BIF_sha512(input1 *Mlrval) *Mlrval { +func BIF_sha512(input1 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } else { - return MlrvalFromString( + return mlrval.FromString( fmt.Sprintf( "%x", - sha512.Sum512([]byte(input1.printrep)), + sha512.Sum512([]byte(input1.AcquireStringValue())), ), ) } diff --git a/internal/pkg/bifs/math.go b/internal/pkg/bifs/math.go index 2a291508e..00e263633 100644 --- a/internal/pkg/bifs/math.go +++ b/internal/pkg/bifs/math.go @@ -1,21 +1,25 @@ package types +import ( + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + // Return error (unary math-library func) -func _math_unary_erro1(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval { - return MLRVAL_ERROR +func _math_unary_erro1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.ERROR } // Return absent (unary math-library func) -func _math_unary_absn1(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval { - return MLRVAL_ABSENT +func _math_unary_absn1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.ABSENT } // Return null (unary math-library func) -func _math_unary_null1(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval { - return MLRVAL_NULL +func _math_unary_null1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.NULL } // Return void (unary math-library func) -func _math_unary_void1(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval { - return MLRVAL_VOID +func _math_unary_void1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.VOID } diff --git a/internal/pkg/bifs/mathlib.go b/internal/pkg/bifs/mathlib.go index 527528200..fa35782cf 100644 --- a/internal/pkg/bifs/mathlib.go +++ b/internal/pkg/bifs/mathlib.go @@ -8,21 +8,22 @@ import ( "math" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ---------------------------------------------------------------- -func math_unary_f_i(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval { - return MlrvalFromFloat64(f(float64(input1.intval))) +func math_unary_f_i(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.FromFloat(f(float64(input1.AcquireIntValue()))) } -func math_unary_i_i(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval { - return MlrvalFromInt(int(f(float64(input1.intval)))) +func math_unary_i_i(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.FromInt(int(f(float64(input1.AcquireIntValue())))) } -func math_unary_f_f(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval { - return MlrvalFromFloat64(f(input1.floatval)) +func math_unary_f_f(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.FromFloat(f(input1.AcquireFloatValue())) } // Disposition vector for unary mathlib functions -var mudispo = [MT_DIM]mathLibUnaryFuncWrapper{ +var mudispo = [mlrval.MT_DIM]mathLibUnaryFuncWrapper{ /*ERROR */ _math_unary_erro1, /*ABSENT */ _math_unary_absn1, /*NULL */ _math_unary_null1, @@ -36,32 +37,32 @@ var mudispo = [MT_DIM]mathLibUnaryFuncWrapper{ /*FUNC */ _math_unary_erro1, } -func BIF_acos(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Acos) } -func BIF_acosh(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Acosh) } -func BIF_asin(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Asin) } -func BIF_asinh(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Asinh) } -func BIF_atan(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Atan) } -func BIF_atanh(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Atanh) } -func BIF_cbrt(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Cbrt) } -func BIF_cos(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Cos) } -func BIF_cosh(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Cosh) } -func BIF_erf(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Erf) } -func BIF_erfc(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Erfc) } -func BIF_exp(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Exp) } -func BIF_expm1(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Expm1) } -func BIF_invqnorm(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, lib.Invqnorm) } -func BIF_log(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Log) } -func BIF_log10(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Log10) } -func BIF_log1p(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Log1p) } -func BIF_qnorm(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, lib.Qnorm) } -func BIF_sin(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Sin) } -func BIF_sinh(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Sinh) } -func BIF_sqrt(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Sqrt) } -func BIF_tan(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Tan) } -func BIF_tanh(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Tanh) } +func BIF_acos(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Acos) } +func BIF_acosh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Acosh) } +func BIF_asin(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Asin) } +func BIF_asinh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Asinh) } +func BIF_atan(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Atan) } +func BIF_atanh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Atanh) } +func BIF_cbrt(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Cbrt) } +func BIF_cos(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Cos) } +func BIF_cosh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Cosh) } +func BIF_erf(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Erf) } +func BIF_erfc(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Erfc) } +func BIF_exp(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Exp) } +func BIF_expm1(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Expm1) } +func BIF_invqnorm(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, lib.Invqnorm) } +func BIF_log(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Log) } +func BIF_log10(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Log10) } +func BIF_log1p(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Log1p) } +func BIF_qnorm(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, lib.Qnorm) } +func BIF_sin(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Sin) } +func BIF_sinh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Sinh) } +func BIF_sqrt(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Sqrt) } +func BIF_tan(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Tan) } +func BIF_tanh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Tanh) } // Disposition vector for unary mathlib functions which are int-preserving -var imudispo = [MT_DIM]mathLibUnaryFuncWrapper{ +var imudispo = [mlrval.MT_DIM]mathLibUnaryFuncWrapper{ /*ERROR */ _math_unary_erro1, /*ABSENT */ _math_unary_absn1, /*NULL */ _math_unary_null1, @@ -76,37 +77,37 @@ var imudispo = [MT_DIM]mathLibUnaryFuncWrapper{ } // Int-preserving -func BIF_abs(input1 *Mlrval) *Mlrval { return imudispo[input1.mvtype](input1, math.Abs) } // xxx -func BIF_ceil(input1 *Mlrval) *Mlrval { return imudispo[input1.mvtype](input1, math.Ceil) } // xxx -func BIF_floor(input1 *Mlrval) *Mlrval { return imudispo[input1.mvtype](input1, math.Floor) } // xxx -func BIF_round(input1 *Mlrval) *Mlrval { return imudispo[input1.mvtype](input1, math.Round) } // xxx -func BIF_sgn(input1 *Mlrval) *Mlrval { return imudispo[input1.mvtype](input1, lib.Sgn) } // xxx +func BIF_abs(input1 *mlrval.Mlrval) *mlrval.Mlrval { return imudispo[input1.Type()](input1, math.Abs) } // xxx +func BIF_ceil(input1 *mlrval.Mlrval) *mlrval.Mlrval { return imudispo[input1.Type()](input1, math.Ceil) } // xxx +func BIF_floor(input1 *mlrval.Mlrval) *mlrval.Mlrval { return imudispo[input1.Type()](input1, math.Floor) } // xxx +func BIF_round(input1 *mlrval.Mlrval) *mlrval.Mlrval { return imudispo[input1.Type()](input1, math.Round) } // xxx +func BIF_sgn(input1 *mlrval.Mlrval) *mlrval.Mlrval { return imudispo[input1.Type()](input1, lib.Sgn) } // xxx // ================================================================ // Exponentiation: DSL operator '**'. See also // https://johnkerl.org/miller6/reference-main-arithmetic.html -func pow_f_ii(input1, input2 *Mlrval) *Mlrval { - foutput := math.Pow(float64(input1.intval), float64(input2.intval)) +func pow_f_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + foutput := math.Pow(float64(input1.AcquireIntValue()), float64(input2.AcquireIntValue())) ioutput := int(foutput) // Int raised to int power should be float if it can be (i.e. unless overflow) if float64(ioutput) == foutput { - return MlrvalFromInt(ioutput) + return mlrval.FromInt(ioutput) } else { - return MlrvalFromFloat64(foutput) + return mlrval.FromFloat(foutput) } } -func pow_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Pow(float64(input1.intval), input2.floatval)) +func pow_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Pow(float64(input1.AcquireIntValue()), input2.AcquireFloatValue())) } -func pow_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Pow(input1.floatval, float64(input2.intval))) +func pow_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Pow(input1.AcquireFloatValue(), float64(input2.AcquireIntValue()))) } -func pow_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Pow(input1.floatval, input2.floatval)) +func pow_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Pow(input1.AcquireFloatValue(), input2.AcquireFloatValue())) } -var pow_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var pow_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _i0__, _f0__, _erro, _absn, _absn, _erro}, @@ -121,25 +122,25 @@ var pow_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_pow(input1, input2 *Mlrval) *Mlrval { - return pow_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_pow(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return pow_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ -func atan2_f_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Atan2(float64(input1.intval), float64(input2.intval))) +func atan2_f_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Atan2(float64(input1.AcquireIntValue()), float64(input2.AcquireIntValue()))) } -func atan2_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Atan2(float64(input1.intval), input2.floatval)) +func atan2_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Atan2(float64(input1.AcquireIntValue()), input2.AcquireFloatValue())) } -func atan2_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Atan2(input1.floatval, float64(input2.intval))) +func atan2_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Atan2(input1.AcquireFloatValue(), float64(input2.AcquireIntValue()))) } -func atan2_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Atan2(input1.floatval, input2.floatval)) +func atan2_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Atan2(input1.AcquireFloatValue(), input2.AcquireFloatValue())) } -var atan2_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var atan2_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _i0__, _f0__, _erro, _absn, _absn, _erro}, @@ -154,8 +155,8 @@ var atan2_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_atan2(input1, input2 *Mlrval) *Mlrval { - return atan2_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_atan2(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return atan2_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ @@ -163,20 +164,20 @@ func mlr_roundm(x, m float64) float64 { return math.Round(x/m) * m } -func roundm_f_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(int(mlr_roundm(float64(input1.intval), float64(input2.intval)))) +func roundm_f_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(int(mlr_roundm(float64(input1.AcquireIntValue()), float64(input2.AcquireIntValue())))) } -func roundm_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(mlr_roundm(float64(input1.intval), input2.floatval)) +func roundm_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(mlr_roundm(float64(input1.AcquireIntValue()), input2.AcquireFloatValue())) } -func roundm_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(mlr_roundm(input1.floatval, float64(input2.intval))) +func roundm_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(mlr_roundm(input1.AcquireFloatValue(), float64(input2.AcquireIntValue()))) } -func roundm_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(mlr_roundm(input1.floatval, input2.floatval)) +func roundm_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(mlr_roundm(input1.AcquireFloatValue(), input2.AcquireFloatValue())) } -var roundm_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var roundm_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _i0__, _f0__, _erro, _absn, _absn, _erro}, @@ -191,12 +192,12 @@ var roundm_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_roundm(input1, input2 *Mlrval) *Mlrval { - return roundm_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_roundm(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return roundm_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ -func BIF_logifit(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_logifit(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsLegit() { return input1 } @@ -210,16 +211,16 @@ func BIF_logifit(input1, input2, input3 *Mlrval) *Mlrval { // int/float OK; rest not x, xok := input1.GetNumericToFloatValue() if !xok { - return MLRVAL_ERROR + return mlrval.ERROR } m, mok := input2.GetNumericToFloatValue() if !mok { - return MLRVAL_ERROR + return mlrval.ERROR } b, bok := input3.GetNumericToFloatValue() if !bok { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromFloat64(1.0 / (1.0 + math.Exp(-m*x-b))) + return mlrval.FromFloat(1.0 / (1.0 + math.Exp(-m*x-b))) } diff --git a/internal/pkg/bifs/random.go b/internal/pkg/bifs/random.go index 6e5a4b9e3..43d9997ed 100644 --- a/internal/pkg/bifs/random.go +++ b/internal/pkg/bifs/random.go @@ -4,16 +4,17 @@ import ( "math" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) -func BIF_urand() *Mlrval { - return MlrvalFromFloat64( +func BIF_urand() *mlrval.Mlrval { + return mlrval.FromFloat( lib.RandFloat64(), ) } -func BIF_urand32() *Mlrval { - return MlrvalFromInt( +func BIF_urand32() *mlrval.Mlrval { + return mlrval.FromInt( int( lib.RandUint32(), ), @@ -21,7 +22,7 @@ func BIF_urand32() *Mlrval { } // TODO: use a disposition matrix -func BIF_urandint(input1, input2 *Mlrval) *Mlrval { +func BIF_urandint(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsLegit() { return input1 } @@ -29,14 +30,14 @@ func BIF_urandint(input1, input2 *Mlrval) *Mlrval { return input2 } if !input1.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } - a := input1.intval - b := input2.intval + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() var lo int = 0 var hi int = 0 @@ -48,10 +49,10 @@ func BIF_urandint(input1, input2 *Mlrval) *Mlrval { hi = a + 1 } u := int(math.Floor(float64(lo) + float64((hi-lo))*lib.RandFloat64())) - return MlrvalFromInt(u) + return mlrval.FromInt(u) } -func BIF_urandrange(input1, input2 *Mlrval) *Mlrval { +func BIF_urandrange(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsLegit() { return input1 } @@ -61,12 +62,12 @@ func BIF_urandrange(input1, input2 *Mlrval) *Mlrval { a, aok := input1.GetNumericToFloatValue() b, bok := input2.GetNumericToFloatValue() if !aok { - return MLRVAL_ERROR + return mlrval.ERROR } if !bok { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromFloat64( + return mlrval.FromFloat( a + (b-a)*lib.RandFloat64(), ) } diff --git a/internal/pkg/bifs/regex.go b/internal/pkg/bifs/regex.go index 20ff71920..c5f30da9e 100644 --- a/internal/pkg/bifs/regex.go +++ b/internal/pkg/bifs/regex.go @@ -4,11 +4,12 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // BIF_ssub implements the ssub function -- no-frills string-replace, no // regexes, no escape sequences. -func BIF_ssub(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_ssub(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsErrorOrAbsent() { return input1 } @@ -19,16 +20,16 @@ func BIF_ssub(input1, input2, input3 *Mlrval) *Mlrval { return input3 } if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input3.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromString( - strings.Replace(input1.printrep, input2.printrep, input3.printrep, 1), + return mlrval.FromString( + strings.Replace(input1.AcquireStringValue(), input2.AcquireStringValue(), input3.AcquireStringValue(), 1), ) } @@ -39,7 +40,7 @@ func BIF_ssub(input1, input2, input3 *Mlrval) *Mlrval { // on each record. Likewise for other regex-using functions in this file. But // first, do a profiling run to see how much time would be saved, and if this // precomputing+caching would be worthwhile. -func BIF_sub(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_sub(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsErrorOrAbsent() { return input1 } @@ -50,26 +51,26 @@ func BIF_sub(input1, input2, input3 *Mlrval) *Mlrval { return input3 } if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input3.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - input := input1.printrep - sregex := input2.printrep - replacement := input3.printrep + input := input1.AcquireStringValue() + sregex := input2.AcquireStringValue() + replacement := input3.AcquireStringValue() stringOutput := lib.RegexSub(input, sregex, replacement) - return MlrvalFromString(stringOutput) + return mlrval.FromString(stringOutput) } // BIF_gsub implements the gsub function, with support for regexes and regex captures // of the form "\1" .. "\9". -func BIF_gsub(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_gsub(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsErrorOrAbsent() { return input1 } @@ -80,26 +81,26 @@ func BIF_gsub(input1, input2, input3 *Mlrval) *Mlrval { return input3 } if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input3.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - input := input1.printrep - sregex := input2.printrep - replacement := input3.printrep + input := input1.AcquireStringValue() + sregex := input2.AcquireStringValue() + replacement := input3.AcquireStringValue() stringOutput := lib.RegexGsub(input, sregex, replacement) - return MlrvalFromString(stringOutput) + return mlrval.FromString(stringOutput) } // BIF_string_matches_regexp implements the =~ operator, with support for // setting regex-captures for later expressions to access using "\1" .. "\9". -func BIF_string_matches_regexp(input1, input2 *Mlrval) (retval *Mlrval, captures []string) { +func BIF_string_matches_regexp(input1, input2 *mlrval.Mlrval) (retval *mlrval.Mlrval, captures []string) { if !input1.IsLegit() { return input1, nil } @@ -108,51 +109,51 @@ func BIF_string_matches_regexp(input1, input2 *Mlrval) (retval *Mlrval, captures } input1string := input1.String() if !input2.IsStringOrVoid() { - return MLRVAL_ERROR, nil + return mlrval.ERROR, nil } - boolOutput, captures := lib.RegexMatches(input1string, input2.printrep) - return MlrvalFromBool(boolOutput), captures + boolOutput, captures := lib.RegexMatches(input1string, input2.AcquireStringValue()) + return mlrval.FromBool(boolOutput), captures } // BIF_string_matches_regexp implements the !=~ operator. -func BIF_string_does_not_match_regexp(input1, input2 *Mlrval) (retval *Mlrval, captures []string) { +func BIF_string_does_not_match_regexp(input1, input2 *mlrval.Mlrval) (retval *mlrval.Mlrval, captures []string) { output, captures := BIF_string_matches_regexp(input1, input2) if output.IsBool() { - return MlrvalFromBool(!output.boolval), captures + return mlrval.FromBool(!output.AcquireBoolValue()), captures } else { // else leave it as error, absent, etc. return output, captures } } -func BIF_regextract(input1, input2 *Mlrval) *Mlrval { +func BIF_regextract(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } - regex := lib.CompileMillerRegexOrDie(input2.printrep) - match := regex.FindStringIndex(input1.printrep) + regex := lib.CompileMillerRegexOrDie(input2.AcquireStringValue()) + match := regex.FindStringIndex(input1.AcquireStringValue()) if match != nil { - return MlrvalFromString(input1.printrep[match[0]:match[1]]) + return mlrval.FromString(input1.AcquireStringValue()[match[0]:match[1]]) } else { - return MLRVAL_ABSENT + return mlrval.ABSENT } } -func BIF_regextract_or_else(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_regextract_or_else(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } - regex := lib.CompileMillerRegexOrDie(input2.printrep) - match := regex.FindStringIndex(input1.printrep) + regex := lib.CompileMillerRegexOrDie(input2.AcquireStringValue()) + match := regex.FindStringIndex(input1.AcquireStringValue()) if match != nil { - return MlrvalFromString(input1.printrep[match[0]:match[1]]) + return mlrval.FromString(input1.AcquireStringValue()[match[0]:match[1]]) } else { return input3 } diff --git a/internal/pkg/bifs/relative_time.go b/internal/pkg/bifs/relative_time.go index fdf014940..9779fb305 100644 --- a/internal/pkg/bifs/relative_time.go +++ b/internal/pkg/bifs/relative_time.go @@ -4,187 +4,189 @@ import ( "fmt" "math" "strings" + + "github.com/johnkerl/miller/internal/pkg/mlrval" ) -func BIF_dhms2sec(input1 *Mlrval) *Mlrval { - if input1.mvtype != MT_STRING { - return MLRVAL_ERROR +func BIF_dhms2sec(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.Type() != MT_STRING { + return mlrval.ERROR } var d, h, m, s int - if strings.HasPrefix(input1.printrep, "-") { + if strings.HasPrefix(input1.AcquireStringValue(), "-") { - n, err := fmt.Sscanf(input1.printrep, "-%dd%dh%dm%ds", &d, &h, &m, &s) + n, err := fmt.Sscanf(input1.AcquireStringValue(), "-%dd%dh%dm%ds", &d, &h, &m, &s) if n == 4 && err == nil { - return MlrvalFromInt(-(s + m*60 + h*60*60 + d*60*60*24)) + return mlrval.FromInt(-(s + m*60 + h*60*60 + d*60*60*24)) } - n, err = fmt.Sscanf(input1.printrep, "-%dh%dm%ds", &h, &m, &s) + n, err = fmt.Sscanf(input1.AcquireStringValue(), "-%dh%dm%ds", &h, &m, &s) if n == 3 && err == nil { - return MlrvalFromInt(-(s + m*60 + h*60*60)) + return mlrval.FromInt(-(s + m*60 + h*60*60)) } - n, err = fmt.Sscanf(input1.printrep, "-%dm%ds", &m, &s) + n, err = fmt.Sscanf(input1.AcquireStringValue(), "-%dm%ds", &m, &s) if n == 2 && err == nil { - return MlrvalFromInt(-(s + m*60)) + return mlrval.FromInt(-(s + m*60)) } - n, err = fmt.Sscanf(input1.printrep, "-%ds", &s) + n, err = fmt.Sscanf(input1.AcquireStringValue(), "-%ds", &s) if n == 1 && err == nil { - return MlrvalFromInt(-(s)) + return mlrval.FromInt(-(s)) } } else { - n, err := fmt.Sscanf(input1.printrep, "%dd%dh%dm%ds", &d, &h, &m, &s) + n, err := fmt.Sscanf(input1.AcquireStringValue(), "%dd%dh%dm%ds", &d, &h, &m, &s) if n == 4 && err == nil { - return MlrvalFromInt(s + m*60 + h*60*60 + d*60*60*24) + return mlrval.FromInt(s + m*60 + h*60*60 + d*60*60*24) } - n, err = fmt.Sscanf(input1.printrep, "%dh%dm%ds", &h, &m, &s) + n, err = fmt.Sscanf(input1.AcquireStringValue(), "%dh%dm%ds", &h, &m, &s) if n == 3 && err == nil { - return MlrvalFromInt(s + m*60 + h*60*60) + return mlrval.FromInt(s + m*60 + h*60*60) } - n, err = fmt.Sscanf(input1.printrep, "%dm%ds", &m, &s) + n, err = fmt.Sscanf(input1.AcquireStringValue(), "%dm%ds", &m, &s) if n == 2 && err == nil { - return MlrvalFromInt(s + m*60) + return mlrval.FromInt(s + m*60) } - n, err = fmt.Sscanf(input1.printrep, "%ds", &s) + n, err = fmt.Sscanf(input1.AcquireStringValue(), "%ds", &s) if n == 1 && err == nil { - return MlrvalFromInt(s) + return mlrval.FromInt(s) } } - return MLRVAL_ERROR + return mlrval.ERROR } -func BIF_dhms2fsec(input1 *Mlrval) *Mlrval { - if input1.mvtype != MT_STRING { - return MLRVAL_ERROR +func BIF_dhms2fsec(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.Type() != MT_STRING { + return mlrval.ERROR } var d, h, m int var s float64 - if strings.HasPrefix(input1.printrep, "-") { + if strings.HasPrefix(input1.AcquireStringValue(), "-") { - n, err := fmt.Sscanf(input1.printrep, "-%dd%dh%dm%fs", &d, &h, &m, &s) + n, err := fmt.Sscanf(input1.AcquireStringValue(), "-%dd%dh%dm%fs", &d, &h, &m, &s) if n == 4 && err == nil { - return MlrvalFromFloat64(-(s + float64(m*60+h*60*60+d*60*60*24))) + return mlrval.FromFloat(-(s + float64(m*60+h*60*60+d*60*60*24))) } - n, err = fmt.Sscanf(input1.printrep, "-%dh%dm%fs", &h, &m, &s) + n, err = fmt.Sscanf(input1.AcquireStringValue(), "-%dh%dm%fs", &h, &m, &s) if n == 3 && err == nil { - return MlrvalFromFloat64(-(s + float64(m*60+h*60*60))) + return mlrval.FromFloat(-(s + float64(m*60+h*60*60))) } - n, err = fmt.Sscanf(input1.printrep, "-%dm%fs", &m, &s) + n, err = fmt.Sscanf(input1.AcquireStringValue(), "-%dm%fs", &m, &s) if n == 2 && err == nil { - return MlrvalFromFloat64(-(s + float64(m*60))) + return mlrval.FromFloat(-(s + float64(m*60))) } - n, err = fmt.Sscanf(input1.printrep, "-%fs", &s) + n, err = fmt.Sscanf(input1.AcquireStringValue(), "-%fs", &s) if n == 1 && err == nil { - return MlrvalFromFloat64(-(s)) + return mlrval.FromFloat(-(s)) } } else { - n, err := fmt.Sscanf(input1.printrep, "%dd%dh%dm%fs", &d, &h, &m, &s) + n, err := fmt.Sscanf(input1.AcquireStringValue(), "%dd%dh%dm%fs", &d, &h, &m, &s) if n == 4 && err == nil { - return MlrvalFromFloat64(s + float64(m*60+h*60*60+d*60*60*24)) + return mlrval.FromFloat(s + float64(m*60+h*60*60+d*60*60*24)) } - n, err = fmt.Sscanf(input1.printrep, "%dh%dm%fs", &h, &m, &s) + n, err = fmt.Sscanf(input1.AcquireStringValue(), "%dh%dm%fs", &h, &m, &s) if n == 3 && err == nil { - return MlrvalFromFloat64(s + float64(m*60+h*60*60)) + return mlrval.FromFloat(s + float64(m*60+h*60*60)) } - n, err = fmt.Sscanf(input1.printrep, "%dm%fs", &m, &s) + n, err = fmt.Sscanf(input1.AcquireStringValue(), "%dm%fs", &m, &s) if n == 2 && err == nil { - return MlrvalFromFloat64(s + float64(m*60)) + return mlrval.FromFloat(s + float64(m*60)) } - n, err = fmt.Sscanf(input1.printrep, "%fs", &s) + n, err = fmt.Sscanf(input1.AcquireStringValue(), "%fs", &s) if n == 1 && err == nil { - return MlrvalFromFloat64(s) + return mlrval.FromFloat(s) } } - return MLRVAL_ERROR + return mlrval.ERROR } -func BIF_hms2sec(input1 *Mlrval) *Mlrval { - if input1.mvtype != MT_STRING { - return MLRVAL_ERROR +func BIF_hms2sec(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.Type() != MT_STRING { + return mlrval.ERROR } - if input1.printrep == "" { - return MLRVAL_ERROR + if input1.AcquireStringValue() == "" { + return mlrval.ERROR } var h, m, s int - if strings.HasPrefix(input1.printrep, "-") { - n, err := fmt.Sscanf(input1.printrep, "-%d:%d:%d", &h, &m, &s) + if strings.HasPrefix(input1.AcquireStringValue(), "-") { + n, err := fmt.Sscanf(input1.AcquireStringValue(), "-%d:%d:%d", &h, &m, &s) if n == 3 && err == nil { - return MlrvalFromInt(-(s + m*60 + h*60*60)) + return mlrval.FromInt(-(s + m*60 + h*60*60)) } } else { - n, err := fmt.Sscanf(input1.printrep, "%d:%d:%d", &h, &m, &s) + n, err := fmt.Sscanf(input1.AcquireStringValue(), "%d:%d:%d", &h, &m, &s) if n == 3 && err == nil { - return MlrvalFromInt(s + m*60 + h*60*60) + return mlrval.FromInt(s + m*60 + h*60*60) } } - return MLRVAL_ERROR + return mlrval.ERROR } -func BIF_hms2fsec(input1 *Mlrval) *Mlrval { - if input1.mvtype != MT_STRING { - return MLRVAL_ERROR +func BIF_hms2fsec(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.Type() != MT_STRING { + return mlrval.ERROR } var h, m int var s float64 - if strings.HasPrefix(input1.printrep, "-") { - n, err := fmt.Sscanf(input1.printrep, "-%d:%d:%f", &h, &m, &s) + if strings.HasPrefix(input1.AcquireStringValue(), "-") { + n, err := fmt.Sscanf(input1.AcquireStringValue(), "-%d:%d:%f", &h, &m, &s) if n == 3 && err == nil { - return MlrvalFromFloat64(-(s + float64(m*60+h*60*60))) + return mlrval.FromFloat(-(s + float64(m*60+h*60*60))) } } else { - n, err := fmt.Sscanf(input1.printrep, "%d:%d:%f", &h, &m, &s) + n, err := fmt.Sscanf(input1.AcquireStringValue(), "%d:%d:%f", &h, &m, &s) if n == 3 && err == nil { - return MlrvalFromFloat64(s + float64(m*60+h*60*60)) + return mlrval.FromFloat(s + float64(m*60+h*60*60)) } } - return MLRVAL_ERROR + return mlrval.ERROR } -func BIF_sec2dhms(input1 *Mlrval) *Mlrval { +func BIF_sec2dhms(input1 *mlrval.Mlrval) *mlrval.Mlrval { isec, ok := input1.GetIntValue() if !ok { - return MLRVAL_ERROR + return mlrval.ERROR } var d, h, m, s int splitIntToDHMS(isec, &d, &h, &m, &s) if d != 0 { - return MlrvalFromString( + return mlrval.FromString( fmt.Sprintf("%dd%02dh%02dm%02ds", d, h, m, s), ) } else if h != 0 { - return MlrvalFromString( + return mlrval.FromString( fmt.Sprintf("%dh%02dm%02ds", h, m, s), ) } else if m != 0 { - return MlrvalFromString( + return mlrval.FromString( fmt.Sprintf("%dm%02ds", m, s), ) } else { - return MlrvalFromString( + return mlrval.FromString( fmt.Sprintf("%ds", s), ) } - return MLRVAL_ERROR + return mlrval.ERROR } -func BIF_sec2hms(input1 *Mlrval) *Mlrval { +func BIF_sec2hms(input1 *mlrval.Mlrval) *mlrval.Mlrval { isec, ok := input1.GetIntValue() if !ok { - return MLRVAL_ERROR + return mlrval.ERROR } sign := "" if isec < 0 { @@ -197,17 +199,17 @@ func BIF_sec2hms(input1 *Mlrval) *Mlrval { splitIntToDHMS(isec, &d, &h, &m, &s) h += d * 24 - return MlrvalFromString( + return mlrval.FromString( fmt.Sprintf("%s%02d:%02d:%02d", sign, h, m, s), ) - return MLRVAL_ERROR + return mlrval.ERROR } -func BIF_fsec2dhms(input1 *Mlrval) *Mlrval { +func BIF_fsec2dhms(input1 *mlrval.Mlrval) *mlrval.Mlrval { fsec, ok := input1.GetNumericToFloatValue() if !ok { - return MLRVAL_ERROR + return mlrval.ERROR } sign := 1 @@ -224,21 +226,21 @@ func BIF_fsec2dhms(input1 *Mlrval) *Mlrval { if d != 0 { d = sign * d - return MlrvalFromString( + return mlrval.FromString( fmt.Sprintf( "%dd%02dh%02dm%09.6fs", d, h, m, float64(s)+fractional), ) } else if h != 0 { h = sign * h - return MlrvalFromString( + return mlrval.FromString( fmt.Sprintf( "%dh%02dm%09.6fs", h, m, float64(s)+fractional), ) } else if m != 0 { m = sign * m - return MlrvalFromString( + return mlrval.FromString( fmt.Sprintf( "%dm%09.6fs", m, float64(s)+fractional), @@ -246,7 +248,7 @@ func BIF_fsec2dhms(input1 *Mlrval) *Mlrval { } else { s = sign * s fractional = float64(sign) * fractional - return MlrvalFromString( + return mlrval.FromString( fmt.Sprintf( "%.6fs", float64(s)+fractional), @@ -254,10 +256,10 @@ func BIF_fsec2dhms(input1 *Mlrval) *Mlrval { } } -func BIF_fsec2hms(input1 *Mlrval) *Mlrval { +func BIF_fsec2hms(input1 *mlrval.Mlrval) *mlrval.Mlrval { fsec, ok := input1.GetNumericToFloatValue() if !ok { - return MLRVAL_ERROR + return mlrval.ERROR } sign := "" @@ -275,16 +277,16 @@ func BIF_fsec2hms(input1 *Mlrval) *Mlrval { // "%02.6f" does not exist so we have to do our own zero-pad if s < 10 { - return MlrvalFromString( + return mlrval.FromString( fmt.Sprintf("%s%02d:%02d:0%.6f", sign, h, m, float64(s)+fractional), ) } else { - return MlrvalFromString( + return mlrval.FromString( fmt.Sprintf("%s%02d:%02d:%.6f", sign, h, m, float64(s)+fractional), ) } - return MLRVAL_ERROR + return mlrval.ERROR } // Helper function diff --git a/internal/pkg/bifs/sort.go b/internal/pkg/bifs/sort.go index def08da45..4fe937030 100644 --- a/internal/pkg/bifs/sort.go +++ b/internal/pkg/bifs/sort.go @@ -8,11 +8,13 @@ import ( "fmt" "os" "strings" + + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // LexicalAscendingComparator is for lexical sort: it stringifies // everything. -func LexicalAscendingComparator(input1 *Mlrval, input2 *Mlrval) int { +func LexicalAscendingComparator(input1 *mlrval.Mlrval, input2 *mlrval.Mlrval) int { sa := input1.String() sb := input2.String() if sa < sb { @@ -26,13 +28,13 @@ func LexicalAscendingComparator(input1 *Mlrval, input2 *Mlrval) int { // LexicalDescendingComparator is for reverse-lexical sort: it stringifies // everything. -func LexicalDescendingComparator(input1 *Mlrval, input2 *Mlrval) int { +func LexicalDescendingComparator(input1 *mlrval.Mlrval, input2 *mlrval.Mlrval) int { return LexicalAscendingComparator(input2, input1) } // CaseFoldAscendingComparator is for case-folded lexical sort: it stringifies // everything. -func CaseFoldAscendingComparator(input1 *Mlrval, input2 *Mlrval) int { +func CaseFoldAscendingComparator(input1 *mlrval.Mlrval, input2 *mlrval.Mlrval) int { sa := input1.String() sb := input2.String() if input1.IsString() { @@ -52,7 +54,7 @@ func CaseFoldAscendingComparator(input1 *Mlrval, input2 *Mlrval) int { // CaseFoldDescendingComparator is for case-folded lexical sort: it stringifies // everything. -func CaseFoldDescendingComparator(input1 *Mlrval, input2 *Mlrval) int { +func CaseFoldDescendingComparator(input1 *mlrval.Mlrval, input2 *mlrval.Mlrval) int { return CaseFoldAscendingComparator(input2, input1) } @@ -65,29 +67,29 @@ func CaseFoldDescendingComparator(input1 *Mlrval, input2 *Mlrval) int { // * numeric compares on numbers // * false < true -func _neg1(input1, input2 *Mlrval) int { +func _neg1(input1, input2 *mlrval.Mlrval) int { return -1 } -func _zero(input1, input2 *Mlrval) int { +func _zero(input1, input2 *mlrval.Mlrval) int { return 0 } -func _pos1(input1, input2 *Mlrval) int { +func _pos1(input1, input2 *mlrval.Mlrval) int { return 1 } -func _scmp(input1, input2 *Mlrval) int { - if input1.printrep < input2.printrep { +func _scmp(input1, input2 *mlrval.Mlrval) int { + if input1.AcquireStringValue() < input2.AcquireStringValue() { return -1 - } else if input1.printrep > input2.printrep { + } else if input1.AcquireStringValue() > input2.AcquireStringValue() { return 1 } else { return 0 } } -func iicmp(input1, input2 *Mlrval) int { - ca := input1.intval - cb := input2.intval +func iicmp(input1, input2 *mlrval.Mlrval) int { + ca := input1.AcquireIntValue() + cb := input2.AcquireIntValue() if ca < cb { return -1 } else if ca > cb { @@ -96,9 +98,9 @@ func iicmp(input1, input2 *Mlrval) int { return 0 } } -func ifcmp(input1, input2 *Mlrval) int { - ca := float64(input1.intval) - cb := input2.floatval +func ifcmp(input1, input2 *mlrval.Mlrval) int { + ca := float64(input1.AcquireIntValue()) + cb := input2.AcquireFloatValue() if ca < cb { return -1 } else if ca > cb { @@ -107,9 +109,9 @@ func ifcmp(input1, input2 *Mlrval) int { return 0 } } -func ficmp(input1, input2 *Mlrval) int { - ca := input1.floatval - cb := float64(input2.intval) +func ficmp(input1, input2 *mlrval.Mlrval) int { + ca := input1.AcquireFloatValue() + cb := float64(input2.AcquireIntValue()) if ca < cb { return -1 } else if ca > cb { @@ -118,9 +120,9 @@ func ficmp(input1, input2 *Mlrval) int { return 0 } } -func ffcmp(input1, input2 *Mlrval) int { - ca := input1.floatval - cb := input2.floatval +func ffcmp(input1, input2 *mlrval.Mlrval) int { + ca := input1.AcquireFloatValue() + cb := input2.AcquireFloatValue() if ca < cb { return -1 } else if ca > cb { @@ -130,9 +132,9 @@ func ffcmp(input1, input2 *Mlrval) int { } } -func bbcmp(input1, input2 *Mlrval) int { - a := input1.boolval - b := input2.boolval +func bbcmp(input1, input2 *mlrval.Mlrval) int { + a := input1.AcquireBoolValue() + b := input2.AcquireBoolValue() if a == false { if b == false { return 0 @@ -148,7 +150,7 @@ func bbcmp(input1, input2 *Mlrval) int { } } -func _xcmp(input1, input2 *Mlrval) int { +func _xcmp(input1, input2 *mlrval.Mlrval) int { fmt.Fprintf(os.Stderr, "mlr: functions cannot be sorted.\n") os.Exit(1) return 0 @@ -165,7 +167,7 @@ func _xcmp(input1, input2 *Mlrval) int { // * false < true // typed_cmp_dispositions is the disposition matrix for numerical sorting of Mlrvals. -var typed_cmp_typedpositions = [MT_DIM][MT_DIM]ComparatorFunc{ +var typed_cmp_typedpositions = [mlrval.MT_DIM][mlrval.MT_DIM]ComparatorFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_zero, _neg1, _neg1, _pos1, _pos1, _pos1, _pos1, _pos1, _zero, _zero, _xcmp}, /*ABSENT */ {_pos1, _zero, _pos1, _pos1, _pos1, _pos1, _pos1, _pos1, _zero, _zero, _xcmp}, @@ -182,12 +184,12 @@ var typed_cmp_typedpositions = [MT_DIM][MT_DIM]ComparatorFunc{ // NumericAscendingComparator is for "numerical" sort: it uses Mlrval sorting // rules by type, including numeric sort for numeric types. -func NumericAscendingComparator(input1 *Mlrval, input2 *Mlrval) int { - return typed_cmp_typedpositions[input1.mvtype][input2.mvtype](input1, input2) +func NumericAscendingComparator(input1 *mlrval.Mlrval, input2 *mlrval.Mlrval) int { + return typed_cmp_typedpositions[input1.Type()][input2.Type()](input1, input2) } // NumericDescendingComparator is for "numerical" sort: it uses Mlrval sorting // rules by type, including numeric sort for numeric types. -func NumericDescendingComparator(input1 *Mlrval, input2 *Mlrval) int { +func NumericDescendingComparator(input1 *mlrval.Mlrval, input2 *mlrval.Mlrval) int { return NumericAscendingComparator(input2, input1) } diff --git a/internal/pkg/bifs/stats.go b/internal/pkg/bifs/stats.go index 3ef436c53..c64b408c3 100644 --- a/internal/pkg/bifs/stats.go +++ b/internal/pkg/bifs/stats.go @@ -4,6 +4,7 @@ import ( "math" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ---------------------------------------------------------------- @@ -23,7 +24,7 @@ import ( // output = [m, b, math.sqrt(var_m), math.sqrt(var_b)] // ---------------------------------------------------------------- -func MlrvalGetVar(mn, msum, msum2 *Mlrval) *Mlrval { +func MlrvalGetVar(mn, msum, msum2 *mlrval.Mlrval) *mlrval.Mlrval { n, isInt := mn.GetIntValue() lib.InternalCodingErrorIf(!isInt) sum, isNumber := msum.GetNumericToFloatValue() @@ -32,7 +33,7 @@ func MlrvalGetVar(mn, msum, msum2 *Mlrval) *Mlrval { lib.InternalCodingErrorIf(!isNumber) if n < 2 { - return MLRVAL_VOID + return mlrval.VOID } mean := float64(sum) / float64(n) @@ -41,11 +42,11 @@ func MlrvalGetVar(mn, msum, msum2 *Mlrval) *Mlrval { numerator = 0.0 } denominator := float64(n - 1) - return MlrvalFromFloat64(numerator / denominator) + return mlrval.FromFloat(numerator / denominator) } // ---------------------------------------------------------------- -func MlrvalGetStddev(mn, msum, msum2 *Mlrval) *Mlrval { +func MlrvalGetStddev(mn, msum, msum2 *mlrval.Mlrval) *mlrval.Mlrval { mvar := MlrvalGetVar(mn, msum, msum2) if mvar.IsVoid() { return mvar @@ -54,7 +55,7 @@ func MlrvalGetStddev(mn, msum, msum2 *Mlrval) *Mlrval { } // ---------------------------------------------------------------- -func MlrvalGetMeanEB(mn, msum, msum2 *Mlrval) *Mlrval { +func MlrvalGetMeanEB(mn, msum, msum2 *mlrval.Mlrval) *mlrval.Mlrval { mvar := MlrvalGetVar(mn, msum, msum2) if mvar.IsVoid() { return mvar @@ -86,11 +87,11 @@ func MlrvalGetMeanEB(mn, msum, msum2 *Mlrval) *Mlrval { // = sumx2 - n mean^2 // ---------------------------------------------------------------- -func MlrvalGetSkewness(mn, msum, msum2, msum3 *Mlrval) *Mlrval { +func MlrvalGetSkewness(mn, msum, msum2, msum3 *mlrval.Mlrval) *mlrval.Mlrval { n, isInt := mn.GetIntValue() lib.InternalCodingErrorIf(!isInt) if n < 2 { - return MLRVAL_VOID + return mlrval.VOID } fn := float64(n) sum, isNumber := msum.GetNumericToFloatValue() @@ -105,7 +106,7 @@ func MlrvalGetSkewness(mn, msum, msum2, msum3 *Mlrval) *Mlrval { numerator = numerator / fn denominator := (sum2 - fn*mean*mean) / (fn - 1.0) denominator = math.Pow(denominator, 1.5) - return MlrvalFromFloat64(numerator / denominator) + return mlrval.FromFloat(numerator / denominator) } // Unbiased: @@ -123,11 +124,11 @@ func MlrvalGetSkewness(mn, msum, msum2, msum3 *Mlrval) *Mlrval { // = sumx4 - mean*(4 sumx3 - mean*(6 sumx2 - 3 n mean^2)) // ---------------------------------------------------------------- -func MlrvalGetKurtosis(mn, msum, msum2, msum3, msum4 *Mlrval) *Mlrval { +func MlrvalGetKurtosis(mn, msum, msum2, msum3, msum4 *mlrval.Mlrval) *mlrval.Mlrval { n, isInt := mn.GetIntValue() lib.InternalCodingErrorIf(!isInt) if n < 2 { - return MLRVAL_VOID + return mlrval.VOID } fn := float64(n) sum, isNumber := msum.GetNumericToFloatValue() @@ -145,6 +146,6 @@ func MlrvalGetKurtosis(mn, msum, msum2, msum3, msum4 *Mlrval) *Mlrval { numerator = numerator / fn denominator := (sum2 - fn*mean*mean) / fn denominator = denominator * denominator - return MlrvalFromFloat64(numerator/denominator - 3.0) + return mlrval.FromFloat(numerator/denominator - 3.0) } diff --git a/internal/pkg/bifs/strings.go b/internal/pkg/bifs/strings.go index d57d04c00..48b13e099 100644 --- a/internal/pkg/bifs/strings.go +++ b/internal/pkg/bifs/strings.go @@ -7,20 +7,21 @@ import ( "unicode/utf8" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ================================================================ -func BIF_strlen(input1 *Mlrval) *Mlrval { +func BIF_strlen(input1 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } else { - return MlrvalFromInt(int(utf8.RuneCountInString(input1.printrep))) + return mlrval.FromInt(int(utf8.RuneCountInString(input1.AcquireStringValue()))) } } // ================================================================ -func BIF_string(input1 *Mlrval) *Mlrval { - return MlrvalFromString(input1.String()) +func BIF_string(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromString(input1.String()) } // ================================================================ @@ -37,11 +38,11 @@ func BIF_string(input1 *Mlrval) *Mlrval { // should be: always the string concatenation of the string representations of // the two arguments. So, we do the string-cast for the user. -func dot_s_xx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromString(input1.String() + input2.String()) +func dot_s_xx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromString(input1.String() + input2.String()) } -var dot_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var dot_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _null, _void, _2___, _s2__, _s2__, _s2__, _absn, _absn, _erro}, @@ -56,38 +57,38 @@ var dot_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_dot(input1, input2 *Mlrval) *Mlrval { - return dot_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_dot(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return dot_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ // substr1(s,m,n) gives substring of s from 1-up position m to n inclusive. // Negative indices -len .. -1 alias to 0 .. len-1. -func BIF_substr_1_up(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_substr_1_up(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - // Handle UTF-8 correctly: len(input1.printrep) will count bytes, not runes. - runes := []rune(input1.printrep) + // Handle UTF-8 correctly: len(input1.AcquireStringValue()) will count bytes, not runes. + runes := []rune(input1.AcquireStringValue()) strlen := int(len(runes)) // For array slices like s[1:2], s[:2], s[1:], when the lower index is // empty in the DSL expression it comes in here as a 1. But when the upper // index is empty in the DSL expression it comes in here as "". if !input2.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } - lowerMindex := input2.intval + lowerMindex := input2.AcquireIntValue() upperMindex := strlen if input3.IsEmpty() { // Keep strlen } else if !input3.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } else { - upperMindex = input3.intval + upperMindex = input3.AcquireIntValue() } // Convert from negative-aliased 1-up to positive-only 0-up @@ -95,14 +96,14 @@ func BIF_substr_1_up(input1, input2, input3 *Mlrval) *Mlrval { n, nok := UnaliasArrayLengthIndex(strlen, upperMindex) if !mok || !nok { - return MLRVAL_VOID + return mlrval.VOID } else if m > n { - return MLRVAL_VOID + return mlrval.VOID } else { // Note Golang slice indices are 0-up, and the 1st index is inclusive // while the 2nd is exclusive. For Miller, indices are 1-up and both // are inclusive. - return MlrvalFromString(string(runes[m : n+1])) + return mlrval.FromString(string(runes[m : n+1])) } } @@ -110,22 +111,22 @@ func BIF_substr_1_up(input1, input2, input3 *Mlrval) *Mlrval { // substr0(s,m,n) gives substring of s from 0-up position m to n inclusive. // Negative indices -len .. -1 alias to 0 .. len-1. -func BIF_substr_0_up(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_substr_0_up(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - // Handle UTF-8 correctly: len(input1.printrep) will count bytes, not runes. - runes := []rune(input1.printrep) + // Handle UTF-8 correctly: len(input1.AcquireStringValue()) will count bytes, not runes. + runes := []rune(input1.AcquireStringValue()) strlen := int(len(runes)) // For array slices like s[1:2], s[:2], s[1:], when the lower index is // empty in the DSL expression it comes in here as a 1. But when the upper // index is empty in the DSL expression it comes in here as "". if !input2.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } - lowerMindex := input2.intval + lowerMindex := input2.AcquireIntValue() if lowerMindex >= 0 { // Make 1-up lowerMindex += 1 @@ -135,9 +136,9 @@ func BIF_substr_0_up(input1, input2, input3 *Mlrval) *Mlrval { if input3.IsEmpty() { // Keep strlen } else if !input3.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } else { - upperMindex = input3.intval + upperMindex = input3.AcquireIntValue() if upperMindex >= 0 { // Make 1-up upperMindex += 1 @@ -149,19 +150,19 @@ func BIF_substr_0_up(input1, input2, input3 *Mlrval) *Mlrval { n, nok := UnaliasArrayLengthIndex(strlen, upperMindex) if !mok || !nok { - return MLRVAL_VOID + return mlrval.VOID } else if m > n { - return MLRVAL_VOID + return mlrval.VOID } else { // Note Golang slice indices are 0-up, and the 1st index is inclusive // while the 2nd is exclusive. For Miller, indices are 1-up and both // are inclusive. - return MlrvalFromString(string(runes[m : n+1])) + return mlrval.FromString(string(runes[m : n+1])) } } // ================================================================ -func BIF_truncate(input1, input2 *Mlrval) *Mlrval { +func BIF_truncate(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsErrorOrAbsent() { return input1 } @@ -169,59 +170,59 @@ func BIF_truncate(input1, input2 *Mlrval) *Mlrval { return input2 } if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } - if input2.intval < 0 { - return MLRVAL_ERROR + if input2.AcquireIntValue() < 0 { + return mlrval.ERROR } - // Handle UTF-8 correctly: len(input1.printrep) will count bytes, not runes. - runes := []rune(input1.printrep) + // Handle UTF-8 correctly: len(input1.AcquireStringValue()) will count bytes, not runes. + runes := []rune(input1.AcquireStringValue()) oldLength := int(len(runes)) - maxLength := input2.intval + maxLength := input2.AcquireIntValue() if oldLength <= maxLength { return input1 } else { - return MlrvalFromString(string(runes[0:maxLength])) + return mlrval.FromString(string(runes[0:maxLength])) } } // ================================================================ -func BIF_lstrip(input1 *Mlrval) *Mlrval { +func BIF_lstrip(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsString() { - return MlrvalFromString(strings.TrimLeft(input1.printrep, " \t")) + return mlrval.FromString(strings.TrimLeft(input1.AcquireStringValue(), " \t")) } else { return input1 } } -func BIF_rstrip(input1 *Mlrval) *Mlrval { +func BIF_rstrip(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsString() { - return MlrvalFromString(strings.TrimRight(input1.printrep, " \t")) + return mlrval.FromString(strings.TrimRight(input1.AcquireStringValue(), " \t")) } else { return input1 } } -func BIF_strip(input1 *Mlrval) *Mlrval { +func BIF_strip(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsString() { - return MlrvalFromString(strings.Trim(input1.printrep, " \t")) + return mlrval.FromString(strings.Trim(input1.AcquireStringValue(), " \t")) } else { return input1 } } // ---------------------------------------------------------------- -func BIF_collapse_whitespace(input1 *Mlrval) *Mlrval { +func BIF_collapse_whitespace(input1 *mlrval.Mlrval) *mlrval.Mlrval { return MlrvalCollapseWhitespaceRegexp(input1, WhitespaceRegexp()) } -func MlrvalCollapseWhitespaceRegexp(input1 *Mlrval, whitespaceRegexp *regexp.Regexp) *Mlrval { +func MlrvalCollapseWhitespaceRegexp(input1 *mlrval.Mlrval, whitespaceRegexp *regexp.Regexp) *mlrval.Mlrval { if input1.IsString() { - return MlrvalFromString(whitespaceRegexp.ReplaceAllString(input1.printrep, " ")) + return mlrval.FromString(whitespaceRegexp.ReplaceAllString(input1.AcquireStringValue(), " ")) } else { return input1 } @@ -232,9 +233,9 @@ func WhitespaceRegexp() *regexp.Regexp { } // ================================================================ -func BIF_toupper(input1 *Mlrval) *Mlrval { +func BIF_toupper(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsString() { - return MlrvalFromString(strings.ToUpper(input1.printrep)) + return mlrval.FromString(strings.ToUpper(input1.AcquireStringValue())) } else if input1.IsVoid() { return input1 } else { @@ -242,9 +243,9 @@ func BIF_toupper(input1 *Mlrval) *Mlrval { } } -func BIF_tolower(input1 *Mlrval) *Mlrval { +func BIF_tolower(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsString() { - return MlrvalFromString(strings.ToLower(input1.printrep)) + return mlrval.FromString(strings.ToLower(input1.AcquireStringValue())) } else if input1.IsVoid() { return input1 } else { @@ -252,17 +253,17 @@ func BIF_tolower(input1 *Mlrval) *Mlrval { } } -func BIF_capitalize(input1 *Mlrval) *Mlrval { +func BIF_capitalize(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsString() { - if input1.printrep == "" { + if input1.AcquireStringValue() == "" { return input1 } else { - runes := []rune(input1.printrep) + runes := []rune(input1.AcquireStringValue()) rfirst := runes[0] rrest := runes[1:] sfirst := strings.ToUpper(string(rfirst)) srest := string(rrest) - return MlrvalFromString(sfirst + srest) + return mlrval.FromString(sfirst + srest) } } else { return input1 @@ -270,7 +271,7 @@ func BIF_capitalize(input1 *Mlrval) *Mlrval { } // ---------------------------------------------------------------- -func BIF_clean_whitespace(input1 *Mlrval) *Mlrval { +func BIF_clean_whitespace(input1 *mlrval.Mlrval) *mlrval.Mlrval { return BIF_strip( MlrvalCollapseWhitespaceRegexp( input1, WhitespaceRegexp(), @@ -279,57 +280,57 @@ func BIF_clean_whitespace(input1 *Mlrval) *Mlrval { } // ================================================================ -func BIF_hexfmt(input1 *Mlrval) *Mlrval { +func BIF_hexfmt(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsInt() { - return MlrvalFromString("0x" + strconv.FormatUint(uint64(input1.intval), 16)) + return mlrval.FromString("0x" + strconv.FormatUint(uint64(input1.AcquireIntValue()), 16)) } else { return input1 } } // ---------------------------------------------------------------- -func fmtnum_is(input1, input2 *Mlrval) *Mlrval { +func fmtnum_is(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input2.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } - formatString := input2.printrep + formatString := input2.AcquireStringValue() formatter, err := GetMlrvalFormatter(formatString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } return formatter.Format(input1) } -func fmtnum_fs(input1, input2 *Mlrval) *Mlrval { +func fmtnum_fs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input2.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } - formatString := input2.printrep + formatString := input2.AcquireStringValue() formatter, err := GetMlrvalFormatter(formatString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } return formatter.Format(input1) } -func fmtnum_bs(input1, input2 *Mlrval) *Mlrval { +func fmtnum_bs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input2.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } - formatString := input2.printrep + formatString := input2.AcquireStringValue() formatter, err := GetMlrvalFormatter(formatString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } - intMv := MlrvalFromInt(lib.BoolToInt(input1.boolval)) + intMv := mlrval.FromInt(lib.BoolToInt(input1.AcquireBoolValue())) return formatter.Format(intMv) } -var fmtnum_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var fmtnum_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _absn, _absn, _absn, _erro, _erro, _erro, _erro}, @@ -344,6 +345,6 @@ var fmtnum_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_fmtnum(input1, input2 *Mlrval) *Mlrval { - return fmtnum_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_fmtnum(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return fmtnum_dispositions[input1.Type()][input2.Type()](input1, input2) } diff --git a/internal/pkg/bifs/system.go b/internal/pkg/bifs/system.go index 26a857ddb..b35c5737d 100644 --- a/internal/pkg/bifs/system.go +++ b/internal/pkg/bifs/system.go @@ -6,40 +6,41 @@ import ( "runtime" "strings" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/platform" "github.com/johnkerl/miller/internal/pkg/version" ) -func BIF_version() *Mlrval { - return MlrvalFromString(version.STRING) +func BIF_version() *mlrval.Mlrval { + return mlrval.FromString(version.STRING) } -func BIF_os() *Mlrval { - return MlrvalFromString(runtime.GOOS) +func BIF_os() *mlrval.Mlrval { + return mlrval.FromString(runtime.GOOS) } -func BIF_hostname() *Mlrval { +func BIF_hostname() *mlrval.Mlrval { hostname, err := os.Hostname() if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } else { - return MlrvalFromString(hostname) + return mlrval.FromString(hostname) } } -func BIF_system(input1 *Mlrval) *Mlrval { +func BIF_system(input1 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - commandString := input1.printrep + commandString := input1.AcquireStringValue() shellRunArray := platform.GetShellRunArray(commandString) outputBytes, err := exec.Command(shellRunArray[0], shellRunArray[1:]...).Output() if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } outputString := strings.TrimRight(string(outputBytes), "\n") - return MlrvalFromString(outputString) + return mlrval.FromString(outputString) } diff --git a/internal/pkg/bifs/types.go b/internal/pkg/bifs/types.go index 0dbed3d34..912af535a 100644 --- a/internal/pkg/bifs/types.go +++ b/internal/pkg/bifs/types.go @@ -5,36 +5,38 @@ import ( "os" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" + "github.com/johnkerl/miller/internal/pkg/types" ) // ================================================================ -func BIF_typeof(input1 *Mlrval) *Mlrval { - return MlrvalFromString(input1.GetTypeName()) +func BIF_typeof(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromString(input1.GetTypeName()) } // ---------------------------------------------------------------- -func string_to_int(input1 *Mlrval) *Mlrval { - i, ok := lib.TryIntFromString(input1.printrep) +func string_to_int(input1 *mlrval.Mlrval) *mlrval.Mlrval { + i, ok := lib.TryIntFromString(input1.AcquireStringValue()) if ok { - return MlrvalFromInt(i) + return mlrval.FromInt(i) } else { - return MLRVAL_ERROR + return mlrval.ERROR } } -func float_to_int(input1 *Mlrval) *Mlrval { - return MlrvalFromInt(int(input1.floatval)) +func float_to_int(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(int(input1.AcquireFloatValue())) } -func bool_to_int(input1 *Mlrval) *Mlrval { - if input1.boolval == true { - return MlrvalFromInt(1) +func bool_to_int(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.AcquireBoolValue() == true { + return mlrval.FromInt(1) } else { - return MlrvalFromInt(0) + return mlrval.FromInt(0) } } -var to_int_dispositions = [MT_DIM]UnaryFunc{ +var to_int_dispositions = [mlrval.MT_DIM]UnaryFunc{ /*ERROR */ _erro1, /*ABSENT */ _absn1, /*NULL */ _null1, @@ -48,33 +50,33 @@ var to_int_dispositions = [MT_DIM]UnaryFunc{ /*FUNC */ _erro1, } -func BIF_int(input1 *Mlrval) *Mlrval { - return to_int_dispositions[input1.mvtype](input1) +func BIF_int(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return to_int_dispositions[input1.Type()](input1) } // ---------------------------------------------------------------- -func string_to_float(input1 *Mlrval) *Mlrval { - f, ok := lib.TryFloat64FromString(input1.printrep) +func string_to_float(input1 *mlrval.Mlrval) *mlrval.Mlrval { + f, ok := lib.TryFloat64FromString(input1.AcquireStringValue()) if ok { - return MlrvalFromFloat64(f) + return mlrval.FromFloat(f) } else { - return MLRVAL_ERROR + return mlrval.ERROR } } -func int_to_float(input1 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval)) +func int_to_float(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue())) } -func bool_to_float(input1 *Mlrval) *Mlrval { - if input1.boolval == true { - return MlrvalFromFloat64(1.0) +func bool_to_float(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.AcquireBoolValue() == true { + return mlrval.FromFloat(1.0) } else { - return MlrvalFromFloat64(0.0) + return mlrval.FromFloat(0.0) } } -var to_float_dispositions = [MT_DIM]UnaryFunc{ +var to_float_dispositions = [mlrval.MT_DIM]UnaryFunc{ /*ERROR */ _erro1, /*ABSENT */ _absn1, /*NULL */ _null1, @@ -88,29 +90,29 @@ var to_float_dispositions = [MT_DIM]UnaryFunc{ /*FUNC */ _erro1, } -func BIF_float(input1 *Mlrval) *Mlrval { - return to_float_dispositions[input1.mvtype](input1) +func BIF_float(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return to_float_dispositions[input1.Type()](input1) } // ---------------------------------------------------------------- -func string_to_boolean(input1 *Mlrval) *Mlrval { - b, ok := lib.TryBoolFromBoolString(input1.printrep) +func string_to_boolean(input1 *mlrval.Mlrval) *mlrval.Mlrval { + b, ok := lib.TryBoolFromBoolString(input1.AcquireStringValue()) if ok { - return MlrvalFromBool(b) + return mlrval.FromBool(b) } else { - return MLRVAL_ERROR + return mlrval.ERROR } } -func int_to_bool(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.intval != 0) +func int_to_bool(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireIntValue() != 0) } -func float_to_bool(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval != 0.0) +func float_to_bool(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() != 0.0) } -var to_boolean_dispositions = [MT_DIM]UnaryFunc{ +var to_boolean_dispositions = [mlrval.MT_DIM]UnaryFunc{ /*ERROR */ _erro1, /*ABSENT */ _absn1, /*NULL */ _null1, @@ -124,91 +126,91 @@ var to_boolean_dispositions = [MT_DIM]UnaryFunc{ /*FUNC */ _erro1, } -func BIF_boolean(input1 *Mlrval) *Mlrval { - return to_boolean_dispositions[input1.mvtype](input1) +func BIF_boolean(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return to_boolean_dispositions[input1.Type()](input1) } // ---------------------------------------------------------------- -func BIF_is_absent(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.IsAbsent()) +func BIF_is_absent(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsAbsent()) } -func BIF_is_error(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.IsError()) +func BIF_is_error(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsError()) } -func BIF_is_bool(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.IsBool()) +func BIF_is_bool(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsBool()) } -func BIF_is_boolean(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.IsBool()) +func BIF_is_boolean(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsBool()) } -func BIF_is_empty(input1 *Mlrval) *Mlrval { +func BIF_is_empty(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsVoid() { - return MLRVAL_TRUE + return mlrval.TRUE } else if input1.IsString() { - if input1.printrep == "" { - return MLRVAL_TRUE + if input1.AcquireStringValue() == "" { + return mlrval.TRUE } else { - return MLRVAL_FALSE + return mlrval.FALSE } } else { - return MLRVAL_FALSE + return mlrval.FALSE } } -func BIF_is_emptymap(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.IsMap() && input1.mapval.IsEmpty()) +func BIF_is_emptymap(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsMap() && input1.AcquireArrayValue().(types.Mlrmap).IsEmpty()) } -func BIF_is_float(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.IsFloat()) +func BIF_is_float(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsFloat()) } -func BIF_is_int(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.IsInt()) +func BIF_is_int(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsInt()) } -func BIF_is_map(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.IsMap()) +func BIF_is_map(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsMap()) } -func BIF_is_array(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.IsArray()) +func BIF_is_array(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsArray()) } -func BIF_is_nonemptymap(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.IsMap() && input1.mapval.FieldCount != 0) +func BIF_is_nonemptymap(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsMap() && input1.AcquireArrayValue().(types.Mlrmap).FieldCount != 0) } -func BIF_is_notempty(input1 *Mlrval) *Mlrval { +func BIF_is_notempty(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsVoid() { - return MLRVAL_FALSE + return mlrval.FALSE } else if input1.IsString() { - if input1.printrep == "" { - return MLRVAL_FALSE + if input1.AcquireStringValue() == "" { + return mlrval.FALSE } else { - return MLRVAL_TRUE + return mlrval.TRUE } } else { - return MLRVAL_TRUE + return mlrval.TRUE } } -func BIF_is_notmap(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype != MT_MAP) +func BIF_is_notmap(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.Type() != MT_MAP) } -func BIF_is_notarray(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype != MT_ARRAY) +func BIF_is_notarray(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.Type() != MT_ARRAY) } -func BIF_is_notnull(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype != MT_ABSENT && input1.mvtype != MT_VOID) +func BIF_is_notnull(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.Type() != MT_ABSENT && input1.Type() != MT_VOID) } -func BIF_is_null(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.IsAbsent() || input1.IsVoid()) +func BIF_is_null(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsAbsent() || input1.IsVoid()) } -func BIF_is_numeric(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.IsInt() || input1.IsFloat()) +func BIF_is_numeric(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsInt() || input1.IsFloat()) } -func BIF_is_present(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype != MT_ABSENT) +func BIF_is_present(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.Type() != MT_ABSENT) } -func BIF_is_string(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.IsString() || input1.IsVoid()) +func BIF_is_string(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsString() || input1.IsVoid()) } // ---------------------------------------------------------------- -func assertingCommon(input1, check *Mlrval, description string, context *Context) *Mlrval { +func assertingCommon(input1, check *mlrval.Mlrval, description string, context *types.Context) *mlrval.Mlrval { if check.IsFalse() { // TODO: get context as in the C impl //fprintf(stderr, "%s: %s type-assertion failed at NR=%lld FNR=%lld FILENAME=%s\n", @@ -227,60 +229,60 @@ func assertingCommon(input1, check *Mlrval, description string, context *Context return input1 } -func BIF_asserting_absent(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_absent(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_absent(input1), "is_absent", context) } -func BIF_asserting_error(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_error(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_error(input1), "is_error", context) } -func BIF_asserting_bool(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_bool(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_bool(input1), "is_bool", context) } -func BIF_asserting_boolean(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_boolean(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_boolean(input1), "is_boolean", context) } -func BIF_asserting_empty(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_empty(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_empty(input1), "is_empty", context) } -func BIF_asserting_emptyMap(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_emptyMap(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_emptymap(input1), "is_empty_map", context) } -func BIF_asserting_float(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_float(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_float(input1), "is_float", context) } -func BIF_asserting_int(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_int(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_int(input1), "is_int", context) } -func BIF_asserting_map(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_map(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_map(input1), "is_map", context) } -func BIF_asserting_array(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_array(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_array(input1), "is_array", context) } -func BIF_asserting_nonempty_map(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_nonempty_map(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_nonemptymap(input1), "is_non_empty_map", context) } -func BIF_asserting_not_empty(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_not_empty(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_notempty(input1), "is_not_empty", context) } -func BIF_asserting_not_map(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_not_map(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_notmap(input1), "is_not_map", context) } -func BIF_asserting_not_array(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_not_array(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_notarray(input1), "is_not_array", context) } -func BIF_asserting_not_null(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_not_null(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_notnull(input1), "is_not_null", context) } -func BIF_asserting_null(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_null(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_null(input1), "is_null", context) } -func BIF_asserting_numeric(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_numeric(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_numeric(input1), "is_numeric", context) } -func BIF_asserting_present(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_present(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_present(input1), "is_present", context) } -func BIF_asserting_string(input1 *Mlrval, context *Context) *Mlrval { +func BIF_asserting_string(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { return assertingCommon(input1, BIF_is_string(input1), "is_string", context) } diff --git a/internal/pkg/input/pseudo_reader_gen.go b/internal/pkg/input/pseudo_reader_gen.go index cb1a1d3f1..0f20a6270 100644 --- a/internal/pkg/input/pseudo_reader_gen.go +++ b/internal/pkg/input/pseudo_reader_gen.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" @@ -63,9 +64,9 @@ func (reader *PseudoReaderGen) process( return } - var doneComparator types.BinaryFunc = types.BIF_greater_than + var doneComparator types.BinaryFunc = bifs.BIF_greater_than if step.GetNumericNegativeorDie() { - doneComparator = types.BIF_less_than + doneComparator = bifs.BIF_less_than } key := reader.readerOptions.GeneratorOptions.FieldName @@ -110,7 +111,7 @@ func (reader *PseudoReaderGen) process( } - value = types.BIF_plus_binary(value, step) + value = bifs.BIF_plus_binary(value, step) } if recordsAndContexts.Len() > 0 { diff --git a/internal/pkg/input/record_reader_xtab.go b/internal/pkg/input/record_reader_xtab.go index dfd6c5eca..4f6456be2 100644 --- a/internal/pkg/input/record_reader_xtab.go +++ b/internal/pkg/input/record_reader_xtab.go @@ -9,6 +9,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) diff --git a/internal/pkg/mlrval/copy.go b/internal/pkg/mlrval/copy.go new file mode 100644 index 000000000..8d3775c9f --- /dev/null +++ b/internal/pkg/mlrval/copy.go @@ -0,0 +1,13 @@ +package mlrval + +// TODO: comment about mvtype; deferrence; copying of deferrence. +func (mv *Mlrval) Copy() *Mlrval { + if mv.mvtype == MT_MAP { + panic("mlrval map-valued copy unimplemented, pending refactor") + } else if mv.mvtype == MT_ARRAY { + panic("mlrval array-valued copy unimplemented, pending refactor") + } else { + other := *mv + return &other + } +} diff --git a/internal/pkg/mlrval/get.go b/internal/pkg/mlrval/get.go index b76a61e0e..c5597dc5d 100644 --- a/internal/pkg/mlrval/get.go +++ b/internal/pkg/mlrval/get.go @@ -8,7 +8,7 @@ import ( // predicate, or another Get...(), since types are JIT-computed on first access // for most data-file values. See type.go for more information. -func (mv *Mlrval) GetString() (stringValue string, isString bool) { +func (mv *Mlrval) GetStringValue() (stringValue string, isString bool) { if mv.Type() == MT_STRING || mv.Type() == MT_VOID { return mv.printrep, true } else { @@ -83,3 +83,39 @@ func (mv *Mlrval) GetFunction() interface{} { func (mv *Mlrval) GetTypeName() string { return TYPE_NAMES[mv.Type()] } + + +// These are for built-in functions operating within type-keyed +// disposition-vector/disposition-matrix context. They've already computed +// mv.Type() -- it's a fatal error if they haven't -- and they need the typed +// value. + +func (mv *Mlrval) AcquireStringValue() string { + lib.InternalCodingErrorIf(mv.mvtype != MT_STRING && mv.mvtype != MT_VOID) + return mv.printrep +} + +func (mv *Mlrval) AcquireIntValue() int { + lib.InternalCodingErrorIf(mv.mvtype != MT_INT) + return mv.intval +} + +func (mv *Mlrval) AcquireFloatValue() float64 { + lib.InternalCodingErrorIf(mv.mvtype != MT_FLOAT) + return mv.floatval +} + +func (mv *Mlrval) AcquireBoolValue() bool { + lib.InternalCodingErrorIf(mv.mvtype != MT_BOOL) + return mv.boolval +} + +func (mv *Mlrval) AcquireArrayValue() interface{} { + lib.InternalCodingErrorIf(mv.mvtype != MT_ARRAY) + return mv.arrayval +} + +func (mv *Mlrval) AcquireMapValue() interface{} { + lib.InternalCodingErrorIf(mv.mvtype != MT_MAP) + return mv.mapval +} diff --git a/internal/pkg/mlrval/get_test.go b/internal/pkg/mlrval/get_test.go index 43ca70c6a..33598c39a 100644 --- a/internal/pkg/mlrval/get_test.go +++ b/internal/pkg/mlrval/get_test.go @@ -12,38 +12,38 @@ import ( func TestGetString(t *testing.T) { mv := FromInferredType("234") - stringval, ok := mv.GetString() + stringval, ok := mv.GetStringValue() assert.False(t, ok) mv = FromDeferredType("234") - stringval, ok = mv.GetString() + stringval, ok = mv.GetStringValue() assert.False(t, ok) mv = FromInferredType("234.5") - stringval, ok = mv.GetString() + stringval, ok = mv.GetStringValue() assert.False(t, ok) mv = FromDeferredType("234.5") - stringval, ok = mv.GetString() + stringval, ok = mv.GetStringValue() assert.False(t, ok) mv = FromInferredType("abc") - stringval, ok = mv.GetString() + stringval, ok = mv.GetStringValue() assert.Equal(t, "abc", stringval) assert.True(t, ok) mv = FromDeferredType("abc") - stringval, ok = mv.GetString() + stringval, ok = mv.GetStringValue() assert.Equal(t, "abc", stringval) assert.True(t, ok) mv = FromInferredType("") - stringval, ok = mv.GetString() + stringval, ok = mv.GetStringValue() assert.Equal(t, "", stringval) assert.True(t, ok) mv = FromDeferredType("") - stringval, ok = mv.GetString() + stringval, ok = mv.GetStringValue() assert.Equal(t, "", stringval) assert.True(t, ok) } diff --git a/internal/pkg/mlrval/output.go b/internal/pkg/mlrval/output.go index 47afa9f22..0adcc69dd 100644 --- a/internal/pkg/mlrval/output.go +++ b/internal/pkg/mlrval/output.go @@ -19,36 +19,41 @@ func (mv Mlrval) String() string { // See mlrval.go for more about JIT-formatting of string backings func (mv *Mlrval) setPrintRep() { if !mv.printrepValid { - // xxx do it -- disposition vector - // xxx temp temp temp temp temp switch mv.mvtype { + case MT_PENDING: // Should not have gotten outside of the JSON decoder, so flag this // clearly visually if it should (buggily) slip through to // user-level visibility. mv.printrep = "(bug-if-you-see-this)" // xxx constdef at top of file break + case MT_ERROR: mv.printrep = "(error)" // xxx constdef at top of file break + case MT_ABSENT: // Callsites should be using absence to do non-assigns, so flag // this clearly visually if it should (buggily) slip through to // user-level visibility. mv.printrep = "(bug-if-you-see-this)" // xxx constdef at top of file break + case MT_VOID: mv.printrep = "" // xxx constdef at top of file break + case MT_STRING: - // panic i suppose break + case MT_INT: mv.printrep = strconv.Itoa(mv.intval) break + case MT_FLOAT: mv.printrep = strconv.FormatFloat(mv.floatval, 'f', -1, 64) break + case MT_BOOL: if mv.boolval == true { mv.printrep = "true" @@ -56,10 +61,9 @@ func (mv *Mlrval) setPrintRep() { mv.printrep = "false" } break - // TODO: handling indentation - case MT_ARRAY: - panic("mlrval array-to-string unimplemented") + case MT_ARRAY: + panic("mlrval array-to-string unimplemented, pending refactor") //bytes, err := mv.MarshalJSON(JSON_MULTILINE, false) //// maybe just InternalCodingErrorIf(err != nil) //if err != nil { @@ -69,9 +73,9 @@ func (mv *Mlrval) setPrintRep() { //mv.printrep = string(bytes) break - case MT_MAP: - panic("mlrval map-to-string unimplemented") + case MT_MAP: + panic("mlrval map-to-string unimplemented, pending refactor") //bytes, err := mv.MarshalJSON(JSON_MULTILINE, false) //// maybe just InternalCodingErrorIf(err != nil) //if err != nil { diff --git a/internal/pkg/runtime/state.go b/internal/pkg/runtime/state.go index e7f725412..1fca8ebad 100644 --- a/internal/pkg/runtime/state.go +++ b/internal/pkg/runtime/state.go @@ -20,7 +20,7 @@ type State struct { Inrec *types.Mlrmap Context *types.Context Oosvars *types.Mlrmap - FilterExpression *types.Mlrval + FilterExpression *mlrval.Mlrval Stack *Stack OutputRecordsAndContexts *list.List // list of *types.RecordAndContext @@ -36,7 +36,7 @@ func NewEmptyState(options *cli.TOptions) *State { Inrec: nil, Context: nil, Oosvars: oosvars, - FilterExpression: types.MLRVAL_TRUE, + FilterExpression: mlrval.TRUE, Stack: NewStack(), // OutputRecordsAndContexts is assigned after construction diff --git a/internal/pkg/transformers/bar.go b/internal/pkg/transformers/bar.go index 70a15440e..ed425b75c 100644 --- a/internal/pkg/transformers/bar.go +++ b/internal/pkg/transformers/bar.go @@ -252,7 +252,7 @@ func (tr *TransformerBar) processNoAuto( if idx > tr.width { idx = tr.width } - inrec.PutReference(fieldName, mlrval.MlrvalFromString(tr.bars[idx])) + inrec.PutReference(fieldName, mlrval.mlrval.FromString(tr.bars[idx])) } outputRecordsAndContexts.PushBack(inrecAndContext) @@ -340,7 +340,7 @@ func (tr *TransformerBar) processAuto( buffer.WriteString("[") buffer.WriteString(shi) buffer.WriteString("]") - record.PutReference(fieldName, mlrval.MlrvalFromString(buffer.String())) + record.PutReference(fieldName, mlrval.mlrval.FromString(buffer.String())) } } diff --git a/internal/pkg/transformers/clean-whitespace.go b/internal/pkg/transformers/clean-whitespace.go index 4e6b236db..93d465f48 100644 --- a/internal/pkg/transformers/clean-whitespace.go +++ b/internal/pkg/transformers/clean-whitespace.go @@ -153,10 +153,10 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeysAndValues( newrec := types.NewMlrmapAsRecord() for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { - oldKey := mlrval.MlrvalFromString(pe.Key) + oldKey := mlrval.mlrval.FromString(pe.Key) // xxx temp - newKey := types.BIF_clean_whitespace(oldKey) - newValue := types.BIF_clean_whitespace(pe.Value) + newKey := bifs.BIF_clean_whitespace(oldKey) + newValue := bifs.BIF_clean_whitespace(pe.Value) // Transferring ownership from old record to new record; no copy needed newrec.PutReference(newKey.String(), newValue) } @@ -178,8 +178,8 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeys( newrec := types.NewMlrmapAsRecord() for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { - oldKey := mlrval.MlrvalFromString(pe.Key) - newKey := types.BIF_clean_whitespace(oldKey) + oldKey := mlrval.mlrval.FromString(pe.Key) + newKey := bifs.BIF_clean_whitespace(oldKey) // Transferring ownership from old record to new record; no copy needed newrec.PutReference(newKey.String(), pe.Value) } @@ -199,7 +199,7 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInValues( ) { if !inrecAndContext.EndOfStream { for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { - pe.Value = types.BIF_clean_whitespace(pe.Value) + pe.Value = bifs.BIF_clean_whitespace(pe.Value) } outputRecordsAndContexts.PushBack(inrecAndContext) } else { diff --git a/internal/pkg/transformers/fill-empty.go b/internal/pkg/transformers/fill-empty.go index 2e619c7be..de41ccee1 100644 --- a/internal/pkg/transformers/fill-empty.go +++ b/internal/pkg/transformers/fill-empty.go @@ -105,7 +105,7 @@ func NewTransformerFillEmpty( if inferType { tr.fillValue = mlrval.MlrvalFromInferredType(fillString) } else { - tr.fillValue = mlrval.MlrvalFromString(fillString) + tr.fillValue = mlrval.mlrval.FromString(fillString) } return tr, nil } diff --git a/internal/pkg/transformers/fraction.go b/internal/pkg/transformers/fraction.go index 7d474257e..5074c627c 100644 --- a/internal/pkg/transformers/fraction.go +++ b/internal/pkg/transformers/fraction.go @@ -225,7 +225,7 @@ func (tr *TransformerFraction) Transform( sumsForGroup[fractionFieldName] = value.Copy() cumusForGroup[fractionFieldName] = tr.zero } else { - sumsForGroup[fractionFieldName] = types.BIF_plus_binary(sum, value) + sumsForGroup[fractionFieldName] = bifs.BIF_plus_binary(sum, value) } } } @@ -261,15 +261,15 @@ func (tr *TransformerFraction) Transform( if tr.doCumu { cumu = cumusForGroup[fractionFieldName] - numerator = types.BIF_plus_binary(value, cumu) + numerator = bifs.BIF_plus_binary(value, cumu) } else { numerator = value } denominator := sumsForGroup[fractionFieldName] if !mlrval.MlrvalEqualsAsBool(value, tr.zero) { - outputValue = types.BIF_divide(numerator, denominator) - outputValue = types.BIF_times(outputValue, tr.multiplier) + outputValue = bifs.BIF_divide(numerator, denominator) + outputValue = bifs.BIF_times(outputValue, tr.multiplier) } else { outputValue = types.MLRVAL_ERROR } @@ -280,7 +280,7 @@ func (tr *TransformerFraction) Transform( ) if tr.doCumu { - cumusForGroup[fractionFieldName] = types.BIF_plus_binary(cumu, value) + cumusForGroup[fractionFieldName] = bifs.BIF_plus_binary(cumu, value) } } } diff --git a/internal/pkg/transformers/nest.go b/internal/pkg/transformers/nest.go index f1a81bfec..e58d7f580 100644 --- a/internal/pkg/transformers/nest.go +++ b/internal/pkg/transformers/nest.go @@ -336,7 +336,7 @@ func (tr *TransformerNest) explodeValuesAcrossFields( i := 1 for _, piece := range pieces { key := tr.fieldName + "_" + strconv.Itoa(i) - value := mlrval.MlrvalFromString(piece) + value := mlrval.mlrval.FromString(piece) recordEntry = inrec.PutReferenceAfter(recordEntry, key, value) i++ } @@ -369,7 +369,7 @@ func (tr *TransformerNest) explodeValuesAcrossRecords( pieces := strings.Split(svalue, tr.nestedFS) for _, piece := range pieces { outrec := inrec.Copy() - outrec.PutReference(tr.fieldName, mlrval.MlrvalFromString(piece)) + outrec.PutReference(tr.fieldName, mlrval.mlrval.FromString(piece)) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } @@ -405,13 +405,13 @@ func (tr *TransformerNest) explodePairsAcrossFields( recordEntry = inrec.PutReferenceAfter( recordEntry, pair[0], - mlrval.MlrvalFromString(pair[1]), + mlrval.mlrval.FromString(pair[1]), ) } else { // there is not a pair recordEntry = inrec.PutReferenceAfter( recordEntry, tr.fieldName, - mlrval.MlrvalFromString(piece), + mlrval.mlrval.FromString(piece), ) } } @@ -450,9 +450,9 @@ func (tr *TransformerNest) explodePairsAcrossRecords( // name, in which case replace its value. pair := strings.SplitN(piece, tr.nestedPS, 2) if len(pair) == 2 { // there is a pair - outrec.PutReferenceAfter(originalEntry, pair[0], mlrval.MlrvalFromString(pair[1])) + outrec.PutReferenceAfter(originalEntry, pair[0], mlrval.mlrval.FromString(pair[1])) } else { // there is not a pair - outrec.PutReferenceAfter(originalEntry, tr.fieldName, mlrval.MlrvalFromString(piece)) + outrec.PutReferenceAfter(originalEntry, tr.fieldName, mlrval.mlrval.FromString(piece)) } outrec.Unlink(originalEntry) @@ -498,7 +498,7 @@ func (tr *TransformerNest) implodeValuesAcrossFields( } if fieldCount > 0 { - newValue := mlrval.MlrvalFromString(buffer.String()) + newValue := mlrval.mlrval.FromString(buffer.String()) if previousEntry == nil { // No record before the unlinked one, i.e. list-head. inrec.PrependReference(tr.fieldName, newValue) } else { @@ -577,7 +577,7 @@ func (tr *TransformerNest) implodeValueAcrossRecords( } // tr.fieldName was already present so we'll overwrite it in-place here. - outrec.PutReference(tr.fieldName, mlrval.MlrvalFromString(buffer.String())) + outrec.PutReference(tr.fieldName, mlrval.mlrval.FromString(buffer.String())) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } } diff --git a/internal/pkg/transformers/reshape.go b/internal/pkg/transformers/reshape.go index 8323a2a51..15119cce2 100644 --- a/internal/pkg/transformers/reshape.go +++ b/internal/pkg/transformers/reshape.go @@ -321,7 +321,7 @@ func (tr *TransformerReshape) wideToLongNoRegex( } else { for pf := pairs.Head; pf != nil; pf = pf.Next { outrec := inrec.Copy() - outrec.PutReference(tr.outputKeyFieldName, mlrval.MlrvalFromString(pf.Key)) + outrec.PutReference(tr.outputKeyFieldName, mlrval.mlrval.FromString(pf.Key)) outrec.PutReference(tr.outputValueFieldName, pf.Value) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } @@ -363,7 +363,7 @@ func (tr *TransformerReshape) wideToLongRegex( } else { for pf := pairs.Head; pf != nil; pf = pf.Next { outrec := inrec.Copy() - outrec.PutReference(tr.outputKeyFieldName, mlrval.MlrvalFromString(pf.Key)) + outrec.PutReference(tr.outputKeyFieldName, mlrval.mlrval.FromString(pf.Key)) outrec.PutReference(tr.outputValueFieldName, pf.Value) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } diff --git a/internal/pkg/transformers/sec2gmt.go b/internal/pkg/transformers/sec2gmt.go index 4b9c8f640..9bb064474 100644 --- a/internal/pkg/transformers/sec2gmt.go +++ b/internal/pkg/transformers/sec2gmt.go @@ -164,7 +164,7 @@ func (tr *TransformerSec2GMT) Transform( if value != nil { floatval, ok := value.GetNumericToFloatValue() if ok { - newValue := mlrval.MlrvalFromString(lib.Sec2GMT( + newValue := mlrval.mlrval.FromString(lib.Sec2GMT( floatval/tr.preDivide, tr.numDecimalPlaces, )) diff --git a/internal/pkg/transformers/sec2gmtdate.go b/internal/pkg/transformers/sec2gmtdate.go index d76ae16ef..ea5cc3940 100644 --- a/internal/pkg/transformers/sec2gmtdate.go +++ b/internal/pkg/transformers/sec2gmtdate.go @@ -118,7 +118,7 @@ func (tr *TransformerSec2GMTDate) Transform( for _, fieldName := range tr.fieldNameList { value := inrec.Get(fieldName) if value != nil { - inrec.PutReference(fieldName, types.BIF_sec2gmtdate(value)) + inrec.PutReference(fieldName, bifs.BIF_sec2gmtdate(value)) } } outputRecordsAndContexts.PushBack(inrecAndContext) diff --git a/internal/pkg/transformers/seqgen.go b/internal/pkg/transformers/seqgen.go index ab73725dd..059ed221b 100644 --- a/internal/pkg/transformers/seqgen.go +++ b/internal/pkg/transformers/seqgen.go @@ -168,12 +168,12 @@ func NewTransformerSeqgen( } if fstep > 0 { - doneComparator = types.BIF_greater_than + doneComparator = bifs.BIF_greater_than } else if fstep < 0 { - doneComparator = types.BIF_less_than + doneComparator = bifs.BIF_less_than } else { if fstart == fstop { - doneComparator = types.BIF_equals + doneComparator = bifs.BIF_equals } else { return nil, errors.New( "mlr seqgen: step must not be zero unless start == stop.", @@ -235,7 +235,7 @@ func (tr *TransformerSeqgen) Transform( outrecAndContext := types.NewRecordAndContext(outrec, context) outputRecordsAndContexts.PushBack(outrecAndContext) - counter = types.BIF_plus_binary(counter, tr.step) + counter = bifs.BIF_plus_binary(counter, tr.step) } outputRecordsAndContexts.PushBack(types.NewEndOfStreamMarker(context)) diff --git a/internal/pkg/transformers/step.go b/internal/pkg/transformers/step.go index d22c163ba..e62e98e16 100644 --- a/internal/pkg/transformers/step.go +++ b/internal/pkg/transformers/step.go @@ -385,7 +385,7 @@ func (stepper *tStepperDelta) process( delta := mlrval.MlrvalFromInt(0) if stepper.previous != nil { - delta = types.BIF_minus_binary(valueFieldValue, stepper.previous) + delta = bifs.BIF_minus_binary(valueFieldValue, stepper.previous) } inrec.PutCopy(stepper.outputFieldName, delta) @@ -448,7 +448,7 @@ func (stepper *tStepperFromFirst) process( if stepper.first == nil { stepper.first = valueFieldValue.Copy() } else { - fromFirst = types.BIF_minus_binary(valueFieldValue, stepper.first) + fromFirst = bifs.BIF_minus_binary(valueFieldValue, stepper.first) } inrec.PutCopy(stepper.outputFieldName, fromFirst) } @@ -481,7 +481,7 @@ func (stepper *tStepperRatio) process( ratio := mlrval.MlrvalFromInt(1) if stepper.previous != nil { - ratio = types.BIF_divide(valueFieldValue, stepper.previous) + ratio = bifs.BIF_divide(valueFieldValue, stepper.previous) } inrec.PutCopy(stepper.outputFieldName, ratio) @@ -512,7 +512,7 @@ func (stepper *tStepperRsum) process( if valueFieldValue.IsEmpty() { inrec.PutCopy(stepper.outputFieldName, types.MLRVAL_VOID) } else { - stepper.rsum = types.BIF_plus_binary(valueFieldValue, stepper.rsum) + stepper.rsum = bifs.BIF_plus_binary(valueFieldValue, stepper.rsum) inrec.PutCopy(stepper.outputFieldName, stepper.rsum) } } @@ -543,7 +543,7 @@ func (stepper *tStepperCounter) process( if valueFieldValue.IsEmpty() { inrec.PutCopy(stepper.outputFieldName, types.MLRVAL_VOID) } else { - stepper.counter = types.BIF_plus_binary(stepper.counter, stepper.one) + stepper.counter = bifs.BIF_plus_binary(stepper.counter, stepper.one) inrec.PutCopy(stepper.outputFieldName, stepper.counter) } } @@ -621,9 +621,9 @@ func (stepper *tStepperEWMA) process( for i := range stepper.alphas { curr := valueFieldValue.Copy() // xxx pending pointer-output refactor - product1 := types.BIF_times(curr, stepper.alphas[i]) - product2 := types.BIF_times(stepper.prevs[i], stepper.oneMinusAlphas[i]) - next := types.BIF_plus_binary(product1, product2) + product1 := bifs.BIF_times(curr, stepper.alphas[i]) + product2 := bifs.BIF_times(stepper.prevs[i], stepper.oneMinusAlphas[i]) + next := bifs.BIF_plus_binary(product1, product2) inrec.PutCopy(stepper.outputFieldNames[i], next) stepper.prevs[i] = next } diff --git a/internal/pkg/transformers/template.go b/internal/pkg/transformers/template.go index ac1716052..27bd8f36c 100644 --- a/internal/pkg/transformers/template.go +++ b/internal/pkg/transformers/template.go @@ -130,7 +130,7 @@ func NewTransformerTemplate( return &TransformerTemplate{ fieldNameList: fieldNames, fieldNameSet: lib.StringListToSet(fieldNames), - fillWith: mlrval.MlrvalFromString(fillWith), + fillWith: mlrval.mlrval.FromString(fillWith), }, nil } diff --git a/internal/pkg/transformers/uniq.go b/internal/pkg/transformers/uniq.go index 5fa100424..78b8d320c 100644 --- a/internal/pkg/transformers/uniq.go +++ b/internal/pkg/transformers/uniq.go @@ -475,7 +475,7 @@ func (tr *TransformerUniq) transformUnlashed( for pf := countsForFieldName.Head; pf != nil; pf = pf.Next { fieldValueString := pf.Key outrec := types.NewMlrmapAsRecord() - outrec.PutReference("field", mlrval.MlrvalFromString(fieldName)) + outrec.PutReference("field", mlrval.mlrval.FromString(fieldName)) outrec.PutCopy( "value", tr.unlashedCountValues.Get(fieldName).(*lib.OrderedMap).Get(fieldValueString).(*mlrval.Mlrval), diff --git a/internal/pkg/transformers/unsparsify.go b/internal/pkg/transformers/unsparsify.go index 33970f192..cebbaeb35 100644 --- a/internal/pkg/transformers/unsparsify.go +++ b/internal/pkg/transformers/unsparsify.go @@ -128,7 +128,7 @@ func NewTransformerUnsparsify( } tr := &TransformerUnsparsify{ - fillerMlrval: mlrval.MlrvalFromString(fillerString), + fillerMlrval: mlrval.mlrval.FromString(fillerString), recordsAndContexts: list.New(), fieldNamesSeen: fieldNamesSeen, } diff --git a/internal/pkg/types-parked/mlrmap_json.go b/internal/pkg/types-parked/mlrmap_json.go index 1a75edec7..60624f18e 100644 --- a/internal/pkg/types-parked/mlrmap_json.go +++ b/internal/pkg/types-parked/mlrmap_json.go @@ -155,7 +155,7 @@ func (entry *MlrmapEntry) JSONStringifyInPlace( if err != nil { entry.Value = MLRVAL_ERROR } else { - entry.Value = MlrvalFromString(string(outputBytes)) + entry.Value = mlrval.FromString(string(outputBytes)) } } diff --git a/internal/pkg/types-parked/mlrval_accessors.go b/internal/pkg/types-parked/mlrval_accessors.go index 86d80a313..b4af1bf5e 100644 --- a/internal/pkg/types-parked/mlrval_accessors.go +++ b/internal/pkg/types-parked/mlrval_accessors.go @@ -57,7 +57,7 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { // values would disappear entirely in a JSON-to-CSV conversion. if mv.mapval.IsEmpty() { if prefix != "" { - retval.PutCopy(prefix, MlrvalFromString("{}")) + retval.PutCopy(prefix, mlrval.FromString("{}")) } } @@ -82,7 +82,7 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { // values would disappear entirely in a JSON-to-CSV conversion. if len(mv.arrayval) == 0 { if prefix != "" { - retval.PutCopy(prefix, MlrvalFromString("[]")) + retval.PutCopy(prefix, mlrval.FromString("[]")) } } diff --git a/internal/pkg/types-parked/mlrval_json.go b/internal/pkg/types-parked/mlrval_json.go index be2e95cbc..3c21b3c8c 100644 --- a/internal/pkg/types-parked/mlrval_json.go +++ b/internal/pkg/types-parked/mlrval_json.go @@ -140,7 +140,7 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) ( sval, ok := startToken.(string) if ok { - mlrval := MlrvalFromString(sval) + mlrval := mlrval.FromString(sval) return mlrval, false, nil } diff --git a/internal/pkg/types/mlrmap_accessors.go b/internal/pkg/types/mlrmap_accessors.go index ff21cc6a9..a75389e49 100644 --- a/internal/pkg/types/mlrmap_accessors.go +++ b/internal/pkg/types/mlrmap_accessors.go @@ -50,29 +50,29 @@ func (mlrmap *Mlrmap) PutReference(key string, value *mlrval.Mlrval) { } } -//// PutCopy copies the key and value (deep-copying in case the value is array/map). -//// This is safe for DSL use. See also PutReference. -//func (mlrmap *Mlrmap) PutCopy(key string, value *mlrval.Mlrval) { -// pe := mlrmap.findEntry(key) -// if pe == nil { -// pe = newMlrmapEntry(key, value.Copy()) -// if mlrmap.Head == nil { -// mlrmap.Head = pe -// mlrmap.Tail = pe -// } else { -// pe.Prev = mlrmap.Tail -// pe.Next = nil -// mlrmap.Tail.Next = pe -// mlrmap.Tail = pe -// } -// if mlrmap.keysToEntries != nil { -// mlrmap.keysToEntries[key] = pe -// } -// mlrmap.FieldCount++ -// } else { -// pe.Value = value.Copy() -// } -//} +// PutCopy copies the key and value (deep-copying in case the value is array/map). +// This is safe for DSL use. See also PutReference. +func (mlrmap *Mlrmap) PutCopy(key string, value *mlrval.Mlrval) { + pe := mlrmap.findEntry(key) + if pe == nil { + pe = newMlrmapEntry(key, value.Copy()) + if mlrmap.Head == nil { + mlrmap.Head = pe + mlrmap.Tail = pe + } else { + pe.Prev = mlrmap.Tail + pe.Next = nil + mlrmap.Tail.Next = pe + mlrmap.Tail = pe + } + if mlrmap.keysToEntries != nil { + mlrmap.keysToEntries[key] = pe + } + mlrmap.FieldCount++ + } else { + pe.Value = value.Copy() + } +} // PrependReference is the same as PutReference, but puts a new entry first, not last. func (mlrmap *Mlrmap) PrependReference(key string, value *mlrval.Mlrval) { @@ -220,10 +220,10 @@ func (mlrmap *Mlrmap) findEntryByPositionalIndex(position int) *MlrmapEntry { // } //} -//// ---------------------------------------------------------------- -//func (mlrmap *Mlrmap) PrependCopy(key string, value *mlrval.Mlrval) { -// mlrmap.PrependReference(key, value.Copy()) -//} +// ---------------------------------------------------------------- +func (mlrmap *Mlrmap) PrependCopy(key string, value *mlrval.Mlrval) { + mlrmap.PrependReference(key, value.Copy()) +} //// ---------------------------------------------------------------- //// Merges that into mlrmap. diff --git a/todo.txt b/todo.txt index 364a94f46..7eda5d923 100644 --- a/todo.txt +++ b/todo.txt @@ -3,7 +3,6 @@ PUNCHDOWN LIST * --ifs-regex & --ips-regex -- guessing is not safe as evidence by '.' and '|' * JIT next: - o mlrval copy o mlrmap copy * big-picture item @ Rmd (csv memes; and beyond); also webdoc intro page From 511a1bf3ee7842472a91524286ec0ce22e4fad8a Mon Sep 17 00:00:00 2001 From: John Kerl Date: Tue, 14 Dec 2021 22:11:40 -0500 Subject: [PATCH 07/19] bifs/arithmetic_test.go --- .vimrc | 3 +- Makefile | 29 +++++- internal/pkg/bifs/arithmetic.go | 43 ++++----- internal/pkg/bifs/arithmetic_test.go | 90 +++++++++++++++++++ internal/pkg/bifs/base.go | 2 +- internal/pkg/bifs/bits.go | 2 +- internal/pkg/bifs/bits_test.go | 19 ++++ internal/pkg/bifs/booleans.go | 26 ++++-- internal/pkg/bifs/collections.go | 8 +- internal/pkg/bifs/collections_test.go | 56 ++++++++++++ internal/pkg/bifs/datetime.go | 2 +- internal/pkg/bifs/hashing.go | 2 +- internal/pkg/bifs/hashing_test.go | 25 ++++++ internal/pkg/bifs/math.go | 2 +- internal/pkg/bifs/mathlib.go | 2 +- internal/pkg/bifs/random.go | 2 +- internal/pkg/bifs/regex.go | 2 +- internal/pkg/bifs/relative_time.go | 2 +- internal/pkg/bifs/sort.go | 2 +- .../pkg/bifs/{bifs_test.go => sort_test.go} | 9 +- internal/pkg/bifs/stats.go | 16 ++-- internal/pkg/bifs/strings.go | 8 +- internal/pkg/bifs/system.go | 2 +- internal/pkg/bifs/types.go | 2 +- .../pkg/dsl/cst/builtin_function_manager.go | 4 +- internal/pkg/dsl/cst/builtin_functions.go | 4 +- internal/pkg/dsl/cst/hofs.go | 4 +- internal/pkg/transformers/fraction.go | 2 +- .../transformers/utils/percentile-keeper.go | 2 +- .../transformers/utils/stats1-accumulators.go | 14 +-- .../pkg/types-parked/mlrval_collections.go | 24 ++--- todo.txt | 23 +++-- 32 files changed, 325 insertions(+), 108 deletions(-) create mode 100644 internal/pkg/bifs/arithmetic_test.go create mode 100644 internal/pkg/bifs/bits_test.go create mode 100644 internal/pkg/bifs/collections_test.go create mode 100644 internal/pkg/bifs/hashing_test.go rename internal/pkg/bifs/{bifs_test.go => sort_test.go} (91%) diff --git a/.vimrc b/.vimrc index 1a413f44a..562ce7c7f 100644 --- a/.vimrc +++ b/.vimrc @@ -3,4 +3,5 @@ map \r :w:!clear;echo Building ...; echo; make tests-in-order "map \f :w:!clear;echo Building ...; echo; make mlrval-tests "map \f :w:!clear;echo Building ...; echo; make mlrmap-tests "map \f :w:!clear;echo Building ...; echo; make input-tests -map \f :w:!clear;echo Building ...; echo; make mlrval-format-test +"map \f :w:!clear;echo Building ...; echo; make mlrval-format-test +map \f :w:!clear;echo Building ...; echo; make bifs-tests diff --git a/Makefile b/Makefile index c89b2acd7..4c8de0225 100644 --- a/Makefile +++ b/Makefile @@ -96,12 +96,39 @@ input-dkvp-test: internal/pkg/input/record_reader_dkvp_nidx.go input-tests: input-dkvp-test +bifs-arithmetic-test: + go test internal/pkg/bifs/arithmetic_test.go \ + internal/pkg/bifs/base.go \ + internal/pkg/bifs/arithmetic.go +bifs-bits-test: + go test internal/pkg/bifs/bits_test.go \ + internal/pkg/bifs/base.go \ + internal/pkg/bifs/arithmetic.go \ + internal/pkg/bifs/bits.go +bifs-collections-test: + go test internal/pkg/bifs/bits_test.go \ + internal/pkg/bifs/base.go \ + internal/pkg/bifs/arithmetic.go \ + internal/pkg/bifs/collections.go +bifs-hashing-test: + go test internal/pkg/bifs/hashing_test.go \ + internal/pkg/bifs/base.go \ + internal/pkg/bifs/arithmetic.go \ + internal/pkg/bifs/hashing.go +bifs-sort-test: + go test internal/pkg/bifs/sort_test.go \ + internal/pkg/bifs/base.go \ + internal/pkg/bifs/arithmetic.go \ + internal/pkg/bifs/sort.go + +bifs-tests: bifs-arithmetic-test bifs-bits-test bifs-collections-test bifs-hashing-test bifs-sort-test + #mlrval_functions_test: # go test internal/pkg/types/mlrval_functions_test.go $(ls internal/pkg/types/*.go | grep -v test) #mlrval_format_test: # go test internal/pkg/types/mlrval_format_test.go $(ls internal/pkg/types/*.go|grep -v test) -tests-in-order: mlrval-tests mlrmap-tests input-tests +tests-in-order: mlrval-tests mlrmap-tests input-tests bifs-tests # ---------------------------------------------------------------- # Regression tests (large number) diff --git a/internal/pkg/bifs/arithmetic.go b/internal/pkg/bifs/arithmetic.go index ad830656d..03aaacdc6 100644 --- a/internal/pkg/bifs/arithmetic.go +++ b/internal/pkg/bifs/arithmetic.go @@ -1,4 +1,4 @@ -package types +package bifs import ( "math" @@ -56,20 +56,9 @@ func BIF_minus_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { return uneg_dispositions[input1.Type()](input1) } -// ================================================================ -// Logical NOT operator - -func BIF_logicalnot(input1 *mlrval.Mlrval) *mlrval.Mlrval { - if input1.IsBool() { - return mlrval.FromBool(!input1.AcquireBoolValue()) - } else { - return mlrval.ERROR - } -} - // ================================================================ // Addition with auto-overflow from int to float when necessary. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic // Auto-overflows up to float. Additions & subtractions overflow by at most // one bit so it suffices to check sign-changes. @@ -127,7 +116,7 @@ func BIF_plus_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { // ================================================================ // Subtraction with auto-overflow from int to float when necessary. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic // Adds & subtracts overflow by at most one bit so it suffices to check // sign-changes. @@ -185,7 +174,7 @@ func BIF_minus_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { // ================================================================ // Multiplication with auto-overflow from int to float when necessary. See -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic // Auto-overflows up to float. // @@ -259,7 +248,7 @@ func BIF_times(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { // ================================================================ // Pythonic division. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic // // Int/int pairings don't produce overflow. // @@ -330,7 +319,7 @@ func BIF_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { // ================================================================ // Integer division: DSL operator '//' as in Python. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic func int_divide_n_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { a := input1.AcquireIntValue() @@ -391,7 +380,7 @@ func BIF_int_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { // ================================================================ // Non-auto-overflowing addition: DSL operator '.+'. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic func dotplus_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return mlrval.FromInt(input1.AcquireIntValue() + input2.AcquireIntValue()) @@ -427,7 +416,7 @@ func BIF_dot_plus(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { // ================================================================ // Non-auto-overflowing subtraction: DSL operator '.-'. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic func dotminus_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return mlrval.FromInt(input1.AcquireIntValue() - input2.AcquireIntValue()) @@ -463,7 +452,7 @@ func BIF_dot_minus(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { // ---------------------------------------------------------------- // Non-auto-overflowing multiplication: DSL operator '.*'. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic func dottimes_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return mlrval.FromInt(input1.AcquireIntValue() * input2.AcquireIntValue()) @@ -499,7 +488,7 @@ func BIF_dot_times(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { // ---------------------------------------------------------------- // 64-bit integer division: DSL operator './'. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic func dotdivide_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return mlrval.FromInt(input1.AcquireIntValue() / input2.AcquireIntValue()) @@ -535,7 +524,7 @@ func BIF_dot_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { // ---------------------------------------------------------------- // 64-bit integer division: DSL operator './/'. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic func dotidivide_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { a := input1.AcquireIntValue() @@ -590,7 +579,7 @@ var dotidivide_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func MlrvalDotIntDivide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { +func BIF_dot_int_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return dotidivide_dispositions[input1.Type()][input2.Type()](input1, input2) } @@ -826,7 +815,7 @@ var min_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func MlrvalBinaryMin(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { +func BIF_min_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return (min_dispositions[input1.Type()][input2.Type()])(input1, input2) } @@ -837,7 +826,7 @@ func BIF_min_variadic(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { retval := mlrvals[0] for i := range mlrvals { if i > 0 { - retval = MlrvalBinaryMin(retval, mlrvals[i]) + retval = BIF_min_binary(retval, mlrvals[i]) } } return retval @@ -910,7 +899,7 @@ var max_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func MlrvalBinaryMax(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { +func BIF_max_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return (max_dispositions[input1.Type()][input2.Type()])(input1, input2) } @@ -921,7 +910,7 @@ func BIF_max_variadic(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { retval := mlrvals[0] for i := range mlrvals { if i > 0 { - retval = MlrvalBinaryMax(retval, mlrvals[i]) + retval = BIF_max_binary(retval, mlrvals[i]) } } return retval diff --git a/internal/pkg/bifs/arithmetic_test.go b/internal/pkg/bifs/arithmetic_test.go new file mode 100644 index 000000000..5935854ac --- /dev/null +++ b/internal/pkg/bifs/arithmetic_test.go @@ -0,0 +1,90 @@ +package bifs + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + +func TestBIF_plus_unary(t *testing.T) { + input := mlrval.FromDeferredType("123") + output := BIF_plus_unary(input) + intval, ok := output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 123, intval) + + input = mlrval.FromDeferredType("-123.5") + output = BIF_plus_unary(input) + floatval, ok := output.GetFloatValue() + assert.True(t, ok) + assert.Equal(t, -123.5, floatval) +} + +func TestBIF_minus_unary(t *testing.T) { + input := mlrval.FromDeferredType("123") + output := BIF_minus_unary(input) + intval, ok := output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, -123, intval) + + input = mlrval.FromDeferredType("-123.5") + output = BIF_minus_unary(input) + floatval, ok := output.GetFloatValue() + assert.True(t, ok) + assert.Equal(t, 123.5, floatval) +} + +func TestBIF_plus_binary(t *testing.T) { + input1 := mlrval.FromDeferredType("123") + input2 := mlrval.FromDeferredType("456") + output := BIF_plus_binary(input1, input2) + intval, ok := output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 579, intval) + + input1 = mlrval.FromDeferredType("123.5") + input2 = mlrval.FromDeferredType("456") + output = BIF_plus_binary(input1, input2) + floatval, ok := output.GetFloatValue() + assert.True(t, ok) + assert.Equal(t, 579.5, floatval) +} + +func TestBIF_plus_binary_overflow(t *testing.T) { + input1 := mlrval.FromInt(0x07ffffffffffffff) + input2 := mlrval.FromInt(0x07fffffffffffffe) + output := BIF_plus_binary(input1, input2) + intval, ok := output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 0x0ffffffffffffffd, intval) + + input1 = mlrval.FromInt(0x7fffffffffffffff) + input2 = mlrval.FromInt(0x7ffffffffffffffe) + output = BIF_plus_binary(input1, input2) + floatval, ok := output.GetFloatValue() + assert.True(t, ok) + assert.Equal(t, 18446744073709552000.0, floatval) +} + +// TODO: copy in more unit-test cases from existing regression-test data + +//func BIF_minus_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_times(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_int_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_dot_plus(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_dot_minus(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_dot_times(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_dot_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_dot_int_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_modulus(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_mod_add(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_mod_sub(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_mod_mul(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_mod_exp(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_min_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_min_variadic(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval +//func BIF_max_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_max_variadic(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval diff --git a/internal/pkg/bifs/base.go b/internal/pkg/bifs/base.go index 435117993..cd7c04160 100644 --- a/internal/pkg/bifs/base.go +++ b/internal/pkg/bifs/base.go @@ -45,7 +45,7 @@ // ('s') to anything else ('x'). // ================================================================ -package types +package bifs import ( "github.com/johnkerl/miller/internal/pkg/mlrval" diff --git a/internal/pkg/bifs/bits.go b/internal/pkg/bifs/bits.go index 78bdbbbd3..71be7c9a6 100644 --- a/internal/pkg/bifs/bits.go +++ b/internal/pkg/bifs/bits.go @@ -1,4 +1,4 @@ -package types +package bifs import ( "github.com/johnkerl/miller/internal/pkg/mlrval" diff --git a/internal/pkg/bifs/bits_test.go b/internal/pkg/bifs/bits_test.go new file mode 100644 index 000000000..1c5bec2a4 --- /dev/null +++ b/internal/pkg/bifs/bits_test.go @@ -0,0 +1,19 @@ +package bifs + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + +func TestBIF_bitcount(t *testing.T) { + input1 := mlrval.FromDeferredType("0xcafe") + output := BIF_bitcount(input1) + intval, ok := output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 11, intval) +} + +// TODO: copy in more unit-test cases from existing regression-test data diff --git a/internal/pkg/bifs/booleans.go b/internal/pkg/bifs/booleans.go index 89cd8b3fd..8228750fc 100644 --- a/internal/pkg/bifs/booleans.go +++ b/internal/pkg/bifs/booleans.go @@ -2,7 +2,7 @@ // Boolean expressions for ==, !=, >, >=, <, <= // ================================================================ -package types +package bifs import ( "github.com/johnkerl/miller/internal/pkg/lib" @@ -400,7 +400,7 @@ func BIF_cmp(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { } // For Go's sort.Slice. -func MlrvalLessThanAsBool(input1, input2 *mlrval.Mlrval) bool { +func BIF_less_than_as_bool(input1, input2 *mlrval.Mlrval) bool { // TODO refactor to avoid copy // This is a hot path for sort GC and is worth significant hand-optimization mretval := lt_dispositions[input1.Type()][input2.Type()](input1, input2) @@ -410,7 +410,7 @@ func MlrvalLessThanAsBool(input1, input2 *mlrval.Mlrval) bool { } // For Go's sort.Slice. -func MlrvalLessThanOrEqualsAsBool(input1, input2 *mlrval.Mlrval) bool { +func BIF_less_than_or_equals_as_bool(input1, input2 *mlrval.Mlrval) bool { // TODO refactor to avoid copy // This is a hot path for sort GC and is worth significant hand-optimization mretval := le_dispositions[input1.Type()][input2.Type()](input1, input2) @@ -420,7 +420,7 @@ func MlrvalLessThanOrEqualsAsBool(input1, input2 *mlrval.Mlrval) bool { } // For top-keeper -func MlrvalGreaterThanAsBool(input1, input2 *mlrval.Mlrval) bool { +func BIF_greater_than_as_bool(input1, input2 *mlrval.Mlrval) bool { // TODO refactor to avoid copy // This is a hot path for sort GC and is worth significant hand-optimization mretval := gt_dispositions[input1.Type()][input2.Type()](input1, input2) @@ -430,7 +430,7 @@ func MlrvalGreaterThanAsBool(input1, input2 *mlrval.Mlrval) bool { } // For top-keeper -func MlrvalGreaterThanOrEqualsAsBool(input1, input2 *mlrval.Mlrval) bool { +func BIF_greater_than_or_equals_as_bool(input1, input2 *mlrval.Mlrval) bool { // TODO refactor to avoid copy // This is a hot path for sort GC and is worth significant hand-optimization mretval := ge_dispositions[input1.Type()][input2.Type()](input1, input2) @@ -440,7 +440,7 @@ func MlrvalGreaterThanOrEqualsAsBool(input1, input2 *mlrval.Mlrval) bool { } // Convenience wrapper for non-DSL callsites that just want a bool -func MlrvalEqualsAsBool(input1, input2 *mlrval.Mlrval) bool { +func BIF_equals_as_bool(input1, input2 *mlrval.Mlrval) bool { mretval := eq_dispositions[input1.Type()][input2.Type()](input1, input2) retval, ok := mretval.GetBoolValue() lib.InternalCodingErrorIf(!ok) @@ -448,7 +448,15 @@ func MlrvalEqualsAsBool(input1, input2 *mlrval.Mlrval) bool { } // ---------------------------------------------------------------- -func MlrvalLogicalAND(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { +func BIF_logical_NOT(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsBool() { + return mlrval.FromBool(!input1.AcquireBoolValue()) + } else { + return mlrval.ERROR + } +} + +func BIF_logical_AND(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsBool() && input2.IsBool() { return mlrval.FromBool(input1.AcquireBoolValue() && input2.AcquireBoolValue()) } else { @@ -456,7 +464,7 @@ func MlrvalLogicalAND(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { } } -func MlrvalLogicalOR(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { +func BIF_logical_OR(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsBool() && input2.IsBool() { return mlrval.FromBool(input1.AcquireBoolValue() || input2.AcquireBoolValue()) } else { @@ -464,7 +472,7 @@ func MlrvalLogicalOR(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { } } -func BIF_logicalxor(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { +func BIF_logical_XOR(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsBool() && input2.IsBool() { return mlrval.FromBool(input1.AcquireBoolValue() != input2.AcquireBoolValue()) } else { diff --git a/internal/pkg/bifs/collections.go b/internal/pkg/bifs/collections.go index 9d003f5c6..649e1d66f 100644 --- a/internal/pkg/bifs/collections.go +++ b/internal/pkg/bifs/collections.go @@ -1,4 +1,4 @@ -package types +package bifs import ( "bytes" @@ -573,12 +573,12 @@ func BIF_splitax(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { input := input1.AcquireStringValue() fieldSeparator := input2.AcquireStringValue() - return mlrvalSplitAXHelper(input, fieldSeparator) + return bif_splitax_helper(input, fieldSeparator) } -// mlrvalSplitAXHelper is Split out for the benefit of BIF_splitax and +// bif_splitax_helper is split out for the benefit of BIF_splitax and // BIF_unflatten. -func mlrvalSplitAXHelper(input string, separator string) *mlrval.Mlrval { +func bif_splitax_helper(input string, separator string) *mlrval.Mlrval { fields := lib.SplitString(input, separator) output := NewSizedMlrvalArray(int(len(fields))) diff --git a/internal/pkg/bifs/collections_test.go b/internal/pkg/bifs/collections_test.go new file mode 100644 index 000000000..c3e479a25 --- /dev/null +++ b/internal/pkg/bifs/collections_test.go @@ -0,0 +1,56 @@ +package bifs + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + +func TestBIF_length(t *testing.T) { + input1 := mlrval.FromInt(123) + output := BIF_length(input1) + intval, ok := output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 1, intval) +} + +// TODO: copy in more unit-test cases from existing regression-test data + +// func BIF_length(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func depth_from_array(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func depth_from_map(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func depth_from_scalar(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_depth(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func leafcount_from_array(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func leafcount_from_map(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func leafcount_from_scalar(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_leafcount(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func has_key_in_array(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func has_key_in_map(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_haskey(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_mapselect(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval +// func BIF_mapexcept(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval +// func BIF_mapsum(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval +// func BIF_mapdiff(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval +// func BIF_joink(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_joinv(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_joinkv(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_splitkv(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_splitkvx(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_splitnv(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_splitnvx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_splita(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_splitax(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func mlrvalSplitAXHelper(input string, separator string) *mlrval.Mlrval +// func BIF_get_keys(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_get_values(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_append(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_flatten(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_flatten_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_unflatten(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_arrayify(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_json_parse(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_json_stringify_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_json_stringify_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval diff --git a/internal/pkg/bifs/datetime.go b/internal/pkg/bifs/datetime.go index a35c10c3d..af39fc971 100644 --- a/internal/pkg/bifs/datetime.go +++ b/internal/pkg/bifs/datetime.go @@ -1,4 +1,4 @@ -package types +package bifs import ( "fmt" diff --git a/internal/pkg/bifs/hashing.go b/internal/pkg/bifs/hashing.go index 976202bb9..6c6c0ce39 100644 --- a/internal/pkg/bifs/hashing.go +++ b/internal/pkg/bifs/hashing.go @@ -1,4 +1,4 @@ -package types +package bifs import ( "crypto/md5" diff --git a/internal/pkg/bifs/hashing_test.go b/internal/pkg/bifs/hashing_test.go new file mode 100644 index 000000000..3e736c0a0 --- /dev/null +++ b/internal/pkg/bifs/hashing_test.go @@ -0,0 +1,25 @@ +package bifs + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + +func TestBIF_md5(t *testing.T) { + input1 := mlrval.FromDeferredType("") + output := BIF_md5(input1) + stringval, ok := output.GetStringValue() + assert.True(t, ok) + assert.Equal(t, "d41d8cd98f00b204e9800998ecf8427e", stringval) + + input1 = mlrval.FromDeferredType("miller") + output = BIF_md5(input1) + stringval, ok = output.GetStringValue() + assert.True(t, ok) + assert.Equal(t, "f0af962ddbc82430e947390b2f3f6e49", stringval) +} + +// TODO: copy in more unit-test cases from existing regression-test data diff --git a/internal/pkg/bifs/math.go b/internal/pkg/bifs/math.go index 00e263633..e92712c4f 100644 --- a/internal/pkg/bifs/math.go +++ b/internal/pkg/bifs/math.go @@ -1,4 +1,4 @@ -package types +package bifs import ( "github.com/johnkerl/miller/internal/pkg/mlrval" diff --git a/internal/pkg/bifs/mathlib.go b/internal/pkg/bifs/mathlib.go index fa35782cf..f98fd7b51 100644 --- a/internal/pkg/bifs/mathlib.go +++ b/internal/pkg/bifs/mathlib.go @@ -2,7 +2,7 @@ // Go math-library functions // ================================================================ -package types +package bifs import ( "math" diff --git a/internal/pkg/bifs/random.go b/internal/pkg/bifs/random.go index 43d9997ed..b345e6d65 100644 --- a/internal/pkg/bifs/random.go +++ b/internal/pkg/bifs/random.go @@ -1,4 +1,4 @@ -package types +package bifs import ( "math" diff --git a/internal/pkg/bifs/regex.go b/internal/pkg/bifs/regex.go index c5f30da9e..448a62278 100644 --- a/internal/pkg/bifs/regex.go +++ b/internal/pkg/bifs/regex.go @@ -1,4 +1,4 @@ -package types +package bifs import ( "strings" diff --git a/internal/pkg/bifs/relative_time.go b/internal/pkg/bifs/relative_time.go index 9779fb305..8801017ef 100644 --- a/internal/pkg/bifs/relative_time.go +++ b/internal/pkg/bifs/relative_time.go @@ -1,4 +1,4 @@ -package types +package bifs import ( "fmt" diff --git a/internal/pkg/bifs/sort.go b/internal/pkg/bifs/sort.go index 4fe937030..d7d0ddafe 100644 --- a/internal/pkg/bifs/sort.go +++ b/internal/pkg/bifs/sort.go @@ -2,7 +2,7 @@ // For sorting // ================================================================ -package types +package bifs import ( "fmt" diff --git a/internal/pkg/bifs/bifs_test.go b/internal/pkg/bifs/sort_test.go similarity index 91% rename from internal/pkg/bifs/bifs_test.go rename to internal/pkg/bifs/sort_test.go index 67ee073c2..a47574c80 100644 --- a/internal/pkg/bifs/bifs_test.go +++ b/internal/pkg/bifs/sort_test.go @@ -1,12 +1,9 @@ -// ================================================================ -// Most Miller tests (thousands of them) are command-line-driven via -// mlr regtest. Here are some cases needing special focus. -// ================================================================ - -package types +package bifs import ( "testing" + + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ---------------------------------------------------------------- diff --git a/internal/pkg/bifs/stats.go b/internal/pkg/bifs/stats.go index c64b408c3..efcabec76 100644 --- a/internal/pkg/bifs/stats.go +++ b/internal/pkg/bifs/stats.go @@ -1,4 +1,4 @@ -package types +package bifs import ( "math" @@ -24,7 +24,7 @@ import ( // output = [m, b, math.sqrt(var_m), math.sqrt(var_b)] // ---------------------------------------------------------------- -func MlrvalGetVar(mn, msum, msum2 *mlrval.Mlrval) *mlrval.Mlrval { +func BIF_get_var(mn, msum, msum2 *mlrval.Mlrval) *mlrval.Mlrval { n, isInt := mn.GetIntValue() lib.InternalCodingErrorIf(!isInt) sum, isNumber := msum.GetNumericToFloatValue() @@ -46,8 +46,8 @@ func MlrvalGetVar(mn, msum, msum2 *mlrval.Mlrval) *mlrval.Mlrval { } // ---------------------------------------------------------------- -func MlrvalGetStddev(mn, msum, msum2 *mlrval.Mlrval) *mlrval.Mlrval { - mvar := MlrvalGetVar(mn, msum, msum2) +func BIF_get_stddev(mn, msum, msum2 *mlrval.Mlrval) *mlrval.Mlrval { + mvar := BIF_get_var(mn, msum, msum2) if mvar.IsVoid() { return mvar } @@ -55,8 +55,8 @@ func MlrvalGetStddev(mn, msum, msum2 *mlrval.Mlrval) *mlrval.Mlrval { } // ---------------------------------------------------------------- -func MlrvalGetMeanEB(mn, msum, msum2 *mlrval.Mlrval) *mlrval.Mlrval { - mvar := MlrvalGetVar(mn, msum, msum2) +func BIF_get_mean_EB(mn, msum, msum2 *mlrval.Mlrval) *mlrval.Mlrval { + mvar := BIF_get_var(mn, msum, msum2) if mvar.IsVoid() { return mvar } @@ -87,7 +87,7 @@ func MlrvalGetMeanEB(mn, msum, msum2 *mlrval.Mlrval) *mlrval.Mlrval { // = sumx2 - n mean^2 // ---------------------------------------------------------------- -func MlrvalGetSkewness(mn, msum, msum2, msum3 *mlrval.Mlrval) *mlrval.Mlrval { +func BIF_get_skewness(mn, msum, msum2, msum3 *mlrval.Mlrval) *mlrval.Mlrval { n, isInt := mn.GetIntValue() lib.InternalCodingErrorIf(!isInt) if n < 2 { @@ -124,7 +124,7 @@ func MlrvalGetSkewness(mn, msum, msum2, msum3 *mlrval.Mlrval) *mlrval.Mlrval { // = sumx4 - mean*(4 sumx3 - mean*(6 sumx2 - 3 n mean^2)) // ---------------------------------------------------------------- -func MlrvalGetKurtosis(mn, msum, msum2, msum3, msum4 *mlrval.Mlrval) *mlrval.Mlrval { +func BIF_get_kurtosis(mn, msum, msum2, msum3, msum4 *mlrval.Mlrval) *mlrval.Mlrval { n, isInt := mn.GetIntValue() lib.InternalCodingErrorIf(!isInt) if n < 2 { diff --git a/internal/pkg/bifs/strings.go b/internal/pkg/bifs/strings.go index 48b13e099..95a893427 100644 --- a/internal/pkg/bifs/strings.go +++ b/internal/pkg/bifs/strings.go @@ -1,4 +1,4 @@ -package types +package bifs import ( "regexp" @@ -217,10 +217,10 @@ func BIF_strip(input1 *mlrval.Mlrval) *mlrval.Mlrval { // ---------------------------------------------------------------- func BIF_collapse_whitespace(input1 *mlrval.Mlrval) *mlrval.Mlrval { - return MlrvalCollapseWhitespaceRegexp(input1, WhitespaceRegexp()) + return BIF_collapse_whitespace_regexp(input1, WhitespaceRegexp()) } -func MlrvalCollapseWhitespaceRegexp(input1 *mlrval.Mlrval, whitespaceRegexp *regexp.Regexp) *mlrval.Mlrval { +func BIF_collapse_whitespace_regexp(input1 *mlrval.Mlrval, whitespaceRegexp *regexp.Regexp) *mlrval.Mlrval { if input1.IsString() { return mlrval.FromString(whitespaceRegexp.ReplaceAllString(input1.AcquireStringValue(), " ")) } else { @@ -273,7 +273,7 @@ func BIF_capitalize(input1 *mlrval.Mlrval) *mlrval.Mlrval { // ---------------------------------------------------------------- func BIF_clean_whitespace(input1 *mlrval.Mlrval) *mlrval.Mlrval { return BIF_strip( - MlrvalCollapseWhitespaceRegexp( + BIF_collapse_whitespace_regexp( input1, WhitespaceRegexp(), ), ) diff --git a/internal/pkg/bifs/system.go b/internal/pkg/bifs/system.go index b35c5737d..974325b8e 100644 --- a/internal/pkg/bifs/system.go +++ b/internal/pkg/bifs/system.go @@ -1,4 +1,4 @@ -package types +package bifs import ( "os" diff --git a/internal/pkg/bifs/types.go b/internal/pkg/bifs/types.go index 912af535a..bd2347421 100644 --- a/internal/pkg/bifs/types.go +++ b/internal/pkg/bifs/types.go @@ -1,4 +1,4 @@ -package types +package bifs import ( "fmt" diff --git a/internal/pkg/dsl/cst/builtin_function_manager.go b/internal/pkg/dsl/cst/builtin_function_manager.go index d90eec326..2367349fd 100644 --- a/internal/pkg/dsl/cst/builtin_function_manager.go +++ b/internal/pkg/dsl/cst/builtin_function_manager.go @@ -263,7 +263,7 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { name: "!", class: FUNC_CLASS_BOOLEAN, help: `Logical negation.`, - unaryFunc: types.BIF_logicalnot, + unaryFunc: types.BIF_logical_NOT, }, { @@ -348,7 +348,7 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { name: "^^", class: FUNC_CLASS_BOOLEAN, help: `Logical XOR.`, - binaryFunc: types.BIF_logicalxor, + binaryFunc: types.BIF_logical_XOR, }, { diff --git a/internal/pkg/dsl/cst/builtin_functions.go b/internal/pkg/dsl/cst/builtin_functions.go index 760303019..4d4cc9d33 100644 --- a/internal/pkg/dsl/cst/builtin_functions.go +++ b/internal/pkg/dsl/cst/builtin_functions.go @@ -805,7 +805,7 @@ func (node *LogicalANDOperatorNode) Evaluate( return types.MLRVAL_ABSENT } - return mlrval.MlrvalLogicalAND(aout, bout) + return mlrval.BIF_logical_AND(aout, bout) } // ================================================================ @@ -852,7 +852,7 @@ func (node *LogicalOROperatorNode) Evaluate( if btype == types.MT_ABSENT { return types.MLRVAL_ABSENT } - return mlrval.MlrvalLogicalOR(aout, bout) + return mlrval.BIF_logical_OR(aout, bout) } // ================================================================ diff --git a/internal/pkg/dsl/cst/hofs.go b/internal/pkg/dsl/cst/hofs.go index 12cd187a4..ef2bd7fe4 100644 --- a/internal/pkg/dsl/cst/hofs.go +++ b/internal/pkg/dsl/cst/hofs.go @@ -616,11 +616,11 @@ func sortA( func sortANumerical(array []mlrval.Mlrval, reverse bool) { if !reverse { sort.Slice(array, func(i, j int) bool { - return mlrval.MlrvalLessThanAsBool(&array[i], &array[j]) + return mlrval.BIF_less_than_as_bool(&array[i], &array[j]) }) } else { sort.Slice(array, func(i, j int) bool { - return mlrval.MlrvalGreaterThanAsBool(&array[i], &array[j]) + return mlrval.BIF_greater_than_as_bool(&array[i], &array[j]) }) } } diff --git a/internal/pkg/transformers/fraction.go b/internal/pkg/transformers/fraction.go index 5074c627c..96cb03f22 100644 --- a/internal/pkg/transformers/fraction.go +++ b/internal/pkg/transformers/fraction.go @@ -267,7 +267,7 @@ func (tr *TransformerFraction) Transform( } denominator := sumsForGroup[fractionFieldName] - if !mlrval.MlrvalEqualsAsBool(value, tr.zero) { + if !mlrval.BIF_equals_as_bool(value, tr.zero) { outputValue = bifs.BIF_divide(numerator, denominator) outputValue = bifs.BIF_times(outputValue, tr.multiplier) } else { diff --git a/internal/pkg/transformers/utils/percentile-keeper.go b/internal/pkg/transformers/utils/percentile-keeper.go index 3219bccd7..a13127fb4 100644 --- a/internal/pkg/transformers/utils/percentile-keeper.go +++ b/internal/pkg/transformers/utils/percentile-keeper.go @@ -258,7 +258,7 @@ func getPercentileLinearlyInterpolated(array []*mlrval.Mlrval, n int, p float64) func (keeper *PercentileKeeper) sortIfNecessary() { if !keeper.sorted { sort.Slice(keeper.data, func(i, j int) bool { - return mlrval.MlrvalLessThanAsBool(keeper.data[i], keeper.data[j]) + return mlrval.BIF_less_than_as_bool(keeper.data[i], keeper.data[j]) }) keeper.sorted = true } diff --git a/internal/pkg/transformers/utils/stats1-accumulators.go b/internal/pkg/transformers/utils/stats1-accumulators.go index c0fee9e84..0bd4f9d07 100644 --- a/internal/pkg/transformers/utils/stats1-accumulators.go +++ b/internal/pkg/transformers/utils/stats1-accumulators.go @@ -440,7 +440,7 @@ func NewStats1MinAccumulator() IStats1Accumulator { } } func (acc *Stats1MinAccumulator) Ingest(value *mlrval.Mlrval) { - acc.min = mlrval.MlrvalBinaryMin(acc.min, value) + acc.min = mlrval.BIF_min_binary(acc.min, value) } func (acc *Stats1MinAccumulator) Emit() *mlrval.Mlrval { if acc.min.IsAbsent() { @@ -464,7 +464,7 @@ func NewStats1MaxAccumulator() IStats1Accumulator { } } func (acc *Stats1MaxAccumulator) Ingest(value *mlrval.Mlrval) { - acc.max = mlrval.MlrvalBinaryMax(acc.max, value) + acc.max = mlrval.BIF_max_binary(acc.max, value) } func (acc *Stats1MaxAccumulator) Emit() *mlrval.Mlrval { if acc.max.IsAbsent() { @@ -498,7 +498,7 @@ func (acc *Stats1VarAccumulator) Ingest(value *mlrval.Mlrval) { acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) } func (acc *Stats1VarAccumulator) Emit() *mlrval.Mlrval { - return mlrval.MlrvalGetVar(mlrval.MlrvalFromInt(acc.count), acc.sum, acc.sum2) + return mlrval.BIF_get_var(mlrval.MlrvalFromInt(acc.count), acc.sum, acc.sum2) } func (acc *Stats1VarAccumulator) Reset() { acc.count = 0 @@ -527,7 +527,7 @@ func (acc *Stats1StddevAccumulator) Ingest(value *mlrval.Mlrval) { acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) } func (acc *Stats1StddevAccumulator) Emit() *mlrval.Mlrval { - return mlrval.MlrvalGetStddev(mlrval.MlrvalFromInt(acc.count), acc.sum, acc.sum2) + return mlrval.BIF_get_stddev(mlrval.MlrvalFromInt(acc.count), acc.sum, acc.sum2) } func (acc *Stats1StddevAccumulator) Reset() { acc.count = 0 @@ -557,7 +557,7 @@ func (acc *Stats1MeanEBAccumulator) Ingest(value *mlrval.Mlrval) { } func (acc *Stats1MeanEBAccumulator) Emit() *mlrval.Mlrval { mcount := mlrval.MlrvalFromInt(acc.count) - return mlrval.MlrvalGetMeanEB(mcount, acc.sum, acc.sum2) + return mlrval.BIF_get_mean_EB(mcount, acc.sum, acc.sum2) } func (acc *Stats1MeanEBAccumulator) Reset() { acc.count = 0 @@ -591,7 +591,7 @@ func (acc *Stats1SkewnessAccumulator) Ingest(value *mlrval.Mlrval) { } func (acc *Stats1SkewnessAccumulator) Emit() *mlrval.Mlrval { mcount := mlrval.MlrvalFromInt(acc.count) - return mlrval.MlrvalGetSkewness(mcount, acc.sum, acc.sum2, acc.sum3) + return mlrval.BIF_get_skewness(mcount, acc.sum, acc.sum2, acc.sum3) } func (acc *Stats1SkewnessAccumulator) Reset() { acc.count = 0 @@ -630,7 +630,7 @@ func (acc *Stats1KurtosisAccumulator) Ingest(value *mlrval.Mlrval) { } func (acc *Stats1KurtosisAccumulator) Emit() *mlrval.Mlrval { mcount := mlrval.MlrvalFromInt(acc.count) - return mlrval.MlrvalGetKurtosis(mcount, acc.sum, acc.sum2, acc.sum3, acc.sum4) + return mlrval.BIF_get_kurtosis(mcount, acc.sum, acc.sum2, acc.sum3, acc.sum4) } func (acc *Stats1KurtosisAccumulator) Reset() { acc.count = 0 diff --git a/internal/pkg/types-parked/mlrval_collections.go b/internal/pkg/types-parked/mlrval_collections.go index d12f5b017..3cd7037bd 100644 --- a/internal/pkg/types-parked/mlrval_collections.go +++ b/internal/pkg/types-parked/mlrval_collections.go @@ -575,18 +575,18 @@ func BsearchMlrvalArrayForDescendingInsert( return 0 } - if MlrvalGreaterThanAsBool(value, (*array)[0]) { + if BIF_greater_than_as_bool(value, (*array)[0]) { return 0 } - if MlrvalLessThanAsBool(value, (*array)[hi]) { + if BIF_less_than_as_bool(value, (*array)[hi]) { return size } for lo < hi { middleElement := (*array)[mid] - if MlrvalEqualsAsBool(value, middleElement) { + if BIF_equals_as_bool(value, middleElement) { return mid - } else if MlrvalGreaterThanAsBool(value, middleElement) { + } else if BIF_greater_than_as_bool(value, middleElement) { hi = mid newmid = (hi + lo) / 2 } else { @@ -594,9 +594,9 @@ func BsearchMlrvalArrayForDescendingInsert( newmid = (hi + lo) / 2 } if mid == newmid { - if MlrvalGreaterThanOrEqualsAsBool(value, (*array)[lo]) { + if BIF_greater_than_or_equals_as_bool(value, (*array)[lo]) { return lo - } else if MlrvalGreaterThanOrEqualsAsBool(value, (*array)[hi]) { + } else if BIF_greater_than_or_equals_as_bool(value, (*array)[hi]) { return hi } else { return hi + 1 @@ -622,18 +622,18 @@ func BsearchMlrvalArrayForAscendingInsert( return 0 } - if MlrvalLessThanAsBool(value, (*array)[0]) { + if BIF_less_than_as_bool(value, (*array)[0]) { return 0 } - if MlrvalGreaterThanAsBool(value, (*array)[hi]) { + if BIF_greater_than_as_bool(value, (*array)[hi]) { return size } for lo < hi { middleElement := (*array)[mid] - if MlrvalEqualsAsBool(value, middleElement) { + if BIF_equals_as_bool(value, middleElement) { return mid - } else if MlrvalLessThanAsBool(value, middleElement) { + } else if BIF_less_than_as_bool(value, middleElement) { hi = mid newmid = (hi + lo) / 2 } else { @@ -641,9 +641,9 @@ func BsearchMlrvalArrayForAscendingInsert( newmid = (hi + lo) / 2 } if mid == newmid { - if MlrvalLessThanOrEqualsAsBool(value, (*array)[lo]) { + if BIF_less_than_or_equals_as_bool(value, (*array)[lo]) { return lo - } else if MlrvalLessThanOrEqualsAsBool(value, (*array)[hi]) { + } else if BIF_less_than_or_equals_as_bool(value, (*array)[hi]) { return hi } else { return hi + 1 diff --git a/todo.txt b/todo.txt index 7eda5d923..6590403aa 100644 --- a/todo.txt +++ b/todo.txt @@ -4,6 +4,12 @@ PUNCHDOWN LIST * --ifs-regex & --ips-regex -- guessing is not safe as evidence by '.' and '|' * JIT next: o mlrmap copy + o krepl miller6 new doclink-comments + +* perf next: + o batchify source-quench + ! hash-records back off + ? further channelize (CSV-first focus) mlrval infer vs record-put ? * big-picture item @ Rmd (csv memes; and beyond); also webdoc intro page @@ -100,8 +106,9 @@ PUNCHDOWN LIST ---------------------------------------------------------------- * JIT refactor o x 'tens/dozens of UT cases' + o populate each bifs/X_test.go for each bifs/X.go from existing regression-test data m internal/pkg/mlrval - - UT-driven mlrval defer/infer + k UT-driven mlrval defer/infer - pkg/mlrval doc.go & README.md o internal/pkg/types k UT-driven mlrmap constructor refactor @@ -112,9 +119,8 @@ PUNCHDOWN LIST mlrmap.PutReference(key, value) k UT-driven DKVP-from-line - - UT-driven X-from-line: nidx, xtab, csvlite, csv - - UT-driven JSON-from-line - - defer channelizers to a separate PR + ~ UT-driven X-from-line: nidx, xtab, csvlite, csv + - UT-driven JSON-from-string o internal/pkg/output - basic mlrval.String() @@ -145,17 +151,16 @@ PUNCHDOWN LIST ---------------------------------------------------------------- * perf more - o see other branch o nim6 note on perf: baseline/cat; processor/pipelining; o update https://miller.readthedocs.io/en/latest/cpu/, and/or https://miller.readthedocs.io/en/latest/streaming-and-memory/, and/or make a new page w/ info on per o JIT: maasure - o pipelined reader - o batched reader separately - o buffered output + k pipelined reader + k batched reader separately + k buffered output o re-check hashed/unhashed and do so periodically - - env-var back door + - CLI switch - check with more than just abixy ---------------------------------------------------------------- From 6bcfe84ab41cc81d916333854a78b74083b60929 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Wed, 15 Dec 2021 09:26:40 -0500 Subject: [PATCH 08/19] iterate on bifs/collections_test.go --- .vimrc | 3 +- Makefile | 102 +-- cmd/mprof/main.go | 14 +- cmd/mprof2/main.go | 4 +- cmd/mprof3/main.go | 4 +- internal/pkg/auxents/repl/session.go | 2 +- internal/pkg/auxents/repl/verbs.go | 2 +- internal/pkg/bifs/booleans.go | 9 +- internal/pkg/bifs/collections.go | 628 ++++++++++-------- internal/pkg/bifs/collections_test.go | 29 +- internal/pkg/bifs/datetime.go | 20 +- internal/pkg/bifs/relative_time.go | 8 +- internal/pkg/bifs/strings.go | 18 +- internal/pkg/bifs/types.go | 14 +- internal/pkg/cli/option_parse.go | 5 +- internal/pkg/dsl/cst/emit_emitp.go | 54 +- internal/pkg/dsl/cst/emitf.go | 8 +- internal/pkg/dsl/cst/hofs.go | 14 +- internal/pkg/dsl/cst/root.go | 4 +- internal/pkg/dsl/cst/tee.go | 4 +- internal/pkg/input/pseudo_reader_gen.go | 2 +- internal/pkg/input/record_reader.go | 2 +- internal/pkg/input/record_reader_csv.go | 2 +- internal/pkg/input/record_reader_csvlite.go | 4 +- internal/pkg/input/record_reader_dkvp_nidx.go | 10 +- internal/pkg/input/record_reader_xtab.go | 4 +- internal/pkg/{types => mlrval}/mlrmap.go | 12 +- .../pkg/{types => mlrval}/mlrmap_accessors.go | 85 +-- .../mlrmap_accessors_test.go | 8 +- .../pkg/{types => mlrval}/mlrmap_new_test.go | 2 +- .../{constants.go => mlrval_constants.go} | 0 .../pkg/mlrval/{copy.go => mlrval_copy.go} | 0 .../mlrval/{format.go => mlrval_format.go} | 0 .../{format_test.go => mlrval_format_test.go} | 0 internal/pkg/mlrval/{get.go => mlrval_get.go} | 8 +- .../{get_test.go => mlrval_get_test.go} | 0 .../pkg/mlrval/{infer.go => mlrval_infer.go} | 0 internal/pkg/mlrval/{is.go => mlrval_is.go} | 0 .../mlrval/{is_test.go => mlrval_is_test.go} | 4 +- internal/pkg/mlrval/{new.go => mlrval_new.go} | 4 +- .../{new_test.go => mlrval_new_test.go} | 8 +- .../mlrval/{output.go => mlrval_output.go} | 0 .../{output_test.go => mlrval_output_test.go} | 0 .../pkg/mlrval/{type.go => mlrval_type.go} | 20 +- internal/pkg/mlrval/tbf.go | 13 - internal/pkg/output/record_writer.go | 4 +- internal/pkg/output/record_writer_csv.go | 4 +- internal/pkg/output/record_writer_csvlite.go | 4 +- internal/pkg/output/record_writer_dkvp.go | 4 +- internal/pkg/output/record_writer_json.go | 14 +- internal/pkg/output/record_writer_markdown.go | 4 +- internal/pkg/output/record_writer_nidx.go | 4 +- internal/pkg/output/record_writer_pprint.go | 10 +- internal/pkg/output/record_writer_xtab.go | 8 +- internal/pkg/runtime/stack.go | 8 +- internal/pkg/runtime/state.go | 8 +- internal/pkg/transformers/altkv.go | 2 +- internal/pkg/transformers/clean-whitespace.go | 4 +- internal/pkg/transformers/count.go | 6 +- internal/pkg/transformers/cut.go | 6 +- internal/pkg/transformers/gap.go | 4 +- internal/pkg/transformers/histogram.go | 4 +- internal/pkg/transformers/join.go | 2 +- .../transformers/most-or-least-frequent.go | 2 +- internal/pkg/transformers/nest.go | 10 +- internal/pkg/transformers/regularize.go | 2 +- .../pkg/transformers/remove-empty-columns.go | 2 +- internal/pkg/transformers/reorder.go | 4 +- internal/pkg/transformers/reshape.go | 12 +- internal/pkg/transformers/seqgen.go | 2 +- internal/pkg/transformers/stats1.go | 14 +- internal/pkg/transformers/stats2.go | 4 +- internal/pkg/transformers/step.go | 16 +- internal/pkg/transformers/template.go | 2 +- internal/pkg/transformers/top.go | 2 +- internal/pkg/transformers/uniq.go | 10 +- internal/pkg/transformers/unsparsify.go | 2 +- .../transformers/utils/stats2-accumulators.go | 20 +- internal/pkg/types/context.go | 6 +- .../{types-parked => types}/mlrval_typing.go | 16 +- scripts/time-big-files | 6 + test/cases/verb-fill-empty/x | 1 - todo.txt | 7 + 83 files changed, 746 insertions(+), 628 deletions(-) rename internal/pkg/{types => mlrval}/mlrmap.go (96%) rename internal/pkg/{types => mlrval}/mlrmap_accessors.go (92%) rename internal/pkg/{types => mlrval}/mlrmap_accessors_test.go (87%) rename internal/pkg/{types => mlrval}/mlrmap_new_test.go (95%) rename internal/pkg/mlrval/{constants.go => mlrval_constants.go} (100%) rename internal/pkg/mlrval/{copy.go => mlrval_copy.go} (100%) rename internal/pkg/mlrval/{format.go => mlrval_format.go} (100%) rename internal/pkg/mlrval/{format_test.go => mlrval_format_test.go} (100%) rename internal/pkg/mlrval/{get.go => mlrval_get.go} (94%) rename internal/pkg/mlrval/{get_test.go => mlrval_get_test.go} (100%) rename internal/pkg/mlrval/{infer.go => mlrval_infer.go} (100%) rename internal/pkg/mlrval/{is.go => mlrval_is.go} (100%) rename internal/pkg/mlrval/{is_test.go => mlrval_is_test.go} (98%) rename internal/pkg/mlrval/{new.go => mlrval_new.go} (98%) rename internal/pkg/mlrval/{new_test.go => mlrval_new_test.go} (96%) rename internal/pkg/mlrval/{output.go => mlrval_output.go} (100%) rename internal/pkg/mlrval/{output_test.go => mlrval_output_test.go} (100%) rename internal/pkg/mlrval/{type.go => mlrval_type.go} (93%) rename internal/pkg/{types-parked => types}/mlrval_typing.go (83%) delete mode 100644 test/cases/verb-fill-empty/x diff --git a/.vimrc b/.vimrc index 562ce7c7f..aa4890dc9 100644 --- a/.vimrc +++ b/.vimrc @@ -4,4 +4,5 @@ map \r :w:!clear;echo Building ...; echo; make tests-in-order "map \f :w:!clear;echo Building ...; echo; make mlrmap-tests "map \f :w:!clear;echo Building ...; echo; make input-tests "map \f :w:!clear;echo Building ...; echo; make mlrval-format-test -map \f :w:!clear;echo Building ...; echo; make bifs-tests +"map \f :w:!clear;echo Building ...; echo; make bifs-tests +map \f :w:!clear;echo Building ...; echo; make bifs-collections-test diff --git a/Makefile b/Makefile index 4c8de0225..3c3bd31bc 100644 --- a/Makefile +++ b/Makefile @@ -36,58 +36,78 @@ lib-unbackslash-test: go test internal/pkg/lib/unbackslash_test.go internal/pkg/lib/unbackslash.go lib_regex_test: go test internal/pkg/lib/regex_test.go internal/pkg/lib/regex.go -lib_tests: lib_unbackslash_test lib_regex_test +lib-tests: + go test github.com/johnkerl/miller/internal/pkg/lib/... mlrval-new-test: go test internal/pkg/mlrval/new_test.go \ - internal/pkg/mlrval/type.go \ - internal/pkg/mlrval/constants.go \ - internal/pkg/mlrval/new.go \ - internal/pkg/mlrval/infer.go + internal/pkg/mlrval/mlrval_type.go \ + internal/pkg/mlrval/mlrval_constants.go \ + internal/pkg/mlrval/mlrval_new.go \ + internal/pkg/mlrval/mlrval_infer.go mlrval-is-test: go test internal/pkg/mlrval/is_test.go \ - internal/pkg/mlrval/type.go \ - internal/pkg/mlrval/constants.go \ - internal/pkg/mlrval/new.go \ - internal/pkg/mlrval/infer.go \ - internal/pkg/mlrval/is.go + internal/pkg/mlrval/mlrval_type.go \ + internal/pkg/mlrval/mlrval_constants.go \ + internal/pkg/mlrval/mlrval_new.go \ + internal/pkg/mlrval/mlrval_infer.go \ + internal/pkg/mlrval/mlrval_is.go mlrval-get-test: go test internal/pkg/mlrval/get_test.go \ - internal/pkg/mlrval/type.go \ - internal/pkg/mlrval/constants.go \ - internal/pkg/mlrval/new.go \ - internal/pkg/mlrval/infer.go \ - internal/pkg/mlrval/is.go \ - internal/pkg/mlrval/get.go + internal/pkg/mlrval/mlrval_type.go \ + internal/pkg/mlrval/mlrval_constants.go \ + internal/pkg/mlrval/mlrval_new.go \ + internal/pkg/mlrval/mlrval_infer.go \ + internal/pkg/mlrval/mlrval_is.go \ + internal/pkg/mlrval/mlrval_get.go mlrval-output-test: go test internal/pkg/mlrval/output_test.go \ - internal/pkg/mlrval/type.go \ - internal/pkg/mlrval/constants.go \ - internal/pkg/mlrval/new.go \ - internal/pkg/mlrval/infer.go \ - internal/pkg/mlrval/is.go \ - internal/pkg/mlrval/get.go \ - internal/pkg/mlrval/output.go \ - internal/pkg/mlrval/format.go + internal/pkg/mlrval/mlrval_type.go \ + internal/pkg/mlrval/mlrval_constants.go \ + internal/pkg/mlrval/mlrval_new.go \ + internal/pkg/mlrval/mlrval_infer.go \ + internal/pkg/mlrval/mlrval_is.go \ + internal/pkg/mlrval/mlrval_get.go \ + internal/pkg/mlrval/mlrval_output.go \ + internal/pkg/mlrval/mlrval_format.go mlrval-format-test: go test internal/pkg/mlrval/format_test.go \ - internal/pkg/mlrval/type.go \ - internal/pkg/mlrval/constants.go \ - internal/pkg/mlrval/new.go \ - internal/pkg/mlrval/infer.go \ - internal/pkg/mlrval/is.go \ - internal/pkg/mlrval/get.go \ - internal/pkg/mlrval/output.go \ - internal/pkg/mlrval/format.go -mlrval-tests: mlrval-new-test mlrval-is-test mlrval-get-test mlrval-output-test mlrval-format-test + internal/pkg/mlrval/mlrval_type.go \ + internal/pkg/mlrval/mlrval_constants.go \ + internal/pkg/mlrval/mlrval_new.go \ + internal/pkg/mlrval/mlrval_infer.go \ + internal/pkg/mlrval/mlrval_is.go \ + internal/pkg/mlrval/mlrval_get.go \ + internal/pkg/mlrval/mlrval_output.go \ + internal/pkg/mlrval/mlrval_format.go +mlrval-tests: + go test github.com/johnkerl/miller/internal/pkg/mlrval/... mlrmap-new-test: - go test internal/pkg/types/mlrmap_new_test.go \ - internal/pkg/types/mlrmap.go + go test internal/pkg/mlrval/mlrmap_new_test.go \ + internal/pkg/mlrval/mlrmap.go \ + internal/pkg/mlrval/mlrval_type.go \ + internal/pkg/mlrval/mlrval_constants.go \ + internal/pkg/mlrval/mlrval_new.go \ + internal/pkg/mlrval/mlrval_infer.go \ + internal/pkg/mlrval/mlrval_is.go \ + internal/pkg/mlrval/mlrval_get.go \ + internal/pkg/mlrval/mlrval_output.go \ + internal/pkg/mlrval/mlrval_format.go mlrmap-accessors-test: - go test internal/pkg/types/mlrmap_accessors_test.go \ - internal/pkg/types/mlrmap.go \ - internal/pkg/types/mlrmap_accessors.go + go test internal/pkg/mlrval/mlrmap_accessors_test.go \ + internal/pkg/mlrval/mlrmap.go \ + internal/pkg/mlrval/mlrmap_accessors.go \ + internal/pkg/mlrval/mlrval_type.go \ + internal/pkg/mlrval/mlrval_constants.go \ + internal/pkg/mlrval/mlrval_new.go \ + internal/pkg/mlrval/mlrval_copy.go \ + internal/pkg/mlrval/mlrval_infer.go \ + internal/pkg/mlrval/mlrval_is.go \ + internal/pkg/mlrval/mlrval_get.go \ + internal/pkg/mlrval/mlrval_output.go \ + internal/pkg/mlrval/mlrval_format.go + mlrmap-tests: mlrmap-new-test mlrmap-accessors-test input-dkvp-test: @@ -106,7 +126,7 @@ bifs-bits-test: internal/pkg/bifs/arithmetic.go \ internal/pkg/bifs/bits.go bifs-collections-test: - go test internal/pkg/bifs/bits_test.go \ + go test internal/pkg/bifs/collections_test.go \ internal/pkg/bifs/base.go \ internal/pkg/bifs/arithmetic.go \ internal/pkg/bifs/collections.go @@ -124,9 +144,9 @@ bifs-sort-test: bifs-tests: bifs-arithmetic-test bifs-bits-test bifs-collections-test bifs-hashing-test bifs-sort-test #mlrval_functions_test: -# go test internal/pkg/types/mlrval_functions_test.go $(ls internal/pkg/types/*.go | grep -v test) +# go test internal/pkg/mlrval/mlrval_functions_test.go $(ls internal/pkg/types/*.go | grep -v test) #mlrval_format_test: -# go test internal/pkg/types/mlrval_format_test.go $(ls internal/pkg/types/*.go|grep -v test) +# go test internal/pkg/mlrval/mlrval_format_test.go $(ls internal/pkg/types/*.go|grep -v test) tests-in-order: mlrval-tests mlrmap-tests input-tests bifs-tests diff --git a/cmd/mprof/main.go b/cmd/mprof/main.go index 1bd60daab..8f0fc8805 100644 --- a/cmd/mprof/main.go +++ b/cmd/mprof/main.go @@ -243,8 +243,8 @@ func (reader *RecordReaderDKVPNonPipelined) processHandle( func (reader *RecordReaderDKVPNonPipelined) recordFromDKVPLine( line string, -) *types.Mlrmap { - record := types.NewMlrmap() +) *mlrval.Mlrmap { + record := mlrval.NewMlrmap() var pairs []string if reader.readerOptions.IFSRegex == nil { // e.g. --no-ifs-regex @@ -379,8 +379,8 @@ func (reader *RecordReaderDKVPListPipelined) processHandle( func (reader *RecordReaderDKVPListPipelined) recordFromDKVPLine( line string, -) *types.Mlrmap { - record := types.NewMlrmap() +) *mlrval.Mlrmap { + record := mlrval.NewMlrmap() var pairs []string if reader.readerOptions.IFSRegex == nil { // e.g. --no-ifs-regex @@ -531,8 +531,8 @@ func (reader *RecordReaderDKVPChanPipelined) processHandle( func (reader *RecordReaderDKVPChanPipelined) recordFromDKVPLine( line string, -) *types.Mlrmap { - record := types.NewMlrmap() +) *mlrval.Mlrmap { + record := mlrval.NewMlrmap() var pairs []string if reader.readerOptions.IFSRegex == nil { // e.g. --no-ifs-regex @@ -643,7 +643,7 @@ func NewRecordWriterDKVP2(writerOptions *cli.TWriterOptions) (*RecordWriterDKVP2 } func (writer *RecordWriterDKVP2) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ostream *bufio.Writer, ) { // End of record stream: nothing special for this output format diff --git a/cmd/mprof2/main.go b/cmd/mprof2/main.go index b680e48a9..2addd5d2f 100644 --- a/cmd/mprof2/main.go +++ b/cmd/mprof2/main.go @@ -255,8 +255,8 @@ func (reader *RecordReaderDKVPChanPipelined) processHandle( func (reader *RecordReaderDKVPChanPipelined) recordFromDKVPLine( line string, -) *types.Mlrmap { - record := types.NewMlrmap() +) *mlrval.Mlrmap { + record := mlrval.NewMlrmap() var pairs []string if reader.readerOptions.IFSRegex == nil { // e.g. --no-ifs-regex diff --git a/cmd/mprof3/main.go b/cmd/mprof3/main.go index fe30e8c52..b1f6c7631 100644 --- a/cmd/mprof3/main.go +++ b/cmd/mprof3/main.go @@ -269,8 +269,8 @@ func (reader *RecordReaderDKVPChanPipelined) getRecordBatch( func (reader *RecordReaderDKVPChanPipelined) recordFromDKVPLine( line string, -) *types.Mlrmap { - record := types.NewMlrmapAsRecord() +) *mlrval.Mlrmap { + record := mlrval.NewMlrmapAsRecord() var pairs []string if reader.readerOptions.IFSRegex == nil { // e.g. --no-ifs-regex diff --git a/internal/pkg/auxents/repl/session.go b/internal/pkg/auxents/repl/session.go index ee5043745..01f7829f7 100644 --- a/internal/pkg/auxents/repl/session.go +++ b/internal/pkg/auxents/repl/session.go @@ -59,7 +59,7 @@ func NewRepl( } // $* is the empty map {} until/unless the user opens a file and reads records from it. - inrec := types.NewMlrmapAsRecord() + inrec := mlrval.NewMlrmapAsRecord() // NR is 0, etc until/unless the user opens a file and reads records from it. context := types.NewContext() diff --git a/internal/pkg/auxents/repl/verbs.go b/internal/pkg/auxents/repl/verbs.go index 5197dc8b6..f889069ac 100644 --- a/internal/pkg/auxents/repl/verbs.go +++ b/internal/pkg/auxents/repl/verbs.go @@ -608,7 +608,7 @@ func handleWrite(repl *Repl, args []string) bool { return true } -func writeRecord(repl *Repl, outrec *types.Mlrmap) { +func writeRecord(repl *Repl, outrec *mlrval.Mlrmap) { if outrec != nil { // E.g. '{"req": {"method": "GET", "path": "/api/check"}}' becomes // req.method=GET,req.path=/api/check. diff --git a/internal/pkg/bifs/booleans.go b/internal/pkg/bifs/booleans.go index 8228750fc..37402907e 100644 --- a/internal/pkg/bifs/booleans.go +++ b/internal/pkg/bifs/booleans.go @@ -7,7 +7,6 @@ package bifs import ( "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" - "github.com/johnkerl/miller/internal/pkg/types" ) // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -234,8 +233,8 @@ func cmp_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - func eq_b_aa(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { - a := input1.AcquireArrayValue().([]mlrval.Mlrval) - b := input2.AcquireArrayValue().([]mlrval.Mlrval) + a := input1.AcquireArrayValue() + b := input2.AcquireArrayValue() // Different-length arrays are not equal if len(a) != len(b) { @@ -260,10 +259,10 @@ func ne_b_aa(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - func eq_b_mm(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { - return mlrval.FromBool(input1.AcquireMapValue().(types.Mlrmap).Equals(input2.AcquireMapValue().(types.Mlrmap))) + return mlrval.FromBool(input1.AcquireMapValue().Equals(input2.AcquireMapValue())) } func ne_b_mm(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { - return mlrval.FromBool(!input1.AcquireArrayValue().(types.Mlrmap).Equals(input2.AcquireArrayValue().(types.Mlrmap))) + return mlrval.FromBool(!input1.AcquireMapValue().Equals(input2.AcquireArrayValue())) } // We get a Golang "initialization loop" due to recursive depth computation diff --git a/internal/pkg/bifs/collections.go b/internal/pkg/bifs/collections.go index 649e1d66f..7754ff6d8 100644 --- a/internal/pkg/bifs/collections.go +++ b/internal/pkg/bifs/collections.go @@ -7,11 +7,10 @@ import ( "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" - "github.com/johnkerl/miller/internal/pkg/types" ) // ================================================================ -// Map/array count. Scalars (including strings) have length 1. +// Map/array count. Scalars (including strings) have length 1; strlen is for string length. func BIF_length(input1 *mlrval.Mlrval) *mlrval.Mlrval { switch input1.Type() { case mlrval.MT_ERROR: @@ -21,10 +20,12 @@ func BIF_length(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mlrval.FromInt(0) break case mlrval.MT_ARRAY: - return mlrval.FromInt(int(len(input1.AcquireArrayValue().([]mlrval.Mlrval)))) + arrayval := input1.AcquireArrayValue() + return mlrval.FromInt(int(len(arrayval))) break case mlrval.MT_MAP: - return mlrval.FromInt(int(input1.AcquireArrayValue().(types.Mlrmap).FieldCount)) + mapval := input1.AcquireMapValue() + return mlrval.FromInt(int(mapval.FieldCount)) break } return mlrval.FromInt(1) @@ -33,7 +34,8 @@ func BIF_length(input1 *mlrval.Mlrval) *mlrval.Mlrval { // ================================================================ func depth_from_array(input1 *mlrval.Mlrval) *mlrval.Mlrval { maxChildDepth := 0 - for _, child := range input1.AcquireArrayValue().([]mlrval.Mlrval) { + arrayval := input1.AcquireArrayValue() + for _, child := range arrayval { childDepth := BIF_depth(&child) lib.InternalCodingErrorIf(!childDepth.IsInt()) iChildDepth := int(childDepth.AcquireIntValue()) @@ -46,7 +48,8 @@ func depth_from_array(input1 *mlrval.Mlrval) *mlrval.Mlrval { func depth_from_map(input1 *mlrval.Mlrval) *mlrval.Mlrval { maxChildDepth := 0 - for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { + mapval := input1.AcquireMapValue() + for pe := mapval.Head; pe != nil; pe = pe.Next { child := pe.Value childDepth := BIF_depth(child) lib.InternalCodingErrorIf(!childDepth.IsInt()) @@ -89,7 +92,8 @@ func BIF_depth(input1 *mlrval.Mlrval) *mlrval.Mlrval { // ================================================================ func leafcount_from_array(input1 *mlrval.Mlrval) *mlrval.Mlrval { sumChildLeafCount := 0 - for _, child := range input1.AcquireArrayValue().([]mlrval.Mlrval) { + arrayval := input1.AcquireArrayValue() + for _, child := range arrayval { // Golang initialization loop if we do this :( // childLeafCount := BIF_leafcount(&child) @@ -109,7 +113,8 @@ func leafcount_from_array(input1 *mlrval.Mlrval) *mlrval.Mlrval { func leafcount_from_map(input1 *mlrval.Mlrval) *mlrval.Mlrval { sumChildLeafCount := 0 - for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { + mapval := input1.AcquireMapValue() + for pe := mapval.Head; pe != nil; pe = pe.Next { child := pe.Value // Golang initialization loop if we do this :( @@ -159,13 +164,14 @@ func has_key_in_array(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input2.IsInt() { return mlrval.ERROR } - _, ok := UnaliasArrayIndex(&input1.AcquireArrayValue().([]mlrval.Mlrval), input2.AcquireIntValue()) + arrayval := input1.AcquireArrayValue() + _, ok := unaliasArrayIndex(&arrayval, input2.AcquireIntValue()) return mlrval.FromBool(ok) } func has_key_in_map(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if input2.IsString() || input2.IsInt() { - return mlrval.FromBool(input1.AcquireArrayValue().(types.Mlrmap).Has(input2.String())) + return mlrval.FromBool(input1.AcquireMapValue().Has(input2.String())) } else { return mlrval.ERROR } @@ -189,8 +195,8 @@ func BIF_mapselect(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { if !mlrvals[0].IsMap() { return mlrval.ERROR } - oldmap := mlrvals[0].AcquireArrayValue().(types.Mlrmap) - newMap := NewMlrmap() + oldmap := mlrvals[0].AcquireMapValue() + newMap := mlrval.NewMlrmap() newKeys := make(map[string]bool) for _, selectArg := range mlrvals[1:] { @@ -199,7 +205,7 @@ func BIF_mapselect(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { } else if selectArg.IsInt() { newKeys[selectArg.String()] = true } else if selectArg.IsArray() { - for _, element := range selectArg.AcquireArrayValue().([]mlrval.Mlrval) { + for _, element := range selectArg.AcquireArrayValue() { if element.IsString() { newKeys[element.AcquireStringValue()] = true } else { @@ -222,101 +228,101 @@ func BIF_mapselect(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { return mlrval.FromMap(newMap) } -// ---------------------------------------------------------------- -func BIF_mapexcept(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { - if len(mlrvals) < 1 { - return mlrval.ERROR - } - if !mlrvals[0].IsMap() { - return mlrval.ERROR - } - newMap := mlrvals[0].AcquireArrayValue().(types.Mlrmap).Copy() +//// ---------------------------------------------------------------- +//func BIF_mapexcept(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { +// if len(mlrvals) < 1 { +// return mlrval.ERROR +// } +// if !mlrvals[0].IsMap() { +// return mlrval.ERROR +// } +// newMap := mlrvals[0].AcquireMapValue().Copy() +// +// for _, exceptArg := range mlrvals[1:] { +// if exceptArg.IsString() { +// newMap.Remove(exceptArg.AcquireStringValue()) +// } else if exceptArg.IsInt() { +// newMap.Remove(exceptArg.String()) +// } else if exceptArg.IsArray() { +// for _, element := range exceptArg.AcquireArrayValue() { +// if element.IsString() { +// newMap.Remove(element.AcquireStringValue()) +// } else { +// return mlrval.ERROR +// } +// } +// } else { +// return mlrval.ERROR +// } +// } +// +// return mlrval.FromMap(newMap) +//} - for _, exceptArg := range mlrvals[1:] { - if exceptArg.IsString() { - newMap.Remove(exceptArg.AcquireStringValue()) - } else if exceptArg.IsInt() { - newMap.Remove(exceptArg.String()) - } else if exceptArg.IsArray() { - for _, element := range exceptArg.AcquireArrayValue().([]mlrval.Mlrval) { - if element.IsString() { - newMap.Remove(element.AcquireStringValue()) - } else { - return mlrval.ERROR - } - } - } else { - return mlrval.ERROR - } - } +//// ---------------------------------------------------------------- +//func BIF_mapsum(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { +// if len(mlrvals) == 0 { +// return mlrval.FromEmptyMap() +// } +// if len(mlrvals) == 1 { +// return mlrvals[0] +// } +// if mlrvals[0].Type() != MT_MAP { +// return mlrval.ERROR +// } +// newMap := mlrvals[0].AcquireMapValue().Copy() +// +// for _, otherMapArg := range mlrvals[1:] { +// if otherMapArg.Type() != MT_MAP { +// return mlrval.ERROR +// } +// +// for pe := otherMapArg.AcquireMapValue().Head; pe != nil; pe = pe.Next { +// newMap.PutCopy(pe.Key, pe.Value) +// } +// } +// +// return mlrval.FromMap(newMap) +//} - return mlrval.FromMap(newMap) -} - -// ---------------------------------------------------------------- -func BIF_mapsum(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { - if len(mlrvals) == 0 { - return mlrval.FromEmptyMap() - } - if len(mlrvals) == 1 { - return mlrvals[0] - } - if mlrvals[0].Type() != MT_MAP { - return mlrval.ERROR - } - newMap := mlrvals[0].AcquireArrayValue().(types.Mlrmap).Copy() - - for _, otherMapArg := range mlrvals[1:] { - if otherMapArg.Type() != MT_MAP { - return mlrval.ERROR - } - - for pe := otherMapArg.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { - newMap.PutCopy(pe.Key, pe.Value) - } - } - - return mlrval.FromMap(newMap) -} - -// ---------------------------------------------------------------- -func BIF_mapdiff(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { - if len(mlrvals) == 0 { - return mlrval.FromEmptyMap() - } - if len(mlrvals) == 1 { - return mlrvals[0] - } - if mlrvals[0].Type() != MT_MAP { - return mlrval.ERROR - } - newMap := mlrvals[0].AcquireArrayValue().(types.Mlrmap).Copy() - - for _, otherMapArg := range mlrvals[1:] { - if otherMapArg.Type() != MT_MAP { - return mlrval.ERROR - } - - for pe := otherMapArg.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { - newMap.Remove(pe.Key) - } - } - - return mlrval.FromMap(newMap) -} +//// ---------------------------------------------------------------- +//func BIF_mapdiff(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { +// if len(mlrvals) == 0 { +// return mlrval.FromEmptyMap() +// } +// if len(mlrvals) == 1 { +// return mlrvals[0] +// } +// if mlrvals[0].Type() != MT_MAP { +// return mlrval.ERROR +// } +// newMap := mlrvals[0].AcquireMapValue().Copy() +// +// for _, otherMapArg := range mlrvals[1:] { +// if otherMapArg.Type() != MT_MAP { +// return mlrval.ERROR +// } +// +// for pe := otherMapArg.AcquireMapValue().Head; pe != nil; pe = pe.Next { +// newMap.Remove(pe.Key) +// } +// } +// +// return mlrval.FromMap(newMap) +//} // ================================================================ // joink([1,2,3], ",") -> "1,2,3" // joink({"a":3,"b":4,"c":5}, ",") -> "a,b,c" func BIF_joink(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { - if input2.Type() != MT_STRING { + if !input2.IsString() { return mlrval.ERROR } fieldSeparator := input2.AcquireStringValue() if input1.IsMap() { var buffer bytes.Buffer - for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { + for pe := input1.AcquireMapValue().Head; pe != nil; pe = pe.Next { buffer.WriteString(pe.Key) if pe.Next != nil { buffer.WriteString(fieldSeparator) @@ -327,7 +333,7 @@ func BIF_joink(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { } else if input1.IsArray() { var buffer bytes.Buffer - for i := range input1.AcquireArrayValue().([]mlrval.Mlrval) { + for i := range input1.AcquireArrayValue() { if i > 0 { buffer.WriteString(fieldSeparator) } @@ -345,7 +351,7 @@ func BIF_joink(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { // joinv([3,4,5], ",") -> "3,4,5" // joinv({"a":3,"b":4,"c":5}, ",") -> "3,4,5" func BIF_joinv(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { - if input2.Type() != MT_STRING { + if !input2.IsString() { return mlrval.ERROR } fieldSeparator := input2.AcquireStringValue() @@ -353,7 +359,7 @@ func BIF_joinv(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsMap() { var buffer bytes.Buffer - for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { + for pe := input1.AcquireMapValue().Head; pe != nil; pe = pe.Next { buffer.WriteString(pe.Value.String()) if pe.Next != nil { buffer.WriteString(fieldSeparator) @@ -364,7 +370,7 @@ func BIF_joinv(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { } else if input1.IsArray() { var buffer bytes.Buffer - for i, element := range input1.AcquireArrayValue().([]mlrval.Mlrval) { + for i, element := range input1.AcquireArrayValue() { if i > 0 { buffer.WriteString(fieldSeparator) } @@ -381,11 +387,11 @@ func BIF_joinv(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { // joinkv([3,4,5], "=", ",") -> "1=3,2=4,3=5" // joinkv({"a":3,"b":4,"c":5}, "=", ",") -> "a=3,b=4,c=5" func BIF_joinkv(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { - if input2.Type() != MT_STRING { + if !input2.IsString() { return mlrval.ERROR } pairSeparator := input2.AcquireStringValue() - if input3.Type() != MT_STRING { + if !input3.IsString() { return mlrval.ERROR } fieldSeparator := input3.AcquireStringValue() @@ -393,7 +399,7 @@ func BIF_joinkv(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsMap() { var buffer bytes.Buffer - for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { + for pe := input1.AcquireMapValue().Head; pe != nil; pe = pe.Next { buffer.WriteString(pe.Key) buffer.WriteString(pairSeparator) buffer.WriteString(pe.Value.String()) @@ -406,7 +412,7 @@ func BIF_joinkv(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { } else if input1.IsArray() { var buffer bytes.Buffer - for i, element := range input1.AcquireArrayValue().([]mlrval.Mlrval) { + for i, element := range input1.AcquireArrayValue() { if i > 0 { buffer.WriteString(fieldSeparator) } @@ -428,16 +434,16 @@ func BIF_splitkv(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { return mlrval.ERROR } - if input2.Type() != MT_STRING { + if !input2.IsString() { return mlrval.ERROR } pairSeparator := input2.AcquireStringValue() - if input3.Type() != MT_STRING { + if !input3.IsString() { return mlrval.ERROR } fieldSeparator := input3.AcquireStringValue() - output := mlrval.FromEmptyMap() + output := mlrval.FromMap(mlrval.NewMlrmap()) fields := lib.SplitString(input1.AcquireStringValue(), fieldSeparator) for i, field := range fields { @@ -445,11 +451,11 @@ func BIF_splitkv(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if len(pair) == 1 { key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up value := mlrval.FromInferredType(pair[0]) - output.AcquireArrayValue().(types.Mlrmap).PutReference(key, value) + output.AcquireMapValue().PutReference(key, value) } else if len(pair) == 2 { key := pair[0] value := mlrval.FromInferredType(pair[1]) - output.AcquireArrayValue().(types.Mlrmap).PutReference(key, value) + output.AcquireMapValue().PutReference(key, value) } else { lib.InternalCodingErrorIf(true) } @@ -463,16 +469,16 @@ func BIF_splitkvx(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { return mlrval.ERROR } - if input2.Type() != MT_STRING { + if !input2.IsString() { return mlrval.ERROR } pairSeparator := input2.AcquireStringValue() - if input3.Type() != MT_STRING { + if !input3.IsString() { return mlrval.ERROR } fieldSeparator := input3.AcquireStringValue() - output := mlrval.FromEmptyMap() + output := mlrval.FromMap(mlrval.NewMlrmap()) fields := lib.SplitString(input1.AcquireStringValue(), fieldSeparator) for i, field := range fields { @@ -480,11 +486,11 @@ func BIF_splitkvx(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if len(pair) == 1 { key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up value := mlrval.FromString(pair[0]) - output.AcquireArrayValue().(types.Mlrmap).PutReference(key, value) + output.AcquireMapValue().PutReference(key, value) } else if len(pair) == 2 { key := pair[0] value := mlrval.FromString(pair[1]) - output.AcquireArrayValue().(types.Mlrmap).PutReference(key, value) + output.AcquireMapValue().PutReference(key, value) } else { lib.InternalCodingErrorIf(true) } @@ -499,17 +505,17 @@ func BIF_splitnv(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { return mlrval.ERROR } - if input2.Type() != MT_STRING { + if !input2.IsString() { return mlrval.ERROR } - output := mlrval.FromEmptyMap() + output := mlrval.FromMap(mlrval.NewMlrmap()) fields := lib.SplitString(input1.AcquireStringValue(), input2.AcquireStringValue()) for i, field := range fields { key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up value := mlrval.FromInferredType(field) - output.AcquireArrayValue().(types.Mlrmap).PutReference(key, value) + output.AcquireMapValue().PutReference(key, value) } return output @@ -521,17 +527,17 @@ func BIF_splitnvx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { return mlrval.ERROR } - if input2.Type() != MT_STRING { + if !input2.IsString() { return mlrval.ERROR } - output := mlrval.FromEmptyMap() + output := mlrval.FromMap(mlrval.NewMlrmap()) fields := lib.SplitString(input1.AcquireStringValue(), input2.AcquireStringValue()) for i, field := range fields { key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up value := mlrval.FromString(field) - output.AcquireArrayValue().(types.Mlrmap).PutReference(key, value) + output.AcquireMapValue().PutReference(key, value) } return output @@ -550,14 +556,14 @@ func BIF_splita(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { fields := lib.SplitString(input1.AcquireStringValue(), fieldSeparator) - output := NewSizedMlrvalArray(int(len(fields))) + arrayval := make([]mlrval.Mlrval, len(fields)) for i, field := range fields { value := mlrval.FromInferredType(field) - output.AcquireArrayValue().([]mlrval.Mlrval)[i] = *value + arrayval[i] = *value } - return output + return mlrval.FromArray(arrayval) } // ---------------------------------------------------------------- @@ -567,7 +573,7 @@ func BIF_splitax(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { return mlrval.ERROR } - if input2.Type() != MT_STRING { + if !input2.IsString() { return mlrval.ERROR } input := input1.AcquireStringValue() @@ -581,33 +587,35 @@ func BIF_splitax(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { func bif_splitax_helper(input string, separator string) *mlrval.Mlrval { fields := lib.SplitString(input, separator) - output := NewSizedMlrvalArray(int(len(fields))) + arrayval := make([]mlrval.Mlrval, len(fields)) for i, field := range fields { - output.AcquireArrayValue().([]mlrval.Mlrval)[i] = *mlrval.FromString(field) + arrayval[i] = *mlrval.FromString(field) } - return output + return mlrval.FromArray(arrayval) } // ---------------------------------------------------------------- func BIF_get_keys(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsMap() { // TODO: make a ReferenceFrom with comments - output := NewSizedMlrvalArray(input1.AcquireArrayValue().(types.Mlrmap).FieldCount) + mapval := input1.AcquireMapValue() + arrayval := make([]mlrval.Mlrval, mapval.FieldCount) i := 0 - for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { - output.AcquireArrayValue().([]mlrval.Mlrval)[i] = *mlrval.FromString(pe.Key) + for pe := mapval.Head; pe != nil; pe = pe.Next { + arrayval[i] = *mlrval.FromString(pe.Key) i++ } - return output + return mlrval.FromArray(arrayval) } else if input1.IsArray() { - output := NewSizedMlrvalArray(int(len(input1.AcquireArrayValue().([]mlrval.Mlrval)))) - for i := range input1.AcquireArrayValue().([]mlrval.Mlrval) { - output.AcquireArrayValue().([]mlrval.Mlrval)[i] = *mlrval.FromInt(int(i + 1)) // Miller user-space indices are 1-up + inputarrayval := input1.AcquireArrayValue() + arrayval := make([]mlrval.Mlrval, len(inputarrayval)) + for i := range inputarrayval { + arrayval[i] = *mlrval.FromInt(int(i + 1)) // Miller user-space indices are 1-up } - return output + return mlrval.FromArray(arrayval) } else { return mlrval.ERROR @@ -617,168 +625,228 @@ func BIF_get_keys(input1 *mlrval.Mlrval) *mlrval.Mlrval { // ---------------------------------------------------------------- func BIF_get_values(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsMap() { - // TODO: make a ReferenceFrom with commenbs - output := NewSizedMlrvalArray(input1.AcquireArrayValue().(types.Mlrmap).FieldCount) + mapval := input1.AcquireMapValue() + arrayval := make([]mlrval.Mlrval, mapval.FieldCount) i := 0 - for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { - output.AcquireArrayValue().([]mlrval.Mlrval)[i] = *pe.Value.Copy() + for pe := mapval.Head; pe != nil; pe = pe.Next { + arrayval[i] = *pe.Value.Copy() i++ } - return output + return mlrval.FromArray(arrayval) } else if input1.IsArray() { - output := NewSizedMlrvalArray(int(len(input1.AcquireArrayValue().([]mlrval.Mlrval)))) - for i, value := range input1.AcquireArrayValue().([]mlrval.Mlrval) { - output.AcquireArrayValue().([]mlrval.Mlrval)[i] = *value.Copy() + inputarrayval := input1.AcquireArrayValue() + arrayval := make([]mlrval.Mlrval, len(inputarrayval)) + for i, value := range inputarrayval { + arrayval[i] = *value.Copy() } - return output + return mlrval.FromArray(arrayval) } else { return mlrval.ERROR } } -// ---------------------------------------------------------------- -func BIF_append(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { - if input1.Type() != MT_ARRAY { - return mlrval.ERROR - } +//// ---------------------------------------------------------------- +//func BIF_append(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { +// if !input1.IsArray() { +// return mlrval.ERROR +// } +// +// output := input1.Copy() +// output.ArrayAppend(input2.Copy()) +// return output +//} - output := input1.Copy() - output.ArrayAppend(input2.Copy()) - return output +//// ---------------------------------------------------------------- +//// First argumemnt is prefix. +//// Second argument is delimiter. +//// Third argument is map or array. +//// flatten("a", ".", {"b": { "c": 4 }}) is {"a.b.c" : 4}. +//// flatten("", ".", {"a": { "b": 3 }}) is {"a.b" : 3}. +//func BIF_flatten(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { +// if input3.IsMap() || input3.IsArray() { +// if !input1.IsString() && input1.Type() != MT_VOID { +// return mlrval.ERROR +// } +// prefix := input1.AcquireStringValue() +// if !input2.IsString() { +// return mlrval.ERROR +// } +// delimiter := input2.AcquireStringValue() +// +// retval := input3.FlattenToMap(prefix, delimiter) +// return &retval +// } else { +// return input3 +// } +//} +// +//// flatten($*, ".") is the same as flatten("", ".", $*) +//func BIF_flatten_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { +// return BIF_flatten(mlrval.VOID, input2, input1) +//} +// +//// ---------------------------------------------------------------- +//// First argument is a map. +//// Second argument is a delimiter string. +//// unflatten({"a.b.c", ".") is {"a": { "b": { "c": 4}}}. +//func BIF_unflatten(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { +// if !input2.IsString() { +// return mlrval.ERROR +// } +// if input1.Type() != MT_MAP { +// return input1 +// } +// oldmap := input1.AcquireMapValue() +// separator := input2.AcquireStringValue() +// newmap := oldmap.CopyUnflattened(separator) +// return mlrval.FromMapReferenced(newmap) +//} +// +//// ---------------------------------------------------------------- +//// Converts maps with "1", "2", ... keys into arrays. Recurses nested data structures. +//func BIF_arrayify(input1 *mlrval.Mlrval) *mlrval.Mlrval { +// if input1.IsMap() { +// if input1.AcquireMapValue().IsEmpty() { +// return input1 +// } +// +// convertible := true +// i := 0 +// for pe := input1.AcquireMapValue().Head; pe != nil; pe = pe.Next { +// sval := strconv.Itoa(i + 1) // Miller user-space indices are 1-up +// i++ +// if pe.Key != sval { +// convertible = false +// } +// pe.Value = BIF_arrayify(pe.Value) +// } +// +// if convertible { +// mapval := input1.AcquireMapValue() +// arrayval := make([]mlrval.Mlrval, input1.AcquireMapValue().FieldCount) +// i := 0 +// for pe := mapval.Head; pe != nil; pe = pe.Next { +// AcquireArrayValue()[i] = *pe.Value.Copy() +// i++ +// } +// return mlrval.FromArrayReference(AcquireArrayValue()) +// +// } else { +// return input1 +// } +// +// } else if input1.IsArray() { +// // TODO: comment (or rethink) that this modifies its inputs!! +// output := input1.Copy() +// for i := range input1.AcquireArrayValue() { +// output.AcquireArrayValue()[i] = *BIF_arrayify(&output.AcquireArrayValue()[i]) +// } +// return output +// +// } else { +// return input1 +// } +//} +// +//// ---------------------------------------------------------------- +//func BIF_json_parse(input1 *mlrval.Mlrval) *mlrval.Mlrval { +// if input1.IsVoid() { +// return input1 +// } else if !input1.IsString() { +// return mlrval.ERROR +// } else { +// output := mlrval.FromPending() +// err := output.UnmarshalJSON([]byte(input1.AcquireStringValue())) +// if err != nil { +// return mlrval.ERROR +// } +// return &output +// } +//} +// +//func BIF_json_stringify_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { +// outputBytes, err := input1.MarshalJSON(JSON_SINGLE_LINE, false) +// if err != nil { +// return mlrval.ERROR +// } else { +// return mlrval.FromString(string(outputBytes)) +// } +//} +// +//func BIF_json_stringify_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { +// var jsonFormatting TJSONFormatting = JSON_SINGLE_LINE +// useMultiline, ok := input2.GetBoolValue() +// if !ok { +// return mlrval.ERROR +// } +// if useMultiline { +// jsonFormatting = JSON_MULTILINE +// } +// +// outputBytes, err := input1.MarshalJSON(jsonFormatting, false) +// if err != nil { +// return mlrval.ERROR +// } else { +// return mlrval.FromString(string(outputBytes)) +// } +//} + +func unaliasArrayIndex(array *[]mlrval.Mlrval, mindex int) (int, bool) { + n := int(len(*array)) + return unaliasArrayLengthIndex(n, mindex) } -// ---------------------------------------------------------------- -// First argumemnt is prefix. -// Second argument is delimiter. -// Third argument is map or array. -// flatten("a", ".", {"b": { "c": 4 }}) is {"a.b.c" : 4}. -// flatten("", ".", {"a": { "b": 3 }}) is {"a.b" : 3}. -func BIF_flatten(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { - if input3.IsMap() || input3.IsArray() { - if input1.Type() != MT_STRING && input1.Type() != MT_VOID { - return mlrval.ERROR - } - prefix := input1.AcquireStringValue() - if input2.Type() != MT_STRING { - return mlrval.ERROR - } - delimiter := input2.AcquireStringValue() +// Input "mindex" is a Miller DSL array index. These are 1-up, so 1..n where n +// is the length of the array. Also, -n..-1 are aliases to 1..n. 0 is never a +// valid index. +// +// Output "zindex" is a Golang array index. These are 0-up, so 0..(n-1). +// +// The second return value indicates whether the Miller index is in-bounds. +// Even if it's out of bounds, while the second return value is false, the +// first return is correctly de-aliased. E.g. if the array has length 5 and the +// mindex is 8, zindex is 7 and valid=false. This is so in array-slice +// operations like 'v = myarray[2:8]' the callsite can hand back slots 2-5 of +// the array (which is the same way Python handles beyond-the-end indexing). - retval := input3.FlattenToMap(prefix, delimiter) - return &retval +// Examples with n = 5: +// +// mindex zindex ok +// -7 -2 false +// -6 -1 false +// -5 0 true +// -4 1 true +// -3 2 true +// -2 3 true +// -1 4 true +// 0 -1 false +// 1 0 true +// 2 1 true +// 3 2 true +// 4 3 true +// 5 4 true +// 6 5 false +// 7 6 false + +func unaliasArrayLengthIndex(n int, mindex int) (int, bool) { + if 1 <= mindex { + zindex := mindex - 1 + if mindex <= n { // in bounds + return zindex, true + } else { // out of bounds + return zindex, false + } + } else if mindex <= -1 { + zindex := mindex + n + if -n <= mindex { // in bounds + return zindex, true + } else { // out of bounds + return zindex, false + } } else { - return input3 - } -} - -// flatten($*, ".") is the same as flatten("", ".", $*) -func BIF_flatten_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { - return BIF_flatten(mlrval.VOID, input2, input1) -} - -// ---------------------------------------------------------------- -// First argument is a map. -// Second argument is a delimiter string. -// unflatten({"a.b.c", ".") is {"a": { "b": { "c": 4}}}. -func BIF_unflatten(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { - if input2.Type() != MT_STRING { - return mlrval.ERROR - } - if input1.Type() != MT_MAP { - return input1 - } - oldmap := input1.AcquireArrayValue().(types.Mlrmap) - separator := input2.AcquireStringValue() - newmap := oldmap.CopyUnflattened(separator) - return mlrval.FromMapReferenced(newmap) -} - -// ---------------------------------------------------------------- -// Converts maps with "1", "2", ... keys into arrays. Recurses nested data structures. -func BIF_arrayify(input1 *mlrval.Mlrval) *mlrval.Mlrval { - if input1.IsMap() { - if input1.AcquireArrayValue().(types.Mlrmap).IsEmpty() { - return input1 - } - - convertible := true - i := 0 - for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { - sval := strconv.Itoa(i + 1) // Miller user-space indices are 1-up - i++ - if pe.Key != sval { - convertible = false - } - pe.Value = BIF_arrayify(pe.Value) - } - - if convertible { - AcquireArrayValue().([]mlrval.Mlrval) := make([]mlrval.Mlrval, input1.AcquireArrayValue().(types.Mlrmap).FieldCount) - i := 0 - for pe := input1.AcquireArrayValue().(types.Mlrmap).Head; pe != nil; pe = pe.Next { - AcquireArrayValue().([]mlrval.Mlrval)[i] = *pe.Value.Copy() - i++ - } - return mlrval.FromArrayReference(AcquireArrayValue().([]mlrval.Mlrval)) - - } else { - return input1 - } - - } else if input1.IsArray() { - // TODO: comment (or rethink) that this modifies its inputs!! - output := input1.Copy() - for i := range input1.AcquireArrayValue().([]mlrval.Mlrval) { - output.AcquireArrayValue().([]mlrval.Mlrval)[i] = *BIF_arrayify(&output.AcquireArrayValue().([]mlrval.Mlrval)[i]) - } - return output - - } else { - return input1 - } -} - -// ---------------------------------------------------------------- -func BIF_json_parse(input1 *mlrval.Mlrval) *mlrval.Mlrval { - if input1.IsVoid() { - return input1 - } else if input1.Type() != MT_STRING { - return mlrval.ERROR - } else { - output := mlrval.FromPending() - err := output.UnmarshalJSON([]byte(input1.AcquireStringValue())) - if err != nil { - return mlrval.ERROR - } - return &output - } -} - -func BIF_json_stringify_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { - outputBytes, err := input1.MarshalJSON(JSON_SINGLE_LINE, false) - if err != nil { - return mlrval.ERROR - } else { - return mlrval.FromString(string(outputBytes)) - } -} - -func BIF_json_stringify_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { - var jsonFormatting TJSONFormatting = JSON_SINGLE_LINE - useMultiline, ok := input2.GetBoolValue() - if !ok { - return mlrval.ERROR - } - if useMultiline { - jsonFormatting = JSON_MULTILINE - } - - outputBytes, err := input1.MarshalJSON(jsonFormatting, false) - if err != nil { - return mlrval.ERROR - } else { - return mlrval.FromString(string(outputBytes)) + // mindex is 0 + return -1, false } } diff --git a/internal/pkg/bifs/collections_test.go b/internal/pkg/bifs/collections_test.go index c3e479a25..08b25061b 100644 --- a/internal/pkg/bifs/collections_test.go +++ b/internal/pkg/bifs/collections_test.go @@ -16,13 +16,32 @@ func TestBIF_length(t *testing.T) { assert.Equal(t, 1, intval) } +func TestBIF_depth(t *testing.T) { + input1 := mlrval.FromInt(123) + output := BIF_depth(input1) + intval, ok := output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 0, intval) + + mapval := mlrval.NewMlrmap() + mapval.PutReference("key", mlrval.FromString("value")) + input1 = mlrval.FromMap(mapval) + output = BIF_depth(input1) + intval, ok = output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 1, intval) + + arrayval := make([]mlrval.Mlrval, 1) + arrayval[0] = *mlrval.FromString("value") + input1 = mlrval.FromArray(arrayval) + output = BIF_depth(input1) + intval, ok = output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 1, intval) +} + // TODO: copy in more unit-test cases from existing regression-test data -// func BIF_length(input1 *mlrval.Mlrval) *mlrval.Mlrval -// func depth_from_array(input1 *mlrval.Mlrval) *mlrval.Mlrval -// func depth_from_map(input1 *mlrval.Mlrval) *mlrval.Mlrval -// func depth_from_scalar(input1 *mlrval.Mlrval) *mlrval.Mlrval -// func BIF_depth(input1 *mlrval.Mlrval) *mlrval.Mlrval // func leafcount_from_array(input1 *mlrval.Mlrval) *mlrval.Mlrval // func leafcount_from_map(input1 *mlrval.Mlrval) *mlrval.Mlrval // func leafcount_from_scalar(input1 *mlrval.Mlrval) *mlrval.Mlrval diff --git a/internal/pkg/bifs/datetime.go b/internal/pkg/bifs/datetime.go index af39fc971..aaac7555b 100644 --- a/internal/pkg/bifs/datetime.go +++ b/internal/pkg/bifs/datetime.go @@ -102,7 +102,7 @@ func BIF_sec2localtime_ternary(input1, input2, input3 *mlrval.Mlrval) *mlrval.Ml return mlrval.ERROR } - locationString, isString := input3.GetString() + locationString, isString := input3.GetStringValue() if !isString { return mlrval.ERROR } @@ -181,7 +181,7 @@ func BIF_strftime_local_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { } func BIF_strftime_local_ternary(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { - locationString, isString := input3.GetString() + locationString, isString := input3.GetStringValue() if !isString { return mlrval.ERROR } @@ -202,7 +202,7 @@ func strftimeHelper(input1, input2 *mlrval.Mlrval, doLocal bool, location *time. if !ok { return mlrval.ERROR } - if input2.Type() != MT_STRING { + if !input2.IsString() { return mlrval.ERROR } @@ -301,10 +301,10 @@ func init() { // Argument 1 is formatted date string like "2021-03-04 02:59:50". // Argument 2 is format string like "%Y-%m-%d %H:%M:%S". func BIF_strptime(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { - if input1.Type() != MT_STRING { + if !input1.IsString() { return mlrval.ERROR } - if input2.Type() != MT_STRING { + if !input2.IsString() { return mlrval.ERROR } timeString := input1.AcquireStringValue() @@ -319,10 +319,10 @@ func BIF_strptime(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { } func BIF_strptime_local_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { - if input1.Type() != MT_STRING { + if !input1.IsString() { return mlrval.ERROR } - if input2.Type() != MT_STRING { + if !input2.IsString() { return mlrval.ERROR } timeString := input1.AcquireStringValue() @@ -337,13 +337,13 @@ func BIF_strptime_local_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { } func BIF_strptime_local_ternary(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { - if input1.Type() != MT_STRING { + if !input1.IsString() { return mlrval.ERROR } - if input2.Type() != MT_STRING { + if !input2.IsString() { return mlrval.ERROR } - if input3.Type() != MT_STRING { + if !input3.IsString() { return mlrval.ERROR } diff --git a/internal/pkg/bifs/relative_time.go b/internal/pkg/bifs/relative_time.go index 8801017ef..4f09130d1 100644 --- a/internal/pkg/bifs/relative_time.go +++ b/internal/pkg/bifs/relative_time.go @@ -9,7 +9,7 @@ import ( ) func BIF_dhms2sec(input1 *mlrval.Mlrval) *mlrval.Mlrval { - if input1.Type() != MT_STRING { + if !input1.IsString() { return mlrval.ERROR } var d, h, m, s int @@ -57,7 +57,7 @@ func BIF_dhms2sec(input1 *mlrval.Mlrval) *mlrval.Mlrval { } func BIF_dhms2fsec(input1 *mlrval.Mlrval) *mlrval.Mlrval { - if input1.Type() != MT_STRING { + if !input1.IsString() { return mlrval.ERROR } @@ -107,7 +107,7 @@ func BIF_dhms2fsec(input1 *mlrval.Mlrval) *mlrval.Mlrval { } func BIF_hms2sec(input1 *mlrval.Mlrval) *mlrval.Mlrval { - if input1.Type() != MT_STRING { + if !input1.IsString() { return mlrval.ERROR } if input1.AcquireStringValue() == "" { @@ -131,7 +131,7 @@ func BIF_hms2sec(input1 *mlrval.Mlrval) *mlrval.Mlrval { } func BIF_hms2fsec(input1 *mlrval.Mlrval) *mlrval.Mlrval { - if input1.Type() != MT_STRING { + if !input1.IsString() { return mlrval.ERROR } diff --git a/internal/pkg/bifs/strings.go b/internal/pkg/bifs/strings.go index 95a893427..5f4dea02c 100644 --- a/internal/pkg/bifs/strings.go +++ b/internal/pkg/bifs/strings.go @@ -83,7 +83,7 @@ func BIF_substr_1_up(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { lowerMindex := input2.AcquireIntValue() upperMindex := strlen - if input3.IsEmpty() { + if input3.IsVoid() { // Keep strlen } else if !input3.IsInt() { return mlrval.ERROR @@ -92,8 +92,8 @@ func BIF_substr_1_up(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { } // Convert from negative-aliased 1-up to positive-only 0-up - m, mok := UnaliasArrayLengthIndex(strlen, lowerMindex) - n, nok := UnaliasArrayLengthIndex(strlen, upperMindex) + m, mok := unaliasArrayLengthIndex(strlen, lowerMindex) + n, nok := unaliasArrayLengthIndex(strlen, upperMindex) if !mok || !nok { return mlrval.VOID @@ -133,7 +133,7 @@ func BIF_substr_0_up(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { } upperMindex := strlen - if input3.IsEmpty() { + if input3.IsVoid() { // Keep strlen } else if !input3.IsInt() { return mlrval.ERROR @@ -146,8 +146,8 @@ func BIF_substr_0_up(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { } // Convert from negative-aliased 1-up to positive-only 0-up - m, mok := UnaliasArrayLengthIndex(strlen, lowerMindex) - n, nok := UnaliasArrayLengthIndex(strlen, upperMindex) + m, mok := unaliasArrayLengthIndex(strlen, lowerMindex) + n, nok := unaliasArrayLengthIndex(strlen, upperMindex) if !mok || !nok { return mlrval.VOID @@ -294,7 +294,7 @@ func fmtnum_is(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return mlrval.ERROR } formatString := input2.AcquireStringValue() - formatter, err := GetMlrvalFormatter(formatString) + formatter, err := mlrval.GetFormatter(formatString) if err != nil { return mlrval.ERROR } @@ -307,7 +307,7 @@ func fmtnum_fs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return mlrval.ERROR } formatString := input2.AcquireStringValue() - formatter, err := GetMlrvalFormatter(formatString) + formatter, err := mlrval.GetFormatter(formatString) if err != nil { return mlrval.ERROR } @@ -320,7 +320,7 @@ func fmtnum_bs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return mlrval.ERROR } formatString := input2.AcquireStringValue() - formatter, err := GetMlrvalFormatter(formatString) + formatter, err := mlrval.GetFormatter(formatString) if err != nil { return mlrval.ERROR } diff --git a/internal/pkg/bifs/types.go b/internal/pkg/bifs/types.go index bd2347421..e11021fa3 100644 --- a/internal/pkg/bifs/types.go +++ b/internal/pkg/bifs/types.go @@ -157,7 +157,7 @@ func BIF_is_empty(input1 *mlrval.Mlrval) *mlrval.Mlrval { } } func BIF_is_emptymap(input1 *mlrval.Mlrval) *mlrval.Mlrval { - return mlrval.FromBool(input1.IsMap() && input1.AcquireArrayValue().(types.Mlrmap).IsEmpty()) + return mlrval.FromBool(input1.IsMap() && input1.AcquireMapValue().IsEmpty()) } func BIF_is_float(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mlrval.FromBool(input1.IsFloat()) @@ -172,7 +172,7 @@ func BIF_is_array(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mlrval.FromBool(input1.IsArray()) } func BIF_is_nonemptymap(input1 *mlrval.Mlrval) *mlrval.Mlrval { - return mlrval.FromBool(input1.IsMap() && input1.AcquireArrayValue().(types.Mlrmap).FieldCount != 0) + return mlrval.FromBool(input1.IsMap() && input1.AcquireMapValue().FieldCount != 0) } func BIF_is_notempty(input1 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsVoid() { @@ -188,13 +188,13 @@ func BIF_is_notempty(input1 *mlrval.Mlrval) *mlrval.Mlrval { } } func BIF_is_notmap(input1 *mlrval.Mlrval) *mlrval.Mlrval { - return mlrval.FromBool(input1.Type() != MT_MAP) + return mlrval.FromBool(!input1.IsMap()) } func BIF_is_notarray(input1 *mlrval.Mlrval) *mlrval.Mlrval { - return mlrval.FromBool(input1.Type() != MT_ARRAY) + return mlrval.FromBool(!input1.IsArray()) } func BIF_is_notnull(input1 *mlrval.Mlrval) *mlrval.Mlrval { - return mlrval.FromBool(input1.Type() != MT_ABSENT && input1.Type() != MT_VOID) + return mlrval.FromBool(!input1.IsAbsent() && !input1.IsVoid()) } func BIF_is_null(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mlrval.FromBool(input1.IsAbsent() || input1.IsVoid()) @@ -203,10 +203,10 @@ func BIF_is_numeric(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mlrval.FromBool(input1.IsInt() || input1.IsFloat()) } func BIF_is_present(input1 *mlrval.Mlrval) *mlrval.Mlrval { - return mlrval.FromBool(input1.Type() != MT_ABSENT) + return mlrval.FromBool(!input1.IsAbsent()) } func BIF_is_string(input1 *mlrval.Mlrval) *mlrval.Mlrval { - return mlrval.FromBool(input1.IsString() || input1.IsVoid()) + return mlrval.FromBool(input1.IsStringOrVoid()) } // ---------------------------------------------------------------- diff --git a/internal/pkg/cli/option_parse.go b/internal/pkg/cli/option_parse.go index 665803e18..68a680dbc 100644 --- a/internal/pkg/cli/option_parse.go +++ b/internal/pkg/cli/option_parse.go @@ -16,7 +16,6 @@ import ( "github.com/johnkerl/miller/internal/pkg/colorizer" "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" - "github.com/johnkerl/miller/internal/pkg/types" ) // FinalizeReaderOptions does a few things. @@ -2585,7 +2584,7 @@ In general --no-hash-records is faster, and is the default. For specific use-cas data having many fields, and many of them being processed during a given processing run, --hash-records might offer a slight performance benefit.`, parser: func(args []string, argc int, pargi *int, options *TOptions) { - types.HashRecords(true) + mlrval.HashRecords(true) *pargi += 1 }, }, @@ -2594,7 +2593,7 @@ data having many fields, and many of them being processed during a given process name: "--no-hash-records", help: `See --hash-records.`, parser: func(args []string, argc int, pargi *int, options *TOptions) { - types.HashRecords(false) + mlrval.HashRecords(false) *pargi += 1 }, }, diff --git a/internal/pkg/dsl/cst/emit_emitp.go b/internal/pkg/dsl/cst/emit_emitp.go index e3b8ec743..fb4af1797 100644 --- a/internal/pkg/dsl/cst/emit_emitp.go +++ b/internal/pkg/dsl/cst/emit_emitp.go @@ -55,7 +55,7 @@ import ( // Shared by emit and emitp type tEmitToRedirectFunc func( - newrec *types.Mlrmap, + newrec *mlrval.Mlrmap, state *runtime.State, ) error @@ -374,7 +374,7 @@ func (node *EmitXStatementNode) executeNonIndexedNonLashedEmit( valueAsMap := value.GetMap() // nil if not a map if valueAsMap == nil { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() newrec.PutCopy(names[i], value) err := node.emitToRedirectFunc(newrec, state) if err != nil { @@ -384,7 +384,7 @@ func (node *EmitXStatementNode) executeNonIndexedNonLashedEmit( } else { recurse := valueAsMap.IsNested() if !recurse { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for pe := valueAsMap.Head; pe != nil; pe = pe.Next { newrec.PutCopy(pe.Key, pe.Value) } @@ -417,7 +417,7 @@ func (node *EmitXStatementNode) executeNonIndexedNonLashedEmitP( if value.IsAbsent() { continue } - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() newrec.PutCopy(names[i], value) err := node.emitToRedirectFunc(newrec, state) if err != nil { @@ -441,7 +441,7 @@ func (node *EmitXStatementNode) executeNonIndexedLashedEmit( leadingValueAsMap := values[0].GetMap() if leadingValueAsMap == nil { // Emit a record like a=1,b=2 - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for i, value := range values { if value.IsAbsent() { continue @@ -463,7 +463,7 @@ func (node *EmitXStatementNode) executeNonIndexedLashedEmit( valueAsMap := value.GetMap() // nil if not a map if valueAsMap == nil { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() newrec.PutCopy(names[i], value) err := node.emitToRedirectFunc(newrec, state) if err != nil { @@ -473,7 +473,7 @@ func (node *EmitXStatementNode) executeNonIndexedLashedEmit( } else { recurse := valueAsMap.IsNested() if !recurse { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for pe := valueAsMap.Head; pe != nil; pe = pe.Next { newrec.PutCopy(pe.Key, pe.Value) } @@ -503,7 +503,7 @@ func (node *EmitXStatementNode) executeNonIndexedLashedEmitP( values []*mlrval.Mlrval, state *runtime.State, ) error { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for i, value := range values { if value.IsAbsent() { @@ -523,7 +523,7 @@ func (node *EmitXStatementNode) executeIndexed( state *runtime.State, ) error { - emittableMaps := make([]*types.Mlrmap, len(values)) + emittableMaps := make([]*mlrval.Mlrmap, len(values)) for i, value := range values { if value.IsAbsent() { return nil @@ -551,7 +551,7 @@ func (node *EmitXStatementNode) executeIndexed( if !node.isEmitP { if !node.isLashed { return node.executeIndexedNonLashedEmitAux( - types.NewMlrmapAsRecord(), + mlrval.NewMlrmapAsRecord(), names, emittableMaps, indices, @@ -559,7 +559,7 @@ func (node *EmitXStatementNode) executeIndexed( ) } else { return node.executeIndexedLashedEmitAux( - types.NewMlrmapAsRecord(), + mlrval.NewMlrmapAsRecord(), names, emittableMaps, indices, @@ -569,7 +569,7 @@ func (node *EmitXStatementNode) executeIndexed( } else { if !node.isLashed { return node.executeIndexedNonLashedEmitPAux( - types.NewMlrmapAsRecord(), + mlrval.NewMlrmapAsRecord(), names, emittableMaps, indices, @@ -577,7 +577,7 @@ func (node *EmitXStatementNode) executeIndexed( ) } else { return node.executeIndexedLashedEmitPAux( - types.NewMlrmapAsRecord(), + mlrval.NewMlrmapAsRecord(), names, emittableMaps, indices, @@ -651,9 +651,9 @@ func (node *EmitXStatementNode) executeIndexed( // * indices = ["b"] func (node *EmitXStatementNode) executeIndexedNonLashedEmitAux( - templateRecord *types.Mlrmap, + templateRecord *mlrval.Mlrmap, names []string, - emittableMaps []*types.Mlrmap, + emittableMaps []*mlrval.Mlrmap, indices []*mlrval.Mlrval, state *runtime.State, ) error { @@ -691,7 +691,7 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitAux( node.executeIndexedNonLashedEmitPAux( newrec, []string{names[i]}, - []*types.Mlrmap{valueAsMap}, + []*mlrval.Mlrmap{valueAsMap}, indices[1:], state, ) @@ -797,9 +797,9 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitAux( // o indices ["b"] func (node *EmitXStatementNode) executeIndexedLashedEmitAux( - templateRecord *types.Mlrmap, + templateRecord *mlrval.Mlrmap, names []string, - emittableMaps []*types.Mlrmap, + emittableMaps []*mlrval.Mlrmap, indices []*mlrval.Mlrval, state *runtime.State, ) error { @@ -815,7 +815,7 @@ func (node *EmitXStatementNode) executeIndexedLashedEmitAux( newrec.PutCopy(indexString, indexValue) nextLevelValues := make([]*mlrval.Mlrval, len(emittableMaps)) - nextLevelMaps := make([]*types.Mlrmap, len(emittableMaps)) + nextLevelMaps := make([]*mlrval.Mlrmap, len(emittableMaps)) for i, emittableMap := range emittableMaps { if emittableMap != nil { nextLevelValues[i] = emittableMaps[i].Get(pe.Key) @@ -866,9 +866,9 @@ func (node *EmitXStatementNode) executeIndexedLashedEmitAux( // Recurses over indices. func (node *EmitXStatementNode) executeIndexedNonLashedEmitPAux( - templateRecord *types.Mlrmap, + templateRecord *mlrval.Mlrmap, names []string, - emittableMaps []*types.Mlrmap, + emittableMaps []*mlrval.Mlrmap, indices []*mlrval.Mlrval, state *runtime.State, ) error { @@ -899,7 +899,7 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitPAux( node.executeIndexedNonLashedEmitPAux( newrec, []string{names[i]}, - []*types.Mlrmap{valueAsMap}, + []*mlrval.Mlrmap{valueAsMap}, indices[1:], state, ) @@ -912,9 +912,9 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitPAux( } func (node *EmitXStatementNode) executeIndexedLashedEmitPAux( - templateRecord *types.Mlrmap, + templateRecord *mlrval.Mlrmap, names []string, - emittableMaps []*types.Mlrmap, + emittableMaps []*mlrval.Mlrmap, indices []*mlrval.Mlrval, state *runtime.State, ) error { @@ -932,7 +932,7 @@ func (node *EmitXStatementNode) executeIndexedLashedEmitPAux( indexValueString := indexValue.String() nextLevels := make([]*mlrval.Mlrval, len(emittableMaps)) - nextLevelMaps := make([]*types.Mlrmap, len(emittableMaps)) + nextLevelMaps := make([]*mlrval.Mlrmap, len(emittableMaps)) for i, emittableMap := range emittableMaps { if emittableMap != nil { nextLevel := emittableMap.Get(indexValueString) @@ -980,7 +980,7 @@ func (node *EmitXStatementNode) executeIndexedLashedEmitPAux( // ---------------------------------------------------------------- func (node *EmitXStatementNode) emitRecordToRecordStream( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, state *runtime.State, ) error { // The output channel is always non-nil, except for the Miller REPL. @@ -994,7 +994,7 @@ func (node *EmitXStatementNode) emitRecordToRecordStream( // ---------------------------------------------------------------- func (node *EmitXStatementNode) emitRecordToFileOrPipe( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, state *runtime.State, ) error { redirectorTarget := node.redirectorTargetEvaluable.Evaluate(state) diff --git a/internal/pkg/dsl/cst/emitf.go b/internal/pkg/dsl/cst/emitf.go index 05a91d7a0..947b73d08 100644 --- a/internal/pkg/dsl/cst/emitf.go +++ b/internal/pkg/dsl/cst/emitf.go @@ -27,7 +27,7 @@ import ( // in the CST logic, to keep this parser/AST logic simpler. type tEmitFToRedirectFunc func( - newrec *types.Mlrmap, + newrec *mlrval.Mlrmap, state *runtime.State, ) error @@ -139,7 +139,7 @@ func (root *RootNode) BuildEmitFStatementNode(astNode *dsl.ASTNode) (IExecutable } func (node *EmitFStatementNode) Execute(state *runtime.State) (*BlockExitPayload, error) { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for i, emitfEvaluable := range node.emitfEvaluables { emitfName := node.emitfNames[i] emitfValue := emitfEvaluable.Evaluate(state) @@ -178,7 +178,7 @@ func getNameFromNamedNode(astNode *dsl.ASTNode, description string) (string, err // ---------------------------------------------------------------- func (node *EmitFStatementNode) emitfToRecordStream( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, state *runtime.State, ) error { // The output channel is always non-nil, except for the Miller REPL. @@ -192,7 +192,7 @@ func (node *EmitFStatementNode) emitfToRecordStream( // ---------------------------------------------------------------- func (node *EmitFStatementNode) emitfToFileOrPipe( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, state *runtime.State, ) error { redirectorTarget := node.redirectorTargetEvaluable.Evaluate(state) diff --git a/internal/pkg/dsl/cst/hofs.go b/internal/pkg/dsl/cst/hofs.go index ef2bd7fe4..d7e83bd66 100644 --- a/internal/pkg/dsl/cst/hofs.go +++ b/internal/pkg/dsl/cst/hofs.go @@ -141,7 +141,7 @@ func isNonAbsentOrDie(mlrval *mlrval.Mlrval, hofName string) *mlrval.Mlrval { // getKVPairForAccumulatorOrDie checks that a user-supplied accumulator value // for a map fold is indeed a single-element map. -func getKVPairForAccumulatorOrDie(mlrval *mlrval.Mlrval, hofName string) *types.Mlrmap { +func getKVPairForAccumulatorOrDie(mlrval *mlrval.Mlrval, hofName string) *mlrval.Mlrmap { kvPair := getKVPair(mlrval) if kvPair == nil { hofCheckDie(mlrval, hofName, "accumulator value must be a single-element map") @@ -151,7 +151,7 @@ func getKVPairForAccumulatorOrDie(mlrval *mlrval.Mlrval, hofName string) *types. // getKVPairForCallbackOrDie checks that a return value from a UDF for map // reduce/fold/apply is indeed a single-element map. -func getKVPairForCallbackOrDie(mlrval *mlrval.Mlrval, hofName string) *types.Mlrmap { +func getKVPairForCallbackOrDie(mlrval *mlrval.Mlrval, hofName string) *mlrval.Mlrmap { kvPair := getKVPair(mlrval) if kvPair == nil { hofCheckDie(mlrval, hofName, "second-argument function must return single-element map") @@ -173,7 +173,7 @@ func hofCheckDie(mlrval *mlrval.Mlrval, hofName string, message string) { } // getKVPair is a helper function getKVPairOrDie. -func getKVPair(mlrval *mlrval.Mlrval) *types.Mlrmap { +func getKVPair(mlrval *mlrval.Mlrval) *mlrval.Mlrmap { mapval := mlrval.GetMap() if mapval == nil { return nil @@ -261,7 +261,7 @@ func selectMap( udfCallsite := hofSpace.udfCallsite argsArray := hofSpace.argsArray - outputMap := types.NewMlrmap() + outputMap := mlrval.NewMlrmap() for pe := inputMap.Head; pe != nil; pe = pe.Next { argsArray[0] = mlrval.MlrvalFromString(pe.Key) @@ -342,7 +342,7 @@ func applyMap( udfCallsite := hofSpace.udfCallsite argsArray := hofSpace.argsArray - outputMap := types.NewMlrmap() + outputMap := mlrval.NewMlrmap() for pe := inputMap.Head; pe != nil; pe = pe.Next { argsArray[0] = mlrval.MlrvalFromString(pe.Key) @@ -683,7 +683,7 @@ func sortMK( } // Make a new map with keys in the new sort order. - outmap := types.NewMlrmap() + outmap := mlrval.NewMlrmap() for i := 0; i < n; i++ { key := keys[i] outmap.PutCopy(key, inmap.Get(key)) @@ -828,7 +828,7 @@ func sortMF( return nret < 0 }) - sortedMap := types.MlrmapFromPairsArray(pairsArray) + sortedMap := mlrval.MlrmapFromPairsArray(pairsArray) return mlrval.MlrvalFromMapReferenced(sortedMap) } diff --git a/internal/pkg/dsl/cst/root.go b/internal/pkg/dsl/cst/root.go index 350efd720..8b0066830 100644 --- a/internal/pkg/dsl/cst/root.go +++ b/internal/pkg/dsl/cst/root.go @@ -464,7 +464,7 @@ func (root *RootNode) ExecuteBeginBlocks(state *runtime.State) error { return nil } -func (root *RootNode) ExecuteMainBlock(state *runtime.State) (outrec *types.Mlrmap, err error) { +func (root *RootNode) ExecuteMainBlock(state *runtime.State) (outrec *mlrval.Mlrmap, err error) { _, err = root.mainBlock.Execute(state) return state.Inrec, err } @@ -488,7 +488,7 @@ func (root *RootNode) ExecuteEndBlocks(state *runtime.State) error { // executed once, and then discarded. // This is the 'execute once' part of that. -func (root *RootNode) ExecuteREPLImmediate(state *runtime.State) (outrec *types.Mlrmap, err error) { +func (root *RootNode) ExecuteREPLImmediate(state *runtime.State) (outrec *mlrval.Mlrmap, err error) { _, err = root.replImmediateBlock.ExecuteFrameless(state) return state.Inrec, err } diff --git a/internal/pkg/dsl/cst/tee.go b/internal/pkg/dsl/cst/tee.go index 2f2fdfdbf..550e7ce64 100644 --- a/internal/pkg/dsl/cst/tee.go +++ b/internal/pkg/dsl/cst/tee.go @@ -58,7 +58,7 @@ import ( // ================================================================ type tTeeToRedirectFunc func( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, state *runtime.State, ) error @@ -156,7 +156,7 @@ func (node *TeeStatementNode) Execute(state *runtime.State) (*BlockExitPayload, // ---------------------------------------------------------------- func (node *TeeStatementNode) teeToFileOrPipe( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, state *runtime.State, ) error { redirectorTarget := node.redirectorTargetEvaluable.Evaluate(state) diff --git a/internal/pkg/input/pseudo_reader_gen.go b/internal/pkg/input/pseudo_reader_gen.go index 0f20a6270..aec04916a 100644 --- a/internal/pkg/input/pseudo_reader_gen.go +++ b/internal/pkg/input/pseudo_reader_gen.go @@ -83,7 +83,7 @@ func (reader *PseudoReaderGen) process( break } - record := types.NewMlrmap() + record := mlrval.NewMlrmap() record.PutCopy(key, value) context.UpdateForInputRecord() diff --git a/internal/pkg/input/record_reader.go b/internal/pkg/input/record_reader.go index f8a140ae1..bf6542314 100644 --- a/internal/pkg/input/record_reader.go +++ b/internal/pkg/input/record_reader.go @@ -14,7 +14,7 @@ const CSV_BOM = "\xef\xbb\xbf" // FILENAME, NF, NF, FNR, etc.) needs to be duplicated and passed through the // channels along with each record. Hence the initial context, which readers // update on each new file/record, and the channel of types.RecordAndContext -// rather than channel of types.Mlrmap. +// rather than channel of mlrval.Mlrmap. type IRecordReader interface { Read( diff --git a/internal/pkg/input/record_reader_csv.go b/internal/pkg/input/record_reader_csv.go index a03f96bf0..05c945c11 100644 --- a/internal/pkg/input/record_reader_csv.go +++ b/internal/pkg/input/record_reader_csv.go @@ -217,7 +217,7 @@ func (reader *RecordReaderCSV) getRecordBatch( } } - record := types.NewMlrmapAsRecord() + record := mlrval.NewMlrmapAsRecord() nh := len(reader.header) nd := len(csvRecord) diff --git a/internal/pkg/input/record_reader_csvlite.go b/internal/pkg/input/record_reader_csvlite.go index 6ab3dcbe4..6e6939400 100644 --- a/internal/pkg/input/record_reader_csvlite.go +++ b/internal/pkg/input/record_reader_csvlite.go @@ -243,7 +243,7 @@ func getRecordBatchExplicitCSVHeader( return } - record := types.NewMlrmapAsRecord() + record := mlrval.NewMlrmapAsRecord() if !reader.readerOptions.AllowRaggedCSVInput { for i, field := range fields { value := mlrval.FromDeferredType(field) @@ -360,7 +360,7 @@ func getRecordBatchImplicitCSVHeader( } } - record := types.NewMlrmapAsRecord() + record := mlrval.NewMlrmapAsRecord() if !reader.readerOptions.AllowRaggedCSVInput { for i, field := range fields { value := mlrval.FromDeferredType(field) diff --git a/internal/pkg/input/record_reader_dkvp_nidx.go b/internal/pkg/input/record_reader_dkvp_nidx.go index 7283c6100..0c65e7f6d 100644 --- a/internal/pkg/input/record_reader_dkvp_nidx.go +++ b/internal/pkg/input/record_reader_dkvp_nidx.go @@ -16,7 +16,7 @@ import ( // splitter_DKVP_NIDX is a function type for the one bit of code differing // between the DKVP reader and the NIDX reader, namely, how it splits lines. -type splitter_DKVP_NIDX func(reader *RecordReaderDKVPNIDX, line string) *types.Mlrmap +type splitter_DKVP_NIDX func(reader *RecordReaderDKVPNIDX, line string) *mlrval.Mlrmap type RecordReaderDKVPNIDX struct { readerOptions *cli.TReaderOptions @@ -151,8 +151,8 @@ func (reader *RecordReaderDKVPNIDX) getRecordBatch( return recordsAndContexts, false } -func recordFromDKVPLine(reader *RecordReaderDKVPNIDX, line string) *types.Mlrmap { - record := types.NewMlrmapAsRecord() +func recordFromDKVPLine(reader *RecordReaderDKVPNIDX, line string) *mlrval.Mlrmap { + record := mlrval.NewMlrmapAsRecord() var pairs []string // TODO: func-pointer this away @@ -191,8 +191,8 @@ func recordFromDKVPLine(reader *RecordReaderDKVPNIDX, line string) *types.Mlrmap return record } -func recordFromNIDXLine(reader *RecordReaderDKVPNIDX, line string) *types.Mlrmap { - record := types.NewMlrmapAsRecord() +func recordFromNIDXLine(reader *RecordReaderDKVPNIDX, line string) *mlrval.Mlrmap { + record := mlrval.NewMlrmapAsRecord() var values []string // TODO: func-pointer this away diff --git a/internal/pkg/input/record_reader_xtab.go b/internal/pkg/input/record_reader_xtab.go index 4f6456be2..ec7530e58 100644 --- a/internal/pkg/input/record_reader_xtab.go +++ b/internal/pkg/input/record_reader_xtab.go @@ -255,8 +255,8 @@ func (reader *RecordReaderXTAB) getRecordBatch( func (reader *RecordReaderXTAB) recordFromXTABLines( stanza *list.List, -) (*types.Mlrmap, error) { - record := types.NewMlrmapAsRecord() +) (*mlrval.Mlrmap, error) { + record := mlrval.NewMlrmapAsRecord() for e := stanza.Front(); e != nil; e = e.Next() { line := e.Value.(string) diff --git a/internal/pkg/types/mlrmap.go b/internal/pkg/mlrval/mlrmap.go similarity index 96% rename from internal/pkg/types/mlrmap.go rename to internal/pkg/mlrval/mlrmap.go index 821e85f07..28e571df0 100644 --- a/internal/pkg/types/mlrmap.go +++ b/internal/pkg/mlrval/mlrmap.go @@ -51,11 +51,7 @@ // trivial. // ================================================================ -package types - -import ( - "github.com/johnkerl/miller/internal/pkg/mlrval" -) +package mlrval // For the C port having this off was a noticeable performance improvement (10-15%). // For the Go port having it off is a less-noticeable performance improvement (5%). @@ -82,7 +78,7 @@ type Mlrmap struct { type MlrmapEntry struct { Key string - Value *mlrval.Mlrval + Value *Mlrval Prev *MlrmapEntry Next *MlrmapEntry } @@ -90,7 +86,7 @@ type MlrmapEntry struct { // Only used for sorting, map-to-pairs-array and pairs-array-to-map contexts. type MlrmapPair struct { Key string - Value *mlrval.Mlrval + Value *Mlrval } // ---------------------------------------------------------------- @@ -141,7 +137,7 @@ func (mlrmap *Mlrmap) isHashed() bool { // ---------------------------------------------------------------- // Value-copy is up to the caller -- PutReference and PutCopy // are in the public Mlrmap API. -func newMlrmapEntry(key string, value *mlrval.Mlrval) *MlrmapEntry { +func newMlrmapEntry(key string, value *Mlrval) *MlrmapEntry { return &MlrmapEntry{ key, value, diff --git a/internal/pkg/types/mlrmap_accessors.go b/internal/pkg/mlrval/mlrmap_accessors.go similarity index 92% rename from internal/pkg/types/mlrmap_accessors.go rename to internal/pkg/mlrval/mlrmap_accessors.go index a75389e49..cb00d9b67 100644 --- a/internal/pkg/types/mlrmap_accessors.go +++ b/internal/pkg/mlrval/mlrmap_accessors.go @@ -1,8 +1,9 @@ -package types +package mlrval import ( + "bytes" + "github.com/johnkerl/miller/internal/pkg/lib" - "github.com/johnkerl/miller/internal/pkg/mlrval" ) // IsEmpty determines if an map is empty. @@ -14,7 +15,7 @@ func (mlrmap *Mlrmap) Has(key string) bool { return mlrmap.findEntry(key) != nil } -func (mlrmap *Mlrmap) Get(key string) *mlrval.Mlrval { +func (mlrmap *Mlrmap) Get(key string) *Mlrval { pe := mlrmap.findEntry(key) if pe == nil { return nil @@ -28,7 +29,7 @@ func (mlrmap *Mlrmap) Get(key string) *mlrval.Mlrval { // where we could create undesired references between different objects. Only // intended to be used at callsites which allocate a mlrval solely for the // purpose of putting into a map, e.g. input-record readers. -func (mlrmap *Mlrmap) PutReference(key string, value *mlrval.Mlrval) { +func (mlrmap *Mlrmap) PutReference(key string, value *Mlrval) { pe := mlrmap.findEntry(key) if pe == nil { pe = newMlrmapEntry(key, value) @@ -52,7 +53,7 @@ func (mlrmap *Mlrmap) PutReference(key string, value *mlrval.Mlrval) { // PutCopy copies the key and value (deep-copying in case the value is array/map). // This is safe for DSL use. See also PutReference. -func (mlrmap *Mlrmap) PutCopy(key string, value *mlrval.Mlrval) { +func (mlrmap *Mlrmap) PutCopy(key string, value *Mlrval) { pe := mlrmap.findEntry(key) if pe == nil { pe = newMlrmapEntry(key, value.Copy()) @@ -75,7 +76,7 @@ func (mlrmap *Mlrmap) PutCopy(key string, value *mlrval.Mlrval) { } // PrependReference is the same as PutReference, but puts a new entry first, not last. -func (mlrmap *Mlrmap) PrependReference(key string, value *mlrval.Mlrval) { +func (mlrmap *Mlrmap) PrependReference(key string, value *Mlrval) { pe := mlrmap.findEntry(key) if pe == nil { pe = newMlrmapEntry(key, value) @@ -101,7 +102,7 @@ func (mlrmap *Mlrmap) PrependReference(key string, value *mlrval.Mlrval) { func (mlrmap *Mlrmap) PutReferenceAfter( pe *MlrmapEntry, key string, - value *mlrval.Mlrval, + value *Mlrval, ) *MlrmapEntry { if pe == nil || pe.Next == nil { // New entry is supposed to go 'after' old, but there is no such old @@ -206,7 +207,7 @@ func (mlrmap *Mlrmap) findEntryByPositionalIndex(position int) *MlrmapEntry { return nil } -//func (mlrmap *Mlrmap) PutCopyWithMlrvalIndex(key *mlrval.Mlrval, value *mlrval.Mlrval) error { +//func (mlrmap *Mlrmap) PutCopyWithMlrvalIndex(key *Mlrval, value *Mlrval) error { // if key.IsString() { // mlrmap.PutCopy(key.printrep, value) // return nil @@ -221,7 +222,7 @@ func (mlrmap *Mlrmap) findEntryByPositionalIndex(position int) *MlrmapEntry { //} // ---------------------------------------------------------------- -func (mlrmap *Mlrmap) PrependCopy(key string, value *mlrval.Mlrval) { +func (mlrmap *Mlrmap) PrependCopy(key string, value *Mlrval) { mlrmap.PrependReference(key, value.Copy()) } @@ -270,7 +271,7 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { // ---------------------------------------------------------------- // TODO: put error-return into this API -//func (mlrmap *Mlrmap) PutNameWithPositionalIndex(position int, name *mlrval.Mlrval) { +//func (mlrmap *Mlrmap) PutNameWithPositionalIndex(position int, name *Mlrval) { // positionalEntry := mlrmap.findEntryByPositionalIndex(position) // // if positionalEntry == nil { @@ -304,7 +305,7 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { // } //} -//func (mlrmap *Mlrmap) GetWithPositionalIndex(position int) *mlrval.Mlrval { +//func (mlrmap *Mlrmap) GetWithPositionalIndex(position int) *Mlrval { // mapEntry := mlrmap.findEntryByPositionalIndex(position) // if mapEntry == nil { // return nil @@ -312,7 +313,7 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { // return mapEntry.Value //} -//func (mlrmap *Mlrmap) GetWithMlrvalIndex(index *mlrval.Mlrval) (*mlrval.Mlrval, error) { +//func (mlrmap *Mlrmap) GetWithMlrvalIndex(index *Mlrval) (*Mlrval, error) { // if index.IsArray() { // return mlrmap.getWithMlrvalArrayIndex(index) // } else { @@ -322,9 +323,9 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { //// This lets the user do '$y = $x[ ["a", "b", "c"] ]' in lieu of //// '$y = $x["a"]["b"]["c"]'. -//func (mlrmap *Mlrmap) getWithMlrvalArrayIndex(index *mlrval.Mlrval) (*mlrval.Mlrval, error) { +//func (mlrmap *Mlrmap) getWithMlrvalArrayIndex(index *Mlrval) (*Mlrval, error) { // current := mlrmap -// var retval *mlrval.Mlrval = nil +// var retval *Mlrval = nil // lib.InternalCodingErrorIf(!index.IsArray()) // array := index.arrayval // n := len(array) @@ -348,7 +349,7 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { // return retval, nil //} -//func (mlrmap *Mlrmap) getWithMlrvalSingleIndex(index *mlrval.Mlrval) (*mlrval.Mlrval, error) { +//func (mlrmap *Mlrmap) getWithMlrvalSingleIndex(index *Mlrval) (*Mlrval, error) { // if index.IsString() { // return mlrmap.Get(index.printrep), nil // } else if index.IsInt() { @@ -380,7 +381,7 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { //// ---------------------------------------------------------------- //// TODO: put error-return into this API -//func (mlrmap *Mlrmap) PutNameWithPositionalIndex(position int, name *mlrval.Mlrval) { +//func (mlrmap *Mlrmap) PutNameWithPositionalIndex(position int, name *Mlrval) { // positionalEntry := mlrmap.findEntryByPositionalIndex(position) // // if positionalEntry == nil { @@ -415,7 +416,7 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { // This is safe for DSL use. See also PutReference. // TODO: put error-return into this API -//func (mlrmap *Mlrmap) PutCopyWithPositionalIndex(position int, value *mlrval.Mlrval) { +//func (mlrmap *Mlrmap) PutCopyWithPositionalIndex(position int, value *Mlrval) { // mapEntry := mlrmap.findEntryByPositionalIndex(position) // // if mapEntry != nil { @@ -445,7 +446,7 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { // } // return true //} -// + //// True if this contains other, i.e. if other is contained by mlrmap. //// * If any key of other is not a key of this, return false. //// * If any key of other has a value unequal to this' value at the same key, return false. @@ -457,7 +458,7 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { // } // thisval := mlrmap.Get(pe.Key) // thatval := pe.Value -// meq := BIF_equals(thisval, thatval) +// meq := bifs.BIF_equals(thisval, thatval) // eq, ok := meq.GetBoolValue() // lib.InternalCodingErrorIf(!ok) // if !eq { @@ -522,31 +523,31 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { // In the former case the indices are ["name", 1, "foo"] and in the latter case // the indices are ["foo", 1]. See also indexed-lvalues.md. // -// This is a Mlrmap (from string to mlrval.Mlrval) so we handle the first level of -// indexing here, then pass the remaining indices to the mlrval.Mlrval at the desired +// This is a Mlrmap (from string to Mlrval) so we handle the first level of +// indexing here, then pass the remaining indices to the Mlrval at the desired // slot. -//func (mlrmap *Mlrmap) PutIndexed(indices []*mlrval.Mlrval, rvalue *mlrval.Mlrval) error { +//func (mlrmap *Mlrmap) PutIndexed(indices []*Mlrval, rvalue *Mlrval) error { // return putIndexedOnMap(mlrmap, indices, rvalue) //} -//func (mlrmap *Mlrmap) RemoveIndexed(indices []*mlrval.Mlrval) error { +//func (mlrmap *Mlrmap) RemoveIndexed(indices []*Mlrval) error { // return removeIndexedOnMap(mlrmap, indices) //} -//// ---------------------------------------------------------------- -//func (mlrmap *Mlrmap) GetKeysJoined() string { -// var buffer bytes.Buffer -// i := 0 -// for pe := mlrmap.Head; pe != nil; pe = pe.Next { -// if i > 0 { -// buffer.WriteString(",") -// } -// i++ -// buffer.WriteString(pe.Key) -// } -// return buffer.String() -//} +// ---------------------------------------------------------------- +func (mlrmap *Mlrmap) GetKeysJoined() string { + var buffer bytes.Buffer + i := 0 + for pe := mlrmap.Head; pe != nil; pe = pe.Next { + if i > 0 { + buffer.WriteString(",") + } + i++ + buffer.WriteString(pe.Key) + } + return buffer.String() +} //// For mlr reshape //func (mlrmap *Mlrmap) GetValuesJoined() string { @@ -602,10 +603,10 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { //// TODO: put 'Copy' into the method name //func (mlrmap *Mlrmap) GetSelectedValuesAndJoined(selectedFieldNames []string) ( // string, -// []*mlrval.Mlrval, +// []*Mlrval, // bool, //) { -// mlrvals := make([]*mlrval.Mlrval, 0, len(selectedFieldNames)) +// mlrvals := make([]*Mlrval, 0, len(selectedFieldNames)) // // if len(selectedFieldNames) == 0 { // // The fall-through is functionally correct, but this is quicker with @@ -633,9 +634,9 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { //// As above but only returns the array. Also, these are references, NOT copies. //// For step and join. -//func (mlrmap *Mlrmap) ReferenceSelectedValues(selectedFieldNames []string) ([]*mlrval.Mlrval, bool) { +//func (mlrmap *Mlrmap) ReferenceSelectedValues(selectedFieldNames []string) ([]*Mlrval, bool) { // allFound := true -// mlrvals := make([]*mlrval.Mlrval, 0, len(selectedFieldNames)) +// mlrvals := make([]*Mlrval, 0, len(selectedFieldNames)) // // for _, selectedFieldName := range selectedFieldNames { // entry := mlrmap.findEntry(selectedFieldName) @@ -651,9 +652,9 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { //// TODO: rename to CopySelectedValues //// As previous but with copying. For stats1. -//func (mlrmap *Mlrmap) GetSelectedValues(selectedFieldNames []string) ([]*mlrval.Mlrval, bool) { +//func (mlrmap *Mlrmap) GetSelectedValues(selectedFieldNames []string) ([]*Mlrval, bool) { // allFound := true -// mlrvals := make([]*mlrval.Mlrval, 0, len(selectedFieldNames)) +// mlrvals := make([]*Mlrval, 0, len(selectedFieldNames)) // // for _, selectedFieldName := range selectedFieldNames { // entry := mlrmap.findEntry(selectedFieldName) diff --git a/internal/pkg/types/mlrmap_accessors_test.go b/internal/pkg/mlrval/mlrmap_accessors_test.go similarity index 87% rename from internal/pkg/types/mlrmap_accessors_test.go rename to internal/pkg/mlrval/mlrmap_accessors_test.go index ae4678d7f..5460c3feb 100644 --- a/internal/pkg/types/mlrmap_accessors_test.go +++ b/internal/pkg/mlrval/mlrmap_accessors_test.go @@ -1,11 +1,9 @@ -package types +package mlrval import ( "testing" "github.com/stretchr/testify/assert" - - "github.com/johnkerl/miller/internal/pkg/mlrval" ) func TestIsEmpty(t *testing.T) { @@ -16,7 +14,7 @@ func TestIsEmpty(t *testing.T) { func TestPutReference(t *testing.T) { mlrmap := NewMlrmap() key1 := "a" - val1 := mlrval.FromInt(1) + val1 := FromInt(1) mlrmap.PutReference(key1, val1) assert.False(t, mlrmap.IsEmpty()) @@ -35,7 +33,7 @@ func TestPutReference(t *testing.T) { assert.Equal(t, 1, intval) key2 := "b" - val2 := mlrval.FromBool(true) + val2 := FromBool(true) mlrmap.PutReference(key2, val2) assert.True(t, mlrmap.Has("a")) diff --git a/internal/pkg/types/mlrmap_new_test.go b/internal/pkg/mlrval/mlrmap_new_test.go similarity index 95% rename from internal/pkg/types/mlrmap_new_test.go rename to internal/pkg/mlrval/mlrmap_new_test.go index 5671bb836..2695daa58 100644 --- a/internal/pkg/types/mlrmap_new_test.go +++ b/internal/pkg/mlrval/mlrmap_new_test.go @@ -1,4 +1,4 @@ -package types +package mlrval import ( "testing" diff --git a/internal/pkg/mlrval/constants.go b/internal/pkg/mlrval/mlrval_constants.go similarity index 100% rename from internal/pkg/mlrval/constants.go rename to internal/pkg/mlrval/mlrval_constants.go diff --git a/internal/pkg/mlrval/copy.go b/internal/pkg/mlrval/mlrval_copy.go similarity index 100% rename from internal/pkg/mlrval/copy.go rename to internal/pkg/mlrval/mlrval_copy.go diff --git a/internal/pkg/mlrval/format.go b/internal/pkg/mlrval/mlrval_format.go similarity index 100% rename from internal/pkg/mlrval/format.go rename to internal/pkg/mlrval/mlrval_format.go diff --git a/internal/pkg/mlrval/format_test.go b/internal/pkg/mlrval/mlrval_format_test.go similarity index 100% rename from internal/pkg/mlrval/format_test.go rename to internal/pkg/mlrval/mlrval_format_test.go diff --git a/internal/pkg/mlrval/get.go b/internal/pkg/mlrval/mlrval_get.go similarity index 94% rename from internal/pkg/mlrval/get.go rename to internal/pkg/mlrval/mlrval_get.go index c5597dc5d..892a4390d 100644 --- a/internal/pkg/mlrval/get.go +++ b/internal/pkg/mlrval/mlrval_get.go @@ -8,6 +8,10 @@ import ( // predicate, or another Get...(), since types are JIT-computed on first access // for most data-file values. See type.go for more information. +func (mv *Mlrval) GetTypeBit() int { + return 1 << mv.Type() +} + func (mv *Mlrval) GetStringValue() (stringValue string, isString bool) { if mv.Type() == MT_STRING || mv.Type() == MT_VOID { return mv.printrep, true @@ -110,12 +114,12 @@ func (mv *Mlrval) AcquireBoolValue() bool { return mv.boolval } -func (mv *Mlrval) AcquireArrayValue() interface{} { +func (mv *Mlrval) AcquireArrayValue() []Mlrval { lib.InternalCodingErrorIf(mv.mvtype != MT_ARRAY) return mv.arrayval } -func (mv *Mlrval) AcquireMapValue() interface{} { +func (mv *Mlrval) AcquireMapValue() *Mlrmap { lib.InternalCodingErrorIf(mv.mvtype != MT_MAP) return mv.mapval } diff --git a/internal/pkg/mlrval/get_test.go b/internal/pkg/mlrval/mlrval_get_test.go similarity index 100% rename from internal/pkg/mlrval/get_test.go rename to internal/pkg/mlrval/mlrval_get_test.go diff --git a/internal/pkg/mlrval/infer.go b/internal/pkg/mlrval/mlrval_infer.go similarity index 100% rename from internal/pkg/mlrval/infer.go rename to internal/pkg/mlrval/mlrval_infer.go diff --git a/internal/pkg/mlrval/is.go b/internal/pkg/mlrval/mlrval_is.go similarity index 100% rename from internal/pkg/mlrval/is.go rename to internal/pkg/mlrval/mlrval_is.go diff --git a/internal/pkg/mlrval/is_test.go b/internal/pkg/mlrval/mlrval_is_test.go similarity index 98% rename from internal/pkg/mlrval/is_test.go rename to internal/pkg/mlrval/mlrval_is_test.go index 6b4d9232e..5e89c7e3c 100644 --- a/internal/pkg/mlrval/is_test.go +++ b/internal/pkg/mlrval/mlrval_is_test.go @@ -19,8 +19,8 @@ func TestIsLegit(t *testing.T) { assert.True(t, FromInt(123).IsLegit()) assert.True(t, FromFloat(123.4).IsLegit()) assert.True(t, FromBool(true).IsLegit()) - assert.True(t, FromArray("test data").IsLegit()) - assert.True(t, FromMap("test data").IsLegit()) + assert.True(t, FromArray([]Mlrval{*FromInt(10)}).IsLegit()) + assert.True(t, FromMap(NewMlrmap()).IsLegit()) } func TestIsErrorOrAbsent(t *testing.T) { diff --git a/internal/pkg/mlrval/new.go b/internal/pkg/mlrval/mlrval_new.go similarity index 98% rename from internal/pkg/mlrval/new.go rename to internal/pkg/mlrval/mlrval_new.go index 1a4e9157b..b6c50395d 100644 --- a/internal/pkg/mlrval/new.go +++ b/internal/pkg/mlrval/mlrval_new.go @@ -198,7 +198,7 @@ func FromFunction(funcval interface{}, name string) *Mlrval { } } -func FromArray(arrayval interface{}) *Mlrval { +func FromArray(arrayval []Mlrval) *Mlrval { return &Mlrval{ mvtype: MT_ARRAY, printrep: INVALID_PRINTREP, @@ -207,7 +207,7 @@ func FromArray(arrayval interface{}) *Mlrval { } } -func FromMap(mapval interface{}) *Mlrval { +func FromMap(mapval *Mlrmap) *Mlrval { return &Mlrval{ mvtype: MT_MAP, printrep: INVALID_PRINTREP, diff --git a/internal/pkg/mlrval/new_test.go b/internal/pkg/mlrval/mlrval_new_test.go similarity index 96% rename from internal/pkg/mlrval/new_test.go rename to internal/pkg/mlrval/mlrval_new_test.go index 48214e9b1..26dfeadc3 100644 --- a/internal/pkg/mlrval/new_test.go +++ b/internal/pkg/mlrval/mlrval_new_test.go @@ -128,15 +128,15 @@ func TestFromFunction(t *testing.T) { } func TestFromArray(t *testing.T) { - mv := FromArray("test data") + mv := FromArray([]Mlrval{*FromInt(10)}) assert.Equal(t, MVType(MT_ARRAY), MVType(mv.mvtype)) assert.True(t, mv.printrepValid) - assert.Equal(t, "test data", mv.arrayval.(string)) + assert.Equal(t, 1, len(mv.arrayval)) } func TestFromMap(t *testing.T) { - mv := FromMap("test data") + mv := FromMap(NewMlrmap()) assert.Equal(t, MVType(MT_MAP), MVType(mv.mvtype)) assert.True(t, mv.printrepValid) - assert.Equal(t, "test data", mv.mapval.(string)) + assert.True(t, mv.mapval.IsEmpty()) } diff --git a/internal/pkg/mlrval/output.go b/internal/pkg/mlrval/mlrval_output.go similarity index 100% rename from internal/pkg/mlrval/output.go rename to internal/pkg/mlrval/mlrval_output.go diff --git a/internal/pkg/mlrval/output_test.go b/internal/pkg/mlrval/mlrval_output_test.go similarity index 100% rename from internal/pkg/mlrval/output_test.go rename to internal/pkg/mlrval/mlrval_output_test.go diff --git a/internal/pkg/mlrval/type.go b/internal/pkg/mlrval/mlrval_type.go similarity index 93% rename from internal/pkg/mlrval/type.go rename to internal/pkg/mlrval/mlrval_type.go index a43e5a010..3b46aa610 100644 --- a/internal/pkg/mlrval/type.go +++ b/internal/pkg/mlrval/mlrval_type.go @@ -1,5 +1,5 @@ // ================================================================ -// The `mlrval.Mlrval` structure includes **string, int, float, boolean, void, +// The `Mlrval` structure includes **string, int, float, boolean, void, // absent, and error** types (not unlike PHP's `zval`) as well as // type-conversion logic for various operators. // @@ -65,10 +65,11 @@ type Mlrval struct { floatval float64 boolval bool - // Interfaced here to avoid package-dependency cycles: - arrayval interface{} // []Mlrval - mapval interface{} // *Mlrmap - funcval interface{} // first-class-function literals from internal/pkg/dsl/cst + arrayval []Mlrval + mapval *Mlrmap + // First-class-function literals from internal/pkg/dsl/cst. + // Interfaced here to avoid package-dependency cycles. + funcval interface{} } const INVALID_PRINTREP = "(bug-if-you-see-this)" @@ -182,3 +183,12 @@ var typeNameToMaskMap = map[string]int{ "any": MT_TYPE_MASK_ANY, "funct": MT_TYPE_MASK_FUNC, } + +func TypeNameToMask(typeName string) (mask int, present bool) { + retval := typeNameToMaskMap[typeName] + if retval != 0 { + return retval, true + } else { + return 0, false + } +} diff --git a/internal/pkg/mlrval/tbf.go b/internal/pkg/mlrval/tbf.go index 822052dbb..9ee8a8b05 100644 --- a/internal/pkg/mlrval/tbf.go +++ b/internal/pkg/mlrval/tbf.go @@ -2,10 +2,6 @@ package mlrval // things to be filed -//func (mv *Mlrval) GetTypeBit() int { -// return 1 << mv.mvtype -//} - //// NewMlrvalForAutoDeepen is for auto-deepen of nested maps in things like //// //// $foo[1]["a"][2]["b"] = 3 @@ -152,15 +148,6 @@ package mlrval // } //} -//func TypeNameToMask(typeName string) (mask int, present bool) { -// retval := typeNameToMaskMap[typeName] -// if retval != 0 { -// return retval, true -// } else { -// return 0, false -// } -//} - // TODO: FILE // //// MlrvalFromInferredTypeForDataFiles is for parsing field values directly from diff --git a/internal/pkg/output/record_writer.go b/internal/pkg/output/record_writer.go index 667687132..30b17badb 100644 --- a/internal/pkg/output/record_writer.go +++ b/internal/pkg/output/record_writer.go @@ -3,7 +3,7 @@ package output import ( "bufio" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // IRecordWriter is the abstract interface for all record-writers. They are @@ -17,7 +17,7 @@ import ( // format. type IRecordWriter interface { Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) diff --git a/internal/pkg/output/record_writer_csv.go b/internal/pkg/output/record_writer_csv.go index 3f8196282..2ebdfd3e2 100644 --- a/internal/pkg/output/record_writer_csv.go +++ b/internal/pkg/output/record_writer_csv.go @@ -8,7 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/colorizer" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) type RecordWriterCSV struct { @@ -37,7 +37,7 @@ func NewRecordWriterCSV(writerOptions *cli.TWriterOptions) (*RecordWriterCSV, er } func (writer *RecordWriterCSV) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { diff --git a/internal/pkg/output/record_writer_csvlite.go b/internal/pkg/output/record_writer_csvlite.go index 40b6cd1f6..6acf0976c 100644 --- a/internal/pkg/output/record_writer_csvlite.go +++ b/internal/pkg/output/record_writer_csvlite.go @@ -6,7 +6,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/colorizer" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) type RecordWriterCSVLite struct { @@ -26,7 +26,7 @@ func NewRecordWriterCSVLite(writerOptions *cli.TWriterOptions) (*RecordWriterCSV } func (writer *RecordWriterCSVLite) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { diff --git a/internal/pkg/output/record_writer_dkvp.go b/internal/pkg/output/record_writer_dkvp.go index 5fd70362a..674388130 100644 --- a/internal/pkg/output/record_writer_dkvp.go +++ b/internal/pkg/output/record_writer_dkvp.go @@ -5,7 +5,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/colorizer" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) type RecordWriterDKVP struct { @@ -19,7 +19,7 @@ func NewRecordWriterDKVP(writerOptions *cli.TWriterOptions) (*RecordWriterDKVP, } func (writer *RecordWriterDKVP) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { diff --git a/internal/pkg/output/record_writer_json.go b/internal/pkg/output/record_writer_json.go index 3673a517e..b962f2dfa 100644 --- a/internal/pkg/output/record_writer_json.go +++ b/internal/pkg/output/record_writer_json.go @@ -6,14 +6,14 @@ import ( "os" "github.com/johnkerl/miller/internal/pkg/cli" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ---------------------------------------------------------------- type RecordWriterJSON struct { // Parameters: writerOptions *cli.TWriterOptions - jsonFormatting types.TJSONFormatting + jsonFormatting mlrval.TJSONFormatting // State: onFirst bool @@ -21,9 +21,9 @@ type RecordWriterJSON struct { // ---------------------------------------------------------------- func NewRecordWriterJSON(writerOptions *cli.TWriterOptions) (*RecordWriterJSON, error) { - var jsonFormatting types.TJSONFormatting = types.JSON_SINGLE_LINE + var jsonFormatting mlrval.TJSONFormatting = mlrval.JSON_SINGLE_LINE if writerOptions.JSONOutputMultiline { - jsonFormatting = types.JSON_MULTILINE + jsonFormatting = mlrval.JSON_MULTILINE } return &RecordWriterJSON{ writerOptions: writerOptions, @@ -34,7 +34,7 @@ func NewRecordWriterJSON(writerOptions *cli.TWriterOptions) (*RecordWriterJSON, // ---------------------------------------------------------------- func (writer *RecordWriterJSON) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { @@ -47,7 +47,7 @@ func (writer *RecordWriterJSON) Write( // ---------------------------------------------------------------- func (writer *RecordWriterJSON) writeWithListWrap( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { @@ -82,7 +82,7 @@ func (writer *RecordWriterJSON) writeWithListWrap( // ---------------------------------------------------------------- func (writer *RecordWriterJSON) writeWithoutListWrap( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { diff --git a/internal/pkg/output/record_writer_markdown.go b/internal/pkg/output/record_writer_markdown.go index 695cdd11c..c7bdf280d 100644 --- a/internal/pkg/output/record_writer_markdown.go +++ b/internal/pkg/output/record_writer_markdown.go @@ -6,7 +6,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/colorizer" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) type RecordWriterMarkdown struct { @@ -28,7 +28,7 @@ func NewRecordWriterMarkdown(writerOptions *cli.TWriterOptions) (*RecordWriterMa // ---------------------------------------------------------------- func (writer *RecordWriterMarkdown) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { diff --git a/internal/pkg/output/record_writer_nidx.go b/internal/pkg/output/record_writer_nidx.go index 1a224407b..d7625ca4b 100644 --- a/internal/pkg/output/record_writer_nidx.go +++ b/internal/pkg/output/record_writer_nidx.go @@ -4,7 +4,7 @@ import ( "bufio" "github.com/johnkerl/miller/internal/pkg/cli" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) type RecordWriterNIDX struct { @@ -20,7 +20,7 @@ func NewRecordWriterNIDX(writerOptions *cli.TWriterOptions) (*RecordWriterNIDX, } func (writer *RecordWriterNIDX) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { diff --git a/internal/pkg/output/record_writer_pprint.go b/internal/pkg/output/record_writer_pprint.go index c293aa97e..96be7f957 100644 --- a/internal/pkg/output/record_writer_pprint.go +++ b/internal/pkg/output/record_writer_pprint.go @@ -9,7 +9,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/colorizer" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) type RecordWriterPPRINT struct { @@ -34,7 +34,7 @@ func NewRecordWriterPPRINT(writerOptions *cli.TWriterOptions) (*RecordWriterPPRI // ---------------------------------------------------------------- func (writer *RecordWriterPPRINT) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { @@ -97,7 +97,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousList( var maxNR int = 0 for e := records.Front(); e != nil; e = e.Next() { - outrec := e.Value.(*types.Mlrmap) + outrec := e.Value.(*mlrval.Mlrmap) nr := outrec.FieldCount if maxNR < nr { maxNR = nr @@ -152,7 +152,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousListNonBarred( onFirst := true for e := records.Front(); e != nil; e = e.Next() { - outrec := e.Value.(*types.Mlrmap) + outrec := e.Value.(*mlrval.Mlrmap) // Print header line if onFirst && !writer.writerOptions.HeaderlessCSVOutput { @@ -254,7 +254,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousListBarred( onFirst := true for e := records.Front(); e != nil; e = e.Next() { - outrec := e.Value.(*types.Mlrmap) + outrec := e.Value.(*mlrval.Mlrmap) // Print header line if onFirst && !writer.writerOptions.HeaderlessCSVOutput { diff --git a/internal/pkg/output/record_writer_xtab.go b/internal/pkg/output/record_writer_xtab.go index 63a0f2fc2..a13536c1e 100644 --- a/internal/pkg/output/record_writer_xtab.go +++ b/internal/pkg/output/record_writer_xtab.go @@ -7,7 +7,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/colorizer" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ---------------------------------------------------------------- @@ -42,7 +42,7 @@ func NewRecordWriterXTAB(writerOptions *cli.TWriterOptions) (*RecordWriterXTAB, } func (writer *RecordWriterXTAB) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { @@ -67,7 +67,7 @@ func (writer *RecordWriterXTAB) Write( } func (writer *RecordWriterXTAB) writeWithLeftAlignedValues( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, maxKeyLength int, @@ -100,7 +100,7 @@ func (writer *RecordWriterXTAB) writeWithLeftAlignedValues( } func (writer *RecordWriterXTAB) writeWithRightAlignedValues( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, maxKeyLength int, diff --git a/internal/pkg/runtime/stack.go b/internal/pkg/runtime/stack.go index 038c107e2..d3db2b0a5 100644 --- a/internal/pkg/runtime/stack.go +++ b/internal/pkg/runtime/stack.go @@ -420,15 +420,15 @@ func (frame *StackFrame) defineTyped( func (frame *StackFrame) setIndexed( stackVariable *StackVariable, indices []*mlrval.Mlrval, - mlrval *mlrval.Mlrval, + mv *mlrval.Mlrval, ) error { value := frame.get(stackVariable) if value == nil { lib.InternalCodingErrorIf(len(indices) < 1) leadingIndex := indices[0] if leadingIndex.IsString() || leadingIndex.IsInt() { - newval := mlrval.MlrvalFromEmptyMap() - newval.PutIndexed(indices, mlrval) + newval := mlrval.FromMap(mlrval.NewMlrmap()) + newval.PutIndexed(indices, mv) return frame.set(stackVariable, newval) } else { return errors.New( @@ -441,7 +441,7 @@ func (frame *StackFrame) setIndexed( } else { // For example maybe the variable exists and is an array but the // leading index is a string. - return value.PutIndexed(indices, mlrval) + return value.PutIndexed(indices, mv) } } diff --git a/internal/pkg/runtime/state.go b/internal/pkg/runtime/state.go index 1fca8ebad..aca8fcee2 100644 --- a/internal/pkg/runtime/state.go +++ b/internal/pkg/runtime/state.go @@ -17,9 +17,9 @@ import ( type State struct { - Inrec *types.Mlrmap + Inrec *mlrval.Mlrmap Context *types.Context - Oosvars *types.Mlrmap + Oosvars *mlrval.Mlrmap FilterExpression *mlrval.Mlrval Stack *Stack OutputRecordsAndContexts *list.List // list of *types.RecordAndContext @@ -31,7 +31,7 @@ type State struct { } func NewEmptyState(options *cli.TOptions) *State { - oosvars := types.NewMlrmap() + oosvars := mlrval.NewMlrmap() return &State{ Inrec: nil, Context: nil, @@ -48,7 +48,7 @@ func NewEmptyState(options *cli.TOptions) *State { } func (state *State) Update( - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, context *types.Context, ) { state.Inrec = inrec diff --git a/internal/pkg/transformers/altkv.go b/internal/pkg/transformers/altkv.go index 2fdde54ac..88bbba1f8 100644 --- a/internal/pkg/transformers/altkv.go +++ b/internal/pkg/transformers/altkv.go @@ -99,7 +99,7 @@ func (tr *TransformerAltkv) Transform( HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel) if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() outputFieldNumber := 1 for pe := inrec.Head; pe != nil; /* increment in loop body */ { diff --git a/internal/pkg/transformers/clean-whitespace.go b/internal/pkg/transformers/clean-whitespace.go index 93d465f48..1f2d4bdb9 100644 --- a/internal/pkg/transformers/clean-whitespace.go +++ b/internal/pkg/transformers/clean-whitespace.go @@ -150,7 +150,7 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeysAndValues( outputDownstreamDoneChannel chan<- bool, ) { if !inrecAndContext.EndOfStream { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { oldKey := mlrval.mlrval.FromString(pe.Key) @@ -175,7 +175,7 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeys( outputDownstreamDoneChannel chan<- bool, ) { if !inrecAndContext.EndOfStream { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { oldKey := mlrval.mlrval.FromString(pe.Key) diff --git a/internal/pkg/transformers/count.go b/internal/pkg/transformers/count.go index 83a79ad87..9aef1ba00 100644 --- a/internal/pkg/transformers/count.go +++ b/internal/pkg/transformers/count.go @@ -172,7 +172,7 @@ func (tr *TransformerCount) countUngrouped( if !inrecAndContext.EndOfStream { tr.ungroupedCount++ } else { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() newrec.PutCopy(tr.outputFieldName, mlrval.MlrvalFromInt(tr.ungroupedCount)) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) @@ -210,7 +210,7 @@ func (tr *TransformerCount) countGrouped( } else { if tr.showCountsOnly { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() newrec.PutCopy(tr.outputFieldName, mlrval.MlrvalFromInt(tr.groupedCounts.FieldCount)) outrecAndContext := types.NewRecordAndContext(newrec, &inrecAndContext.Context) @@ -219,7 +219,7 @@ func (tr *TransformerCount) countGrouped( } else { for outer := tr.groupedCounts.Head; outer != nil; outer = outer.Next { groupingKey := outer.Key - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() // Example: // * Suppose group-by fields are a,b. diff --git a/internal/pkg/transformers/cut.go b/internal/pkg/transformers/cut.go index 8f90ee902..51b038c1d 100644 --- a/internal/pkg/transformers/cut.go +++ b/internal/pkg/transformers/cut.go @@ -201,7 +201,7 @@ func (tr *TransformerCut) includeWithInputOrder( ) { if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - outrec := types.NewMlrmap() + outrec := mlrval.NewMlrmap() for pe := inrec.Head; pe != nil; pe = pe.Next { fieldName := pe.Key _, wanted := tr.fieldNameSet[fieldName] @@ -226,7 +226,7 @@ func (tr *TransformerCut) includeWithArgOrder( ) { if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - outrec := types.NewMlrmap() + outrec := mlrval.NewMlrmap() for _, fieldName := range tr.fieldNameList { value := inrec.Get(fieldName) if value != nil { @@ -268,7 +268,7 @@ func (tr *TransformerCut) processWithRegexes( ) { if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for pe := inrec.Head; pe != nil; pe = pe.Next { matchesAny := false for _, regex := range tr.regexes { diff --git a/internal/pkg/transformers/gap.go b/internal/pkg/transformers/gap.go index 61e31b4d3..d7ec544fd 100644 --- a/internal/pkg/transformers/gap.go +++ b/internal/pkg/transformers/gap.go @@ -156,7 +156,7 @@ func (tr *TransformerGap) transformUnkeyed( ) { if !inrecAndContext.EndOfStream { if tr.recordCount > 0 && tr.recordCount%tr.gapCount == 0 { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } outputRecordsAndContexts.PushBack(inrecAndContext) @@ -183,7 +183,7 @@ func (tr *TransformerGap) transformKeyed( } if groupingKey != tr.previousGroupingKey && tr.recordCount > 0 { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } diff --git a/internal/pkg/transformers/histogram.go b/internal/pkg/transformers/histogram.go index 83ba99d98..5471efa8c 100644 --- a/internal/pkg/transformers/histogram.go +++ b/internal/pkg/transformers/histogram.go @@ -258,7 +258,7 @@ func (tr *TransformerHistogram) emitNonAuto( countFieldNames[valueFieldName] = tr.outputPrefix + valueFieldName + "_count" } for i := 0; i < tr.nbins; i++ { - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() outrec.PutReference( tr.outputPrefix+"bin_lo", @@ -364,7 +364,7 @@ func (tr *TransformerHistogram) emitAuto( } for i := 0; i < nbins; i++ { - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() outrec.PutReference( tr.outputPrefix+"bin_lo", diff --git a/internal/pkg/transformers/join.go b/internal/pkg/transformers/join.go index 7f9eca2e2..794a15a47 100644 --- a/internal/pkg/transformers/join.go +++ b/internal/pkg/transformers/join.go @@ -543,7 +543,7 @@ func (tr *TransformerJoin) formAndEmitPairs( rightrec := rightRecordAndContext.Record // Allocate a new output record which is the join of the left and right records. - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() // Add the joined-on fields to the new output record n := len(tr.opts.leftJoinFieldNames) diff --git a/internal/pkg/transformers/most-or-least-frequent.go b/internal/pkg/transformers/most-or-least-frequent.go index c0e6bda07..f5b75b5cf 100644 --- a/internal/pkg/transformers/most-or-least-frequent.go +++ b/internal/pkg/transformers/most-or-least-frequent.go @@ -275,7 +275,7 @@ func (tr *TransformerMostOrLeastFrequent) Transform( outputLength = tr.maxOutputLength } for i := 0; i < outputLength; i++ { - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() groupByFieldValues := tr.valuesForGroup[sortPairs[i].groupingKey] for j := range tr.groupByFieldNames { outrec.PutCopy( diff --git a/internal/pkg/transformers/nest.go b/internal/pkg/transformers/nest.go index e58d7f580..220ec6917 100644 --- a/internal/pkg/transformers/nest.go +++ b/internal/pkg/transformers/nest.go @@ -474,7 +474,7 @@ func (tr *TransformerNest) implodeValuesAcrossFields( if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - var previousEntry *types.MlrmapEntry = nil + var previousEntry *mlrval.MlrmapEntry = nil fieldCount := 0 var buffer bytes.Buffer for pe := inrec.Head; pe != nil; /* increment in loop */ { @@ -552,7 +552,7 @@ func (tr *TransformerNest) implodeValueAcrossRecords( bucket = iBucket.(*tNestBucket) } - pair := types.NewMlrmapAsRecord() + pair := mlrval.NewMlrmapAsRecord() pair.PutReference(tr.fieldName, fieldValueCopy) bucket.pairs.PushBack(pair) @@ -568,7 +568,7 @@ func (tr *TransformerNest) implodeValueAcrossRecords( i := 0 for pg := bucket.pairs.Front(); pg != nil; pg = pg.Next() { - pr := pg.Value.(*types.Mlrmap) + pr := pg.Value.(*mlrval.Mlrmap) if i > 0 { buffer.WriteString(tr.nestedFS) } @@ -587,11 +587,11 @@ func (tr *TransformerNest) implodeValueAcrossRecords( } type tNestBucket struct { - representative *types.Mlrmap + representative *mlrval.Mlrmap pairs *list.List } -func newNestBucket(representative *types.Mlrmap) *tNestBucket { +func newNestBucket(representative *mlrval.Mlrmap) *tNestBucket { return &tNestBucket{ representative: representative, pairs: list.New(), diff --git a/internal/pkg/transformers/regularize.go b/internal/pkg/transformers/regularize.go index 22ab2fbf8..1e20a7d5b 100644 --- a/internal/pkg/transformers/regularize.go +++ b/internal/pkg/transformers/regularize.go @@ -112,7 +112,7 @@ func (tr *TransformerRegularize) Transform( tr.sortedToOriginal[currentSortedFieldNamesJoined] = currentFieldNames outputRecordsAndContexts.PushBack(inrecAndContext) } else { - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() for _, fieldName := range previousSortedFieldNames { outrec.PutReference(fieldName, inrec.Get(fieldName)) // inrec will be GC'ed } diff --git a/internal/pkg/transformers/remove-empty-columns.go b/internal/pkg/transformers/remove-empty-columns.go index d6b60f8bd..411e635a9 100644 --- a/internal/pkg/transformers/remove-empty-columns.go +++ b/internal/pkg/transformers/remove-empty-columns.go @@ -118,7 +118,7 @@ func (tr *TransformerRemoveEmptyColumns) Transform( outrecAndContext := e.Value.(*types.RecordAndContext) outrec := outrecAndContext.Record - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for pe := outrec.Head; pe != nil; pe = pe.Next { _, ok := tr.namesWithNonEmptyValues[pe.Key] diff --git a/internal/pkg/transformers/reorder.go b/internal/pkg/transformers/reorder.go index df1b419d7..9c75bcf4e 100644 --- a/internal/pkg/transformers/reorder.go +++ b/internal/pkg/transformers/reorder.go @@ -232,7 +232,7 @@ func (tr *TransformerReorder) reorderBefore( return } - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() pe := inrec.Head // * inrec will be GC'ed @@ -289,7 +289,7 @@ func (tr *TransformerReorder) reorderAfter( return } - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() pe := inrec.Head // * inrec will be GC'ed diff --git a/internal/pkg/transformers/reshape.go b/internal/pkg/transformers/reshape.go index 15119cce2..99ce0e0c7 100644 --- a/internal/pkg/transformers/reshape.go +++ b/internal/pkg/transformers/reshape.go @@ -302,7 +302,7 @@ func (tr *TransformerReshape) wideToLongNoRegex( ) { if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - pairs := types.NewMlrmap() + pairs := mlrval.NewMlrmap() for _, inputFieldName := range tr.inputFieldNames { value := inrec.Get(inputFieldName) if value != nil { @@ -341,7 +341,7 @@ func (tr *TransformerReshape) wideToLongRegex( ) { if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - pairs := types.NewMlrmap() + pairs := mlrval.NewMlrmap() for pd := inrec.Head; pd != nil; pd = pd.Next { for _, inputFieldRegex := range tr.inputFieldRegexes { @@ -440,13 +440,13 @@ func (tr *TransformerReshape) longToWide( } type tReshapeBucket struct { - representative *types.Mlrmap - pairs *types.Mlrmap + representative *mlrval.Mlrmap + pairs *mlrval.Mlrmap } -func newReshapeBucket(representative *types.Mlrmap) *tReshapeBucket { +func newReshapeBucket(representative *mlrval.Mlrmap) *tReshapeBucket { return &tReshapeBucket{ representative: representative, - pairs: types.NewMlrmap(), + pairs: mlrval.NewMlrmap(), } } diff --git a/internal/pkg/transformers/seqgen.go b/internal/pkg/transformers/seqgen.go index 059ed221b..aeab63af1 100644 --- a/internal/pkg/transformers/seqgen.go +++ b/internal/pkg/transformers/seqgen.go @@ -227,7 +227,7 @@ func (tr *TransformerSeqgen) Transform( break } - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() outrec.PutCopy(tr.fieldName, counter) context.UpdateForInputRecord() diff --git a/internal/pkg/transformers/stats1.go b/internal/pkg/transformers/stats1.go index 03b875b78..d047c6bfb 100644 --- a/internal/pkg/transformers/stats1.go +++ b/internal/pkg/transformers/stats1.go @@ -426,7 +426,7 @@ func (tr *TransformerStats1) handleInputRecord( // regexed group-by field names, the group-by field names/values are the same // on every record. func (tr *TransformerStats1) getGroupByFieldNamesWithoutRegexes( - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, ) ( groupingKey string, groupByFieldValues *lib.OrderedMap, // OrderedMap[string]*mlrval.Mlrval, @@ -449,7 +449,7 @@ func (tr *TransformerStats1) getGroupByFieldNamesWithoutRegexes( // regexed group-by field names, the group-by field names/values may or may not // be the same on every record. func (tr *TransformerStats1) getGroupByFieldNamesWithRegexes( - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, ) ( groupingKey string, groupByFieldValues *lib.OrderedMap, // OrderedMap[string]*mlrval.Mlrval, @@ -481,7 +481,7 @@ func (tr *TransformerStats1) getGroupByFieldNamesWithRegexes( } func (tr *TransformerStats1) ingestWithoutValueFieldRegexes( - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, groupingKey string, level2 *lib.OrderedMap, ) { @@ -518,7 +518,7 @@ func (tr *TransformerStats1) ingestWithoutValueFieldRegexes( } func (tr *TransformerStats1) ingestWithValueFieldRegexes( - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, groupingKey string, level2 *lib.OrderedMap, ) { @@ -600,7 +600,7 @@ func (tr *TransformerStats1) handleEndOfRecordStream( level2 := pa.Value.(*lib.OrderedMap) groupByFieldValues := tr.groupingKeysToGroupByFieldValues[groupingKey] - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() tr.emitIntoOutputRecord( inrecAndContext.Record, @@ -616,10 +616,10 @@ func (tr *TransformerStats1) handleEndOfRecordStream( } func (tr *TransformerStats1) emitIntoOutputRecord( - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, groupByFieldValues *lib.OrderedMap, // OrderedMap[string]*mlrval.Mlrval, level2accumulators *lib.OrderedMap, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { for pa := tr.groupByFieldNamesForOutput.Head; pa != nil; pa = pa.Next { diff --git a/internal/pkg/transformers/stats2.go b/internal/pkg/transformers/stats2.go index 067268296..ca7b9be1f 100644 --- a/internal/pkg/transformers/stats2.go +++ b/internal/pkg/transformers/stats2.go @@ -394,7 +394,7 @@ func (tr *TransformerStats2) emit( context *types.Context, ) { for pa := tr.namedAccumulators.Head; pa != nil; pa = pa.Next { - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() // Add in a=s,b=t fields: groupingKey := pa.Key @@ -428,7 +428,7 @@ func (tr *TransformerStats2) emit( } func (tr *TransformerStats2) populateRecord( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, valueFieldName1 string, valueFieldName2 string, valueFieldsToAccumulator *lib.OrderedMap, diff --git a/internal/pkg/transformers/step.go b/internal/pkg/transformers/step.go index e62e98e16..c2238f810 100644 --- a/internal/pkg/transformers/step.go +++ b/internal/pkg/transformers/step.go @@ -317,7 +317,7 @@ type tStepperAllocator func( ) tStepper type tStepper interface { - process(valueFieldValue *mlrval.Mlrval, inputRecord *types.Mlrmap) + process(valueFieldValue *mlrval.Mlrval, inputRecord *mlrval.Mlrmap) } type tStepperLookup struct { @@ -376,7 +376,7 @@ func stepperDeltaAlloc( func (stepper *tStepperDelta) process( valueFieldValue *mlrval.Mlrval, - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, ) { if valueFieldValue.IsEmpty() { inrec.PutCopy(stepper.outputFieldName, types.MLRVAL_VOID) @@ -411,7 +411,7 @@ func stepperShiftAlloc( func (stepper *tStepperShift) process( valueFieldValue *mlrval.Mlrval, - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, ) { if stepper.previous == nil { shift := types.MLRVAL_VOID @@ -442,7 +442,7 @@ func stepperFromFirstAlloc( func (stepper *tStepperFromFirst) process( valueFieldValue *mlrval.Mlrval, - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, ) { fromFirst := mlrval.MlrvalFromInt(0) if stepper.first == nil { @@ -472,7 +472,7 @@ func stepperRatioAlloc( func (stepper *tStepperRatio) process( valueFieldValue *mlrval.Mlrval, - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, ) { if valueFieldValue.IsEmpty() { inrec.PutCopy(stepper.outputFieldName, types.MLRVAL_VOID) @@ -507,7 +507,7 @@ func stepperRsumAlloc( func (stepper *tStepperRsum) process( valueFieldValue *mlrval.Mlrval, - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, ) { if valueFieldValue.IsEmpty() { inrec.PutCopy(stepper.outputFieldName, types.MLRVAL_VOID) @@ -538,7 +538,7 @@ func stepperCounterAlloc( func (stepper *tStepperCounter) process( valueFieldValue *mlrval.Mlrval, - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, ) { if valueFieldValue.IsEmpty() { inrec.PutCopy(stepper.outputFieldName, types.MLRVAL_VOID) @@ -609,7 +609,7 @@ func stepperEWMAAlloc( func (stepper *tStepperEWMA) process( valueFieldValue *mlrval.Mlrval, - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, ) { if !stepper.havePrevs { for i := range stepper.alphas { diff --git a/internal/pkg/transformers/template.go b/internal/pkg/transformers/template.go index 27bd8f36c..285ad307e 100644 --- a/internal/pkg/transformers/template.go +++ b/internal/pkg/transformers/template.go @@ -145,7 +145,7 @@ func (tr *TransformerTemplate) Transform( HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel) if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - outrec := types.NewMlrmap() + outrec := mlrval.NewMlrmap() for _, fieldName := range tr.fieldNameList { value := inrec.Get(fieldName) if value != nil { diff --git a/internal/pkg/transformers/top.go b/internal/pkg/transformers/top.go index c2012dd24..5a8199d19 100644 --- a/internal/pkg/transformers/top.go +++ b/internal/pkg/transformers/top.go @@ -260,7 +260,7 @@ func (tr *TransformerTop) emit( } else { for i := 0; i < tr.topCount; i++ { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() // Add in a=s,b=t fields: for j := range tr.groupByFieldNames { diff --git a/internal/pkg/transformers/uniq.go b/internal/pkg/transformers/uniq.go index 78b8d320c..3be2de7ea 100644 --- a/internal/pkg/transformers/uniq.go +++ b/internal/pkg/transformers/uniq.go @@ -400,7 +400,7 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowNumDistinctOnly( } } else { // end of record stream - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() outrec.PutReference( tr.outputFieldName, mlrval.MlrvalFromInt(tr.uniqifiedRecordCounts.FieldCount), @@ -474,7 +474,7 @@ func (tr *TransformerUniq) transformUnlashed( countsForFieldName := pe.Value.(*lib.OrderedMap) for pf := countsForFieldName.Head; pf != nil; pf = pf.Next { fieldValueString := pf.Key - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() outrec.PutReference("field", mlrval.mlrval.FromString(fieldName)) outrec.PutCopy( "value", @@ -510,7 +510,7 @@ func (tr *TransformerUniq) transformNumDistinctOnly( } } else { - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() outrec.PutReference( "count", mlrval.MlrvalFromInt(tr.countsByGroup.FieldCount), @@ -545,7 +545,7 @@ func (tr *TransformerUniq) transformWithCounts( } else { // end of record stream for pa := tr.countsByGroup.Head; pa != nil; pa = pa.Next { - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() valuesForGroup := tr.valuesByGroup.Get(pa.Key).([]*mlrval.Mlrval) for i, fieldName := range tr.fieldNames { outrec.PutCopy( @@ -585,7 +585,7 @@ func (tr *TransformerUniq) transformWithoutCounts( if !present { tr.countsByGroup.Put(groupingKey, 1) tr.valuesByGroup.Put(groupingKey, selectedValues) - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() for i, fieldName := range tr.fieldNames { outrec.PutCopy( diff --git a/internal/pkg/transformers/unsparsify.go b/internal/pkg/transformers/unsparsify.go index cebbaeb35..b8a3850c4 100644 --- a/internal/pkg/transformers/unsparsify.go +++ b/internal/pkg/transformers/unsparsify.go @@ -175,7 +175,7 @@ func (tr *TransformerUnsparsify) transformNonStreaming( outrecAndContext := e.Value.(*types.RecordAndContext) outrec := outrecAndContext.Record - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for pe := tr.fieldNamesSeen.Head; pe != nil; pe = pe.Next { fieldName := pe.Key if !outrec.Has(fieldName) { diff --git a/internal/pkg/transformers/utils/stats2-accumulators.go b/internal/pkg/transformers/utils/stats2-accumulators.go index 3859c0680..458a411a2 100644 --- a/internal/pkg/transformers/utils/stats2-accumulators.go +++ b/internal/pkg/transformers/utils/stats2-accumulators.go @@ -24,13 +24,13 @@ type IStats2Accumulator interface { Populate( valueFieldName1 string, valueFieldName2 string, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) Fit( x float64, y float64, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) } @@ -178,7 +178,7 @@ func (acc *Stats2LinRegOLSAccumulator) Ingest( func (acc *Stats2LinRegOLSAccumulator) Populate( valueFieldName1 string, valueFieldName2 string, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { if acc.count < 2 { outrec.PutCopy(acc.mOutputFieldName, types.MLRVAL_VOID) @@ -196,7 +196,7 @@ func (acc *Stats2LinRegOLSAccumulator) Populate( func (acc *Stats2LinRegOLSAccumulator) Fit( x float64, y float64, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { if !acc.fitReady { @@ -263,7 +263,7 @@ func (acc *Stats2LogiRegAccumulator) Ingest( func (acc *Stats2LogiRegAccumulator) Populate( valueFieldName1 string, valueFieldName2 string, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { if len(acc.xs) < 2 { @@ -280,7 +280,7 @@ func (acc *Stats2LogiRegAccumulator) Populate( func (acc *Stats2LogiRegAccumulator) Fit( x float64, y float64, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { if !acc.fitReady { @@ -348,7 +348,7 @@ func (acc *Stats2R2Accumulator) Ingest( func (acc *Stats2R2Accumulator) Populate( valueFieldName1 string, valueFieldName2 string, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { if acc.count < 2 { @@ -372,7 +372,7 @@ func (acc *Stats2R2Accumulator) Populate( func (acc *Stats2R2Accumulator) Fit( x float64, y float64, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { } @@ -489,7 +489,7 @@ func (acc *Stats2CorrCovAccumulator) Ingest( func (acc *Stats2CorrCovAccumulator) Populate( valueFieldName1 string, valueFieldName2 string, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { if acc.doWhich == DO_COVX { @@ -597,7 +597,7 @@ func (acc *Stats2CorrCovAccumulator) Populate( func (acc *Stats2CorrCovAccumulator) Fit( x float64, y float64, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { if acc.doWhich != DO_LINREG_PCA { return diff --git a/internal/pkg/types/context.go b/internal/pkg/types/context.go index 311edb520..91a82540b 100644 --- a/internal/pkg/types/context.go +++ b/internal/pkg/types/context.go @@ -4,6 +4,8 @@ import ( "bytes" "container/list" "strconv" + + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // Since Go is concurrent, the context struct (AWK-like variables such as @@ -17,14 +19,14 @@ import ( // output ordering. type RecordAndContext struct { - Record *Mlrmap + Record *mlrval.Mlrmap Context Context OutputString string EndOfStream bool } func NewRecordAndContext( - record *Mlrmap, + record *mlrval.Mlrmap, context *Context, ) *RecordAndContext { return &RecordAndContext{ diff --git a/internal/pkg/types-parked/mlrval_typing.go b/internal/pkg/types/mlrval_typing.go similarity index 83% rename from internal/pkg/types-parked/mlrval_typing.go rename to internal/pkg/types/mlrval_typing.go index a9ba8550e..334241c3c 100644 --- a/internal/pkg/types-parked/mlrval_typing.go +++ b/internal/pkg/types/mlrval_typing.go @@ -8,6 +8,8 @@ package types import ( "errors" "fmt" + + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ---------------------------------------------------------------- @@ -21,7 +23,7 @@ func NewTypeGatedMlrvalName( name string, // e.g. "x" typeName string, // e.g. "num" ) (*TypeGatedMlrvalName, error) { - typeMask, ok := TypeNameToMask(typeName) + typeMask, ok := mlrval.TypeNameToMask(typeName) if !ok { return nil, errors.New( fmt.Sprintf( @@ -36,7 +38,7 @@ func NewTypeGatedMlrvalName( }, nil } -func (tname *TypeGatedMlrvalName) Check(value *Mlrval) error { +func (tname *TypeGatedMlrvalName) Check(value *mlrval.Mlrval) error { bit := value.GetTypeBit() if bit&tname.TypeMask != 0 { return nil @@ -53,13 +55,13 @@ func (tname *TypeGatedMlrvalName) Check(value *Mlrval) error { // ---------------------------------------------------------------- type TypeGatedMlrvalVariable struct { typeGatedMlrvalName *TypeGatedMlrvalName - value *Mlrval + value *mlrval.Mlrval } func NewTypeGatedMlrvalVariable( name string, // e.g. "x" typeName string, // e.g. "num" - value *Mlrval, + value *mlrval.Mlrval, ) (*TypeGatedMlrvalVariable, error) { typeGatedMlrvalName, err := NewTypeGatedMlrvalName(name, typeName) if err != nil { @@ -81,7 +83,7 @@ func (tvar *TypeGatedMlrvalVariable) GetName() string { return tvar.typeGatedMlrvalName.Name } -func (tvar *TypeGatedMlrvalVariable) GetValue() *Mlrval { +func (tvar *TypeGatedMlrvalVariable) GetValue() *mlrval.Mlrval { return tvar.value } @@ -89,7 +91,7 @@ func (tvar *TypeGatedMlrvalVariable) ValueString() string { return tvar.value.String() } -func (tvar *TypeGatedMlrvalVariable) Assign(value *Mlrval) error { +func (tvar *TypeGatedMlrvalVariable) Assign(value *mlrval.Mlrval) error { err := tvar.typeGatedMlrvalName.Check(value) if err != nil { return err @@ -101,5 +103,5 @@ func (tvar *TypeGatedMlrvalVariable) Assign(value *Mlrval) error { } func (tvar *TypeGatedMlrvalVariable) Unassign() { - tvar.value = MLRVAL_ABSENT + tvar.value = mlrval.ABSENT } diff --git a/scripts/time-big-files b/scripts/time-big-files index 3de09bf1c..c334e233c 100755 --- a/scripts/time-big-files +++ b/scripts/time-big-files @@ -3,24 +3,30 @@ ourdir=$(dirname $0) $ourdir/time-big-file dkvp mlr5 +$ourdir/time-big-file dkvp mlr $ourdir/time-big-file dkvp 'mlr -S' echo $ourdir/time-big-file nidx mlr5 +$ourdir/time-big-file nidx mlr $ourdir/time-big-file nidx 'mlr -S' echo $ourdir/time-big-file xtab mlr5 +$ourdir/time-big-file xtab mlr $ourdir/time-big-file xtab 'mlr -S' echo $ourdir/time-big-file csv mlr5 +$ourdir/time-big-file csv mlr $ourdir/time-big-file csv 'mlr -S' echo $ourdir/time-big-file csvlite mlr5 +$ourdir/time-big-file csvlite mlr $ourdir/time-big-file csvlite 'mlr -S' echo $ourdir/time-big-file json mlr5 +$ourdir/time-big-file json mlr $ourdir/time-big-file json 'mlr -S' diff --git a/test/cases/verb-fill-empty/x b/test/cases/verb-fill-empty/x deleted file mode 100644 index f4bdb8b24..000000000 --- a/test/cases/verb-fill-empty/x +++ /dev/null @@ -1 +0,0 @@ -$tb = typeof($b) diff --git a/todo.txt b/todo.txt index 6590403aa..e90f9393d 100644 --- a/todo.txt +++ b/todo.txt @@ -3,8 +3,11 @@ PUNCHDOWN LIST * --ifs-regex & --ips-regex -- guessing is not safe as evidence by '.' and '|' * JIT next: + o mv.Equals & mv.Copy o mlrmap copy + o JSON I/O UT o krepl miller6 new doclink-comments + ? []Mlrval -> []*Mlrval ? * perf next: o batchify source-quench @@ -42,6 +45,8 @@ PUNCHDOWN LIST - make a 2nd/3rd cmd main w/ simple model & tweak that o dkvp-reader factor-out ... o mods: + - replace rac-list with batch-array -- ? profile to see what value. + - unaliasArrayIndex in multiple packages for cycdep break -- ? - randsel for arrays - downstreamDone batchify > head @@ -265,6 +270,8 @@ PUNCHDOWN LIST ================================================================ NON-BLOCKERS +* handling for --cpuprofile not at args[1] slot + * try simpler-than-regex-split-string for repeated-single -- especially for XTAB reader * UT-per-se of XTAB channelizedStanzaScanner From f18983c221bb149cf6a0242d1855a666c1e2fc1c Mon Sep 17 00:00:00 2001 From: John Kerl Date: Thu, 16 Dec 2021 17:42:10 -0500 Subject: [PATCH 09/19] mlrval_cmp.go --- .vimrc | 14 +- Makefile | 1 + internal/pkg/bifs/booleans.go | 2 +- internal/pkg/input/pseudo_reader_gen.go | 8 +- internal/pkg/mlrval/mlrmap_accessors.go | 63 ++-- internal/pkg/mlrval/mlrval_cmp.go | 439 ++++++++++++++++++++++++ todo.txt | 3 + 7 files changed, 484 insertions(+), 46 deletions(-) create mode 100644 internal/pkg/mlrval/mlrval_cmp.go diff --git a/.vimrc b/.vimrc index aa4890dc9..bef8ce9ef 100644 --- a/.vimrc +++ b/.vimrc @@ -1,8 +1,8 @@ map \d :w:!clear;echo Building ...; echo; make mlr -map \r :w:!clear;echo Building ...; echo; make tests-in-order -"map \f :w:!clear;echo Building ...; echo; make mlrval-tests -"map \f :w:!clear;echo Building ...; echo; make mlrmap-tests -"map \f :w:!clear;echo Building ...; echo; make input-tests -"map \f :w:!clear;echo Building ...; echo; make mlrval-format-test -"map \f :w:!clear;echo Building ...; echo; make bifs-tests -map \f :w:!clear;echo Building ...; echo; make bifs-collections-test +map \f :w:!clear;echo Building ...; echo; make tests-in-order +"map \r :w:!clear;echo Building ...; echo; make mlrval-tests +"map \r :w:!clear;echo Building ...; echo; make mlrmap-tests +"map \r :w:!clear;echo Building ...; echo; make input-tests +"map \r :w:!clear;echo Building ...; echo; make mlrval-format-test +"map \r :w:!clear;echo Building ...; echo; make bifs-tests +map \r :w:!clear;echo Building ...; echo; make bifs-collections-test diff --git a/Makefile b/Makefile index 3c3bd31bc..310e562c7 100644 --- a/Makefile +++ b/Makefile @@ -101,6 +101,7 @@ mlrmap-accessors-test: internal/pkg/mlrval/mlrval_type.go \ internal/pkg/mlrval/mlrval_constants.go \ internal/pkg/mlrval/mlrval_new.go \ + internal/pkg/mlrval/mlrval_cmp.go \ internal/pkg/mlrval/mlrval_copy.go \ internal/pkg/mlrval/mlrval_infer.go \ internal/pkg/mlrval/mlrval_is.go \ diff --git a/internal/pkg/bifs/booleans.go b/internal/pkg/bifs/booleans.go index 37402907e..ce7f513db 100644 --- a/internal/pkg/bifs/booleans.go +++ b/internal/pkg/bifs/booleans.go @@ -262,7 +262,7 @@ func eq_b_mm(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return mlrval.FromBool(input1.AcquireMapValue().Equals(input2.AcquireMapValue())) } func ne_b_mm(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { - return mlrval.FromBool(!input1.AcquireMapValue().Equals(input2.AcquireArrayValue())) + return mlrval.FromBool(!input1.AcquireMapValue().Equals(input2.AcquireMapValue())) } // We get a Golang "initialization loop" due to recursive depth computation diff --git a/internal/pkg/input/pseudo_reader_gen.go b/internal/pkg/input/pseudo_reader_gen.go index aec04916a..5aad82399 100644 --- a/internal/pkg/input/pseudo_reader_gen.go +++ b/internal/pkg/input/pseudo_reader_gen.go @@ -64,9 +64,9 @@ func (reader *PseudoReaderGen) process( return } - var doneComparator types.BinaryFunc = bifs.BIF_greater_than + var doneComparator mlrval.CmpFuncBool = mlrval.GreaterThan if step.GetNumericNegativeorDie() { - doneComparator = bifs.BIF_less_than + doneComparator = mlrval.LessThan } key := reader.readerOptions.GeneratorOptions.FieldName @@ -77,9 +77,7 @@ func (reader *PseudoReaderGen) process( eof := false for !eof { - mdone := doneComparator(value, stop) - done, _ := mdone.GetBoolValue() - if done { + if doneComparator(value, stop) { break } diff --git a/internal/pkg/mlrval/mlrmap_accessors.go b/internal/pkg/mlrval/mlrmap_accessors.go index cb00d9b67..07aff6b14 100644 --- a/internal/pkg/mlrval/mlrmap_accessors.go +++ b/internal/pkg/mlrval/mlrmap_accessors.go @@ -433,40 +433,37 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { // } //} -//// ---------------------------------------------------------------- -//func (mlrmap *Mlrmap) Equals(other *Mlrmap) bool { -// if mlrmap.FieldCount != mlrmap.FieldCount { -// return false -// } -// if !mlrmap.Contains(other) { -// return false -// } -// if !other.Contains(mlrmap) { -// return false -// } -// return true -//} +// ---------------------------------------------------------------- +func (mlrmap *Mlrmap) Equals(other *Mlrmap) bool { + if mlrmap.FieldCount != mlrmap.FieldCount { + return false + } + if !mlrmap.Contains(other) { + return false + } + if !other.Contains(mlrmap) { + return false + } + return true +} -//// True if this contains other, i.e. if other is contained by mlrmap. -//// * If any key of other is not a key of this, return false. -//// * If any key of other has a value unequal to this' value at the same key, return false. -//// * Else return true -//func (mlrmap *Mlrmap) Contains(other *Mlrmap) bool { -// for pe := other.Head; pe != nil; pe = pe.Next { -// if !mlrmap.Has(pe.Key) { -// return false -// } -// thisval := mlrmap.Get(pe.Key) -// thatval := pe.Value -// meq := bifs.BIF_equals(thisval, thatval) -// eq, ok := meq.GetBoolValue() -// lib.InternalCodingErrorIf(!ok) -// if !eq { -// return false -// } -// } -// return true -//} +// True if this contains other, i.e. if other is contained by mlrmap. +// * If any key of other is not a key of this, return false. +// * If any key of other has a value unequal to this' value at the same key, return false. +// * Else return true +func (mlrmap *Mlrmap) Contains(other *Mlrmap) bool { + for pe := other.Head; pe != nil; pe = pe.Next { + if !mlrmap.Has(pe.Key) { + return false + } + thisval := mlrmap.Get(pe.Key) + thatval := pe.Value + if !Equals(thisval, thatval) { + return false + } + } + return true +} //// ---------------------------------------------------------------- //func (mlrmap *Mlrmap) Clear() { diff --git a/internal/pkg/mlrval/mlrval_cmp.go b/internal/pkg/mlrval/mlrval_cmp.go new file mode 100644 index 000000000..940577ddb --- /dev/null +++ b/internal/pkg/mlrval/mlrval_cmp.go @@ -0,0 +1,439 @@ +// ================================================================ +// Boolean expressions for ==, !=, >, >=, <, <=, <=> on Mlrvals +// ================================================================ + +// TODO: comment about mvtype; deferral; copying of deferrence. + +package mlrval + +import ( + "github.com/johnkerl/miller/internal/pkg/lib" +) + +type CmpFuncBool func(input1, input2 *Mlrval) bool +type CmpFuncInt func(input1, input2 *Mlrval) int // -1, 0, 1 for <=> + +// ---------------------------------------------------------------- +// Exported methods + +func Equals(input1, input2 *Mlrval) bool { + return eq_dispositions[input1.mvtype][input2.mvtype](input1, input2) +} +func NotEquals(input1, input2 *Mlrval) bool { + return ne_dispositions[input1.mvtype][input2.mvtype](input1, input2) +} +func GreaterThan(input1, input2 *Mlrval) bool { + return gt_dispositions[input1.mvtype][input2.mvtype](input1, input2) +} +func GreaterThanOrEquals(input1, input2 *Mlrval) bool { + return ge_dispositions[input1.mvtype][input2.mvtype](input1, input2) +} +func LessThan(input1, input2 *Mlrval) bool { + return lt_dispositions[input1.mvtype][input2.mvtype](input1, input2) +} +func LessThanOrEquals(input1, input2 *Mlrval) bool { + return le_dispositions[input1.mvtype][input2.mvtype](input1, input2) +} +func Cmp(input1, input2 *Mlrval) int { + return cmp_dispositions[input1.mvtype][input2.mvtype](input1, input2) +} + +// ---------------------------------------------------------------- +// Support routines for disposition-matrix entries + +// string_cmp implements the spaceship operator for strings. +func string_cmp(a, b string) int { + if a < b { + return -1 + } + if a > b { + return 1 + } + return 0 +} + +// int_cmp implements the spaceship operator for ints. +func int_cmp(a, b int) int { + if a < b { + return -1 + } + if a > b { + return 1 + } + return 0 +} + +// float_cmp implements the spaceship operator for floats. +func float_cmp(a, b float64) int { + if a < b { + return -1 + } + if a > b { + return 1 + } + return 0 +} + +// _true return boolean true as a binary-input function +func _true(input1, input2 *Mlrval) bool { + return true +} + +// _fals return boolean false as a binary-input function +func _fals(input1, input2 *Mlrval) bool { + return false +} + +// _i0__ return int 0 as a binary-input function +func _i0__(input1, input2 *Mlrval) int { + return 0 +} + +// _i1__ return int 1 as a binary-input function +func _i1__(input1, input2 *Mlrval) int { + return 1 +} + +// _n1__ return int -1 as a binary-input function +func _n1__(input1, input2 *Mlrval) int { + return -1 +} + +// ---------------------------------------------------------------- +// Disposition-matrix entries + +func eq_b_ss(input1, input2 *Mlrval) bool { + return input1.printrep == input2.printrep +} +func ne_b_ss(input1, input2 *Mlrval) bool { + return input1.printrep != input2.printrep +} +func gt_b_ss(input1, input2 *Mlrval) bool { + return input1.printrep > input2.printrep +} +func ge_b_ss(input1, input2 *Mlrval) bool { + return input1.printrep >= input2.printrep +} +func lt_b_ss(input1, input2 *Mlrval) bool { + return input1.printrep < input2.printrep +} +func le_b_ss(input1, input2 *Mlrval) bool { + return input1.printrep <= input2.printrep +} +func cmp_b_ss(input1, input2 *Mlrval) int { + return string_cmp(input1.printrep, input2.printrep) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_xs(input1, input2 *Mlrval) bool { + return input1.String() == input2.printrep +} +func ne_b_xs(input1, input2 *Mlrval) bool { + return input1.String() != input2.printrep +} +func gt_b_xs(input1, input2 *Mlrval) bool { + return input1.String() > input2.printrep +} +func ge_b_xs(input1, input2 *Mlrval) bool { + return input1.String() >= input2.printrep +} +func lt_b_xs(input1, input2 *Mlrval) bool { + return input1.String() < input2.printrep +} +func le_b_xs(input1, input2 *Mlrval) bool { + return input1.String() <= input2.printrep +} +func cmp_b_xs(input1, input2 *Mlrval) int { + return string_cmp(input1.String(), input2.printrep) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_sx(input1, input2 *Mlrval) bool { + return input1.printrep == input2.String() +} +func ne_b_sx(input1, input2 *Mlrval) bool { + return input1.printrep != input2.String() +} +func gt_b_sx(input1, input2 *Mlrval) bool { + return input1.printrep > input2.String() +} +func ge_b_sx(input1, input2 *Mlrval) bool { + return input1.printrep >= input2.String() +} +func lt_b_sx(input1, input2 *Mlrval) bool { + return input1.printrep < input2.String() +} +func le_b_sx(input1, input2 *Mlrval) bool { + return input1.printrep <= input2.String() +} +func cmp_b_sx(input1, input2 *Mlrval) int { + return string_cmp(input1.printrep, input2.String()) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_ii(input1, input2 *Mlrval) bool { + return input1.intval == input2.intval +} +func ne_b_ii(input1, input2 *Mlrval) bool { + return input1.intval != input2.intval +} +func gt_b_ii(input1, input2 *Mlrval) bool { + return input1.intval > input2.intval +} +func ge_b_ii(input1, input2 *Mlrval) bool { + return input1.intval >= input2.intval +} +func lt_b_ii(input1, input2 *Mlrval) bool { + return input1.intval < input2.intval +} +func le_b_ii(input1, input2 *Mlrval) bool { + return input1.intval <= input2.intval +} +func cmp_b_ii(input1, input2 *Mlrval) int { + return int_cmp(input1.intval, input2.intval) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_if(input1, input2 *Mlrval) bool { + return float64(input1.intval) == input2.floatval +} +func ne_b_if(input1, input2 *Mlrval) bool { + return float64(input1.intval) != input2.floatval +} +func gt_b_if(input1, input2 *Mlrval) bool { + return float64(input1.intval) > input2.floatval +} +func ge_b_if(input1, input2 *Mlrval) bool { + return float64(input1.intval) >= input2.floatval +} +func lt_b_if(input1, input2 *Mlrval) bool { + return float64(input1.intval) < input2.floatval +} +func le_b_if(input1, input2 *Mlrval) bool { + return float64(input1.intval) <= input2.floatval +} +func cmp_b_if(input1, input2 *Mlrval) int { + return float_cmp(float64(input1.intval), input2.floatval) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_fi(input1, input2 *Mlrval) bool { + return input1.floatval == float64(input2.intval) +} +func ne_b_fi(input1, input2 *Mlrval) bool { + return input1.floatval != float64(input2.intval) +} +func gt_b_fi(input1, input2 *Mlrval) bool { + return input1.floatval > float64(input2.intval) +} +func ge_b_fi(input1, input2 *Mlrval) bool { + return input1.floatval >= float64(input2.intval) +} +func lt_b_fi(input1, input2 *Mlrval) bool { + return input1.floatval < float64(input2.intval) +} +func le_b_fi(input1, input2 *Mlrval) bool { + return input1.floatval <= float64(input2.intval) +} +func cmp_b_fi(input1, input2 *Mlrval) int { + return float_cmp(input1.floatval, float64(input2.intval)) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_ff(input1, input2 *Mlrval) bool { + return input1.floatval == input2.floatval +} +func ne_b_ff(input1, input2 *Mlrval) bool { + return input1.floatval != input2.floatval +} +func gt_b_ff(input1, input2 *Mlrval) bool { + return input1.floatval > input2.floatval +} +func ge_b_ff(input1, input2 *Mlrval) bool { + return input1.floatval >= input2.floatval +} +func lt_b_ff(input1, input2 *Mlrval) bool { + return input1.floatval < input2.floatval +} +func le_b_ff(input1, input2 *Mlrval) bool { + return input1.floatval <= input2.floatval +} +func cmp_b_ff(input1, input2 *Mlrval) int { + return float_cmp(input1.floatval, input2.floatval) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_bb(input1, input2 *Mlrval) bool { + return input1.boolval == input2.boolval +} +func ne_b_bb(input1, input2 *Mlrval) bool { + return input1.boolval != input2.boolval +} + +func gt_b_bb(input1, input2 *Mlrval) bool { + return lib.BoolToInt(input1.boolval) > lib.BoolToInt(input2.boolval) +} +func ge_b_bb(input1, input2 *Mlrval) bool { + return lib.BoolToInt(input1.boolval) >= lib.BoolToInt(input2.boolval) +} +func lt_b_bb(input1, input2 *Mlrval) bool { + return lib.BoolToInt(input1.boolval) < lib.BoolToInt(input2.boolval) +} +func le_b_bb(input1, input2 *Mlrval) bool { + return lib.BoolToInt(input1.boolval) <= lib.BoolToInt(input2.boolval) +} +func cmp_b_bb(input1, input2 *Mlrval) int { + return int_cmp(lib.BoolToInt(input1.boolval), lib.BoolToInt(input2.boolval)) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_aa(input1, input2 *Mlrval) bool { + a := input1.arrayval + b := input2.arrayval + + // Different-length arrays are not equal + if len(a) != len(b) { + return false + } + + // Same-length arrays: return false if any slot is not equal, else true. + for i := range a { + if !Equals(&a[i], &b[i]) { + return false + } + } + + return true +} +func ne_b_aa(input1, input2 *Mlrval) bool { + return !eq_b_aa(input1, input2) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_mm(input1, input2 *Mlrval) bool { + return input1.mapval.Equals(input2.mapval) +} +func ne_b_mm(input1, input2 *Mlrval) bool { + return !input1.mapval.Equals(input2.mapval) +} + +// We get a Golang "initialization loop" due to recursive depth computation +// if this is defined statically. So, we use a "package init" function. +var eq_dispositions = [MT_DIM][MT_DIM]CmpFuncBool{} + +func init() { + eq_dispositions = [MT_DIM][MT_DIM]CmpFuncBool{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*ABSENT */ {_fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*NULL */ {_fals, _fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*VOID */ {_fals, _fals, _fals, eq_b_ss, eq_b_ss, eq_b_sx, eq_b_sx, _fals, _fals, _fals, _fals}, + /*STRING */ {_fals, _fals, _fals, eq_b_ss, eq_b_ss, eq_b_sx, eq_b_sx, _fals, _fals, _fals, _fals}, + /*INT */ {_fals, _fals, _fals, eq_b_xs, eq_b_xs, eq_b_ii, eq_b_if, _fals, _fals, _fals, _fals}, + /*FLOAT */ {_fals, _fals, _fals, eq_b_xs, eq_b_xs, eq_b_fi, eq_b_ff, _fals, _fals, _fals, _fals}, + /*BOOL */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, eq_b_bb, _fals, _fals, _fals}, + /*ARRAY */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, eq_b_aa, _fals, _fals}, + /*MAP */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, eq_b_mm, _fals}, + /*FUNC */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _true}, + } +} + +var ne_dispositions = [MT_DIM][MT_DIM]CmpFuncBool{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*ABSENT */ {_fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*NULL */ {_fals, _fals, _fals, _true, _true, _true, _true, _true, _true, _true, _fals}, + /*VOID */ {_fals, _fals, _true, ne_b_ss, ne_b_ss, ne_b_sx, ne_b_sx, _true, _true, _true, _fals}, + /*STRING */ {_fals, _fals, _true, ne_b_ss, ne_b_ss, ne_b_sx, ne_b_sx, _true, _true, _true, _fals}, + /*INT */ {_fals, _fals, _true, ne_b_xs, ne_b_xs, ne_b_ii, ne_b_if, _true, _true, _true, _fals}, + /*FLOAT */ {_fals, _fals, _true, ne_b_xs, ne_b_xs, ne_b_fi, ne_b_ff, _true, _true, _true, _fals}, + /*BOOL */ {_fals, _fals, _true, _true, _true, _true, _true, ne_b_bb, _true, _true, _fals}, + /*ARRAY */ {_fals, _fals, _true, _true, _true, _true, _true, _true, ne_b_aa, _true, _fals}, + /*MAP */ {_fals, _fals, _true, _true, _true, _true, _true, _true, _true, ne_b_mm, _fals}, + /*FUNC */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _true}, +} + +var gt_dispositions = [MT_DIM][MT_DIM]CmpFuncBool{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*ABSENT */ {_fals, _true, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*NULL */ {_true, _fals, _fals, _true, _true, _true, _true, _true, _fals, _fals, _fals}, + /*VOID */ {_fals, _fals, _fals, gt_b_ss, gt_b_ss, gt_b_sx, gt_b_sx, _fals, _fals, _fals, _fals}, + /*STRING */ {_fals, _fals, _fals, gt_b_ss, gt_b_ss, gt_b_sx, gt_b_sx, _fals, _fals, _fals, _fals}, + /*INT */ {_fals, _fals, _fals, gt_b_xs, gt_b_xs, gt_b_ii, gt_b_if, _fals, _fals, _fals, _fals}, + /*FLOAT */ {_fals, _fals, _fals, gt_b_xs, gt_b_xs, gt_b_fi, gt_b_ff, _fals, _fals, _fals, _fals}, + /*BOOL */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, gt_b_bb, _fals, _fals, _fals}, + /*ARRAY */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*MAP */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*FUNC */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _true}, +} + +var ge_dispositions = [MT_DIM][MT_DIM]CmpFuncBool{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*ABSENT */ {_fals, _true, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*NULL */ {_true, _fals, _true, _true, _true, _true, _true, _true, _fals, _fals, _fals}, + /*VOID */ {_fals, _fals, _fals, ge_b_ss, ge_b_ss, ge_b_sx, ge_b_sx, _fals, _fals, _fals, _fals}, + /*STRING */ {_fals, _fals, _fals, ge_b_ss, ge_b_ss, ge_b_sx, ge_b_sx, _fals, _fals, _fals, _fals}, + /*INT */ {_fals, _fals, _fals, ge_b_xs, ge_b_xs, ge_b_ii, ge_b_if, _fals, _fals, _fals, _fals}, + /*FLOAT */ {_fals, _fals, _fals, ge_b_xs, ge_b_xs, ge_b_fi, ge_b_ff, _fals, _fals, _fals, _fals}, + /*BOOL */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, ge_b_bb, _fals, _fals, _fals}, + /*ARRAY */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*MAP */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*FUNC */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _true}, +} + +var lt_dispositions = [MT_DIM][MT_DIM]CmpFuncBool{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_true, _fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*ABSENT */ {_fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*NULL */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*VOID */ {_fals, _fals, _true, lt_b_ss, lt_b_ss, lt_b_sx, lt_b_sx, _fals, _fals, _fals, _fals}, + /*STRING */ {_fals, _fals, _true, lt_b_ss, lt_b_ss, lt_b_sx, lt_b_sx, _fals, _fals, _fals, _fals}, + /*INT */ {_fals, _fals, _true, lt_b_xs, lt_b_xs, lt_b_ii, lt_b_if, _fals, _fals, _fals, _fals}, + /*FLOAT */ {_fals, _fals, _true, lt_b_xs, lt_b_xs, lt_b_fi, lt_b_ff, _fals, _fals, _fals, _fals}, + /*BOOL */ {_fals, _fals, _true, _fals, _fals, _fals, _fals, lt_b_bb, _fals, _fals, _fals}, + /*ARRAY */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*MAP */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*FUNC */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _true}, +} + +var le_dispositions = [MT_DIM][MT_DIM]CmpFuncBool{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_true, _fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*ABSENT */ {_fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*NULL */ {_fals, _fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*VOID */ {_fals, _fals, _true, le_b_ss, le_b_ss, le_b_sx, le_b_sx, _fals, _fals, _fals, _fals}, + /*STRING */ {_fals, _fals, _true, le_b_ss, le_b_ss, le_b_sx, le_b_sx, _fals, _fals, _fals, _fals}, + /*INT */ {_fals, _fals, _true, le_b_xs, le_b_xs, le_b_ii, le_b_if, _fals, _fals, _fals, _fals}, + /*FLOAT */ {_fals, _fals, _true, le_b_xs, le_b_xs, le_b_fi, le_b_ff, _fals, _fals, _fals, _fals}, + /*BOOL */ {_fals, _fals, _true, _fals, _fals, _fals, _fals, le_b_bb, _fals, _fals, _fals}, + /*ARRAY */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*MAP */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*FUNC */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _true}, +} + +// TODO: flesh these out for array and map +var cmp_dispositions = [MT_DIM][MT_DIM]CmpFuncInt{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_i0__, _i0__, _n1__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__}, + /*ABSENT */ {_i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__}, + /*NULL */ {_i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__}, + /*VOID */ {_i0__, _i0__, _n1__, cmp_b_ss, cmp_b_ss, cmp_b_sx, cmp_b_sx, _i0__, _i0__, _i0__, _i0__}, + /*STRING */ {_i0__, _i0__, _n1__, cmp_b_ss, cmp_b_ss, cmp_b_sx, cmp_b_sx, _i0__, _i0__, _i0__, _i0__}, + /*INT */ {_i0__, _i0__, _n1__, cmp_b_xs, cmp_b_xs, cmp_b_ii, cmp_b_if, _i0__, _i0__, _i0__, _i0__}, + /*FLOAT */ {_i0__, _i0__, _n1__, cmp_b_xs, cmp_b_xs, cmp_b_fi, cmp_b_ff, _i0__, _i0__, _i0__, _i0__}, + /*BOOL */ {_i0__, _i0__, _n1__, _i0__, _i0__, _i0__, _i0__, cmp_b_bb, _i0__, _i0__, _i0__}, + /*ARRAY */ {_i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__}, + /*MAP */ {_i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__}, + /*FUNC */ {_i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__}, +} diff --git a/todo.txt b/todo.txt index e90f9393d..c23183bd0 100644 --- a/todo.txt +++ b/todo.txt @@ -5,6 +5,7 @@ PUNCHDOWN LIST * JIT next: o mv.Equals & mv.Copy o mlrmap copy + o move all but wrappers from $bifs/booleans.go to mlrval; also copy-reduce for sort -- ? o JSON I/O UT o krepl miller6 new doclink-comments ? []Mlrval -> []*Mlrval ? @@ -45,6 +46,8 @@ PUNCHDOWN LIST - make a 2nd/3rd cmd main w/ simple model & tweak that o dkvp-reader factor-out ... o mods: + ! need cmps for array/map -- no absent/error anymore here + ! moreover, proofread all mlrval_cmp.go dispo mxes - replace rac-list with batch-array -- ? profile to see what value. - unaliasArrayIndex in multiple packages for cycdep break -- ? - randsel for arrays From e84897bb755653d1c6fbb629ee9f8a0f7d02bd89 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Thu, 16 Dec 2021 17:52:10 -0500 Subject: [PATCH 10/19] mlrval JSON iterate --- .../{types-parked => mlrval}/mlrmap_json.go | 12 ++++---- .../{types-parked => mlrval}/mlrval_json.go | 30 +++++++++---------- internal/pkg/mlrval/mlrval_new.go | 17 +++++++++++ 3 files changed, 38 insertions(+), 21 deletions(-) rename internal/pkg/{types-parked => mlrval}/mlrmap_json.go (95%) rename internal/pkg/{types-parked => mlrval}/mlrval_json.go (96%) diff --git a/internal/pkg/types-parked/mlrmap_json.go b/internal/pkg/mlrval/mlrmap_json.go similarity index 95% rename from internal/pkg/types-parked/mlrmap_json.go rename to internal/pkg/mlrval/mlrmap_json.go index 60624f18e..7d8715302 100644 --- a/internal/pkg/types-parked/mlrmap_json.go +++ b/internal/pkg/mlrval/mlrmap_json.go @@ -2,7 +2,7 @@ // See mlrval_json.go for details. This is the unmarshal/marshal solely for Mlrmap. // ================================================================ -package types +package mlrval import ( "bytes" @@ -76,7 +76,7 @@ func (mlrmap *Mlrmap) marshalJSONAuxMultiline( // Write the key which is necessarily string-valued in Miller, and in // JSON for that matter :) for i := 0; i < elementNestingDepth; i++ { - buffer.WriteString(MLRVAL_JSON_INDENT_STRING) + buffer.WriteString(JSON_INDENT_STRING) } encoded := string(millerJSONEncodeString(pe.Key)) colorized := colorizer.MaybeColorizeKey(encoded, outputIsStdout) @@ -102,7 +102,7 @@ func (mlrmap *Mlrmap) marshalJSONAuxMultiline( // Write empty map as '{}'. if mlrmap.Head != nil { for i := 0; i < elementNestingDepth-1; i++ { - buffer.WriteString(MLRVAL_JSON_INDENT_STRING) + buffer.WriteString(JSON_INDENT_STRING) } } buffer.WriteString("}") @@ -153,9 +153,9 @@ func (entry *MlrmapEntry) JSONStringifyInPlace( ) { outputBytes, err := entry.Value.MarshalJSON(jsonFormatting, false) if err != nil { - entry.Value = MLRVAL_ERROR + entry.Value = ERROR } else { - entry.Value = mlrval.FromString(string(outputBytes)) + entry.Value = FromString(string(outputBytes)) } } @@ -165,6 +165,6 @@ func (entry *MlrmapEntry) JSONParseInPlace() { input := entry.Value.String() err := entry.Value.UnmarshalJSON([]byte(input)) if err != nil { - entry.Value = MLRVAL_ERROR + entry.Value = ERROR } } diff --git a/internal/pkg/types-parked/mlrval_json.go b/internal/pkg/mlrval/mlrval_json.go similarity index 96% rename from internal/pkg/types-parked/mlrval_json.go rename to internal/pkg/mlrval/mlrval_json.go index 3c21b3c8c..91da499dc 100644 --- a/internal/pkg/types-parked/mlrval_json.go +++ b/internal/pkg/mlrval/mlrval_json.go @@ -6,7 +6,7 @@ // Please see also https://golang.org/pkg/encoding/json/ // ================================================================ -package types +package mlrval import ( "bytes" @@ -18,7 +18,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/lib" ) -const MLRVAL_JSON_INDENT_STRING string = " " +const JSON_INDENT_STRING string = " " type TJSONFormatting int @@ -101,7 +101,7 @@ const ( // ---------------------------------------------------------------- func (mv *Mlrval) UnmarshalJSON(inputBytes []byte) error { - *mv = MlrvalFromPending() + mv = FromPending() decoder := json.NewDecoder(bytes.NewReader(inputBytes)) mlrval, eof, err := MlrvalDecodeFromJSON(decoder) if eof { @@ -135,23 +135,23 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) ( delimiter, isDelim := startToken.(json.Delim) if !isDelim { if startToken == nil { - return MLRVAL_NULL, false, nil + return NULL, false, nil } sval, ok := startToken.(string) if ok { - mlrval := mlrval.FromString(sval) + mlrval := FromString(sval) return mlrval, false, nil } bval, ok := startToken.(bool) if ok { - return MlrvalFromBool(bval), false, nil + return FromBool(bval), false, nil } nval, ok := startToken.(json.Number) if ok { - mlrval := MlrvalFromInferredType(nval.String()) + mlrval := FromInferredType(nval.String()) return mlrval, false, nil } @@ -178,9 +178,9 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) ( ) } - mlrval := MlrvalFromPending() + mlrval := FromPending() if isArray { - mlrval = MlrvalEmptyArray() + mlrval = FromEmptyArray() for decoder.More() { element, eof, err := MlrvalDecodeFromJSON(decoder) @@ -191,11 +191,11 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) ( if err != nil { return nil, false, err } - mlrval.ArrayAppend(element) + ArrayAppend(element) } } else { - mlrval = *MlrvalFromEmptyMap() + mlrval = FromEmptyMap() for decoder.More() { key, eof, err := MlrvalDecodeFromJSON(decoder) @@ -223,7 +223,7 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) ( } // xxx check here string-valued key - mlrval.MapPut(key, value) + MapPut(key, value) } } @@ -252,7 +252,7 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) ( return nil, false, imbalanceError } - return &mlrval, false, nil + return mlrval, false, nil } return nil, false, errors.New("unimplemented") @@ -495,7 +495,7 @@ func (mv *Mlrval) marshalJSONArrayMultipleLines( return "", err } for i := 0; i < elementNestingDepth; i++ { - buffer.WriteString(MLRVAL_JSON_INDENT_STRING) + buffer.WriteString(JSON_INDENT_STRING) } buffer.WriteString(elementString) if i < n-1 { @@ -507,7 +507,7 @@ func (mv *Mlrval) marshalJSONArrayMultipleLines( // Write empty array as '[]' if n > 0 { for i := 0; i < elementNestingDepth-1; i++ { - buffer.WriteString(MLRVAL_JSON_INDENT_STRING) + buffer.WriteString(JSON_INDENT_STRING) } } diff --git a/internal/pkg/mlrval/mlrval_new.go b/internal/pkg/mlrval/mlrval_new.go index b6c50395d..77e89e87f 100644 --- a/internal/pkg/mlrval/mlrval_new.go +++ b/internal/pkg/mlrval/mlrval_new.go @@ -10,6 +10,15 @@ import ( "github.com/johnkerl/miller/internal/pkg/lib" ) +// TODO: comment for JSON-scanner context. +func FromPending() *Mlrval { + return &Mlrval{ + mvtype: MT_PENDING, + printrep: "(bug-if-you-see-this-pending)", + printrepValid: false, + } +} + // TODO: comment JIT context. Some things we already know are typed -- DSL // things, or JSON contents. Others are deferred, e.g. items from any file // format except JSON. @@ -207,6 +216,10 @@ func FromArray(arrayval []Mlrval) *Mlrval { } } +func FromEmptyArray() *Mlrval { + return FromArray(make([]Mlrval, 0)) +} + func FromMap(mapval *Mlrmap) *Mlrval { return &Mlrval{ mvtype: MT_MAP, @@ -215,3 +228,7 @@ func FromMap(mapval *Mlrmap) *Mlrval { mapval: mapval, } } + +func FromEmptyMap() *Mlrval { + return FromMap(NewMlrmap()) +} From d33682073c26c8f68a0ffade4523475b61d9d2fc Mon Sep 17 00:00:00 2001 From: John Kerl Date: Thu, 16 Dec 2021 18:30:34 -0500 Subject: [PATCH 11/19] iterate applying mlrval refactors to dependent packages --- cmd/mprof/main.go | 12 +- cmd/mprof2/main.go | 4 +- cmd/mprof3/main.go | 4 +- internal/pkg/auxents/help/entry.go | 10 +- internal/pkg/auxents/repl/dsl.go | 6 +- internal/pkg/auxents/repl/session.go | 6 +- internal/pkg/bifs/collections.go | 398 +++--- .../pkg/dsl/cst/builtin_function_manager.go | 432 +++---- internal/pkg/dsl/cst/builtin_functions.go | 65 +- internal/pkg/dsl/cst/collections.go | 76 +- internal/pkg/dsl/cst/cond.go | 4 +- internal/pkg/dsl/cst/emit_emitp.go | 8 +- internal/pkg/dsl/cst/emitf.go | 1 + internal/pkg/dsl/cst/env.go | 7 +- internal/pkg/dsl/cst/evaluable.go | 12 +- internal/pkg/dsl/cst/for.go | 15 +- internal/pkg/dsl/cst/hofs.go | 104 +- internal/pkg/dsl/cst/if.go | 4 +- internal/pkg/dsl/cst/leaves.go | 78 +- internal/pkg/dsl/cst/lvalues.go | 4 +- internal/pkg/dsl/cst/root.go | 2 +- internal/pkg/dsl/cst/tee.go | 1 + internal/pkg/dsl/cst/types.go | 1 - internal/pkg/dsl/cst/udf.go | 14 +- internal/pkg/mlrval/mlrmap_accessors.go | 1152 ++++++++--------- .../{types-parked => mlrval}/mlrmap_print.go | 2 +- .../mlrval_collections.go | 46 +- internal/pkg/mlrval/mlrval_get.go | 8 +- internal/pkg/mlrval/mlrval_json.go | 18 +- internal/pkg/mlrval/tbf.go | 94 +- internal/pkg/transformers/cat.go | 4 +- internal/pkg/transformers/count-similar.go | 2 +- internal/pkg/transformers/count.go | 6 +- internal/pkg/transformers/fill-empty.go | 2 +- internal/pkg/transformers/fraction.go | 8 +- internal/pkg/transformers/histogram.go | 12 +- .../transformers/most-or-least-frequent.go | 2 +- internal/pkg/transformers/put-or-filter.go | 2 +- internal/pkg/transformers/seqgen.go | 8 +- internal/pkg/transformers/step.go | 28 +- internal/pkg/transformers/top.go | 6 +- internal/pkg/transformers/uniq.go | 10 +- .../pkg/transformers/utils/join-bucket.go | 1 - .../transformers/utils/percentile-keeper.go | 16 +- .../transformers/utils/stats1-accumulators.go | 154 +-- .../transformers/utils/stats2-accumulators.go | 97 +- internal/pkg/transformers/utils/top-keeper.go | 2 +- 47 files changed, 1447 insertions(+), 1501 deletions(-) rename internal/pkg/{types-parked => mlrval}/mlrmap_print.go (99%) rename internal/pkg/{types-parked => mlrval}/mlrval_collections.go (94%) diff --git a/cmd/mprof/main.go b/cmd/mprof/main.go index 8f0fc8805..e7fa4c082 100644 --- a/cmd/mprof/main.go +++ b/cmd/mprof/main.go @@ -268,11 +268,11 @@ func (reader *RecordReaderDKVPNonPipelined) recordFromDKVPLine( // "a=". Here we use the positional index as the key. This way // DKVP is a generalization of NIDX. key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := types.MlrvalFromInferredTypeForDataFiles(kv[0]) + value := mlrval.FromInferredTypeForDataFiles(kv[0]) record.PutReference(key, value) } else { key := kv[0] - value := types.MlrvalFromInferredTypeForDataFiles(kv[1]) + value := mlrval.FromInferredTypeForDataFiles(kv[1]) record.PutReference(key, value) } } @@ -404,11 +404,11 @@ func (reader *RecordReaderDKVPListPipelined) recordFromDKVPLine( // "a=". Here we use the positional index as the key. This way // DKVP is a generalization of NIDX. key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := types.MlrvalFromInferredTypeForDataFiles(kv[0]) + value := mlrval.FromInferredTypeForDataFiles(kv[0]) record.PutReference(key, value) } else { key := kv[0] - value := types.MlrvalFromInferredTypeForDataFiles(kv[1]) + value := mlrval.FromInferredTypeForDataFiles(kv[1]) record.PutReference(key, value) } } @@ -556,11 +556,11 @@ func (reader *RecordReaderDKVPChanPipelined) recordFromDKVPLine( // "a=". Here we use the positional index as the key. This way // DKVP is a generalization of NIDX. key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := types.MlrvalFromInferredTypeForDataFiles(kv[0]) + value := mlrval.FromInferredTypeForDataFiles(kv[0]) record.PutReference(key, value) } else { key := kv[0] - value := types.MlrvalFromInferredTypeForDataFiles(kv[1]) + value := mlrval.FromInferredTypeForDataFiles(kv[1]) record.PutReference(key, value) } } diff --git a/cmd/mprof2/main.go b/cmd/mprof2/main.go index 2addd5d2f..6619e2b41 100644 --- a/cmd/mprof2/main.go +++ b/cmd/mprof2/main.go @@ -280,11 +280,11 @@ func (reader *RecordReaderDKVPChanPipelined) recordFromDKVPLine( // "a=". Here we use the positional index as the key. This way // DKVP is a generalization of NIDX. key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := types.MlrvalFromInferredTypeForDataFiles(kv[0]) + value := mlrval.FromInferredTypeForDataFiles(kv[0]) record.PutReference(key, value) } else { key := kv[0] - value := types.MlrvalFromInferredTypeForDataFiles(kv[1]) + value := mlrval.FromInferredTypeForDataFiles(kv[1]) record.PutReference(key, value) } } diff --git a/cmd/mprof3/main.go b/cmd/mprof3/main.go index b1f6c7631..a30bc0d65 100644 --- a/cmd/mprof3/main.go +++ b/cmd/mprof3/main.go @@ -294,11 +294,11 @@ func (reader *RecordReaderDKVPChanPipelined) recordFromDKVPLine( // "a=". Here we use the positional index as the key. This way // DKVP is a generalization of NIDX. key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := types.MlrvalFromInferredTypeForDataFiles(kv[0]) + value := mlrval.FromInferredTypeForDataFiles(kv[0]) record.PutReference(key, value) } else { key := kv[0] - value := types.MlrvalFromInferredTypeForDataFiles(kv[1]) + value := mlrval.FromInferredTypeForDataFiles(kv[1]) record.PutReference(key, value) } } diff --git a/internal/pkg/auxents/help/entry.go b/internal/pkg/auxents/help/entry.go index 72dc42db0..932f7dc12 100644 --- a/internal/pkg/auxents/help/entry.go +++ b/internal/pkg/auxents/help/entry.go @@ -449,10 +449,10 @@ func helpOutputColorization() { // ---------------------------------------------------------------- func helpTypeArithmeticInfo() { mlrvals := []*mlrval.Mlrval{ - mlrval.MlrvalFromInt(1), - mlrval.MlrvalFromFloat64(2.5), - types.MLRVAL_ABSENT, - types.MLRVAL_ERROR, + mlrval.FromInt(1), + mlrval.FromFloat(2.5), + mlrval.ABSENT, + mlrval.ERROR, } n := len(mlrvals) @@ -471,7 +471,7 @@ func helpTypeArithmeticInfo() { } else if i == -1 { fmt.Printf(" %-10s", "------") } else { - sum := types.BIF_plus_binary(mlrvals[i], mlrvals[j]) + sum := bifs.BIF_plus_binary(mlrvals[i], mlrvals[j]) fmt.Printf(" %-10s", sum.String()) } } diff --git a/internal/pkg/auxents/repl/dsl.go b/internal/pkg/auxents/repl/dsl.go index 222ce1553..045341bfd 100644 --- a/internal/pkg/auxents/repl/dsl.go +++ b/internal/pkg/auxents/repl/dsl.go @@ -84,8 +84,8 @@ func (repl *Repl) handleDSLStringAux( // The filter expression for the main Miller DSL is any non-assignment // statment like 'true' or '$x > 0.5' etc. For the REPL, we re-use this for // interactive expressions to be printed to the terminal. For the main DSL, - // the default is mlrval.MlrvalFromTrue(); for the REPL, the default is - // types.MLRVAL_VOID. + // the default is mlrval.FromTrue(); for the REPL, the default is + // mlrval.VOID. filterExpression := repl.runtimeState.FilterExpression if filterExpression.IsVoid() { // nothing to print @@ -94,7 +94,7 @@ func (repl *Repl) handleDSLStringAux( } else { fmt.Println(filterExpression.String()) } - repl.runtimeState.FilterExpression = types.MLRVAL_VOID + repl.runtimeState.FilterExpression = mlrval.VOID } return nil diff --git a/internal/pkg/auxents/repl/session.go b/internal/pkg/auxents/repl/session.go index 01f7829f7..20f8dfe10 100644 --- a/internal/pkg/auxents/repl/session.go +++ b/internal/pkg/auxents/repl/session.go @@ -68,9 +68,9 @@ func NewRepl( // The filter expression for the main Miller DSL is any non-assignment // statment like 'true' or '$x > 0.5' etc. For the REPL, we re-use this for // interactive expressions to be printed to the terminal. For the main DSL, - // the default is mlrval.MlrvalFromTrue(); for the REPL, the default is - // types.MLRVAL_VOID. - runtimeState.FilterExpression = types.MLRVAL_VOID + // the default is mlrval.FromTrue(); for the REPL, the default is + // mlrval.VOID. + runtimeState.FilterExpression = mlrval.VOID // For control-C handling sysToSignalHandlerChannel := make(chan os.Signal, 1) // Our signal handler reads system notification here diff --git a/internal/pkg/bifs/collections.go b/internal/pkg/bifs/collections.go index 7754ff6d8..480697f8e 100644 --- a/internal/pkg/bifs/collections.go +++ b/internal/pkg/bifs/collections.go @@ -228,62 +228,62 @@ func BIF_mapselect(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { return mlrval.FromMap(newMap) } -//// ---------------------------------------------------------------- -//func BIF_mapexcept(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { -// if len(mlrvals) < 1 { -// return mlrval.ERROR -// } -// if !mlrvals[0].IsMap() { -// return mlrval.ERROR -// } -// newMap := mlrvals[0].AcquireMapValue().Copy() -// -// for _, exceptArg := range mlrvals[1:] { -// if exceptArg.IsString() { -// newMap.Remove(exceptArg.AcquireStringValue()) -// } else if exceptArg.IsInt() { -// newMap.Remove(exceptArg.String()) -// } else if exceptArg.IsArray() { -// for _, element := range exceptArg.AcquireArrayValue() { -// if element.IsString() { -// newMap.Remove(element.AcquireStringValue()) -// } else { -// return mlrval.ERROR -// } -// } -// } else { -// return mlrval.ERROR -// } -// } -// -// return mlrval.FromMap(newMap) -//} +// ---------------------------------------------------------------- +func BIF_mapexcept(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { + if len(mlrvals) < 1 { + return mlrval.ERROR + } + if !mlrvals[0].IsMap() { + return mlrval.ERROR + } + newMap := mlrvals[0].AcquireMapValue().Copy() -//// ---------------------------------------------------------------- -//func BIF_mapsum(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { -// if len(mlrvals) == 0 { -// return mlrval.FromEmptyMap() -// } -// if len(mlrvals) == 1 { -// return mlrvals[0] -// } -// if mlrvals[0].Type() != MT_MAP { -// return mlrval.ERROR -// } -// newMap := mlrvals[0].AcquireMapValue().Copy() -// -// for _, otherMapArg := range mlrvals[1:] { -// if otherMapArg.Type() != MT_MAP { -// return mlrval.ERROR -// } -// -// for pe := otherMapArg.AcquireMapValue().Head; pe != nil; pe = pe.Next { -// newMap.PutCopy(pe.Key, pe.Value) -// } -// } -// -// return mlrval.FromMap(newMap) -//} + for _, exceptArg := range mlrvals[1:] { + if exceptArg.IsString() { + newMap.Remove(exceptArg.AcquireStringValue()) + } else if exceptArg.IsInt() { + newMap.Remove(exceptArg.String()) + } else if exceptArg.IsArray() { + for _, element := range exceptArg.AcquireArrayValue() { + if element.IsString() { + newMap.Remove(element.AcquireStringValue()) + } else { + return mlrval.ERROR + } + } + } else { + return mlrval.ERROR + } + } + + return mlrval.FromMap(newMap) +} + +// ---------------------------------------------------------------- +func BIF_mapsum(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { + if len(mlrvals) == 0 { + return mlrval.FromEmptyMap() + } + if len(mlrvals) == 1 { + return mlrvals[0] + } + if mlrvals[0].Type() != mlrval.MT_MAP { + return mlrval.ERROR + } + newMap := mlrvals[0].AcquireMapValue().Copy() + + for _, otherMapArg := range mlrvals[1:] { + if otherMapArg.Type() != mlrval.MT_MAP { + return mlrval.ERROR + } + + for pe := otherMapArg.AcquireMapValue().Head; pe != nil; pe = pe.Next { + newMap.PutCopy(pe.Key, pe.Value) + } + } + + return mlrval.FromMap(newMap) +} //// ---------------------------------------------------------------- //func BIF_mapdiff(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { @@ -647,151 +647,151 @@ func BIF_get_values(input1 *mlrval.Mlrval) *mlrval.Mlrval { } } -//// ---------------------------------------------------------------- -//func BIF_append(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { -// if !input1.IsArray() { -// return mlrval.ERROR -// } -// -// output := input1.Copy() -// output.ArrayAppend(input2.Copy()) -// return output -//} +// ---------------------------------------------------------------- +func BIF_append(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsArray() { + return mlrval.ERROR + } -//// ---------------------------------------------------------------- -//// First argumemnt is prefix. -//// Second argument is delimiter. -//// Third argument is map or array. -//// flatten("a", ".", {"b": { "c": 4 }}) is {"a.b.c" : 4}. -//// flatten("", ".", {"a": { "b": 3 }}) is {"a.b" : 3}. -//func BIF_flatten(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { -// if input3.IsMap() || input3.IsArray() { -// if !input1.IsString() && input1.Type() != MT_VOID { -// return mlrval.ERROR -// } -// prefix := input1.AcquireStringValue() -// if !input2.IsString() { -// return mlrval.ERROR -// } -// delimiter := input2.AcquireStringValue() -// -// retval := input3.FlattenToMap(prefix, delimiter) -// return &retval -// } else { -// return input3 -// } -//} -// -//// flatten($*, ".") is the same as flatten("", ".", $*) -//func BIF_flatten_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { -// return BIF_flatten(mlrval.VOID, input2, input1) -//} -// -//// ---------------------------------------------------------------- -//// First argument is a map. -//// Second argument is a delimiter string. -//// unflatten({"a.b.c", ".") is {"a": { "b": { "c": 4}}}. -//func BIF_unflatten(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { -// if !input2.IsString() { -// return mlrval.ERROR -// } -// if input1.Type() != MT_MAP { -// return input1 -// } -// oldmap := input1.AcquireMapValue() -// separator := input2.AcquireStringValue() -// newmap := oldmap.CopyUnflattened(separator) -// return mlrval.FromMapReferenced(newmap) -//} -// -//// ---------------------------------------------------------------- -//// Converts maps with "1", "2", ... keys into arrays. Recurses nested data structures. -//func BIF_arrayify(input1 *mlrval.Mlrval) *mlrval.Mlrval { -// if input1.IsMap() { -// if input1.AcquireMapValue().IsEmpty() { -// return input1 -// } -// -// convertible := true -// i := 0 -// for pe := input1.AcquireMapValue().Head; pe != nil; pe = pe.Next { -// sval := strconv.Itoa(i + 1) // Miller user-space indices are 1-up -// i++ -// if pe.Key != sval { -// convertible = false -// } -// pe.Value = BIF_arrayify(pe.Value) -// } -// -// if convertible { -// mapval := input1.AcquireMapValue() -// arrayval := make([]mlrval.Mlrval, input1.AcquireMapValue().FieldCount) -// i := 0 -// for pe := mapval.Head; pe != nil; pe = pe.Next { -// AcquireArrayValue()[i] = *pe.Value.Copy() -// i++ -// } -// return mlrval.FromArrayReference(AcquireArrayValue()) -// -// } else { -// return input1 -// } -// -// } else if input1.IsArray() { -// // TODO: comment (or rethink) that this modifies its inputs!! -// output := input1.Copy() -// for i := range input1.AcquireArrayValue() { -// output.AcquireArrayValue()[i] = *BIF_arrayify(&output.AcquireArrayValue()[i]) -// } -// return output -// -// } else { -// return input1 -// } -//} -// -//// ---------------------------------------------------------------- -//func BIF_json_parse(input1 *mlrval.Mlrval) *mlrval.Mlrval { -// if input1.IsVoid() { -// return input1 -// } else if !input1.IsString() { -// return mlrval.ERROR -// } else { -// output := mlrval.FromPending() -// err := output.UnmarshalJSON([]byte(input1.AcquireStringValue())) -// if err != nil { -// return mlrval.ERROR -// } -// return &output -// } -//} -// -//func BIF_json_stringify_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { -// outputBytes, err := input1.MarshalJSON(JSON_SINGLE_LINE, false) -// if err != nil { -// return mlrval.ERROR -// } else { -// return mlrval.FromString(string(outputBytes)) -// } -//} -// -//func BIF_json_stringify_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { -// var jsonFormatting TJSONFormatting = JSON_SINGLE_LINE -// useMultiline, ok := input2.GetBoolValue() -// if !ok { -// return mlrval.ERROR -// } -// if useMultiline { -// jsonFormatting = JSON_MULTILINE -// } -// -// outputBytes, err := input1.MarshalJSON(jsonFormatting, false) -// if err != nil { -// return mlrval.ERROR -// } else { -// return mlrval.FromString(string(outputBytes)) -// } -//} + output := input1.Copy() + output.ArrayAppend(input2.Copy()) + return output +} + +// ---------------------------------------------------------------- +// First argumemnt is prefix. +// Second argument is delimiter. +// Third argument is map or array. +// flatten("a", ".", {"b": { "c": 4 }}) is {"a.b.c" : 4}. +// flatten("", ".", {"a": { "b": 3 }}) is {"a.b" : 3}. +func BIF_flatten(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { + if input3.IsMap() || input3.IsArray() { + if !input1.IsString() && input1.Type() != mlrval.MT_VOID { + return mlrval.ERROR + } + prefix := input1.AcquireStringValue() + if !input2.IsString() { + return mlrval.ERROR + } + delimiter := input2.AcquireStringValue() + + retval := input3.FlattenToMap(prefix, delimiter) + return &retval + } else { + return input3 + } +} + +// flatten($*, ".") is the same as flatten("", ".", $*) +func BIF_flatten_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return BIF_flatten(mlrval.VOID, input2, input1) +} + +// ---------------------------------------------------------------- +// First argument is a map. +// Second argument is a delimiter string. +// unflatten({"a.b.c", ".") is {"a": { "b": { "c": 4}}}. +func BIF_unflatten(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if !input2.IsString() { + return mlrval.ERROR + } + if input1.Type() != mlrval.MT_MAP { + return input1 + } + oldmap := input1.AcquireMapValue() + separator := input2.AcquireStringValue() + newmap := oldmap.CopyUnflattened(separator) + return mlrval.FromMap(newmap) +} + +// ---------------------------------------------------------------- +// Converts maps with "1", "2", ... keys into arrays. Recurses nested data structures. +func BIF_arrayify(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsMap() { + if input1.AcquireMapValue().IsEmpty() { + return input1 + } + + convertible := true + i := 0 + for pe := input1.AcquireMapValue().Head; pe != nil; pe = pe.Next { + sval := strconv.Itoa(i + 1) // Miller user-space indices are 1-up + i++ + if pe.Key != sval { + convertible = false + } + pe.Value = BIF_arrayify(pe.Value) + } + + if convertible { + mapval := input1.AcquireMapValue() + arrayval := make([]mlrval.Mlrval, input1.AcquireMapValue().FieldCount) + i := 0 + for pe := mapval.Head; pe != nil; pe = pe.Next { + arrayval[i] = *pe.Value.Copy() + i++ + } + return mlrval.FromArray(arrayval) + + } else { + return input1 + } + + } else if input1.IsArray() { + // TODO: comment (or rethink) that this modifies its inputs!! + output := input1.Copy() + for i := range input1.AcquireArrayValue() { + output.AcquireArrayValue()[i] = *BIF_arrayify(&output.AcquireArrayValue()[i]) + } + return output + + } else { + return input1 + } +} + +// ---------------------------------------------------------------- +func BIF_json_parse(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsVoid() { + return input1 + } else if !input1.IsString() { + return mlrval.ERROR + } else { + output := mlrval.FromPending() + err := output.UnmarshalJSON([]byte(input1.AcquireStringValue())) + if err != nil { + return mlrval.ERROR + } + return output + } +} + +func BIF_json_stringify_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { + outputBytes, err := input1.MarshalJSON(mlrval.JSON_SINGLE_LINE, false) + if err != nil { + return mlrval.ERROR + } else { + return mlrval.FromString(string(outputBytes)) + } +} + +func BIF_json_stringify_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var jsonFormatting mlrval.TJSONFormatting = mlrval.JSON_SINGLE_LINE + useMultiline, ok := input2.GetBoolValue() + if !ok { + return mlrval.ERROR + } + if useMultiline { + jsonFormatting = mlrval.JSON_MULTILINE + } + + outputBytes, err := input1.MarshalJSON(jsonFormatting, false) + if err != nil { + return mlrval.ERROR + } else { + return mlrval.FromString(string(outputBytes)) + } +} func unaliasArrayIndex(array *[]mlrval.Mlrval, mindex int) (int, bool) { n := int(len(*array)) diff --git a/internal/pkg/dsl/cst/builtin_function_manager.go b/internal/pkg/dsl/cst/builtin_function_manager.go index 2367349fd..f1c05f6b9 100644 --- a/internal/pkg/dsl/cst/builtin_function_manager.go +++ b/internal/pkg/dsl/cst/builtin_function_manager.go @@ -19,9 +19,9 @@ import ( "sort" "strings" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/colorizer" "github.com/johnkerl/miller/internal/pkg/lib" - "github.com/johnkerl/miller/internal/pkg/types" ) type TFunctionClass string @@ -53,16 +53,16 @@ type BuiltinFunctionInfo struct { hasMultipleArities bool minimumVariadicArity int maximumVariadicArity int // 0 means no max - zaryFunc types.ZaryFunc - unaryFunc types.UnaryFunc - binaryFunc types.BinaryFunc - ternaryFunc types.TernaryFunc - variadicFunc types.VariadicFunc - unaryFuncWithContext types.UnaryFuncWithContext // asserting_{typename} - regexCaptureBinaryFunc types.RegexCaptureBinaryFunc // =~ and !=~ - binaryFuncWithState BinaryFuncWithState // select, apply, reduce - ternaryFuncWithState TernaryFuncWithState // fold - variadicFuncWithState VariadicFuncWithState // sort + zaryFunc bifs.ZaryFunc + unaryFunc bifs.UnaryFunc + binaryFunc bifs.BinaryFunc + ternaryFunc bifs.TernaryFunc + variadicFunc bifs.VariadicFunc + unaryFuncWithContext bifs.UnaryFuncWithContext // asserting_{typename} + regexCaptureBinaryFunc bifs.RegexCaptureBinaryFunc // =~ and !=~ + binaryFuncWithState BinaryFuncWithState // select, apply, reduce + ternaryFuncWithState TernaryFuncWithState // fold + variadicFuncWithState VariadicFuncWithState // sort } // ================================================================ @@ -88,8 +88,8 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { name: "+", class: FUNC_CLASS_ARITHMETIC, help: `Addition as binary operator; unary plus operator.`, - unaryFunc: types.BIF_plus_unary, - binaryFunc: types.BIF_plus_binary, + unaryFunc: bifs.BIF_plus_unary, + binaryFunc: bifs.BIF_plus_binary, hasMultipleArities: true, }, @@ -97,8 +97,8 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { name: "-", class: FUNC_CLASS_ARITHMETIC, help: `Subtraction as binary operator; unary negation operator.`, - unaryFunc: types.BIF_minus_unary, - binaryFunc: types.BIF_minus_binary, + unaryFunc: bifs.BIF_minus_unary, + binaryFunc: bifs.BIF_minus_binary, hasMultipleArities: true, }, @@ -106,154 +106,154 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { name: "*", class: FUNC_CLASS_ARITHMETIC, help: `Multiplication, with integer*integer overflow to float.`, - binaryFunc: types.BIF_times, + binaryFunc: bifs.BIF_times, }, { name: "/", class: FUNC_CLASS_ARITHMETIC, help: `Division. Integer / integer is floating-point.`, - binaryFunc: types.BIF_divide, + binaryFunc: bifs.BIF_divide, }, { name: "//", class: FUNC_CLASS_ARITHMETIC, help: `Pythonic integer division, rounding toward negative.`, - binaryFunc: types.BIF_int_divide, + binaryFunc: bifs.BIF_int_divide, }, { name: "**", class: FUNC_CLASS_ARITHMETIC, help: `Exponentiation. Same as pow, but as an infix operator.`, - binaryFunc: types.BIF_pow, + binaryFunc: bifs.BIF_pow, }, { name: "pow", class: FUNC_CLASS_ARITHMETIC, help: `Exponentiation. Same as **, but as a function.`, - binaryFunc: types.BIF_pow, + binaryFunc: bifs.BIF_pow, }, { name: ".+", class: FUNC_CLASS_ARITHMETIC, help: `Addition, with integer-to-integer overflow.`, - binaryFunc: types.BIF_dot_plus, + binaryFunc: bifs.BIF_dot_plus, }, { name: ".-", class: FUNC_CLASS_ARITHMETIC, help: `Subtraction, with integer-to-integer overflow.`, - binaryFunc: types.BIF_dot_minus, + binaryFunc: bifs.BIF_dot_minus, }, { name: ".*", class: FUNC_CLASS_ARITHMETIC, help: `Multiplication, with integer-to-integer overflow.`, - binaryFunc: types.BIF_dot_times, + binaryFunc: bifs.BIF_dot_times, }, { name: "./", class: FUNC_CLASS_ARITHMETIC, help: `Integer division; not pythonic.`, - binaryFunc: types.BIF_dot_divide, + binaryFunc: bifs.BIF_dot_divide, }, { name: "%", class: FUNC_CLASS_ARITHMETIC, help: `Remainder; never negative-valued (pythonic).`, - binaryFunc: types.BIF_modulus, + binaryFunc: bifs.BIF_modulus, }, { name: "~", class: FUNC_CLASS_ARITHMETIC, help: `Bitwise NOT. Beware '$y=~$x' since =~ is the regex-match operator: try '$y = ~$x'.`, - unaryFunc: types.BIF_bitwise_not, + unaryFunc: bifs.BIF_bitwise_not, }, { name: "&", class: FUNC_CLASS_ARITHMETIC, help: `Bitwise AND.`, - binaryFunc: types.BIF_bitwise_and, + binaryFunc: bifs.BIF_bitwise_and, }, { name: "|", class: FUNC_CLASS_ARITHMETIC, help: `Bitwise OR.`, - binaryFunc: types.BIF_bitwise_or, + binaryFunc: bifs.BIF_bitwise_or, }, { name: "^", help: `Bitwise XOR.`, class: FUNC_CLASS_ARITHMETIC, - binaryFunc: types.BIF_bitwise_xor, + binaryFunc: bifs.BIF_bitwise_xor, }, { name: "<<", class: FUNC_CLASS_ARITHMETIC, help: `Bitwise left-shift.`, - binaryFunc: types.BIF_left_shift, + binaryFunc: bifs.BIF_left_shift, }, { name: ">>", class: FUNC_CLASS_ARITHMETIC, help: `Bitwise signed right-shift.`, - binaryFunc: types.BIF_signed_right_shift, + binaryFunc: bifs.BIF_signed_right_shift, }, { name: ">>>", class: FUNC_CLASS_ARITHMETIC, help: `Bitwise unsigned right-shift.`, - binaryFunc: types.BIF_unsigned_right_shift, + binaryFunc: bifs.BIF_unsigned_right_shift, }, { name: "bitcount", class: FUNC_CLASS_ARITHMETIC, help: "Count of 1-bits.", - unaryFunc: types.BIF_bitcount, + unaryFunc: bifs.BIF_bitcount, }, { name: "madd", class: FUNC_CLASS_ARITHMETIC, help: `a + b mod m (integers)`, - ternaryFunc: types.BIF_mod_add, + ternaryFunc: bifs.BIF_mod_add, }, { name: "msub", class: FUNC_CLASS_ARITHMETIC, help: `a - b mod m (integers)`, - ternaryFunc: types.BIF_mod_sub, + ternaryFunc: bifs.BIF_mod_sub, }, { name: "mmul", class: FUNC_CLASS_ARITHMETIC, help: `a * b mod m (integers)`, - ternaryFunc: types.BIF_mod_mul, + ternaryFunc: bifs.BIF_mod_mul, }, { name: "mexp", class: FUNC_CLASS_ARITHMETIC, help: `a ** b mod m (integers)`, - ternaryFunc: types.BIF_mod_exp, + ternaryFunc: bifs.BIF_mod_exp, }, // ---------------------------------------------------------------- @@ -263,7 +263,7 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { name: "!", class: FUNC_CLASS_BOOLEAN, help: `Logical negation.`, - unaryFunc: types.BIF_logical_NOT, + unaryFunc: bifs.BIF_logical_NOT, }, { @@ -271,63 +271,63 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { class: FUNC_CLASS_BOOLEAN, help: `String/numeric equality. Mixing number and string results in string compare.`, - binaryFunc: types.BIF_equals, + binaryFunc: bifs.BIF_equals, }, { name: "!=", class: FUNC_CLASS_BOOLEAN, help: `String/numeric inequality. Mixing number and string results in string compare.`, - binaryFunc: types.BIF_not_equals, + binaryFunc: bifs.BIF_not_equals, }, { name: ">", help: `String/numeric greater-than. Mixing number and string results in string compare.`, class: FUNC_CLASS_BOOLEAN, - binaryFunc: types.BIF_greater_than, + binaryFunc: bifs.BIF_greater_than, }, { name: ">=", help: `String/numeric greater-than-or-equals. Mixing number and string results in string compare.`, class: FUNC_CLASS_BOOLEAN, - binaryFunc: types.BIF_greater_than_or_equals, + binaryFunc: bifs.BIF_greater_than_or_equals, }, { name: "<=>", help: `Comparator, nominally for sorting. Given a <=> b, returns <0, 0, >0 as a < b, a == b, or a > b, respectively.`, class: FUNC_CLASS_BOOLEAN, - binaryFunc: types.BIF_cmp, + binaryFunc: bifs.BIF_cmp, }, { name: "<", class: FUNC_CLASS_BOOLEAN, help: `String/numeric less-than. Mixing number and string results in string compare.`, - binaryFunc: types.BIF_less_than, + binaryFunc: bifs.BIF_less_than, }, { name: "<=", class: FUNC_CLASS_BOOLEAN, help: `String/numeric less-than-or-equals. Mixing number and string results in string compare.`, - binaryFunc: types.BIF_less_than_or_equals, + binaryFunc: bifs.BIF_less_than_or_equals, }, { name: "=~", class: FUNC_CLASS_BOOLEAN, help: `String (left-hand side) matches regex (right-hand side), e.g. '$name =~ "^a.*b$"'.`, - regexCaptureBinaryFunc: types.BIF_string_matches_regexp, + regexCaptureBinaryFunc: bifs.BIF_string_matches_regexp, }, { name: "!=~", class: FUNC_CLASS_BOOLEAN, help: `String (left-hand side) does not match regex (right-hand side), e.g. '$name !=~ "^a.*b$"'.`, - regexCaptureBinaryFunc: types.BIF_string_does_not_match_regexp, + regexCaptureBinaryFunc: bifs.BIF_string_does_not_match_regexp, }, { @@ -348,7 +348,7 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { name: "^^", class: FUNC_CLASS_BOOLEAN, help: `Logical XOR.`, - binaryFunc: types.BIF_logical_XOR, + binaryFunc: bifs.BIF_logical_XOR, }, { @@ -379,91 +379,91 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { name: ".", class: FUNC_CLASS_STRING, help: `String concatenation.`, - binaryFunc: types.BIF_dot, + binaryFunc: bifs.BIF_dot, }, { name: "capitalize", class: FUNC_CLASS_STRING, help: "Convert string's first character to uppercase.", - unaryFunc: types.BIF_capitalize, + unaryFunc: bifs.BIF_capitalize, }, { name: "clean_whitespace", class: FUNC_CLASS_STRING, help: "Same as collapse_whitespace and strip.", - unaryFunc: types.BIF_clean_whitespace, + unaryFunc: bifs.BIF_clean_whitespace, }, { name: "collapse_whitespace", class: FUNC_CLASS_STRING, help: "Strip repeated whitespace from string.", - unaryFunc: types.BIF_collapse_whitespace, + unaryFunc: bifs.BIF_collapse_whitespace, }, { name: "gsub", class: FUNC_CLASS_STRING, help: `'$name=gsub($name, "old", "new")' (replace all).`, - ternaryFunc: types.BIF_gsub, + ternaryFunc: bifs.BIF_gsub, }, { name: "lstrip", class: FUNC_CLASS_STRING, help: "Strip leading whitespace from string.", - unaryFunc: types.BIF_lstrip, + unaryFunc: bifs.BIF_lstrip, }, { name: "regextract", class: FUNC_CLASS_STRING, help: `'$name=regextract($name, "[A-Z]{3}[0-9]{2}")'`, - binaryFunc: types.BIF_regextract, + binaryFunc: bifs.BIF_regextract, }, { name: "regextract_or_else", class: FUNC_CLASS_STRING, help: `'$name=regextract_or_else($name, "[A-Z]{3}[0-9]{2}", "default")'`, - ternaryFunc: types.BIF_regextract_or_else, + ternaryFunc: bifs.BIF_regextract_or_else, }, { name: "rstrip", class: FUNC_CLASS_STRING, help: "Strip trailing whitespace from string.", - unaryFunc: types.BIF_rstrip, + unaryFunc: bifs.BIF_rstrip, }, { name: "strip", class: FUNC_CLASS_STRING, help: "Strip leading and trailing whitespace from string.", - unaryFunc: types.BIF_strip, + unaryFunc: bifs.BIF_strip, }, { name: "strlen", class: FUNC_CLASS_STRING, help: "String length.", - unaryFunc: types.BIF_strlen, + unaryFunc: bifs.BIF_strlen, }, { name: "ssub", class: FUNC_CLASS_STRING, help: `Like sub but does no regexing. No characters are special.`, - ternaryFunc: types.BIF_ssub, + ternaryFunc: bifs.BIF_ssub, }, { name: "sub", class: FUNC_CLASS_STRING, help: `'$name=sub($name, "old", "new")' (replace once).`, - ternaryFunc: types.BIF_sub, + ternaryFunc: bifs.BIF_sub, }, { @@ -471,14 +471,14 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { class: FUNC_CLASS_STRING, help: `substr0(s,m,n) gives substring of s from 0-up position m to n inclusive. Negative indices -len .. -1 alias to 0 .. len-1. See also substr and substr1.`, - ternaryFunc: types.BIF_substr_0_up, + ternaryFunc: bifs.BIF_substr_0_up, }, { name: "substr1", class: FUNC_CLASS_STRING, help: `substr1(s,m,n) gives substring of s from 1-up position m to n inclusive. Negative indices -len .. -1 alias to 1 .. len. See also substr and substr0.`, - ternaryFunc: types.BIF_substr_1_up, + ternaryFunc: bifs.BIF_substr_1_up, }, { name: "substr", @@ -486,28 +486,28 @@ Negative indices -len .. -1 alias to 1 .. len. See also substr and substr0.`, help: `substr is an alias for substr0. See also substr1. Miller is generally 1-up with all array and string indices, but, this is a backward-compatibility issue with Miller 5 and below. Arrays are new in Miller 6; the substr function is older.`, - ternaryFunc: types.BIF_substr_0_up, + ternaryFunc: bifs.BIF_substr_0_up, }, { name: "tolower", class: FUNC_CLASS_STRING, help: "Convert string to lowercase.", - unaryFunc: types.BIF_tolower, + unaryFunc: bifs.BIF_tolower, }, { name: "toupper", class: FUNC_CLASS_STRING, help: "Convert string to uppercase.", - unaryFunc: types.BIF_toupper, + unaryFunc: bifs.BIF_toupper, }, { name: "truncate", class: FUNC_CLASS_STRING, help: `Truncates string first argument to max length of int second argument.`, - binaryFunc: types.BIF_truncate, + binaryFunc: bifs.BIF_truncate, }, // ---------------------------------------------------------------- @@ -517,25 +517,25 @@ Arrays are new in Miller 6; the substr function is older.`, name: "md5", class: FUNC_CLASS_HASHING, help: `MD5 hash.`, - unaryFunc: types.BIF_md5, + unaryFunc: bifs.BIF_md5, }, { name: "sha1", class: FUNC_CLASS_HASHING, help: `SHA1 hash.`, - unaryFunc: types.BIF_sha1, + unaryFunc: bifs.BIF_sha1, }, { name: "sha256", class: FUNC_CLASS_HASHING, help: `SHA256 hash.`, - unaryFunc: types.BIF_sha256, + unaryFunc: bifs.BIF_sha256, }, { name: "sha512", class: FUNC_CLASS_HASHING, help: `SHA512 hash.`, - unaryFunc: types.BIF_sha512, + unaryFunc: bifs.BIF_sha512, }, // ---------------------------------------------------------------- @@ -545,119 +545,119 @@ Arrays are new in Miller 6; the substr function is older.`, name: "abs", class: FUNC_CLASS_MATH, help: "Absolute value.", - unaryFunc: types.BIF_abs, + unaryFunc: bifs.BIF_abs, }, { name: "acos", class: FUNC_CLASS_MATH, help: "Inverse trigonometric cosine.", - unaryFunc: types.BIF_acos, + unaryFunc: bifs.BIF_acos, }, { name: "acosh", class: FUNC_CLASS_MATH, help: "Inverse hyperbolic cosine.", - unaryFunc: types.BIF_acosh, + unaryFunc: bifs.BIF_acosh, }, { name: "asin", class: FUNC_CLASS_MATH, help: "Inverse trigonometric sine.", - unaryFunc: types.BIF_asin, + unaryFunc: bifs.BIF_asin, }, { name: "asinh", class: FUNC_CLASS_MATH, help: "Inverse hyperbolic sine.", - unaryFunc: types.BIF_asinh, + unaryFunc: bifs.BIF_asinh, }, { name: "atan", class: FUNC_CLASS_MATH, help: "One-argument arctangent.", - unaryFunc: types.BIF_atan, + unaryFunc: bifs.BIF_atan, }, { name: "atan2", class: FUNC_CLASS_MATH, help: "Two-argument arctangent.", - binaryFunc: types.BIF_atan2, + binaryFunc: bifs.BIF_atan2, }, { name: "atanh", class: FUNC_CLASS_MATH, help: "Inverse hyperbolic tangent.", - unaryFunc: types.BIF_atanh, + unaryFunc: bifs.BIF_atanh, }, { name: "cbrt", class: FUNC_CLASS_MATH, help: "Cube root.", - unaryFunc: types.BIF_cbrt, + unaryFunc: bifs.BIF_cbrt, }, { name: "ceil", class: FUNC_CLASS_MATH, help: "Ceiling: nearest integer at or above.", - unaryFunc: types.BIF_ceil, + unaryFunc: bifs.BIF_ceil, }, { name: "cos", class: FUNC_CLASS_MATH, help: "Trigonometric cosine.", - unaryFunc: types.BIF_cos, + unaryFunc: bifs.BIF_cos, }, { name: "cosh", class: FUNC_CLASS_MATH, help: "Hyperbolic cosine.", - unaryFunc: types.BIF_cosh, + unaryFunc: bifs.BIF_cosh, }, { name: "erf", class: FUNC_CLASS_MATH, help: "Error function.", - unaryFunc: types.BIF_erf, + unaryFunc: bifs.BIF_erf, }, { name: "erfc", class: FUNC_CLASS_MATH, help: "Complementary error function.", - unaryFunc: types.BIF_erfc, + unaryFunc: bifs.BIF_erfc, }, { name: "exp", class: FUNC_CLASS_MATH, help: "Exponential function e**x.", - unaryFunc: types.BIF_exp, + unaryFunc: bifs.BIF_exp, }, { name: "expm1", class: FUNC_CLASS_MATH, help: "e**x - 1.", - unaryFunc: types.BIF_expm1, + unaryFunc: bifs.BIF_expm1, }, { name: "floor", class: FUNC_CLASS_MATH, help: "Floor: nearest integer at or below.", - unaryFunc: types.BIF_floor, + unaryFunc: bifs.BIF_floor, }, { @@ -665,112 +665,112 @@ Arrays are new in Miller 6; the substr function is older.`, class: FUNC_CLASS_MATH, help: `Inverse of normal cumulative distribution function. Note that invqorm(urand()) is normally distributed.`, - unaryFunc: types.BIF_invqnorm, + unaryFunc: bifs.BIF_invqnorm, }, { name: "log", class: FUNC_CLASS_MATH, help: "Natural (base-e) logarithm.", - unaryFunc: types.BIF_log, + unaryFunc: bifs.BIF_log, }, { name: "log10", class: FUNC_CLASS_MATH, help: "Base-10 logarithm.", - unaryFunc: types.BIF_log10, + unaryFunc: bifs.BIF_log10, }, { name: "log1p", class: FUNC_CLASS_MATH, help: "log(1-x).", - unaryFunc: types.BIF_log1p, + unaryFunc: bifs.BIF_log1p, }, { name: "logifit", class: FUNC_CLASS_MATH, help: `Given m and b from logistic regression, compute fit: $yhat=logifit($x,$m,$b).`, - ternaryFunc: types.BIF_logifit, + ternaryFunc: bifs.BIF_logifit, }, { name: "max", class: FUNC_CLASS_MATH, help: `Max of n numbers; null loses.`, - variadicFunc: types.BIF_max_variadic, + variadicFunc: bifs.BIF_max_variadic, }, { name: "min", class: FUNC_CLASS_MATH, help: `Min of n numbers; null loses.`, - variadicFunc: types.BIF_min_variadic, + variadicFunc: bifs.BIF_min_variadic, }, { name: "qnorm", class: FUNC_CLASS_MATH, help: `Normal cumulative distribution function.`, - unaryFunc: types.BIF_qnorm, + unaryFunc: bifs.BIF_qnorm, }, { name: "round", class: FUNC_CLASS_MATH, help: "Round to nearest integer.", - unaryFunc: types.BIF_round, + unaryFunc: bifs.BIF_round, }, { name: "sgn", class: FUNC_CLASS_MATH, help: `+1, 0, -1 for positive, zero, negative input respectively.`, - unaryFunc: types.BIF_sgn, + unaryFunc: bifs.BIF_sgn, }, { name: "sin", class: FUNC_CLASS_MATH, help: "Trigonometric sine.", - unaryFunc: types.BIF_sin, + unaryFunc: bifs.BIF_sin, }, { name: "sinh", class: FUNC_CLASS_MATH, help: "Hyperbolic sine.", - unaryFunc: types.BIF_sinh, + unaryFunc: bifs.BIF_sinh, }, { name: "sqrt", class: FUNC_CLASS_MATH, help: "Square root.", - unaryFunc: types.BIF_sqrt, + unaryFunc: bifs.BIF_sqrt, }, { name: "tan", class: FUNC_CLASS_MATH, help: "Trigonometric tangent.", - unaryFunc: types.BIF_tan, + unaryFunc: bifs.BIF_tan, }, { name: "tanh", class: FUNC_CLASS_MATH, help: "Hyperbolic tangent.", - unaryFunc: types.BIF_tanh, + unaryFunc: bifs.BIF_tanh, }, { name: "roundm", class: FUNC_CLASS_MATH, help: `Round to nearest multiple of m: roundm($x,$m) is the same as round($x/$m)*$m.`, - binaryFunc: types.BIF_roundm, + binaryFunc: bifs.BIF_roundm, }, { @@ -780,28 +780,28 @@ is normally distributed.`, examples: []string{ "Int-valued example: '$n=floor(20+urand()*11)'.", }, - zaryFunc: types.BIF_urand, + zaryFunc: bifs.BIF_urand, }, { name: "urandint", class: FUNC_CLASS_MATH, help: `Integer uniformly distributed between inclusive integer endpoints.`, - binaryFunc: types.BIF_urandint, + binaryFunc: bifs.BIF_urandint, }, { name: "urandrange", class: FUNC_CLASS_MATH, help: `Floating-point numbers uniformly distributed on the interval [a, b).`, - binaryFunc: types.BIF_urandrange, + binaryFunc: bifs.BIF_urandrange, }, { name: "urand32", class: FUNC_CLASS_MATH, help: `Integer uniformly distributed 0 and 2**32-1 inclusive.`, - zaryFunc: types.BIF_urand32, + zaryFunc: bifs.BIF_urand32, }, // ---------------------------------------------------------------- @@ -814,7 +814,7 @@ is normally distributed.`, examples: []string{ `gmt2sec("2001-02-03T04:05:06Z") = 981173106`, }, - unaryFunc: types.BIF_gmt2sec, + unaryFunc: bifs.BIF_gmt2sec, }, { @@ -827,8 +827,8 @@ unless second argument is supplied.`, `localtime2sec("2001-02-03 04:05:06", "Asia/Istanbul") = 981165906"`, }, // TODO: help-string - unaryFunc: types.BIF_localtime2sec_unary, - binaryFunc: types.BIF_localtime2sec_binary, + unaryFunc: bifs.BIF_localtime2sec_unary, + binaryFunc: bifs.BIF_localtime2sec_binary, hasMultipleArities: true, }, @@ -842,8 +842,8 @@ argument n, includes n decimal places for the seconds part.`, `sec2gmt(1234567890.123456) = "2009-02-13T23:31:30Z"`, `sec2gmt(1234567890.123456, 6) = "2009-02-13T23:31:30.123456Z"`, }, - unaryFunc: types.BIF_sec2gmt_unary, - binaryFunc: types.BIF_sec2gmt_binary, + unaryFunc: bifs.BIF_sec2gmt_unary, + binaryFunc: bifs.BIF_sec2gmt_binary, hasMultipleArities: true, }, @@ -859,9 +859,9 @@ includes n decimal places for the seconds part`, `sec2localtime(1234567890.123456, 6) = "2009-02-14 01:31:30.123456" with TZ="Asia/Istanbul"`, `sec2localtime(1234567890.123456, 6, "Asia/Istanbul") = "2009-02-14 01:31:30.123456"`, }, - unaryFunc: types.BIF_sec2localtime_unary, - binaryFunc: types.BIF_sec2localtime_binary, - ternaryFunc: types.BIF_sec2localtime_ternary, + unaryFunc: bifs.BIF_sec2localtime_unary, + binaryFunc: bifs.BIF_sec2localtime_binary, + ternaryFunc: bifs.BIF_sec2localtime_ternary, hasMultipleArities: true, }, @@ -873,7 +873,7 @@ Leaves non-numbers as-is.`, examples: []string{ `sec2gmtdate(1440768801.7) = "2015-08-28".`, }, - unaryFunc: types.BIF_sec2gmtdate, + unaryFunc: bifs.BIF_sec2gmtdate, }, { @@ -885,8 +885,8 @@ Leaves non-numbers as-is. Consults $TZ environment variable unless second argume `sec2localdate(1440768801.7) = "2015-08-28" with TZ="Asia/Istanbul"`, `sec2localdate(1440768801.7, "Asia/Istanbul") = "2015-08-28"`, }, - unaryFunc: types.BIF_sec2localdate_unary, - binaryFunc: types.BIF_sec2localdate_binary, + unaryFunc: bifs.BIF_sec2localdate_unary, + binaryFunc: bifs.BIF_sec2localdate_binary, hasMultipleArities: true, }, @@ -899,8 +899,8 @@ is supplied.`, `localtime2gmt("2000-01-01 00:00:00") = "1999-12-31T22:00:00Z" with TZ="Asia/Istanbul"`, `localtime2gmt("2000-01-01 00:00:00", "Asia/Istanbul") = "1999-12-31T22:00:00Z"`, }, - unaryFunc: types.BIF_localtime2gmt_unary, - binaryFunc: types.BIF_localtime2gmt_binary, + unaryFunc: bifs.BIF_localtime2gmt_unary, + binaryFunc: bifs.BIF_localtime2gmt_binary, hasMultipleArities: true, }, @@ -913,8 +913,8 @@ is supplied.`, `gmt2localtime("1999-12-31T22:00:00Z") = "2000-01-01 00:00:00" with TZ="Asia/Istanbul"`, `gmt2localtime("1999-12-31T22:00:00Z", "Asia/Istanbul") = "2000-01-01 00:00:00"`, }, - unaryFunc: types.BIF_gmt2localtime_unary, - binaryFunc: types.BIF_gmt2localtime_binary, + unaryFunc: bifs.BIF_gmt2localtime_unary, + binaryFunc: bifs.BIF_gmt2localtime_binary, hasMultipleArities: true, }, @@ -929,7 +929,7 @@ strftime_local.`, `strftime(1440768801.7,"%Y-%m-%dT%H:%M:%SZ") = "2015-08-28T13:33:21Z"`, `strftime(1440768801.7,"%Y-%m-%dT%H:%M:%3SZ") = "2015-08-28T13:33:21.700Z"`, }, - binaryFunc: types.BIF_strftime, + binaryFunc: bifs.BIF_strftime, }, { @@ -943,7 +943,7 @@ strftime_local.`, `strptime("1970-01-01 00:00:00 EET", "%Y-%m-%d %H:%M:%S %Z") = -7200`, }, - binaryFunc: types.BIF_strptime, + binaryFunc: bifs.BIF_strptime, }, { @@ -955,8 +955,8 @@ strftime_local.`, `strftime_local(1440768801.7, "%Y-%m-%d %H:%M:%3S %z") = "2015-08-28 16:33:21.700 +0300" with TZ="Asia/Istanbul"`, `strftime_local(1440768801.7, "%Y-%m-%d %H:%M:%3S %z", "Asia/Istanbul") = "2015-08-28 16:33:21.700 +0300"`, }, - binaryFunc: types.BIF_strftime_local_binary, - ternaryFunc: types.BIF_strftime_local_ternary, + binaryFunc: bifs.BIF_strftime_local_binary, + ternaryFunc: bifs.BIF_strftime_local_ternary, hasMultipleArities: true, }, @@ -972,8 +972,8 @@ strftime_local.`, // TODO: fix parse error on decimal part //`strptime_local("2015-08-28 13:33:21.345","%Y-%m-%d %H:%M:%S") = 1440758001.345`, }, - binaryFunc: types.BIF_strptime_local_binary, - ternaryFunc: types.BIF_strptime_local_ternary, + binaryFunc: bifs.BIF_strptime_local_binary, + ternaryFunc: bifs.BIF_strptime_local_ternary, hasMultipleArities: true, }, @@ -981,77 +981,77 @@ strftime_local.`, name: "dhms2fsec", class: FUNC_CLASS_TIME, help: `Recovers floating-point seconds as in dhms2fsec("5d18h53m20.250000s") = 500000.250000`, - unaryFunc: types.BIF_dhms2fsec, + unaryFunc: bifs.BIF_dhms2fsec, }, { name: "dhms2sec", class: FUNC_CLASS_TIME, help: `Recovers integer seconds as in dhms2sec("5d18h53m20s") = 500000`, - unaryFunc: types.BIF_dhms2sec, + unaryFunc: bifs.BIF_dhms2sec, }, { name: "fsec2dhms", class: FUNC_CLASS_TIME, help: `Formats floating-point seconds as in fsec2dhms(500000.25) = "5d18h53m20.250000s"`, - unaryFunc: types.BIF_fsec2dhms, + unaryFunc: bifs.BIF_fsec2dhms, }, { name: "fsec2hms", class: FUNC_CLASS_TIME, help: `Formats floating-point seconds as in fsec2hms(5000.25) = "01:23:20.250000"`, - unaryFunc: types.BIF_fsec2hms, + unaryFunc: bifs.BIF_fsec2hms, }, { name: "hms2fsec", class: FUNC_CLASS_TIME, help: `Recovers floating-point seconds as in hms2fsec("01:23:20.250000") = 5000.250000`, - unaryFunc: types.BIF_hms2fsec, + unaryFunc: bifs.BIF_hms2fsec, }, { name: "hms2sec", class: FUNC_CLASS_TIME, help: `Recovers integer seconds as in hms2sec("01:23:20") = 5000`, - unaryFunc: types.BIF_hms2sec, + unaryFunc: bifs.BIF_hms2sec, }, { name: "sec2dhms", class: FUNC_CLASS_TIME, help: `Formats integer seconds as in sec2dhms(500000) = "5d18h53m20s"`, - unaryFunc: types.BIF_sec2dhms, + unaryFunc: bifs.BIF_sec2dhms, }, { name: "sec2hms", class: FUNC_CLASS_TIME, help: `Formats integer seconds as in sec2hms(5000) = "01:23:20"`, - unaryFunc: types.BIF_sec2hms, + unaryFunc: bifs.BIF_sec2hms, }, { name: "systime", class: FUNC_CLASS_TIME, help: "help string will go here", - zaryFunc: types.BIF_systime, + zaryFunc: bifs.BIF_systime, }, { name: "systimeint", class: FUNC_CLASS_TIME, help: "help string will go here", - zaryFunc: types.BIF_systimeint, + zaryFunc: bifs.BIF_systimeint, }, { name: "uptime", class: FUNC_CLASS_TIME, help: "help string will go here", - zaryFunc: types.BIF_uptime, + zaryFunc: bifs.BIF_uptime, }, // ---------------------------------------------------------------- @@ -1061,273 +1061,273 @@ strftime_local.`, name: "is_absent", class: FUNC_CLASS_TYPING, help: "False if field is present in input, true otherwise", - unaryFunc: types.BIF_is_absent, + unaryFunc: bifs.BIF_is_absent, }, { name: "is_array", class: FUNC_CLASS_TYPING, help: "True if argument is an array.", - unaryFunc: types.BIF_is_array, + unaryFunc: bifs.BIF_is_array, }, { name: "is_bool", class: FUNC_CLASS_TYPING, help: "True if field is present with boolean value. Synonymous with is_boolean.", - unaryFunc: types.BIF_is_bool, + unaryFunc: bifs.BIF_is_bool, }, { name: "is_boolean", class: FUNC_CLASS_TYPING, help: "True if field is present with boolean value. Synonymous with is_bool.", - unaryFunc: types.BIF_is_boolean, + unaryFunc: bifs.BIF_is_boolean, }, { name: "is_empty", class: FUNC_CLASS_TYPING, help: "True if field is present in input with empty string value, false otherwise.", - unaryFunc: types.BIF_is_empty, + unaryFunc: bifs.BIF_is_empty, }, { name: "is_empty_map", class: FUNC_CLASS_TYPING, help: "True if argument is a map which is empty.", - unaryFunc: types.BIF_is_emptymap, + unaryFunc: bifs.BIF_is_emptymap, }, { name: "is_error", class: FUNC_CLASS_TYPING, help: "True if if argument is an error, such as taking string length of an integer.", - unaryFunc: types.BIF_is_error, + unaryFunc: bifs.BIF_is_error, }, { name: "is_float", class: FUNC_CLASS_TYPING, help: "True if field is present with value inferred to be float", - unaryFunc: types.BIF_is_float, + unaryFunc: bifs.BIF_is_float, }, { name: "is_int", class: FUNC_CLASS_TYPING, help: "True if field is present with value inferred to be int", - unaryFunc: types.BIF_is_int, + unaryFunc: bifs.BIF_is_int, }, { name: "is_map", class: FUNC_CLASS_TYPING, help: "True if argument is a map.", - unaryFunc: types.BIF_is_map, + unaryFunc: bifs.BIF_is_map, }, { name: "is_nonempty_map", class: FUNC_CLASS_TYPING, help: "True if argument is a map which is non-empty.", - unaryFunc: types.BIF_is_nonemptymap, + unaryFunc: bifs.BIF_is_nonemptymap, }, { name: "is_not_empty", class: FUNC_CLASS_TYPING, help: "False if field is present in input with empty value, true otherwise", - unaryFunc: types.BIF_is_notempty, + unaryFunc: bifs.BIF_is_notempty, }, { name: "is_not_map", class: FUNC_CLASS_TYPING, help: "True if argument is not a map.", - unaryFunc: types.BIF_is_notmap, + unaryFunc: bifs.BIF_is_notmap, }, { name: "is_not_array", class: FUNC_CLASS_TYPING, help: "True if argument is not an array.", - unaryFunc: types.BIF_is_notarray, + unaryFunc: bifs.BIF_is_notarray, }, { name: "is_not_null", class: FUNC_CLASS_TYPING, help: "False if argument is null (empty or absent), true otherwise.", - unaryFunc: types.BIF_is_notnull, + unaryFunc: bifs.BIF_is_notnull, }, { name: "is_null", class: FUNC_CLASS_TYPING, help: "True if argument is null (empty or absent), false otherwise.", - unaryFunc: types.BIF_is_null, + unaryFunc: bifs.BIF_is_null, }, { name: "is_numeric", class: FUNC_CLASS_TYPING, help: "True if field is present with value inferred to be int or float", - unaryFunc: types.BIF_is_numeric, + unaryFunc: bifs.BIF_is_numeric, }, { name: "is_present", class: FUNC_CLASS_TYPING, help: "True if field is present in input, false otherwise.", - unaryFunc: types.BIF_is_present, + unaryFunc: bifs.BIF_is_present, }, { name: "is_string", class: FUNC_CLASS_TYPING, help: "True if field is present with string (including empty-string) value", - unaryFunc: types.BIF_is_string, + unaryFunc: bifs.BIF_is_string, }, { name: "asserting_absent", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_absent on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_absent, + unaryFuncWithContext: bifs.BIF_asserting_absent, }, { name: "asserting_array", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_array on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_array, + unaryFuncWithContext: bifs.BIF_asserting_array, }, { name: "asserting_bool", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_bool on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_bool, + unaryFuncWithContext: bifs.BIF_asserting_bool, }, { name: "asserting_boolean", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_boolean on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_boolean, + unaryFuncWithContext: bifs.BIF_asserting_boolean, }, { name: "asserting_error", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_error on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_error, + unaryFuncWithContext: bifs.BIF_asserting_error, }, { name: "asserting_empty", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_empty on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_empty, + unaryFuncWithContext: bifs.BIF_asserting_empty, }, { name: "asserting_empty_map", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_empty_map on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_emptyMap, + unaryFuncWithContext: bifs.BIF_asserting_emptyMap, }, { name: "asserting_float", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_float on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_float, + unaryFuncWithContext: bifs.BIF_asserting_float, }, { name: "asserting_int", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_int on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_int, + unaryFuncWithContext: bifs.BIF_asserting_int, }, { name: "asserting_map", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_map on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_map, + unaryFuncWithContext: bifs.BIF_asserting_map, }, { name: "asserting_nonempty_map", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_nonempty_map on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_nonempty_map, + unaryFuncWithContext: bifs.BIF_asserting_nonempty_map, }, { name: "asserting_not_empty", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_not_empty on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_not_empty, + unaryFuncWithContext: bifs.BIF_asserting_not_empty, }, { name: "asserting_not_map", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_not_map on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_not_map, + unaryFuncWithContext: bifs.BIF_asserting_not_map, }, { name: "asserting_not_array", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_not_array on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_not_array, + unaryFuncWithContext: bifs.BIF_asserting_not_array, }, { name: "asserting_not_null", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_not_null on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_not_null, + unaryFuncWithContext: bifs.BIF_asserting_not_null, }, { name: "asserting_null", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_null on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_null, + unaryFuncWithContext: bifs.BIF_asserting_null, }, { name: "asserting_numeric", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_numeric on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_numeric, + unaryFuncWithContext: bifs.BIF_asserting_numeric, }, { name: "asserting_present", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_present on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_present, + unaryFuncWithContext: bifs.BIF_asserting_present, }, { name: "asserting_string", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_string on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_string, + unaryFuncWithContext: bifs.BIF_asserting_string, }, { name: "typeof", class: FUNC_CLASS_TYPING, help: "Convert argument to type of argument (e.g. \"str\"). For debug.", - unaryFunc: types.BIF_typeof, + unaryFunc: bifs.BIF_typeof, }, // ---------------------------------------------------------------- @@ -1337,14 +1337,14 @@ strftime_local.`, name: "boolean", class: FUNC_CLASS_CONVERSION, help: "Convert int/float/bool/string to boolean.", - unaryFunc: types.BIF_boolean, + unaryFunc: bifs.BIF_boolean, }, { name: "float", class: FUNC_CLASS_CONVERSION, help: "Convert int/float/bool/string to float.", - unaryFunc: types.BIF_float, + unaryFunc: bifs.BIF_float, }, { @@ -1352,21 +1352,21 @@ strftime_local.`, class: FUNC_CLASS_CONVERSION, help: `Convert int/float/bool to string using printf-style format string, e.g. '$s = fmtnum($n, "%08d")' or '$t = fmtnum($n, "%.6e")'.`, - binaryFunc: types.BIF_fmtnum, + binaryFunc: bifs.BIF_fmtnum, }, { name: "hexfmt", class: FUNC_CLASS_CONVERSION, help: `Convert int to hex string, e.g. 255 to "0xff".`, - unaryFunc: types.BIF_hexfmt, + unaryFunc: bifs.BIF_hexfmt, }, { name: "int", class: FUNC_CLASS_CONVERSION, help: "Convert int/float/bool/string to int.", - unaryFunc: types.BIF_int, + unaryFunc: bifs.BIF_int, }, { @@ -1377,7 +1377,7 @@ strftime_local.`, `joink({"a":3,"b":4,"c":5}, ",") = "a,b,c".`, `joink([1,2,3], ",") = "1,2,3".`, }, - binaryFunc: types.BIF_joink, + binaryFunc: bifs.BIF_joink, }, { @@ -1388,7 +1388,7 @@ strftime_local.`, `joinv([3,4,5], ",") = "3,4,5"`, `joinv({"a":3,"b":4,"c":5}, ",") = "3,4,5"`, }, - binaryFunc: types.BIF_joinv, + binaryFunc: bifs.BIF_joinv, }, { @@ -1399,7 +1399,7 @@ strftime_local.`, `joinkv([3,4,5], "=", ",") = "1=3,2=4,3=5"`, `joinkv({"a":3,"b":4,"c":5}, "=", ",") = "a=3,b=4,c=5"`, }, - ternaryFunc: types.BIF_joinkv, + ternaryFunc: bifs.BIF_joinkv, }, { @@ -1409,7 +1409,7 @@ strftime_local.`, examples: []string{ `splita("3,4,5", ",") = [3,4,5]`, }, - binaryFunc: types.BIF_splita, + binaryFunc: bifs.BIF_splita, }, { @@ -1419,7 +1419,7 @@ strftime_local.`, examples: []string{ `splita("3,4,5", ",") = ["3","4","5"]`, }, - binaryFunc: types.BIF_splitax, + binaryFunc: bifs.BIF_splitax, }, { @@ -1429,7 +1429,7 @@ strftime_local.`, examples: []string{ `splitkv("a=3,b=4,c=5", "=", ",") = {"a":3,"b":4,"c":5}`, }, - ternaryFunc: types.BIF_splitkv, + ternaryFunc: bifs.BIF_splitkv, }, { @@ -1439,7 +1439,7 @@ strftime_local.`, examples: []string{ `splitkvx("a=3,b=4,c=5", "=", ",") = {"a":"3","b":"4","c":"5"}`, }, - ternaryFunc: types.BIF_splitkvx, + ternaryFunc: bifs.BIF_splitkvx, }, { @@ -1449,7 +1449,7 @@ strftime_local.`, examples: []string{ `splitnv("a,b,c", ",") = {"1":"a","2":"b","3":"c"}`, }, - binaryFunc: types.BIF_splitnv, + binaryFunc: bifs.BIF_splitnv, }, { @@ -1459,14 +1459,14 @@ strftime_local.`, examples: []string{ `splitnvx("3,4,5", ",") = {"1":"3","2":"4","3":"5"}`, }, - binaryFunc: types.BIF_splitnvx, + binaryFunc: bifs.BIF_splitnvx, }, { name: "string", class: FUNC_CLASS_CONVERSION, help: "Convert int/float/bool/string/array/map to string.", - unaryFunc: types.BIF_string, + unaryFunc: bifs.BIF_string, }, // ---------------------------------------------------------------- @@ -1476,7 +1476,7 @@ strftime_local.`, name: "append", class: FUNC_CLASS_COLLECTIONS, help: "Appends second argument to end of first argument, which must be an array.", - binaryFunc: types.BIF_append, + binaryFunc: bifs.BIF_append, }, { @@ -1484,14 +1484,14 @@ strftime_local.`, class: FUNC_CLASS_COLLECTIONS, help: `Walks through a nested map/array, converting any map with consecutive keys "1", "2", ... into an array. Useful to wrap the output of unflatten.`, - unaryFunc: types.BIF_arrayify, + unaryFunc: bifs.BIF_arrayify, }, { name: "depth", class: FUNC_CLASS_COLLECTIONS, help: "Prints maximum depth of map/array. Scalars have depth 0.", - unaryFunc: types.BIF_depth, + unaryFunc: bifs.BIF_depth, }, { @@ -1504,8 +1504,8 @@ for non-JSON file formats like CSV.`, `flatten("", ".", {"a": { "b": 3 }}) is {"a.b" : 3}.`, `Two-argument version: flatten($*, ".") is the same as flatten("", ".", $*).`, }, - binaryFunc: types.BIF_flatten_binary, - ternaryFunc: types.BIF_flatten, + binaryFunc: bifs.BIF_flatten_binary, + ternaryFunc: bifs.BIF_flatten, hasMultipleArities: true, }, @@ -1513,14 +1513,14 @@ for non-JSON file formats like CSV.`, name: "get_keys", class: FUNC_CLASS_COLLECTIONS, help: "Returns array of keys of map or array", - unaryFunc: types.BIF_get_keys, + unaryFunc: bifs.BIF_get_keys, }, { name: "get_values", class: FUNC_CLASS_COLLECTIONS, help: "Returns array of keys of map or array -- in the latter case, returns a copy of the array", - unaryFunc: types.BIF_get_values, + unaryFunc: bifs.BIF_get_values, }, { @@ -1529,22 +1529,22 @@ for non-JSON file formats like CSV.`, help: `True/false if map has/hasn't key, e.g. 'haskey($*, "a")' or 'haskey(mymap, mykey)', or true/false if array index is in bounds / out of bounds. Error if 1st argument is not a map or array. Note -n..-1 alias to 1..n in Miller arrays.`, - binaryFunc: types.BIF_haskey, + binaryFunc: bifs.BIF_haskey, }, { name: "json_parse", class: FUNC_CLASS_COLLECTIONS, help: `Converts value from JSON-formatted string.`, - unaryFunc: types.BIF_json_parse, + unaryFunc: bifs.BIF_json_parse, }, { name: "json_stringify", class: FUNC_CLASS_COLLECTIONS, help: `Converts value to JSON-formatted string. Default output is single-line. With optional second boolean argument set to true, produces multiline output.`, - unaryFunc: types.BIF_json_stringify_unary, - binaryFunc: types.BIF_json_stringify_binary, + unaryFunc: bifs.BIF_json_stringify_unary, + binaryFunc: bifs.BIF_json_stringify_binary, hasMultipleArities: true, }, @@ -1552,14 +1552,14 @@ With optional second boolean argument set to true, produces multiline output.`, name: "leafcount", class: FUNC_CLASS_COLLECTIONS, help: `Counts total number of terminal values in map/array. For single-level map/array, same as length.`, - unaryFunc: types.BIF_leafcount, + unaryFunc: bifs.BIF_leafcount, }, { name: "length", class: FUNC_CLASS_COLLECTIONS, help: "Counts number of top-level entries in array/map. Scalars have length 1.", - unaryFunc: types.BIF_length, + unaryFunc: bifs.BIF_length, }, { @@ -1567,7 +1567,7 @@ With optional second boolean argument set to true, produces multiline output.`, class: FUNC_CLASS_COLLECTIONS, help: `With 0 args, returns empty map. With 1 arg, returns copy of arg. With 2 or more, returns copy of arg 1 with all keys from any of remaining argument maps removed.`, - variadicFunc: types.BIF_mapdiff, + variadicFunc: bifs.BIF_mapdiff, }, { @@ -1576,7 +1576,7 @@ returns copy of arg 1 with all keys from any of remaining argument maps removed. help: `Returns a map with keys from remaining arguments, if any, unset. Remaining arguments can be strings or arrays of string. E.g. 'mapexcept({1:2,3:4,5:6}, 1, 5, 7)' is '{3:4}' and 'mapexcept({1:2,3:4,5:6}, [1, 5, 7])' is '{3:4}'.`, - variadicFunc: types.BIF_mapexcept, + variadicFunc: bifs.BIF_mapexcept, minimumVariadicArity: 1, }, @@ -1586,7 +1586,7 @@ and 'mapexcept({1:2,3:4,5:6}, [1, 5, 7])' is '{3:4}'.`, help: `Returns a map with only keys from remaining arguments set. Remaining arguments can be strings or arrays of string. E.g. 'mapselect({1:2,3:4,5:6}, 1, 5, 7)' is '{1:2,5:6}' and 'mapselect({1:2,3:4,5:6}, [1, 5, 7])' is '{1:2,5:6}'.`, - variadicFunc: types.BIF_mapselect, + variadicFunc: bifs.BIF_mapselect, minimumVariadicArity: 1, }, @@ -1595,7 +1595,7 @@ Remaining arguments can be strings or arrays of string. E.g. 'mapselect({1:2,3: class: FUNC_CLASS_COLLECTIONS, help: `With 0 args, returns empty map. With >= 1 arg, returns a map with key-value pairs from all arguments. Rightmost collisions win, e.g. 'mapsum({1:2,3:4},{1:5})' is '{1:5,3:4}'.`, - variadicFunc: types.BIF_mapsum, + variadicFunc: bifs.BIF_mapsum, }, { @@ -1606,7 +1606,7 @@ See also arrayify.`, examples: []string{ `unflatten({"a.b.c" : 4}, ".") is {"a": "b": { "c": 4 }}.`, }, - binaryFunc: types.BIF_unflatten, + binaryFunc: bifs.BIF_unflatten, }, // ---------------------------------------------------------------- @@ -1727,28 +1727,28 @@ either case it should return a boolean.`, name: "hostname", class: FUNC_CLASS_SYSTEM, help: `Returns the hostname as a string.`, - zaryFunc: types.BIF_hostname, + zaryFunc: bifs.BIF_hostname, }, { name: "os", class: FUNC_CLASS_SYSTEM, help: `Returns the operating-system name as a string.`, - zaryFunc: types.BIF_os, + zaryFunc: bifs.BIF_os, }, { name: "system", class: FUNC_CLASS_SYSTEM, help: `Run command string, yielding its stdout minus final carriage return.`, - unaryFunc: types.BIF_system, + unaryFunc: bifs.BIF_system, }, { name: "version", class: FUNC_CLASS_SYSTEM, help: `Returns the Miller version as a string.`, - zaryFunc: types.BIF_version, + zaryFunc: bifs.BIF_version, }, } diff --git a/internal/pkg/dsl/cst/builtin_functions.go b/internal/pkg/dsl/cst/builtin_functions.go index 4d4cc9d33..2907bb9a1 100644 --- a/internal/pkg/dsl/cst/builtin_functions.go +++ b/internal/pkg/dsl/cst/builtin_functions.go @@ -8,11 +8,11 @@ import ( "errors" "fmt" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -89,7 +89,7 @@ func (root *RootNode) BuildMultipleArityFunctionCallsiteNode( // ---------------------------------------------------------------- type ZaryFunctionCallsiteNode struct { - zaryFunc types.ZaryFunc + zaryFunc bifs.ZaryFunc } func (root *RootNode) BuildZaryFunctionCallsiteNode( @@ -123,7 +123,7 @@ func (node *ZaryFunctionCallsiteNode) Evaluate( // ---------------------------------------------------------------- type UnaryFunctionCallsiteNode struct { - unaryFunc types.UnaryFunc + unaryFunc bifs.UnaryFunc evaluable1 IEvaluable } @@ -164,7 +164,7 @@ func (node *UnaryFunctionCallsiteNode) Evaluate( // ---------------------------------------------------------------- type UnaryFunctionWithContextCallsiteNode struct { - unaryFuncWithContext types.UnaryFuncWithContext + unaryFuncWithContext bifs.UnaryFuncWithContext evaluable1 IEvaluable } @@ -205,7 +205,7 @@ func (node *UnaryFunctionWithContextCallsiteNode) Evaluate( // ---------------------------------------------------------------- type BinaryFunctionCallsiteNode struct { - binaryFunc types.BinaryFunc + binaryFunc bifs.BinaryFunc evaluable1 IEvaluable evaluable2 IEvaluable } @@ -421,7 +421,7 @@ func (node *TernaryFunctionWithStateCallsiteNode) Evaluate( // !=~ callsites only -- not sub/gsub, and not the capture-using replacement // statements like '$y = "\2:\1". type RegexCaptureBinaryFunctionCallsiteNode struct { - regexCaptureBinaryFunc types.RegexCaptureBinaryFunc + regexCaptureBinaryFunc bifs.RegexCaptureBinaryFunc evaluable1 IEvaluable evaluable2 IEvaluable } @@ -525,23 +525,20 @@ func (node *DotCallsiteNode) Evaluate( // Case 1: map.attribute as shorthand for map["attribute"] value2 := mapvalue1.Get(node.string2) if value2 == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { return value2 } } else { // Case 2: string concatenation value2 := node.evaluable2.Evaluate(state) - return types.BIF_dot( - value1, - value2, - ) + return bifs.BIF_dot(value1, value2) } } // ---------------------------------------------------------------- type TernaryFunctionCallsiteNode struct { - ternaryFunc types.TernaryFunc + ternaryFunc bifs.TernaryFunc evaluable1 IEvaluable evaluable2 IEvaluable evaluable3 IEvaluable @@ -607,7 +604,7 @@ func (node *TernaryFunctionCallsiteNode) Evaluate( // ---------------------------------------------------------------- type VariadicFunctionCallsiteNode struct { - variadicFunc types.VariadicFunc + variadicFunc bifs.VariadicFunc evaluables []IEvaluable } @@ -782,11 +779,11 @@ func (node *LogicalANDOperatorNode) Evaluate( ) *mlrval.Mlrval { aout := node.a.Evaluate(state) atype := aout.GetType() - if !(atype == types.MT_ABSENT || atype == types.MT_BOOL) { - return types.MLRVAL_ERROR + if !(atype == mlrval.MT_ABSENT || atype == mlrval.MT_BOOL) { + return mlrval.ERROR } - if atype == types.MT_ABSENT { - return types.MLRVAL_ABSENT + if atype == mlrval.MT_ABSENT { + return mlrval.ABSENT } if aout.IsFalse() { // This means false && bogus type evaluates to true, which is sad but @@ -798,11 +795,11 @@ func (node *LogicalANDOperatorNode) Evaluate( bout := node.b.Evaluate(state) btype := bout.GetType() - if !(btype == types.MT_ABSENT || btype == types.MT_BOOL) { - return types.MLRVAL_ERROR + if !(btype == mlrval.MT_ABSENT || btype == mlrval.MT_BOOL) { + return mlrval.ERROR } - if btype == types.MT_ABSENT { - return types.MLRVAL_ABSENT + if btype == mlrval.MT_ABSENT { + return mlrval.ABSENT } return mlrval.BIF_logical_AND(aout, bout) @@ -830,11 +827,11 @@ func (node *LogicalOROperatorNode) Evaluate( ) *mlrval.Mlrval { aout := node.a.Evaluate(state) atype := aout.GetType() - if !(atype == types.MT_ABSENT || atype == types.MT_BOOL) { - return types.MLRVAL_ERROR + if !(atype == mlrval.MT_ABSENT || atype == mlrval.MT_BOOL) { + return mlrval.ERROR } - if atype == types.MT_ABSENT { - return types.MLRVAL_ABSENT + if atype == mlrval.MT_ABSENT { + return mlrval.ABSENT } if aout.IsTrue() { // This means true || bogus type evaluates to true, which is sad but @@ -846,11 +843,11 @@ func (node *LogicalOROperatorNode) Evaluate( bout := node.b.Evaluate(state) btype := bout.GetType() - if !(btype == types.MT_ABSENT || btype == types.MT_BOOL) { - return types.MLRVAL_ERROR + if !(btype == mlrval.MT_ABSENT || btype == mlrval.MT_BOOL) { + return mlrval.ERROR } - if btype == types.MT_ABSENT { - return types.MLRVAL_ABSENT + if btype == mlrval.MT_ABSENT { + return mlrval.ABSENT } return mlrval.BIF_logical_OR(aout, bout) } @@ -871,7 +868,7 @@ func (node *AbsentCoalesceOperatorNode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { aout := node.a.Evaluate(state) - if aout.GetType() != types.MT_ABSENT { + if aout.GetType() != mlrval.MT_ABSENT { return aout } @@ -895,7 +892,7 @@ func (node *EmptyCoalesceOperatorNode) Evaluate( ) *mlrval.Mlrval { aout := node.a.Evaluate(state) atype := aout.GetType() - if atype == types.MT_ABSENT || atype == types.MT_VOID || (atype == types.MT_STRING && aout.String() == "") { + if atype == mlrval.MT_ABSENT || atype == mlrval.MT_VOID || (atype == mlrval.MT_STRING && aout.String() == "") { return node.b.Evaluate(state) } else { return aout @@ -915,7 +912,7 @@ func (node *StandardTernaryOperatorNode) Evaluate( boolValue, isBool := aout.GetBoolValue() if !isBool { - return types.MLRVAL_ERROR + return mlrval.ERROR } // Short-circuit: defer evaluation unless needed @@ -939,10 +936,10 @@ func (node *StandardTernaryOperatorNode) Evaluate( func BinaryShortCircuitPlaceholder(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { lib.InternalCodingErrorPanic("Short-circuting was not correctly implemented") - return types.MLRVAL_ERROR // not reached + return mlrval.ERROR // not reached } func TernaryShortCircuitPlaceholder(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { lib.InternalCodingErrorPanic("Short-circuting was not correctly implemented") - return types.MLRVAL_ERROR // not reached + return mlrval.ERROR // not reached } diff --git a/internal/pkg/dsl/cst/collections.go b/internal/pkg/dsl/cst/collections.go index 12cf11b15..ef736d984 100644 --- a/internal/pkg/dsl/cst/collections.go +++ b/internal/pkg/dsl/cst/collections.go @@ -48,7 +48,7 @@ func (node *ArrayLiteralNode) Evaluate( for i := range node.evaluables { mlrvals[i] = *node.evaluables[i].Evaluate(state) } - return mlrval.MlrvalFromArrayReference(mlrvals) + return mlrval.FromArrayReference(mlrvals) } // ---------------------------------------------------------------- @@ -97,21 +97,21 @@ func (node *CollectionIndexAccessNode) Evaluate( } else if baseMlrval.IsStringOrVoid() { mindex, isInt := indexMlrval.GetIntValue() if !isInt { - return types.MLRVAL_ERROR + return mlrval.ERROR } // Handle UTF-8 correctly: len(input1.printrep) will count bytes, not runes. runes := []rune(baseMlrval.String()) // Miller uses 1-up, and negatively aliased, indexing for strings and arrays. zindex, inBounds := types.UnaliasArrayLengthIndex(len(runes), mindex) if !inBounds { - return types.MLRVAL_ERROR + return mlrval.ERROR } - return mlrval.MlrvalFromString(string(runes[zindex])) + return mlrval.FromString(string(runes[zindex])) } else if baseMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } @@ -162,22 +162,22 @@ func (node *ArraySliceAccessNode) Evaluate( upperIndexMlrval := node.upperIndexEvaluable.Evaluate(state) if baseMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } if baseMlrval.IsString() { - return types.BIF_substr_1_up(baseMlrval, lowerIndexMlrval, upperIndexMlrval) + return bifs.BIF_substr_1_up(baseMlrval, lowerIndexMlrval, upperIndexMlrval) } array := baseMlrval.GetArray() if array == nil { - return types.MLRVAL_ERROR + return mlrval.ERROR } n := len(array) if lowerIndexMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } if upperIndexMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } lowerIndex, ok := lowerIndexMlrval.GetIntValue() @@ -185,7 +185,7 @@ func (node *ArraySliceAccessNode) Evaluate( if lowerIndexMlrval.IsEmpty() { lowerIndex = 1 } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } upperIndex, ok := upperIndexMlrval.GetIntValue() @@ -193,7 +193,7 @@ func (node *ArraySliceAccessNode) Evaluate( if upperIndexMlrval.IsEmpty() { upperIndex = n } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } @@ -204,7 +204,7 @@ func (node *ArraySliceAccessNode) Evaluate( upperZindex, _ := types.UnaliasArrayIndex(&array, upperIndex) if lowerZindex > upperZindex { - return mlrval.MlrvalFromArrayReference(make([]mlrval.Mlrval, 0)) + return mlrval.FromArrayReference(make([]mlrval.Mlrval, 0)) } // Semantics: say x=[1,2,3,4,5]. Then x[3:10] is [3,4,5]. @@ -221,13 +221,13 @@ func (node *ArraySliceAccessNode) Evaluate( if lowerZindex < 0 { lowerZindex = 0 if lowerZindex > upperZindex { - return mlrval.MlrvalFromArrayReference(make([]mlrval.Mlrval, 0)) + return mlrval.FromArrayReference(make([]mlrval.Mlrval, 0)) } } if upperZindex > n-1 { upperZindex = n - 1 if lowerZindex > upperZindex { - return mlrval.MlrvalFromArrayReference(make([]mlrval.Mlrval, 0)) + return mlrval.FromArrayReference(make([]mlrval.Mlrval, 0)) } } @@ -243,7 +243,7 @@ func (node *ArraySliceAccessNode) Evaluate( di++ } - return mlrval.MlrvalFromArrayReference(retval) + return mlrval.FromArrayReference(retval) } // ================================================================ @@ -277,20 +277,20 @@ func (node *PositionalFieldNameNode) Evaluate( ) *mlrval.Mlrval { indexMlrval := node.indexEvaluable.Evaluate(state) if indexMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } index, ok := indexMlrval.GetIntValue() if !ok { - return types.MLRVAL_ERROR + return mlrval.ERROR } name, ok := state.Inrec.GetNameAtPositionalIndex(index) if !ok { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } - return mlrval.MlrvalFromString(name) + return mlrval.FromString(name) } // ================================================================ @@ -323,17 +323,17 @@ func (node *PositionalFieldValueNode) Evaluate( ) *mlrval.Mlrval { indexMlrval := node.indexEvaluable.Evaluate(state) if indexMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } index, ok := indexMlrval.GetIntValue() if !ok { - return types.MLRVAL_ERROR + return mlrval.ERROR } retval := state.Inrec.GetWithPositionalIndex(index) if retval == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } return retval @@ -378,36 +378,36 @@ func (node *ArrayOrMapPositionalNameAccessNode) Evaluate( indexMlrval := node.indexEvaluable.Evaluate(state) if indexMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } index, ok := indexMlrval.GetIntValue() if !ok { - return types.MLRVAL_ERROR + return mlrval.ERROR } if baseMlrval.IsArray() { n, _ := baseMlrval.GetArrayLength() zindex, ok := types.UnaliasArrayLengthIndex(int(n), index) if ok { - return mlrval.MlrvalFromInt(zindex + 1) // Miller user-space indices are 1-up + return mlrval.FromInt(zindex + 1) // Miller user-space indices are 1-up } else { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } } else if baseMlrval.IsMap() { name, ok := baseMlrval.GetMap().GetNameAtPositionalIndex(index) if !ok { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { - return mlrval.MlrvalFromString(name) + return mlrval.FromString(name) } } else if baseMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } @@ -450,12 +450,12 @@ func (node *ArrayOrMapPositionalValueAccessNode) Evaluate( indexMlrval := node.indexEvaluable.Evaluate(state) if indexMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } index, ok := indexMlrval.GetIntValue() if !ok { - return types.MLRVAL_ERROR + return mlrval.ERROR } if baseMlrval.IsArray() { @@ -466,16 +466,16 @@ func (node *ArrayOrMapPositionalValueAccessNode) Evaluate( } else if baseMlrval.IsMap() { value := baseMlrval.GetMap().GetWithPositionalIndex(index) if value == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } return value } else if baseMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } @@ -538,7 +538,7 @@ func (node *RootNode) BuildMapLiteralNode( func (node *MapLiteralNode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { - output := mlrval.MlrvalFromEmptyMap() + output := mlrval.FromEmptyMap() for i := range node.evaluablePairs { key := node.evaluablePairs[i].Key.Evaluate(state) diff --git a/internal/pkg/dsl/cst/cond.go b/internal/pkg/dsl/cst/cond.go index ba7f251f9..fb79bbcfd 100644 --- a/internal/pkg/dsl/cst/cond.go +++ b/internal/pkg/dsl/cst/cond.go @@ -11,7 +11,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) type CondBlockNode struct { @@ -46,7 +46,7 @@ func (root *RootNode) BuildCondBlockNode(astNode *dsl.ASTNode) (*CondBlockNode, func (node *CondBlockNode) Execute( state *runtime.State, ) (*BlockExitPayload, error) { - condition := types.MLRVAL_TRUE + condition := mlrval.TRUE if node.conditionNode != nil { condition = node.conditionNode.Evaluate(state) } diff --git a/internal/pkg/dsl/cst/emit_emitp.go b/internal/pkg/dsl/cst/emit_emitp.go index fb4af1797..9433ca51b 100644 --- a/internal/pkg/dsl/cst/emit_emitp.go +++ b/internal/pkg/dsl/cst/emit_emitp.go @@ -664,7 +664,7 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitAux( for i, emittableMap := range emittableMaps { for pe := emittableMap.Head; pe != nil; pe = pe.Next { newrec := templateRecord.Copy() - newrec.PutCopy(indexString, mlrval.MlrvalFromString(pe.Key)) + newrec.PutCopy(indexString, mlrval.FromString(pe.Key)) if len(indices) == 1 { valueAsMap := pe.Value.GetMap() @@ -811,7 +811,7 @@ func (node *EmitXStatementNode) executeIndexedLashedEmitAux( for pe := leadingMap.Head; pe != nil; pe = pe.Next { newrec := templateRecord.Copy() - indexValue := mlrval.MlrvalFromString(pe.Key) + indexValue := mlrval.FromString(pe.Key) newrec.PutCopy(indexString, indexValue) nextLevelValues := make([]*mlrval.Mlrval, len(emittableMaps)) @@ -879,7 +879,7 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitPAux( for i, emittableMap := range emittableMaps { for pe := emittableMap.Head; pe != nil; pe = pe.Next { newrec := templateRecord.Copy() - newrec.PutCopy(indexString, mlrval.MlrvalFromString(pe.Key)) + newrec.PutCopy(indexString, mlrval.FromString(pe.Key)) if len(indices) == 1 { newrec.PutCopy(names[i], pe.Value) @@ -927,7 +927,7 @@ func (node *EmitXStatementNode) executeIndexedLashedEmitPAux( for pe := leadingMap.Head; pe != nil; pe = pe.Next { newrec := templateRecord.Copy() - indexValue := mlrval.MlrvalFromString(pe.Key) + indexValue := mlrval.FromString(pe.Key) newrec.PutCopy(indexString, indexValue) indexValueString := indexValue.String() diff --git a/internal/pkg/dsl/cst/emitf.go b/internal/pkg/dsl/cst/emitf.go index 947b73d08..33426f784 100644 --- a/internal/pkg/dsl/cst/emitf.go +++ b/internal/pkg/dsl/cst/emitf.go @@ -11,6 +11,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/output" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" diff --git a/internal/pkg/dsl/cst/env.go b/internal/pkg/dsl/cst/env.go index 0017c6dc7..d53fb80d9 100644 --- a/internal/pkg/dsl/cst/env.go +++ b/internal/pkg/dsl/cst/env.go @@ -14,7 +14,6 @@ import ( "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) type EnvironmentVariableNode struct { @@ -39,11 +38,11 @@ func (node *EnvironmentVariableNode) Evaluate( ) *mlrval.Mlrval { name := node.nameEvaluable.Evaluate(state) if name.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } if !name.IsString() { - return types.MLRVAL_ERROR + return mlrval.ERROR } - return mlrval.MlrvalFromString(os.Getenv(name.String())) + return mlrval.FromString(os.Getenv(name.String())) } diff --git a/internal/pkg/dsl/cst/evaluable.go b/internal/pkg/dsl/cst/evaluable.go index e21aadf70..bcca85559 100644 --- a/internal/pkg/dsl/cst/evaluable.go +++ b/internal/pkg/dsl/cst/evaluable.go @@ -13,8 +13,8 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -110,7 +110,7 @@ func (node *IndirectFieldValueNode) Evaluate( ) *mlrval.Mlrval { // TODO: err fieldName := node.fieldNameEvaluable.Evaluate(state) if fieldName.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } // For normal DSL use the CST validator will prohibit this from being @@ -119,7 +119,7 @@ func (node *IndirectFieldValueNode) Evaluate( // print inrec attributes. Also, a UDF/UDS invoked from begin/end could try // to access the inrec, and that would get past the validator. if state.Inrec == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } value, err := state.Inrec.GetWithMlrvalIndex(fieldName) @@ -130,7 +130,7 @@ func (node *IndirectFieldValueNode) Evaluate( os.Exit(1) } if value == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } return value } @@ -160,12 +160,12 @@ func (node *IndirectOosvarValueNode) Evaluate( ) *mlrval.Mlrval { // TODO: err oosvarName := node.oosvarNameEvaluable.Evaluate(state) if oosvarName.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } value := state.Oosvars.Get(oosvarName.String()) if value == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } return value diff --git a/internal/pkg/dsl/cst/for.go b/internal/pkg/dsl/cst/for.go index 0a9ffb042..2653fc4fa 100644 --- a/internal/pkg/dsl/cst/for.go +++ b/internal/pkg/dsl/cst/for.go @@ -11,7 +11,6 @@ import ( "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -127,7 +126,7 @@ func (node *ForLoopOneVariableNode) Execute(state *runtime.State) (*BlockExitPay state.Stack.PushStackFrame() defer state.Stack.PopStackFrame() for pe := mapval.Head; pe != nil; pe = pe.Next { - mapkey := mlrval.MlrvalFromString(pe.Key) + mapkey := mlrval.FromString(pe.Key) err := state.Stack.SetAtScope(node.indexVariable, mapkey) if err != nil { @@ -312,7 +311,7 @@ func (node *ForLoopTwoVariableNode) Execute(state *runtime.State) (*BlockExitPay state.Stack.PushStackFrame() defer state.Stack.PopStackFrame() for pe := mapval.Head; pe != nil; pe = pe.Next { - mapkey := mlrval.MlrvalFromString(pe.Key) + mapkey := mlrval.FromString(pe.Key) err := state.Stack.SetAtScope(node.keyIndexVariable, mapkey) if err != nil { @@ -353,7 +352,7 @@ func (node *ForLoopTwoVariableNode) Execute(state *runtime.State) (*BlockExitPay state.Stack.PushStackFrame() defer state.Stack.PopStackFrame() for zindex, element := range arrayval { - mindex := mlrval.MlrvalFromInt(int(zindex + 1)) + mindex := mlrval.FromInt(int(zindex + 1)) err := state.Stack.SetAtScope(node.keyIndexVariable, mindex) if err != nil { @@ -538,7 +537,7 @@ func (node *ForLoopMultivariableNode) executeOuter( mapval := mlrval.GetMap() for pe := mapval.Head; pe != nil; pe = pe.Next { - mapkey := mlrval.MlrvalFromString(pe.Key) + mapkey := mlrval.FromString(pe.Key) err := state.Stack.SetAtScope(keyIndexVariables[0], mapkey) if err != nil { @@ -571,7 +570,7 @@ func (node *ForLoopMultivariableNode) executeOuter( // Go storage ("zindex") is 0-up. for zindex, element := range arrayval { - mindex := mlrval.MlrvalFromInt(int(zindex + 1)) + mindex := mlrval.FromInt(int(zindex + 1)) err := state.Stack.SetAtScope(keyIndexVariables[0], mindex) if err != nil { @@ -626,7 +625,7 @@ func (node *ForLoopMultivariableNode) executeInner( mapval := mlrval.GetMap() for pe := mapval.Head; pe != nil; pe = pe.Next { - mapkey := mlrval.MlrvalFromString(pe.Key) + mapkey := mlrval.FromString(pe.Key) err := state.Stack.SetAtScope(keyIndexVariable, mapkey) if err != nil { @@ -664,7 +663,7 @@ func (node *ForLoopMultivariableNode) executeInner( // Go storage ("zindex") is 0-up. for zindex, element := range arrayval { - mindex := mlrval.MlrvalFromInt(int(zindex + 1)) + mindex := mlrval.FromInt(int(zindex + 1)) err := state.Stack.SetAtScope(keyIndexVariable, mindex) if err != nil { diff --git a/internal/pkg/dsl/cst/hofs.go b/internal/pkg/dsl/cst/hofs.go index d7e83bd66..b5960cb13 100644 --- a/internal/pkg/dsl/cst/hofs.go +++ b/internal/pkg/dsl/cst/hofs.go @@ -206,7 +206,7 @@ func SelectHOF( } else if input1.IsMap() { return selectMap(input1, input2, state) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } @@ -217,7 +217,7 @@ func selectArray( ) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "select") @@ -243,7 +243,7 @@ func selectArray( outputArray = append(outputArray, *inputArray[i].Copy()) } } - return mlrval.MlrvalFromArrayReference(outputArray) + return mlrval.FromArrayReference(outputArray) } func selectMap( @@ -253,7 +253,7 @@ func selectMap( ) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "select") @@ -264,7 +264,7 @@ func selectMap( outputMap := mlrval.NewMlrmap() for pe := inputMap.Head; pe != nil; pe = pe.Next { - argsArray[0] = mlrval.MlrvalFromString(pe.Key) + argsArray[0] = mlrval.FromString(pe.Key) argsArray[1] = pe.Value mret := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) bret, ok := mret.GetBoolValue() @@ -281,7 +281,7 @@ func selectMap( } } - return mlrval.MlrvalFromMap(outputMap) + return mlrval.FromMap(outputMap) } // ================================================================ @@ -297,7 +297,7 @@ func ApplyHOF( } else if input1.IsMap() { return applyMap(input1, input2, state) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } @@ -308,7 +308,7 @@ func applyArray( ) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "apply") @@ -324,7 +324,7 @@ func applyArray( isNonAbsentOrDie(&retval, "apply") outputArray[i] = retval } - return mlrval.MlrvalFromArrayReference(outputArray) + return mlrval.FromArrayReference(outputArray) } func applyMap( @@ -334,7 +334,7 @@ func applyMap( ) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "apply") @@ -345,13 +345,13 @@ func applyMap( outputMap := mlrval.NewMlrmap() for pe := inputMap.Head; pe != nil; pe = pe.Next { - argsArray[0] = mlrval.MlrvalFromString(pe.Key) + argsArray[0] = mlrval.FromString(pe.Key) argsArray[1] = pe.Value retval := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) kvPair := getKVPairForCallbackOrDie(retval, "apply") outputMap.PutReference(kvPair.Head.Key, kvPair.Head.Value) } - return mlrval.MlrvalFromMap(outputMap) + return mlrval.FromMap(outputMap) } // ================================================================ @@ -367,7 +367,7 @@ func ReduceHOF( } else if input1.IsMap() { return reduceMap(input1, input2, state) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } @@ -378,7 +378,7 @@ func reduceArray( ) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "reduce") @@ -408,7 +408,7 @@ func reduceMap( ) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "reduce") @@ -422,15 +422,15 @@ func reduceMap( } for pe := inputMap.Head.Next; pe != nil; pe = pe.Next { - argsArray[0] = mlrval.MlrvalFromString(accumulator.Head.Key) + argsArray[0] = mlrval.FromString(accumulator.Head.Key) argsArray[1] = accumulator.Head.Value - argsArray[2] = mlrval.MlrvalFromString(pe.Key) + argsArray[2] = mlrval.FromString(pe.Key) argsArray[3] = pe.Value.Copy() retval := (udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray)) kvPair := getKVPairForCallbackOrDie(retval, "reduce") accumulator = kvPair } - return mlrval.MlrvalFromMap(accumulator) + return mlrval.FromMap(accumulator) } // ================================================================ @@ -447,7 +447,7 @@ func FoldHOF( } else if input1.IsMap() { return foldMap(input1, input2, input3, state) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } @@ -459,7 +459,7 @@ func foldArray( ) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "fold") @@ -486,7 +486,7 @@ func foldMap( ) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "fold") @@ -495,21 +495,21 @@ func foldMap( argsArray := hofSpace.argsArray if inputMap.IsEmpty() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } accumulator := getKVPairForAccumulatorOrDie(input3, "reduce").Copy() for pe := inputMap.Head; pe != nil; pe = pe.Next { - argsArray[0] = mlrval.MlrvalFromString(accumulator.Head.Key) + argsArray[0] = mlrval.FromString(accumulator.Head.Key) argsArray[1] = accumulator.Head.Value - argsArray[2] = mlrval.MlrvalFromString(pe.Key) + argsArray[2] = mlrval.FromString(pe.Key) argsArray[3] = pe.Value.Copy() retval := (udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray)) kvPair := getKVPairForCallbackOrDie(retval, "reduce") accumulator = kvPair } - return mlrval.MlrvalFromMap(accumulator) + return mlrval.FromMap(accumulator) } // ================================================================ @@ -526,7 +526,7 @@ func SortHOF( } else if inputs[0].IsMap() { return sortMK(inputs[0], "") } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } else if inputs[1].IsString() { @@ -535,7 +535,7 @@ func SortHOF( } else if inputs[0].IsMap() { return sortMK(inputs[0], inputs[1].String()) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } else if inputs[1].IsFunction() { @@ -544,7 +544,7 @@ func SortHOF( } else if inputs[0].IsMap() { return sortMF(inputs[0], inputs[1], state) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } else { @@ -553,7 +553,7 @@ func SortHOF( ) os.Exit(1) } - return types.MLRVAL_ERROR + return mlrval.ERROR } // ---------------------------------------------------------------- @@ -593,7 +593,7 @@ func sortA( flags string, ) *mlrval.Mlrval { if input1.GetArray() == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } output := input1.Copy() @@ -656,7 +656,7 @@ func sortMK( ) *mlrval.Mlrval { inmap := input1.GetMap() if inmap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } // Copy the keys to an array for sorting. @@ -689,7 +689,7 @@ func sortMK( outmap.PutCopy(key, inmap.Get(key)) } - return mlrval.MlrvalFromMapReferenced(outmap) + return mlrval.FromMapReferenced(outmap) } func sortMKNumerical(array []string, reverse bool) { @@ -749,10 +749,10 @@ func sortAF( ) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsFunction() { - return types.MLRVAL_ERROR + return mlrval.ERROR } hofSpace := getHOFSpace(input2, 2, "sort", "array") @@ -781,7 +781,7 @@ func sortAF( // Go sort-callback conventions: true if a < b, false otherwise. return nret < 0 }) - return mlrval.MlrvalFromArrayReference(outputArray) + return mlrval.FromArrayReference(outputArray) } // sortAF implements sort on arrays with callback UDF. @@ -792,10 +792,10 @@ func sortMF( ) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsFunction() { - return types.MLRVAL_ERROR + return mlrval.ERROR } pairsArray := inputMap.ToPairsArray() @@ -805,9 +805,9 @@ func sortMF( argsArray := hofSpace.argsArray sort.Slice(pairsArray, func(i, j int) bool { - argsArray[0] = mlrval.MlrvalFromString(pairsArray[i].Key) + argsArray[0] = mlrval.FromString(pairsArray[i].Key) argsArray[1] = pairsArray[i].Value - argsArray[2] = mlrval.MlrvalFromString(pairsArray[j].Key) + argsArray[2] = mlrval.FromString(pairsArray[j].Key) argsArray[3] = pairsArray[j].Value // Call the user's comparator function. @@ -829,7 +829,7 @@ func sortMF( }) sortedMap := mlrval.MlrmapFromPairsArray(pairsArray) - return mlrval.MlrvalFromMapReferenced(sortedMap) + return mlrval.FromMapReferenced(sortedMap) } // ================================================================ @@ -845,7 +845,7 @@ func AnyHOF( } else if input1.IsMap() { return anyMap(input1, input2, state) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } @@ -856,7 +856,7 @@ func anyArray( ) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "any") @@ -882,7 +882,7 @@ func anyArray( break } } - return mlrval.MlrvalFromBool(boolAny) + return mlrval.FromBool(boolAny) } func anyMap( @@ -892,7 +892,7 @@ func anyMap( ) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "any") @@ -903,7 +903,7 @@ func anyMap( boolAny := false for pe := inputMap.Head; pe != nil; pe = pe.Next { - argsArray[0] = mlrval.MlrvalFromString(pe.Key) + argsArray[0] = mlrval.FromString(pe.Key) argsArray[1] = pe.Value mret := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) bret, ok := mret.GetBoolValue() @@ -921,7 +921,7 @@ func anyMap( } } - return mlrval.MlrvalFromBool(boolAny) + return mlrval.FromBool(boolAny) } // ================================================================ @@ -937,7 +937,7 @@ func EveryHOF( } else if input1.IsMap() { return everyMap(input1, input2, state) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } @@ -948,7 +948,7 @@ func everyArray( ) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "every") @@ -974,7 +974,7 @@ func everyArray( break } } - return mlrval.MlrvalFromBool(boolEvery) + return mlrval.FromBool(boolEvery) } func everyMap( @@ -984,7 +984,7 @@ func everyMap( ) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "every") @@ -995,7 +995,7 @@ func everyMap( boolEvery := true for pe := inputMap.Head; pe != nil; pe = pe.Next { - argsArray[0] = mlrval.MlrvalFromString(pe.Key) + argsArray[0] = mlrval.FromString(pe.Key) argsArray[1] = pe.Value mret := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) bret, ok := mret.GetBoolValue() @@ -1013,5 +1013,5 @@ func everyMap( } } - return mlrval.MlrvalFromBool(boolEvery) + return mlrval.FromBool(boolEvery) } diff --git a/internal/pkg/dsl/cst/if.go b/internal/pkg/dsl/cst/if.go index 6b2fa223e..853cb8b95 100644 --- a/internal/pkg/dsl/cst/if.go +++ b/internal/pkg/dsl/cst/if.go @@ -9,8 +9,8 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -120,7 +120,7 @@ func (root *RootNode) BuildIfChainNode(astNode *dsl.ASTNode) (*IfChainNode, erro // ---------------------------------------------------------------- func (node *IfChainNode) Execute(state *runtime.State) (*BlockExitPayload, error) { for _, ifItem := range node.ifItems { - condition := types.MLRVAL_TRUE + condition := mlrval.TRUE if ifItem.conditionNode != nil { condition = ifItem.conditionNode.Evaluate(state) } diff --git a/internal/pkg/dsl/cst/leaves.go b/internal/pkg/dsl/cst/leaves.go index 2c5e965b9..8e802ea92 100644 --- a/internal/pkg/dsl/cst/leaves.go +++ b/internal/pkg/dsl/cst/leaves.go @@ -12,7 +12,6 @@ import ( "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -122,11 +121,11 @@ func (node *DirectFieldRvalueNode) Evaluate( // print inrec attributes. Also, a UDF/UDS invoked from begin/end could try // to access the inrec, and that would get past the validator. if state.Inrec == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } value := state.Inrec.Get(node.fieldName) if value == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { return value } @@ -148,9 +147,9 @@ func (node *FullSrecRvalueNode) Evaluate( // print inrec attributes. Also, a UDF/UDS invoked from begin/end could try // to access the inrec, and that would get past the validator. if state.Inrec == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { - return mlrval.MlrvalFromMap(state.Inrec) + return mlrval.FromMap(state.Inrec) } } @@ -169,7 +168,7 @@ func (node *DirectOosvarRvalueNode) Evaluate( ) *mlrval.Mlrval { value := state.Oosvars.Get(node.variableName) if value == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { return value } @@ -185,7 +184,7 @@ func (root *RootNode) BuildFullOosvarRvalueNode() *FullOosvarRvalueNode { func (node *FullOosvarRvalueNode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { - return mlrval.MlrvalFromMap(state.Oosvars) + return mlrval.FromMap(state.Oosvars) } // ---------------------------------------------------------------- @@ -219,14 +218,14 @@ func (node *LocalVariableNode) Evaluate( udf := node.udfManager.LookUpDisregardingArity(functionName) if udf != nil { - return mlrval.MlrvalFromFunction(udf, functionName) + return mlrval.FromFunction(udf, functionName) } // TODO: allow built-in functions as well. Needs some API-merging as a // prerequisite since UDFs and BIFs are managed in quite different // structures. - return types.MLRVAL_ABSENT + return mlrval.ABSENT } // ---------------------------------------------------------------- @@ -243,7 +242,7 @@ type RegexLiteralNode struct { func (root *RootNode) BuildRegexLiteralNode(literal string) IEvaluable { return &RegexLiteralNode{ - literal: mlrval.MlrvalFromString(literal), + literal: mlrval.FromString(literal), } } @@ -289,7 +288,7 @@ func (root *RootNode) BuildStringLiteralNode(literal string) IEvaluable { hasCaptures, replacementCaptureMatrix := lib.RegexReplacementHasCaptures(literal) if !hasCaptures { return &StringLiteralNode{ - literal: mlrval.MlrvalFromString(literal), + literal: mlrval.FromString(literal), } } else { return &RegexCaptureReplacementNode{ @@ -318,7 +317,7 @@ func (node *StringLiteralNode) Evaluate( func (node *RegexCaptureReplacementNode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { - return mlrval.MlrvalFromString( + return mlrval.FromString( lib.InterpolateCaptures( node.replacementString, node.replacementCaptureMatrix, @@ -334,7 +333,7 @@ type IntLiteralNode struct { func (root *RootNode) BuildIntLiteralNode(literal string) *IntLiteralNode { return &IntLiteralNode{ - literal: mlrval.MlrvalFromPrevalidatedIntString(literal), + literal: mlrval.FromPrevalidatedIntString(literal), } } func (node *IntLiteralNode) Evaluate( @@ -350,7 +349,7 @@ type FloatLiteralNode struct { func (root *RootNode) BuildFloatLiteralNode(literal string) *FloatLiteralNode { return &FloatLiteralNode{ - literal: mlrval.MlrvalFromPrevalidatedFloat64String(literal), + literal: mlrval.FromPrevalidatedFloat64String(literal), } } func (node *FloatLiteralNode) Evaluate( @@ -366,7 +365,7 @@ type BoolLiteralNode struct { func (root *RootNode) BuildBoolLiteralNode(literal string) *BoolLiteralNode { return &BoolLiteralNode{ - literal: mlrval.MlrvalFromBoolString(literal), + literal: mlrval.FromBoolString(literal), } } func (node *BoolLiteralNode) Evaluate( @@ -382,7 +381,7 @@ type NullLiteralNode struct { func (root *RootNode) BuildNullLiteralNode() *NullLiteralNode { return &NullLiteralNode{ - literal: types.MLRVAL_NULL, + literal: mlrval.NULL, } } func (node *NullLiteralNode) Evaluate( @@ -455,7 +454,7 @@ func (root *RootNode) BuildFILENAMENode() *FILENAMENode { func (node *FILENAMENode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { - return mlrval.MlrvalFromString(state.Context.FILENAME) + return mlrval.FromString(state.Context.FILENAME) } // ---------------------------------------------------------------- @@ -468,7 +467,7 @@ func (root *RootNode) BuildFILENUMNode() *FILENUMNode { func (node *FILENUMNode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { - return mlrval.MlrvalFromInt(state.Context.FILENUM) + return mlrval.FromInt(state.Context.FILENUM) } // ---------------------------------------------------------------- @@ -481,7 +480,7 @@ func (root *RootNode) BuildNFNode() *NFNode { func (node *NFNode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { - return mlrval.MlrvalFromInt(state.Inrec.FieldCount) + return mlrval.FromInt(state.Inrec.FieldCount) } // ---------------------------------------------------------------- @@ -494,7 +493,7 @@ func (root *RootNode) BuildNRNode() *NRNode { func (node *NRNode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { - return mlrval.MlrvalFromInt(state.Context.NR) + return mlrval.FromInt(state.Context.NR) } // ---------------------------------------------------------------- @@ -507,7 +506,7 @@ func (root *RootNode) BuildFNRNode() *FNRNode { func (node *FNRNode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { - return mlrval.MlrvalFromInt(state.Context.FNR) + return mlrval.FromInt(state.Context.FNR) } // ---------------------------------------------------------------- @@ -519,9 +518,8 @@ func (root *RootNode) BuildIRSNode() *IRSNode { } func (node *IRSNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Options.ReaderOptions.IRS) ->>>>>>> 8f0ac5233 (mlrmap refactor) +) *mlrval.Mlrval { + return mlrval.FromString(state.Options.ReaderOptions.IRS) } // ---------------------------------------------------------------- @@ -533,8 +531,8 @@ func (root *RootNode) BuildIFSNode() *IFSNode { } func (node *IFSNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Options.ReaderOptions.IFS) +) *mlrval.Mlrval { + return mlrval.FromString(state.Options.ReaderOptions.IFS) } // ---------------------------------------------------------------- @@ -546,8 +544,8 @@ func (root *RootNode) BuildIPSNode() *IPSNode { } func (node *IPSNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Options.ReaderOptions.IPS) +) *mlrval.Mlrval { + return mlrval.FromString(state.Options.ReaderOptions.IPS) } // ---------------------------------------------------------------- @@ -559,8 +557,8 @@ func (root *RootNode) BuildORSNode() *ORSNode { } func (node *ORSNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Options.WriterOptions.ORS) +) *mlrval.Mlrval { + return mlrval.FromString(state.Options.WriterOptions.ORS) } // ---------------------------------------------------------------- @@ -572,8 +570,8 @@ func (root *RootNode) BuildOFSNode() *OFSNode { } func (node *OFSNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Options.WriterOptions.OFS) +) *mlrval.Mlrval { + return mlrval.FromString(state.Options.WriterOptions.OFS) } // ---------------------------------------------------------------- @@ -585,8 +583,8 @@ func (root *RootNode) BuildOPSNode() *OPSNode { } func (node *OPSNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Options.WriterOptions.OPS) +) *mlrval.Mlrval { + return mlrval.FromString(state.Options.WriterOptions.OPS) } // ---------------------------------------------------------------- @@ -598,8 +596,8 @@ func (root *RootNode) BuildFLATSEPNode() *FLATSEPNode { } func (node *FLATSEPNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Options.WriterOptions.FLATSEP) +) *mlrval.Mlrval { + return mlrval.FromString(state.Options.WriterOptions.FLATSEP) } // ================================================================ @@ -633,7 +631,7 @@ func (root *RootNode) BuildMathPINode() *MathPINode { func (node *MathPINode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { - return mlrval.MlrvalFromFloat64(math.Pi) + return mlrval.FromFloat(math.Pi) } // ---------------------------------------------------------------- @@ -646,7 +644,7 @@ func (root *RootNode) BuildMathENode() *MathENode { func (node *MathENode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { - return mlrval.MlrvalFromFloat64(math.E) + return mlrval.FromFloat(math.E) } // ================================================================ @@ -664,7 +662,7 @@ func (root *RootNode) BuildArraySliceEmptyLowerIndexNode( func (node *LiteralOneNode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { - return mlrval.MlrvalFromInt(1) + return mlrval.FromInt(1) } // ================================================================ @@ -684,7 +682,7 @@ func (root *RootNode) BuildArraySliceEmptyUpperIndexNode( func (node *LiteralEmptyStringNode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { - return mlrval.MlrvalFromString("") + return mlrval.FromString("") } // ---------------------------------------------------------------- diff --git a/internal/pkg/dsl/cst/lvalues.go b/internal/pkg/dsl/cst/lvalues.go index 9e531e4f1..55338b669 100644 --- a/internal/pkg/dsl/cst/lvalues.go +++ b/internal/pkg/dsl/cst/lvalues.go @@ -93,7 +93,7 @@ func (root *RootNode) BuildDirectFieldValueLvalueNode( ) (IAssignable, error) { lib.InternalCodingErrorIf(astNode.Type != dsl.NodeTypeDirectFieldValue) - lhsFieldName := mlrval.MlrvalFromString(string(astNode.Token.Lit)) + lhsFieldName := mlrval.FromString(string(astNode.Token.Lit)) return NewDirectFieldValueLvalueNode(lhsFieldName), nil } @@ -586,7 +586,7 @@ type DirectOosvarValueLvalueNode struct { func (root *RootNode) BuildDirectOosvarValueLvalueNode(astNode *dsl.ASTNode) (IAssignable, error) { lib.InternalCodingErrorIf(astNode.Type != dsl.NodeTypeDirectOosvarValue) - lhsOosvarName := mlrval.MlrvalFromString(string(astNode.Token.Lit)) + lhsOosvarName := mlrval.FromString(string(astNode.Token.Lit)) return NewDirectOosvarValueLvalueNode(lhsOosvarName), nil } diff --git a/internal/pkg/dsl/cst/root.go b/internal/pkg/dsl/cst/root.go index 8b0066830..27636bc02 100644 --- a/internal/pkg/dsl/cst/root.go +++ b/internal/pkg/dsl/cst/root.go @@ -14,11 +14,11 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/dsl" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/output" "github.com/johnkerl/miller/internal/pkg/parsing/lexer" "github.com/johnkerl/miller/internal/pkg/parsing/parser" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // NewEmptyRoot sets up an empty CST, before ingesting any DSL strings. For diff --git a/internal/pkg/dsl/cst/tee.go b/internal/pkg/dsl/cst/tee.go index 550e7ce64..c433bce02 100644 --- a/internal/pkg/dsl/cst/tee.go +++ b/internal/pkg/dsl/cst/tee.go @@ -10,6 +10,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/output" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" diff --git a/internal/pkg/dsl/cst/types.go b/internal/pkg/dsl/cst/types.go index b28cb1c19..85dd7e52a 100644 --- a/internal/pkg/dsl/cst/types.go +++ b/internal/pkg/dsl/cst/types.go @@ -11,7 +11,6 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- diff --git a/internal/pkg/dsl/cst/udf.go b/internal/pkg/dsl/cst/udf.go index d466e4929..734f0f72d 100644 --- a/internal/pkg/dsl/cst/udf.go +++ b/internal/pkg/dsl/cst/udf.go @@ -251,34 +251,34 @@ func (site *UDFCallsite) EvaluateWithArguments( // being MT_ERROR should be mapped to MT_ERROR here (nominally, // data-dependent). But error-return could be something not data-dependent. if err != nil { - err = udf.signature.typeGatedReturnValue.Check(types.MLRVAL_ERROR) + err = udf.signature.typeGatedReturnValue.Check(mlrval.ERROR) if err != nil { fmt.Fprint(os.Stderr, err) os.Exit(1) } - return types.MLRVAL_ERROR + return mlrval.ERROR } // Fell off end of function with no return if blockExitPayload == nil { - err = udf.signature.typeGatedReturnValue.Check(types.MLRVAL_ABSENT) + err = udf.signature.typeGatedReturnValue.Check(mlrval.ABSENT) if err != nil { fmt.Fprint(os.Stderr, err) os.Exit(1) } - return types.MLRVAL_ABSENT + return mlrval.ABSENT } // TODO: should be an internal coding error. This would be break or // continue not in a loop, or return-void, both of which should have been // reported as syntax errors during the parsing pass. if blockExitPayload.blockExitStatus != BLOCK_EXIT_RETURN_VALUE { - err = udf.signature.typeGatedReturnValue.Check(types.MLRVAL_ABSENT) + err = udf.signature.typeGatedReturnValue.Check(mlrval.ABSENT) if err != nil { fmt.Fprint(os.Stderr, err) os.Exit(1) } - return types.MLRVAL_ABSENT + return mlrval.ABSENT } // Definitely a Miller internal coding error if the user put 'return x' in @@ -448,7 +448,7 @@ func (root *RootNode) BuildUnnamedUDFNode(astNode *dsl.ASTNode) (IEvaluable, err return nil, err } - udfAsMlrval := mlrval.MlrvalFromFunction(udf, name) + udfAsMlrval := mlrval.FromFunction(udf, name) return &UnnamedUDFNode{ udfAsMlrval: udfAsMlrval, diff --git a/internal/pkg/mlrval/mlrmap_accessors.go b/internal/pkg/mlrval/mlrmap_accessors.go index 07aff6b14..33592f33c 100644 --- a/internal/pkg/mlrval/mlrmap_accessors.go +++ b/internal/pkg/mlrval/mlrmap_accessors.go @@ -2,6 +2,7 @@ package mlrval import ( "bytes" + "errors" "github.com/johnkerl/miller/internal/pkg/lib" ) @@ -207,38 +208,38 @@ func (mlrmap *Mlrmap) findEntryByPositionalIndex(position int) *MlrmapEntry { return nil } -//func (mlrmap *Mlrmap) PutCopyWithMlrvalIndex(key *Mlrval, value *Mlrval) error { -// if key.IsString() { -// mlrmap.PutCopy(key.printrep, value) -// return nil -// } else if key.IsInt() { -// mlrmap.PutCopy(key.String(), value) -// return nil -// } else { -// return errors.New( -// "mlr: record/map indices must be string, int, or array thereof; got " + key.GetTypeName(), -// ) -// } -//} +func (mlrmap *Mlrmap) PutCopyWithMlrvalIndex(key *Mlrval, value *Mlrval) error { + if key.IsString() { + mlrmap.PutCopy(key.printrep, value) + return nil + } else if key.IsInt() { + mlrmap.PutCopy(key.String(), value) + return nil + } else { + return errors.New( + "mlr: record/map indices must be string, int, or array thereof; got " + key.GetTypeName(), + ) + } +} // ---------------------------------------------------------------- func (mlrmap *Mlrmap) PrependCopy(key string, value *Mlrval) { mlrmap.PrependReference(key, value.Copy()) } -//// ---------------------------------------------------------------- -//// Merges that into mlrmap. -//func (mlrmap *Mlrmap) Merge(other *Mlrmap) { -// for pe := other.Head; pe != nil; pe = pe.Next { -// mlrmap.PutCopy(pe.Key, pe.Value) -// } -//} +// ---------------------------------------------------------------- +// Merges that into mlrmap. +func (mlrmap *Mlrmap) Merge(other *Mlrmap) { + for pe := other.Head; pe != nil; pe = pe.Next { + mlrmap.PutCopy(pe.Key, pe.Value) + } +} -//// ---------------------------------------------------------------- -//// Exposed for the 'nest' verb -//func (mlrmap *Mlrmap) GetEntry(key string) *MlrmapEntry { -// return mlrmap.findEntry(key) -//} +// ---------------------------------------------------------------- +// Exposed for the 'nest' verb +func (mlrmap *Mlrmap) GetEntry(key string) *MlrmapEntry { + return mlrmap.findEntry(key) +} func (mlrmap *Mlrmap) GetKeys() []string { keys := make([]string, mlrmap.FieldCount) @@ -251,7 +252,98 @@ func (mlrmap *Mlrmap) GetKeys() []string { } // ---------------------------------------------------------------- -// For '$[[[1]]]' etc. in the DSL. +// TODO: put error-return into this API +func (mlrmap *Mlrmap) PutNameWithPositionalIndex(position int, name *Mlrval) { + positionalEntry := mlrmap.findEntryByPositionalIndex(position) + + if positionalEntry == nil { + // TODO: handle out-of-bounds accesses + return + } + + s := "" + if name.IsString() || name.IsInt() { + s = name.String() + } else { + // TODO: return MlrvalFromError() + return + } + + // E.g. there are fields named 'a' and 'b', as positions 1 and 2, + // and the user does '$[[1]] = $[[2]]'. Then there would be two b's. + mapEntry := mlrmap.findEntry(s) + if mapEntry != nil && mapEntry != positionalEntry { + if mlrmap.keysToEntries != nil { + delete(mlrmap.keysToEntries, positionalEntry.Key) + } + mlrmap.Unlink(mapEntry) + } + + lib.InternalCodingErrorIf(s == "") + positionalEntry.Key = s + + if mlrmap.keysToEntries != nil { + mlrmap.keysToEntries[s] = positionalEntry + } +} + +func (mlrmap *Mlrmap) GetWithPositionalIndex(position int) *Mlrval { + mapEntry := mlrmap.findEntryByPositionalIndex(position) + if mapEntry == nil { + return nil + } + return mapEntry.Value +} + +func (mlrmap *Mlrmap) GetWithMlrvalIndex(index *Mlrval) (*Mlrval, error) { + if index.IsArray() { + return mlrmap.getWithMlrvalArrayIndex(index) + } else { + return mlrmap.getWithMlrvalSingleIndex(index) + } +} + +// This lets the user do '$y = $x[ ["a", "b", "c"] ]' in lieu of +// '$y = $x["a"]["b"]["c"]'. +func (mlrmap *Mlrmap) getWithMlrvalArrayIndex(index *Mlrval) (*Mlrval, error) { + current := mlrmap + var retval *Mlrval = nil + lib.InternalCodingErrorIf(!index.IsArray()) + array := index.arrayval + n := len(array) + for i, piece := range array { + next, err := current.GetWithMlrvalIndex(&piece) + if err != nil { + return nil, err + } + if i < n-1 { + if !next.IsMap() { + return nil, errors.New( + "mlr: cannot multi-index non-map.", + ) + } + current = next.mapval + } else { + retval = next.Copy() + } + } + lib.InternalCodingErrorIf(retval == nil) + return retval, nil +} + +func (mlrmap *Mlrmap) getWithMlrvalSingleIndex(index *Mlrval) (*Mlrval, error) { + if index.IsString() { + return mlrmap.Get(index.printrep), nil + } else if index.IsInt() { + return mlrmap.Get(index.String()), nil + } else { + return nil, errors.New( + "Record/map indices must be string, int, or array thereof; got " + index.GetTypeName(), + ) + } +} + +// For '$[[1]]' etc. in the DSL. // // Notes: // * This is a linear search. @@ -269,169 +361,27 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { return mapEntry.Key, true } -// ---------------------------------------------------------------- -// TODO: put error-return into this API -//func (mlrmap *Mlrmap) PutNameWithPositionalIndex(position int, name *Mlrval) { -// positionalEntry := mlrmap.findEntryByPositionalIndex(position) -// -// if positionalEntry == nil { -// // TODO: handle out-of-bounds accesses -// return -// } -// -// s := "" -// if name.IsString() || name.IsInt() { -// s = name.String() -// } else { -// // TODO: return MlrvalFromError() -// return -// } -// -// // E.g. there are fields named 'a' and 'b', as positions 1 and 2, -// // and the user does '$[[1]] = $[[2]]'. Then there would be two b's. -// mapEntry := mlrmap.findEntry(s) -// if mapEntry != nil && mapEntry != positionalEntry { -// if mlrmap.keysToEntries != nil { -// delete(mlrmap.keysToEntries, positionalEntry.Key) -// } -// mlrmap.Unlink(mapEntry) -// } -// -// lib.InternalCodingErrorIf(s == "") -// positionalEntry.Key = s -// -// if mlrmap.keysToEntries != nil { -// mlrmap.keysToEntries[s] = positionalEntry -// } -//} - -//func (mlrmap *Mlrmap) GetWithPositionalIndex(position int) *Mlrval { -// mapEntry := mlrmap.findEntryByPositionalIndex(position) -// if mapEntry == nil { -// return nil -// } -// return mapEntry.Value -//} - -//func (mlrmap *Mlrmap) GetWithMlrvalIndex(index *Mlrval) (*Mlrval, error) { -// if index.IsArray() { -// return mlrmap.getWithMlrvalArrayIndex(index) -// } else { -// return mlrmap.getWithMlrvalSingleIndex(index) -// } -//} - -//// This lets the user do '$y = $x[ ["a", "b", "c"] ]' in lieu of -//// '$y = $x["a"]["b"]["c"]'. -//func (mlrmap *Mlrmap) getWithMlrvalArrayIndex(index *Mlrval) (*Mlrval, error) { -// current := mlrmap -// var retval *Mlrval = nil -// lib.InternalCodingErrorIf(!index.IsArray()) -// array := index.arrayval -// n := len(array) -// for i, piece := range array { -// next, err := current.GetWithMlrvalIndex(&piece) -// if err != nil { -// return nil, err -// } -// if i < n-1 { -// if !next.IsMap() { -// return nil, errors.New( -// "mlr: cannot multi-index non-map.", -// ) -// } -// current = next.mapval -// } else { -// retval = next.Copy() -// } -// } -// lib.InternalCodingErrorIf(retval == nil) -// return retval, nil -//} - -//func (mlrmap *Mlrmap) getWithMlrvalSingleIndex(index *Mlrval) (*Mlrval, error) { -// if index.IsString() { -// return mlrmap.Get(index.printrep), nil -// } else if index.IsInt() { -// return mlrmap.Get(index.String()), nil -// } else { -// return nil, errors.New( -// "Record/map indices must be string, int, or array thereof; got " + index.GetTypeName(), -// ) -// } -//} - -// For '$[[1]]' etc. in the DSL. -// -// Notes: -// * This is a linear search. -// * Indices are 1-up not 0-up -// * Indices -n..-1 are aliases for 1..n. In particular, it will be faster to -// get the -1st field than the nth. -// * Returns 0 on invalid index: 0, or < -n, or > n where n is the number of -// fields. - -//func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { -// mapEntry := mlrmap.findEntryByPositionalIndex(position) -// if mapEntry == nil { -// return "", false -// } -// return mapEntry.Key, true -//} - -//// ---------------------------------------------------------------- -//// TODO: put error-return into this API -//func (mlrmap *Mlrmap) PutNameWithPositionalIndex(position int, name *Mlrval) { -// positionalEntry := mlrmap.findEntryByPositionalIndex(position) -// -// if positionalEntry == nil { -// // TODO: handle out-of-bounds accesses -// return -// } -// -// // TODO: rekey the hashmap -// s := "" -// if name.IsString() { -// s = name.printrep -// } else if name.IsInt() { -// s = name.String() -// } else { -// // TODO: return MlrvalFromError() -// return -// } -// -// // E.g. there are fields named 'a' and 'b', as positions 1 and 2, -// // and the user does '$[[1]] = $[[2]]'. Then there would be two b's. -// mapEntry := mlrmap.findEntry(s) -// if mapEntry != nil && mapEntry != positionalEntry { -// mlrmap.Unlink(mapEntry) -// } -// -// lib.InternalCodingErrorIf(s == "") -// positionalEntry.Key = s -//} - // ---------------------------------------------------------------- // Copies the key and value (deep-copying in case the value is array/map). // This is safe for DSL use. See also PutReference. // TODO: put error-return into this API -//func (mlrmap *Mlrmap) PutCopyWithPositionalIndex(position int, value *Mlrval) { -// mapEntry := mlrmap.findEntryByPositionalIndex(position) -// -// if mapEntry != nil { -// mapEntry.Value = value.Copy() -// } else { -// return -// } -//} +func (mlrmap *Mlrmap) PutCopyWithPositionalIndex(position int, value *Mlrval) { + mapEntry := mlrmap.findEntryByPositionalIndex(position) -//func (mlrmap *Mlrmap) RemoveWithPositionalIndex(position int) { -// mapEntry := mlrmap.findEntryByPositionalIndex(position) -// if mapEntry != nil { -// mlrmap.Unlink(mapEntry) -// } -//} + if mapEntry != nil { + mapEntry.Value = value.Copy() + } else { + return + } +} + +func (mlrmap *Mlrmap) RemoveWithPositionalIndex(position int) { + mapEntry := mlrmap.findEntryByPositionalIndex(position) + if mapEntry != nil { + mlrmap.Unlink(mapEntry) + } +} // ---------------------------------------------------------------- func (mlrmap *Mlrmap) Equals(other *Mlrmap) bool { @@ -465,55 +415,51 @@ func (mlrmap *Mlrmap) Contains(other *Mlrmap) bool { return true } -//// ---------------------------------------------------------------- -//func (mlrmap *Mlrmap) Clear() { -// mlrmap.FieldCount = 0 -// // Assuming everything unreferenced is getting GC'ed by the Go runtime -// mlrmap.Head = nil -// mlrmap.Tail = nil -// if mlrmap.keysToEntries != nil { -// mlrmap.keysToEntries = make(map[string]*MlrmapEntry) -// } -//} +func (mlrmap *Mlrmap) Clear() { + mlrmap.FieldCount = 0 + // Assuming everything unreferenced is getting GC'ed by the Go runtime + mlrmap.Head = nil + mlrmap.Tail = nil + if mlrmap.keysToEntries != nil { + mlrmap.keysToEntries = make(map[string]*MlrmapEntry) + } +} -//// ---------------------------------------------------------------- -//func (mlrmap *Mlrmap) Copy() *Mlrmap { -// other := NewMlrmapMaybeHashed(mlrmap.isHashed()) -// for pe := mlrmap.Head; pe != nil; pe = pe.Next { -// other.PutCopy(pe.Key, pe.Value) -// } -// return other -//} +// ---------------------------------------------------------------- +func (mlrmap *Mlrmap) Copy() *Mlrmap { + other := NewMlrmapMaybeHashed(mlrmap.isHashed()) + for pe := mlrmap.Head; pe != nil; pe = pe.Next { + other.PutCopy(pe.Key, pe.Value) + } + return other +} -//// ---------------------------------------------------------------- -//// Returns true if it was found and removed -//func (mlrmap *Mlrmap) Remove(key string) bool { -// pe := mlrmap.findEntry(key) -// if pe == nil { -// return false -// } else { -// mlrmap.Unlink(pe) -// return true -// } -//} +// Returns true if it was found and removed +func (mlrmap *Mlrmap) Remove(key string) bool { + pe := mlrmap.findEntry(key) + if pe == nil { + return false + } else { + mlrmap.Unlink(pe) + return true + } +} -//// ---------------------------------------------------------------- -//func (mlrmap *Mlrmap) MoveToHead(key string) { -// pe := mlrmap.findEntry(key) -// if pe != nil { -// mlrmap.Unlink(pe) -// mlrmap.linkAtHead(pe) -// } -//} +func (mlrmap *Mlrmap) MoveToHead(key string) { + pe := mlrmap.findEntry(key) + if pe != nil { + mlrmap.Unlink(pe) + mlrmap.linkAtHead(pe) + } +} -//// ---------------------------------------------------------------- -//func (mlrmap *Mlrmap) MoveToTail(key string) { -// pe := mlrmap.findEntry(key) -// if pe != nil { -// mlrmap.Unlink(pe) -// mlrmap.linkAtTail(pe) -// } -//} +func (mlrmap *Mlrmap) MoveToTail(key string) { + pe := mlrmap.findEntry(key) + if pe != nil { + mlrmap.Unlink(pe) + mlrmap.linkAtTail(pe) + } +} // ---------------------------------------------------------------- // E.g. '$name[1]["foo"] = "bar"' or '$*["foo"][1] = "bar"' @@ -524,13 +470,13 @@ func (mlrmap *Mlrmap) Contains(other *Mlrmap) bool { // indexing here, then pass the remaining indices to the Mlrval at the desired // slot. -//func (mlrmap *Mlrmap) PutIndexed(indices []*Mlrval, rvalue *Mlrval) error { -// return putIndexedOnMap(mlrmap, indices, rvalue) -//} +func (mlrmap *Mlrmap) PutIndexed(indices []*Mlrval, rvalue *Mlrval) error { + return putIndexedOnMap(mlrmap, indices, rvalue) +} -//func (mlrmap *Mlrmap) RemoveIndexed(indices []*Mlrval) error { -// return removeIndexedOnMap(mlrmap, indices) -//} +func (mlrmap *Mlrmap) RemoveIndexed(indices []*Mlrval) error { + return removeIndexedOnMap(mlrmap, indices) +} // ---------------------------------------------------------------- func (mlrmap *Mlrmap) GetKeysJoined() string { @@ -546,19 +492,19 @@ func (mlrmap *Mlrmap) GetKeysJoined() string { return buffer.String() } -//// For mlr reshape -//func (mlrmap *Mlrmap) GetValuesJoined() string { -// var buffer bytes.Buffer -// i := 0 -// for pe := mlrmap.Head; pe != nil; pe = pe.Next { -// if i > 0 { -// buffer.WriteString(",") -// } -// i++ -// buffer.WriteString(pe.Value.String()) -// } -// return buffer.String() -//} +// For mlr reshape +func (mlrmap *Mlrmap) GetValuesJoined() string { + var buffer bytes.Buffer + i := 0 + for pe := mlrmap.Head; pe != nil; pe = pe.Next { + if i > 0 { + buffer.WriteString(",") + } + i++ + buffer.WriteString(pe.Value.String()) + } + return buffer.String() +} // ---------------------------------------------------------------- // For group-by in several transformers. If the record is 'a=x,b=y,c=3,d=4,e=5' and @@ -571,381 +517,379 @@ func (mlrmap *Mlrmap) GetKeysJoined() string { // -n 1' vs 'mlr tail -n 1 -g a,b,c'). In this case the return value is simply // the empty string. -//func (mlrmap *Mlrmap) GetSelectedValuesJoined(selectedFieldNames []string) (string, bool) { -// if len(selectedFieldNames) == 0 { -// // The fall-through is functionally correct, but this is quicker with -// // skipping setting up an empty bytes-buffer and stringifying it. The -// // non-grouped case is quite normal and is worth optimizing for. -// return "", true -// } -// -// var buffer bytes.Buffer -// for i, selectedFieldName := range selectedFieldNames { -// entry := mlrmap.findEntry(selectedFieldName) -// if entry == nil { -// return "", false -// } -// if i > 0 { -// buffer.WriteString(",") -// } -// // This may be an array or map, or just a string/int/etc. Regardless we -// // stringify it. -// buffer.WriteString(entry.Value.String()) -// } -// return buffer.String(), true -//} +func (mlrmap *Mlrmap) GetSelectedValuesJoined(selectedFieldNames []string) (string, bool) { + if len(selectedFieldNames) == 0 { + // The fall-through is functionally correct, but this is quicker with + // skipping setting up an empty bytes-buffer and stringifying it. The + // non-grouped case is quite normal and is worth optimizing for. + return "", true + } -//// As with GetSelectedValuesJoined but also returning the array of mlrvals. -//// For sort. -//// TODO: put 'Copy' into the method name -//func (mlrmap *Mlrmap) GetSelectedValuesAndJoined(selectedFieldNames []string) ( -// string, -// []*Mlrval, -// bool, -//) { -// mlrvals := make([]*Mlrval, 0, len(selectedFieldNames)) -// -// if len(selectedFieldNames) == 0 { -// // The fall-through is functionally correct, but this is quicker with -// // skipping setting up an empty bytes-buffer and stringifying it. The -// // non-grouped case is quite normal and is worth optimizing for. -// return "", mlrvals, true -// } -// -// var buffer bytes.Buffer -// for i, selectedFieldName := range selectedFieldNames { -// entry := mlrmap.findEntry(selectedFieldName) -// if entry == nil { -// return "", mlrvals, false -// } -// if i > 0 { -// buffer.WriteString(",") -// } -// // This may be an array or map, or just a string/int/etc. Regardless we -// // stringify it. -// buffer.WriteString(entry.Value.String()) -// mlrvals = append(mlrvals, entry.Value.Copy()) -// } -// return buffer.String(), mlrvals, true -//} + var buffer bytes.Buffer + for i, selectedFieldName := range selectedFieldNames { + entry := mlrmap.findEntry(selectedFieldName) + if entry == nil { + return "", false + } + if i > 0 { + buffer.WriteString(",") + } + // This may be an array or map, or just a string/int/etc. Regardless we + // stringify it. + buffer.WriteString(entry.Value.String()) + } + return buffer.String(), true +} -//// As above but only returns the array. Also, these are references, NOT copies. -//// For step and join. -//func (mlrmap *Mlrmap) ReferenceSelectedValues(selectedFieldNames []string) ([]*Mlrval, bool) { -// allFound := true -// mlrvals := make([]*Mlrval, 0, len(selectedFieldNames)) -// -// for _, selectedFieldName := range selectedFieldNames { -// entry := mlrmap.findEntry(selectedFieldName) -// if entry != nil { -// mlrvals = append(mlrvals, entry.Value) -// } else { -// mlrvals = append(mlrvals, nil) -// allFound = false -// } -// } -// return mlrvals, allFound -//} +// As with GetSelectedValuesJoined but also returning the array of mlrvals. +// For sort. +// TODO: put 'Copy' into the method name +func (mlrmap *Mlrmap) GetSelectedValuesAndJoined(selectedFieldNames []string) ( + string, + []*Mlrval, + bool, +) { + mlrvals := make([]*Mlrval, 0, len(selectedFieldNames)) -//// TODO: rename to CopySelectedValues -//// As previous but with copying. For stats1. -//func (mlrmap *Mlrmap) GetSelectedValues(selectedFieldNames []string) ([]*Mlrval, bool) { -// allFound := true -// mlrvals := make([]*Mlrval, 0, len(selectedFieldNames)) -// -// for _, selectedFieldName := range selectedFieldNames { -// entry := mlrmap.findEntry(selectedFieldName) -// if entry != nil { -// mlrvals = append(mlrvals, entry.Value.Copy()) -// } else { -// mlrvals = append(mlrvals, nil) -// allFound = false -// } -// } -// return mlrvals, allFound -//} + if len(selectedFieldNames) == 0 { + // The fall-through is functionally correct, but this is quicker with + // skipping setting up an empty bytes-buffer and stringifying it. The + // non-grouped case is quite normal and is worth optimizing for. + return "", mlrvals, true + } -//// Similar to the above but only checks availability. For join. -//func (mlrmap *Mlrmap) HasSelectedKeys(selectedFieldNames []string) bool { -// for _, selectedFieldName := range selectedFieldNames { -// entry := mlrmap.findEntry(selectedFieldName) -// if entry == nil { -// return false -// } -// } -// return true -//} + var buffer bytes.Buffer + for i, selectedFieldName := range selectedFieldNames { + entry := mlrmap.findEntry(selectedFieldName) + if entry == nil { + return "", mlrvals, false + } + if i > 0 { + buffer.WriteString(",") + } + // This may be an array or map, or just a string/int/etc. Regardless we + // stringify it. + buffer.WriteString(entry.Value.String()) + mlrvals = append(mlrvals, entry.Value.Copy()) + } + return buffer.String(), mlrvals, true +} -//// ---------------------------------------------------------------- -//// For mlr nest implode across records. -//func (mlrmap *Mlrmap) GetKeysJoinedExcept(px *MlrmapEntry) string { -// var buffer bytes.Buffer -// i := 0 -// for pe := mlrmap.Head; pe != nil; pe = pe.Next { -// if pe == px { -// continue -// } -// if i > 0 { -// buffer.WriteString(",") -// } -// i++ -// buffer.WriteString(pe.Key) -// } -// return buffer.String() -//} +// As above but only returns the array. Also, these are references, NOT copies. +// For step and join. +func (mlrmap *Mlrmap) ReferenceSelectedValues(selectedFieldNames []string) ([]*Mlrval, bool) { + allFound := true + mlrvals := make([]*Mlrval, 0, len(selectedFieldNames)) -//// For mlr nest implode across records. -//func (mlrmap *Mlrmap) GetValuesJoinedExcept(px *MlrmapEntry) string { -// var buffer bytes.Buffer -// i := 0 -// for pe := mlrmap.Head; pe != nil; pe = pe.Next { -// if pe == px { -// continue -// } -// if i > 0 { -// buffer.WriteString(",") -// } -// i++ -// // This may be an array or map, or just a string/int/etc. Regardless we -// // stringify it. -// buffer.WriteString(pe.Value.String()) -// } -// return buffer.String() -//} + for _, selectedFieldName := range selectedFieldNames { + entry := mlrmap.findEntry(selectedFieldName) + if entry != nil { + mlrvals = append(mlrvals, entry.Value) + } else { + mlrvals = append(mlrvals, nil) + allFound = false + } + } + return mlrvals, allFound +} -//// ---------------------------------------------------------------- -//func (mlrmap *Mlrmap) Rename(oldKey string, newKey string) bool { -// entry := mlrmap.findEntry(oldKey) -// if entry == nil { -// // Rename field from 'a' to 'b' where there is no 'a': no-op -// return false -// } -// -// existing := mlrmap.findEntry(newKey) -// if existing == nil { -// // Rename field from 'a' to 'b' where there is no 'b': simple update -// entry.Key = newKey -// -// if mlrmap.keysToEntries != nil { -// delete(mlrmap.keysToEntries, oldKey) -// mlrmap.keysToEntries[newKey] = entry -// } -// } else { -// // Rename field from 'a' to 'b' where there are both 'a' and 'b': -// // remove old 'a' and put its value into the slot of 'b'. -// existing.Value = entry.Value -// if mlrmap.keysToEntries != nil { -// delete(mlrmap.keysToEntries, oldKey) -// } -// mlrmap.Unlink(entry) -// } -// -// return true -//} +// TODO: rename to CopySelectedValues +// As previous but with copying. For stats1. +func (mlrmap *Mlrmap) GetSelectedValues(selectedFieldNames []string) ([]*Mlrval, bool) { + allFound := true + mlrvals := make([]*Mlrval, 0, len(selectedFieldNames)) -//// ---------------------------------------------------------------- -//func (mlrmap *Mlrmap) Label(newNames []string) { -// other := NewMlrmapAsRecord() -// -// i := 0 -// numNewNames := len(newNames) -// for { -// if i >= numNewNames { -// break -// } -// pe := mlrmap.pop() -// if pe == nil { -// break -// } -// // Old record will be GC'ed: just move pointers -// other.PutReference(newNames[i], pe.Value) -// i++ -// } -// -// for { -// pe := mlrmap.pop() -// if pe == nil { -// break -// } -// // Example: -// // * Input record has keys a,b,i,x,y -// // * Requested labeling is d,x,f -// // * The first three records a,b,i should be renamed to d,x,f -// // * The old x needs to disappear (for key-uniqueness) -// // * The y field is carried through -// if other.Has(pe.Key) { -// continue -// } -// other.PutReference(pe.Key, pe.Value) -// } -// -// *mlrmap = *other -//} + for _, selectedFieldName := range selectedFieldNames { + entry := mlrmap.findEntry(selectedFieldName) + if entry != nil { + mlrvals = append(mlrvals, entry.Value.Copy()) + } else { + mlrvals = append(mlrvals, nil) + allFound = false + } + } + return mlrvals, allFound +} -//// ---------------------------------------------------------------- -//func (mlrmap *Mlrmap) SortByKey() { -// keys := mlrmap.GetKeys() -// -// lib.SortStrings(keys) -// -// other := NewMlrmapAsRecord() -// -// for _, key := range keys { -// // Old record will be GC'ed: just move pointers -// other.PutReference(key, mlrmap.Get(key)) -// } -// -// *mlrmap = *other -//} +// Similar to the above but only checks availability. For join. +func (mlrmap *Mlrmap) HasSelectedKeys(selectedFieldNames []string) bool { + for _, selectedFieldName := range selectedFieldNames { + entry := mlrmap.findEntry(selectedFieldName) + if entry == nil { + return false + } + } + return true +} -//// ---------------------------------------------------------------- -//func (mlrmap *Mlrmap) SortByKeyRecursively() { -// keys := mlrmap.GetKeys() -// -// lib.SortStrings(keys) -// -// other := NewMlrmapAsRecord() -// -// for _, key := range keys { -// // Old record will be GC'ed: just move pointers -// value := mlrmap.Get(key) -// if value.IsMap() { -// value.mapval.SortByKeyRecursively() -// } -// other.PutReference(key, value) -// } -// -// *mlrmap = *other -//} +// ---------------------------------------------------------------- +// For mlr nest implode across records. +func (mlrmap *Mlrmap) GetKeysJoinedExcept(px *MlrmapEntry) string { + var buffer bytes.Buffer + i := 0 + for pe := mlrmap.Head; pe != nil; pe = pe.Next { + if pe == px { + continue + } + if i > 0 { + buffer.WriteString(",") + } + i++ + buffer.WriteString(pe.Key) + } + return buffer.String() +} -//// ---------------------------------------------------------------- -//// Only checks to see if the first entry is a map. For emit/emitp. -//func (mlrmap *Mlrmap) IsNested() bool { -// if mlrmap.Head == nil { -// return false -// } else if mlrmap.Head.Value.GetMap() == nil { -// TODO: check IsArrayOrMap() -// return false -// } else { -// return true -// } -//} +// For mlr nest implode across records. +func (mlrmap *Mlrmap) GetValuesJoinedExcept(px *MlrmapEntry) string { + var buffer bytes.Buffer + i := 0 + for pe := mlrmap.Head; pe != nil; pe = pe.Next { + if pe == px { + continue + } + if i > 0 { + buffer.WriteString(",") + } + i++ + // This may be an array or map, or just a string/int/etc. Regardless we + // stringify it. + buffer.WriteString(pe.Value.String()) + } + return buffer.String() +} + +// ---------------------------------------------------------------- +func (mlrmap *Mlrmap) Rename(oldKey string, newKey string) bool { + entry := mlrmap.findEntry(oldKey) + if entry == nil { + // Rename field from 'a' to 'b' where there is no 'a': no-op + return false + } + + existing := mlrmap.findEntry(newKey) + if existing == nil { + // Rename field from 'a' to 'b' where there is no 'b': simple update + entry.Key = newKey + + if mlrmap.keysToEntries != nil { + delete(mlrmap.keysToEntries, oldKey) + mlrmap.keysToEntries[newKey] = entry + } + } else { + // Rename field from 'a' to 'b' where there are both 'a' and 'b': + // remove old 'a' and put its value into the slot of 'b'. + existing.Value = entry.Value + if mlrmap.keysToEntries != nil { + delete(mlrmap.keysToEntries, oldKey) + } + mlrmap.Unlink(entry) + } + + return true +} + +// ---------------------------------------------------------------- +func (mlrmap *Mlrmap) Label(newNames []string) { + other := NewMlrmapAsRecord() + + i := 0 + numNewNames := len(newNames) + for { + if i >= numNewNames { + break + } + pe := mlrmap.pop() + if pe == nil { + break + } + // Old record will be GC'ed: just move pointers + other.PutReference(newNames[i], pe.Value) + i++ + } + + for { + pe := mlrmap.pop() + if pe == nil { + break + } + // Example: + // * Input record has keys a,b,i,x,y + // * Requested labeling is d,x,f + // * The first three records a,b,i should be renamed to d,x,f + // * The old x needs to disappear (for key-uniqueness) + // * The y field is carried through + if other.Has(pe.Key) { + continue + } + other.PutReference(pe.Key, pe.Value) + } + + *mlrmap = *other +} + +// ---------------------------------------------------------------- +func (mlrmap *Mlrmap) SortByKey() { + keys := mlrmap.GetKeys() + + lib.SortStrings(keys) + + other := NewMlrmapAsRecord() + + for _, key := range keys { + // Old record will be GC'ed: just move pointers + other.PutReference(key, mlrmap.Get(key)) + } + + *mlrmap = *other +} + +// ---------------------------------------------------------------- +func (mlrmap *Mlrmap) SortByKeyRecursively() { + keys := mlrmap.GetKeys() + + lib.SortStrings(keys) + + other := NewMlrmapAsRecord() + + for _, key := range keys { + // Old record will be GC'ed: just move pointers + value := mlrmap.Get(key) + if value.IsMap() { + value.mapval.SortByKeyRecursively() + } + other.PutReference(key, value) + } + + *mlrmap = *other +} + +// ---------------------------------------------------------------- +// Only checks to see if the first entry is a map. For emit/emitp. +func (mlrmap *Mlrmap) IsNested() bool { + if mlrmap.Head == nil { + return false + } else if mlrmap.Head.Value.GetMap() == nil { + //TODO: check IsArrayOrMap() + return false + } else { + return true + } +} // ================================================================ // PRIVATE METHODS -//// ---------------------------------------------------------------- -//func (mlrmap *Mlrmap) Unlink(pe *MlrmapEntry) { -// if pe == mlrmap.Head { -// if pe == mlrmap.Tail { -// mlrmap.Head = nil -// mlrmap.Tail = nil -// } else { -// mlrmap.Head = pe.Next -// pe.Next.Prev = nil -// } -// } else { -// pe.Prev.Next = pe.Next -// if pe == mlrmap.Tail { -// mlrmap.Tail = pe.Prev -// } else { -// pe.Next.Prev = pe.Prev -// } -// } -// if mlrmap.keysToEntries != nil { -// delete(mlrmap.keysToEntries, pe.Key) -// } -// mlrmap.FieldCount-- -//} +func (mlrmap *Mlrmap) Unlink(pe *MlrmapEntry) { + if pe == mlrmap.Head { + if pe == mlrmap.Tail { + mlrmap.Head = nil + mlrmap.Tail = nil + } else { + mlrmap.Head = pe.Next + pe.Next.Prev = nil + } + } else { + pe.Prev.Next = pe.Next + if pe == mlrmap.Tail { + mlrmap.Tail = pe.Prev + } else { + pe.Next.Prev = pe.Prev + } + } + if mlrmap.keysToEntries != nil { + delete(mlrmap.keysToEntries, pe.Key) + } + mlrmap.FieldCount-- +} -//// ---------------------------------------------------------------- -//// Does not check for duplicate keys -//func (mlrmap *Mlrmap) linkAtHead(pe *MlrmapEntry) { -// if mlrmap.Head == nil { -// pe.Prev = nil -// pe.Next = nil -// mlrmap.Head = pe -// mlrmap.Tail = pe -// } else { -// pe.Prev = nil -// pe.Next = mlrmap.Head -// mlrmap.Head.Prev = pe -// mlrmap.Head = pe -// } -// if mlrmap.keysToEntries != nil { -// mlrmap.keysToEntries[pe.Key] = pe -// } -// mlrmap.FieldCount++ -//} +// Does not check for duplicate keys +func (mlrmap *Mlrmap) linkAtHead(pe *MlrmapEntry) { + if mlrmap.Head == nil { + pe.Prev = nil + pe.Next = nil + mlrmap.Head = pe + mlrmap.Tail = pe + } else { + pe.Prev = nil + pe.Next = mlrmap.Head + mlrmap.Head.Prev = pe + mlrmap.Head = pe + } + if mlrmap.keysToEntries != nil { + mlrmap.keysToEntries[pe.Key] = pe + } + mlrmap.FieldCount++ +} -//// Does not check for duplicate keys -//func (mlrmap *Mlrmap) linkAtTail(pe *MlrmapEntry) { -// if mlrmap.Head == nil { -// pe.Prev = nil -// pe.Next = nil -// mlrmap.Head = pe -// mlrmap.Tail = pe -// } else { -// pe.Prev = mlrmap.Tail -// pe.Next = nil -// mlrmap.Tail.Next = pe -// mlrmap.Tail = pe -// } -// if mlrmap.keysToEntries != nil { -// mlrmap.keysToEntries[pe.Key] = pe -// } -// mlrmap.FieldCount++ -//} +// Does not check for duplicate keys +func (mlrmap *Mlrmap) linkAtTail(pe *MlrmapEntry) { + if mlrmap.Head == nil { + pe.Prev = nil + pe.Next = nil + mlrmap.Head = pe + mlrmap.Tail = pe + } else { + pe.Prev = mlrmap.Tail + pe.Next = nil + mlrmap.Tail.Next = pe + mlrmap.Tail = pe + } + if mlrmap.keysToEntries != nil { + mlrmap.keysToEntries[pe.Key] = pe + } + mlrmap.FieldCount++ +} -//// ---------------------------------------------------------------- -//func (mlrmap *Mlrmap) pop() *MlrmapEntry { -// if mlrmap.Head == nil { -// return nil -// } else { -// pe := mlrmap.Head -// mlrmap.Unlink(pe) -// return pe -// } -//} -// -//// ---------------------------------------------------------------- - -//// ToPairsArray is used for sorting maps by key/value/etc, e.g. the sortmf DSL function. -//func (mlrmap *Mlrmap) ToPairsArray() []MlrmapPair { -// pairsArray := make([]MlrmapPair, mlrmap.FieldCount) -// i := 0 -// for pe := mlrmap.Head; pe != nil; pe = pe.Next { -// pairsArray[i].Key = pe.Key -// pairsArray[i].Value = pe.Value.Copy() -// i++ -// } -// -// return pairsArray -//} - -//// MlrmapFromPairsArray is used for sorting maps by key/value/etc, e.g. the sortmf DSL function. -//func MlrmapFromPairsArray(pairsArray []MlrmapPair) *Mlrmap { -// mlrmap := NewMlrmap() -// for i := range pairsArray { -// mlrmap.PutCopy(pairsArray[i].Key, pairsArray[i].Value) -// } -// -// return mlrmap -//} +// ---------------------------------------------------------------- +func (mlrmap *Mlrmap) pop() *MlrmapEntry { + if mlrmap.Head == nil { + return nil + } else { + pe := mlrmap.Head + mlrmap.Unlink(pe) + return pe + } +} // ---------------------------------------------------------------- -//// GetFirstPair returns the first key-value pair as its own map. If the map is -//// empty (i.e. there is no first pair) it returns nil. -//func (mlrmap *Mlrmap) GetFirstPair() *Mlrmap { -// if mlrmap.Head == nil { -// return nil -// } -// pair := NewMlrmap() -// pair.PutCopy(mlrmap.Head.Key, mlrmap.Head.Value) -// return pair -//} -// -//func (mlrmap *Mlrmap) IsSinglePair() bool { -// return mlrmap.FieldCount == 1 -//} +// ToPairsArray is used for sorting maps by key/value/etc, e.g. the sortmf DSL function. +func (mlrmap *Mlrmap) ToPairsArray() []MlrmapPair { + pairsArray := make([]MlrmapPair, mlrmap.FieldCount) + i := 0 + for pe := mlrmap.Head; pe != nil; pe = pe.Next { + pairsArray[i].Key = pe.Key + pairsArray[i].Value = pe.Value.Copy() + i++ + } + + return pairsArray +} + +// MlrmapFromPairsArray is used for sorting maps by key/value/etc, e.g. the sortmf DSL function. +func MlrmapFromPairsArray(pairsArray []MlrmapPair) *Mlrmap { + mlrmap := NewMlrmap() + for i := range pairsArray { + mlrmap.PutCopy(pairsArray[i].Key, pairsArray[i].Value) + } + + return mlrmap +} + +// ---------------------------------------------------------------- + +// GetFirstPair returns the first key-value pair as its own map. If the map is +// empty (i.e. there is no first pair) it returns nil. +func (mlrmap *Mlrmap) GetFirstPair() *Mlrmap { + if mlrmap.Head == nil { + return nil + } + pair := NewMlrmap() + pair.PutCopy(mlrmap.Head.Key, mlrmap.Head.Value) + return pair +} + +func (mlrmap *Mlrmap) IsSinglePair() bool { + return mlrmap.FieldCount == 1 +} diff --git a/internal/pkg/types-parked/mlrmap_print.go b/internal/pkg/mlrval/mlrmap_print.go similarity index 99% rename from internal/pkg/types-parked/mlrmap_print.go rename to internal/pkg/mlrval/mlrmap_print.go index ad9959a0a..9cfd8122a 100644 --- a/internal/pkg/types-parked/mlrmap_print.go +++ b/internal/pkg/mlrval/mlrmap_print.go @@ -1,4 +1,4 @@ -package types +package mlrval import ( "bytes" diff --git a/internal/pkg/types-parked/mlrval_collections.go b/internal/pkg/mlrval/mlrval_collections.go similarity index 94% rename from internal/pkg/types-parked/mlrval_collections.go rename to internal/pkg/mlrval/mlrval_collections.go index 3cd7037bd..9adf56450 100644 --- a/internal/pkg/types-parked/mlrval_collections.go +++ b/internal/pkg/mlrval/mlrval_collections.go @@ -66,7 +66,7 @@ // // ================================================================ -package types +package mlrval import ( "errors" @@ -80,14 +80,14 @@ import ( // TODO: copy-reduction refactor func (mv *Mlrval) ArrayGet(mindex *Mlrval) Mlrval { if !mv.IsArray() { - return *MLRVAL_ERROR + return *ERROR } if !mindex.IsInt() { - return *MLRVAL_ERROR + return *ERROR } value := arrayGetAliased(&mv.arrayval, mindex.intval) if value == nil { - return *MLRVAL_ABSENT + return *ABSENT } else { return *value } @@ -219,15 +219,15 @@ func (mv *Mlrval) ArrayAppend(value *Mlrval) { // ================================================================ func (mv *Mlrval) MapGet(key *Mlrval) Mlrval { if !mv.IsMap() { - return *MLRVAL_ERROR + return *ERROR } mval, err := mv.mapval.GetWithMlrvalIndex(key) if err != nil { // xxx maybe error-return in the API - return *MLRVAL_ERROR + return *ERROR } if mval == nil { - return *MLRVAL_ABSENT + return *ABSENT } // This returns a reference, not a (deep) copy. In general in Miller, we // copy only on write/put. @@ -299,10 +299,10 @@ func (mv *Mlrval) PutIndexed(indices []*Mlrval, rvalue *Mlrval) error { } else { baseIndex := indices[0] if baseIndex.IsString() { - *mv = *MlrvalFromEmptyMap() + mv = FromEmptyMap() return putIndexedOnMap(mv.mapval, indices, rvalue) } else if baseIndex.IsInt() { - *mv = MlrvalEmptyArray() + mv = FromEmptyArray() return putIndexedOnArray(&mv.arrayval, indices, rvalue) } else { return errors.New( @@ -408,11 +408,11 @@ func putIndexedOnArray( // Overwrite what's in this slot if it's the wrong type if nextIndex.IsString() { if (*baseArray)[zindex].mvtype != MT_MAP { - (*baseArray)[zindex] = *MlrvalFromEmptyMap() + (*baseArray)[zindex] = *FromEmptyMap() } } else if nextIndex.IsInt() { if (*baseArray)[zindex].mvtype != MT_ARRAY { - (*baseArray)[zindex] = MlrvalEmptyArray() + (*baseArray)[zindex] = *FromEmptyArray() } } else { return errors.New( @@ -575,18 +575,18 @@ func BsearchMlrvalArrayForDescendingInsert( return 0 } - if BIF_greater_than_as_bool(value, (*array)[0]) { + if GreaterThan(value, (*array)[0]) { return 0 } - if BIF_less_than_as_bool(value, (*array)[hi]) { + if LessThan(value, (*array)[hi]) { return size } for lo < hi { middleElement := (*array)[mid] - if BIF_equals_as_bool(value, middleElement) { + if Equals(value, middleElement) { return mid - } else if BIF_greater_than_as_bool(value, middleElement) { + } else if GreaterThan(value, middleElement) { hi = mid newmid = (hi + lo) / 2 } else { @@ -594,9 +594,9 @@ func BsearchMlrvalArrayForDescendingInsert( newmid = (hi + lo) / 2 } if mid == newmid { - if BIF_greater_than_or_equals_as_bool(value, (*array)[lo]) { + if GreaterThanOrEquals(value, (*array)[lo]) { return lo - } else if BIF_greater_than_or_equals_as_bool(value, (*array)[hi]) { + } else if GreaterThanOrEquals(value, (*array)[hi]) { return hi } else { return hi + 1 @@ -622,18 +622,18 @@ func BsearchMlrvalArrayForAscendingInsert( return 0 } - if BIF_less_than_as_bool(value, (*array)[0]) { + if LessThan(value, (*array)[0]) { return 0 } - if BIF_greater_than_as_bool(value, (*array)[hi]) { + if GreaterThan(value, (*array)[hi]) { return size } for lo < hi { middleElement := (*array)[mid] - if BIF_equals_as_bool(value, middleElement) { + if Equals(value, middleElement) { return mid - } else if BIF_less_than_as_bool(value, middleElement) { + } else if LessThan(value, middleElement) { hi = mid newmid = (hi + lo) / 2 } else { @@ -641,9 +641,9 @@ func BsearchMlrvalArrayForAscendingInsert( newmid = (hi + lo) / 2 } if mid == newmid { - if BIF_less_than_or_equals_as_bool(value, (*array)[lo]) { + if LessThanOrEquals(value, (*array)[lo]) { return lo - } else if BIF_less_than_or_equals_as_bool(value, (*array)[hi]) { + } else if LessThanOrEquals(value, (*array)[hi]) { return hi } else { return hi + 1 diff --git a/internal/pkg/mlrval/mlrval_get.go b/internal/pkg/mlrval/mlrval_get.go index 892a4390d..88706199b 100644 --- a/internal/pkg/mlrval/mlrval_get.go +++ b/internal/pkg/mlrval/mlrval_get.go @@ -60,7 +60,7 @@ func (mv *Mlrval) GetBoolValue() (boolValue bool, isBool bool) { } } -func (mv *Mlrval) GetArray() interface{} { +func (mv *Mlrval) GetArray() []Mlrval { if mv.IsArray() { return mv.arrayval } else { @@ -68,7 +68,7 @@ func (mv *Mlrval) GetArray() interface{} { } } -func (mv *Mlrval) GetMap() interface{} { +func (mv *Mlrval) GetMap() *Mlrmap { if mv.IsMap() { return mv.mapval } else { @@ -88,6 +88,10 @@ func (mv *Mlrval) GetTypeName() string { return TYPE_NAMES[mv.Type()] } +func GetTypeName(mvtype MVType) string { + return TYPE_NAMES[mvtype] +} + // These are for built-in functions operating within type-keyed // disposition-vector/disposition-matrix context. They've already computed diff --git a/internal/pkg/mlrval/mlrval_json.go b/internal/pkg/mlrval/mlrval_json.go index 91da499dc..9e17b49d8 100644 --- a/internal/pkg/mlrval/mlrval_json.go +++ b/internal/pkg/mlrval/mlrval_json.go @@ -101,16 +101,16 @@ const ( // ---------------------------------------------------------------- func (mv *Mlrval) UnmarshalJSON(inputBytes []byte) error { - mv = FromPending() + *mv = *FromPending() decoder := json.NewDecoder(bytes.NewReader(inputBytes)) - mlrval, eof, err := MlrvalDecodeFromJSON(decoder) + pmv, eof, err := MlrvalDecodeFromJSON(decoder) if eof { return errors.New("Miller JSON parser: unexpected premature EOF.") } if err != nil { return err } - *mv = *mlrval + *mv = *pmv return nil } @@ -178,9 +178,9 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) ( ) } - mlrval := FromPending() + mv := FromPending() if isArray { - mlrval = FromEmptyArray() + mv = FromEmptyArray() for decoder.More() { element, eof, err := MlrvalDecodeFromJSON(decoder) @@ -191,11 +191,11 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) ( if err != nil { return nil, false, err } - ArrayAppend(element) + mv.ArrayAppend(element) } } else { - mlrval = FromEmptyMap() + mv = FromEmptyMap() for decoder.More() { key, eof, err := MlrvalDecodeFromJSON(decoder) @@ -223,7 +223,7 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) ( } // xxx check here string-valued key - MapPut(key, value) + mv.MapPut(key, value) } } @@ -252,7 +252,7 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) ( return nil, false, imbalanceError } - return mlrval, false, nil + return mv, false, nil } return nil, false, errors.New("unimplemented") diff --git a/internal/pkg/mlrval/tbf.go b/internal/pkg/mlrval/tbf.go index 9ee8a8b05..21b1c34c4 100644 --- a/internal/pkg/mlrval/tbf.go +++ b/internal/pkg/mlrval/tbf.go @@ -1,25 +1,31 @@ package mlrval +import ( + "errors" + + "github.com/johnkerl/miller/internal/pkg/lib" +) + // things to be filed -//// NewMlrvalForAutoDeepen is for auto-deepen of nested maps in things like -//// -//// $foo[1]["a"][2]["b"] = 3 -//// -//// Autocreated levels are maps. Array levels can be explicitly created e.g. -//// -//// $foo[1]["a"] ??= [] -//// $foo[1]["a"][2]["b"] = 3 -//func NewMlrvalForAutoDeepen(mvtype MVType) (*Mlrval, error) { -// if mvtype == MT_STRING || mvtype == MT_INT { -// empty := MlrvalFromEmptyMap() -// return empty, nil -// } else { -// return nil, errors.New( -// "mlr: indices must be string, int, or array thereof; got " + GetTypeName(mvtype), -// ) -// } -//} +// NewMlrvalForAutoDeepen is for auto-deepen of nested maps in things like +// +// $foo[1]["a"][2]["b"] = 3 +// +// Autocreated levels are maps. Array levels can be explicitly created e.g. +// +// $foo[1]["a"] ??= [] +// $foo[1]["a"][2]["b"] = 3 +func NewMlrvalForAutoDeepen(mvtype MVType) (*Mlrval, error) { + if mvtype == MT_STRING || mvtype == MT_INT { + empty := FromEmptyMap() + return empty, nil + } else { + return nil, errors.New( + "mlr: indices must be string, int, or array thereof; got " + GetTypeName(mvtype), + ) + } +} //func MlrvalFromEmptyMap() *Mlrval { // return &Mlrval{ @@ -103,32 +109,32 @@ package mlrval // arrayval: input, // } //} -// -//func LengthenMlrvalArray(array *[]Mlrval, newLength64 int) { -// newLength := int(newLength64) -// lib.InternalCodingErrorIf(newLength <= len(*array)) -// -// if newLength <= cap(*array) { -// newArray := (*array)[:newLength] -// for zindex := len(*array); zindex < newLength; zindex++ { -// // TODO: comment why not MT_ABSENT or MT_VOID -// newArray[zindex] = *NULL -// } -// *array = newArray -// } else { -// newArray := make([]Mlrval, newLength, 2*newLength) -// zindex := 0 -// for zindex = 0; zindex < len(*array); zindex++ { -// newArray[zindex] = (*array)[zindex] -// } -// for zindex = len(*array); zindex < newLength; zindex++ { -// // TODO: comment why not MT_ABSENT or MT_VOID -// newArray[zindex] = *NULL -// } -// *array = newArray -// } -//} -// + +func LengthenMlrvalArray(array *[]Mlrval, newLength64 int) { + newLength := int(newLength64) + lib.InternalCodingErrorIf(newLength <= len(*array)) + + if newLength <= cap(*array) { + newArray := (*array)[:newLength] + for zindex := len(*array); zindex < newLength; zindex++ { + // TODO: comment why not MT_ABSENT or MT_VOID + newArray[zindex] = *NULL + } + *array = newArray + } else { + newArray := make([]Mlrval, newLength, 2*newLength) + zindex := 0 + for zindex = 0; zindex < len(*array); zindex++ { + newArray[zindex] = (*array)[zindex] + } + for zindex = len(*array); zindex < newLength; zindex++ { + // TODO: comment why not MT_ABSENT or MT_VOID + newArray[zindex] = *NULL + } + *array = newArray + } +} + //// NewMlrvalForAutoDeepen is for auto-deepen of nested maps in things like //// //// $foo[1]["a"][2]["b"] = 3 diff --git a/internal/pkg/transformers/cat.go b/internal/pkg/transformers/cat.go index 3865a578c..c56b56d9c 100644 --- a/internal/pkg/transformers/cat.go +++ b/internal/pkg/transformers/cat.go @@ -179,7 +179,7 @@ func (tr *TransformerCat) countersUngrouped( inrec := inrecAndContext.Record tr.counter++ key := tr.counterFieldName - inrec.PrependCopy(key, mlrval.MlrvalFromInt(tr.counter)) + inrec.PrependCopy(key, mlrval.FromInt(tr.counter)) } outputRecordsAndContexts.PushBack(inrecAndContext) } @@ -211,7 +211,7 @@ func (tr *TransformerCat) countersGrouped( } key := tr.counterFieldName - inrec.PrependCopy(key, mlrval.MlrvalFromInt(counter)) + inrec.PrependCopy(key, mlrval.FromInt(counter)) } outputRecordsAndContexts.PushBack(inrecAndContext) } diff --git a/internal/pkg/transformers/count-similar.go b/internal/pkg/transformers/count-similar.go index 8e94fc629..a3e45e16d 100644 --- a/internal/pkg/transformers/count-similar.go +++ b/internal/pkg/transformers/count-similar.go @@ -155,7 +155,7 @@ func (tr *TransformerCountSimilar) Transform( recordListForGroup := outer.Value.(*list.List) // TODO: make 64-bit friendly groupSize := recordListForGroup.Len() - mgroupSize := mlrval.MlrvalFromInt(int(groupSize)) + mgroupSize := mlrval.FromInt(int(groupSize)) for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() { recordAndContext := inner.Value.(*types.RecordAndContext) recordAndContext.Record.PutCopy(tr.counterFieldName, mgroupSize) diff --git a/internal/pkg/transformers/count.go b/internal/pkg/transformers/count.go index 9aef1ba00..88de43c59 100644 --- a/internal/pkg/transformers/count.go +++ b/internal/pkg/transformers/count.go @@ -173,7 +173,7 @@ func (tr *TransformerCount) countUngrouped( tr.ungroupedCount++ } else { newrec := mlrval.NewMlrmapAsRecord() - newrec.PutCopy(tr.outputFieldName, mlrval.MlrvalFromInt(tr.ungroupedCount)) + newrec.PutCopy(tr.outputFieldName, mlrval.FromInt(tr.ungroupedCount)) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker @@ -211,7 +211,7 @@ func (tr *TransformerCount) countGrouped( } else { if tr.showCountsOnly { newrec := mlrval.NewMlrmapAsRecord() - newrec.PutCopy(tr.outputFieldName, mlrval.MlrvalFromInt(tr.groupedCounts.FieldCount)) + newrec.PutCopy(tr.outputFieldName, mlrval.FromInt(tr.groupedCounts.FieldCount)) outrecAndContext := types.NewRecordAndContext(newrec, &inrecAndContext.Context) outputRecordsAndContexts.PushBack(outrecAndContext) @@ -236,7 +236,7 @@ func (tr *TransformerCount) countGrouped( } countForGroup := outer.Value.(int) - newrec.PutCopy(tr.outputFieldName, mlrval.MlrvalFromInt(countForGroup)) + newrec.PutCopy(tr.outputFieldName, mlrval.FromInt(countForGroup)) outrecAndContext := types.NewRecordAndContext(newrec, &inrecAndContext.Context) outputRecordsAndContexts.PushBack(outrecAndContext) diff --git a/internal/pkg/transformers/fill-empty.go b/internal/pkg/transformers/fill-empty.go index de41ccee1..2b57de4e1 100644 --- a/internal/pkg/transformers/fill-empty.go +++ b/internal/pkg/transformers/fill-empty.go @@ -103,7 +103,7 @@ func NewTransformerFillEmpty( ) (*TransformerFillEmpty, error) { tr := &TransformerFillEmpty{} if inferType { - tr.fillValue = mlrval.MlrvalFromInferredType(fillString) + tr.fillValue = mlrval.FromInferredType(fillString) } else { tr.fillValue = mlrval.mlrval.FromString(fillString) } diff --git a/internal/pkg/transformers/fraction.go b/internal/pkg/transformers/fraction.go index 96cb03f22..5178511d1 100644 --- a/internal/pkg/transformers/fraction.go +++ b/internal/pkg/transformers/fraction.go @@ -159,14 +159,14 @@ func NewTransformerFraction( var multiplier *mlrval.Mlrval var outputFieldNameSuffix string if doPercents { - multiplier = mlrval.MlrvalFromInt(100) + multiplier = mlrval.FromInt(100) if doCumu { outputFieldNameSuffix = "_cumulative_percent" } else { outputFieldNameSuffix = "_percent" } } else { - multiplier = mlrval.MlrvalFromInt(1) + multiplier = mlrval.FromInt(1) if doCumu { outputFieldNameSuffix = "_cumulative_fraction" } else { @@ -174,7 +174,7 @@ func NewTransformerFraction( } } - zero := mlrval.MlrvalFromInt(0) + zero := mlrval.FromInt(0) return &TransformerFraction{ fractionFieldNames: fractionFieldNames, @@ -271,7 +271,7 @@ func (tr *TransformerFraction) Transform( outputValue = bifs.BIF_divide(numerator, denominator) outputValue = bifs.BIF_times(outputValue, tr.multiplier) } else { - outputValue = types.MLRVAL_ERROR + outputValue = mlrval.ERROR } outrec.PutCopy( diff --git a/internal/pkg/transformers/histogram.go b/internal/pkg/transformers/histogram.go index 5471efa8c..57abf529f 100644 --- a/internal/pkg/transformers/histogram.go +++ b/internal/pkg/transformers/histogram.go @@ -262,17 +262,17 @@ func (tr *TransformerHistogram) emitNonAuto( outrec.PutReference( tr.outputPrefix+"bin_lo", - mlrval.MlrvalFromFloat64((tr.lo+float64(i))/tr.mul), + mlrval.FromFloat((tr.lo+float64(i))/tr.mul), ) outrec.PutReference( tr.outputPrefix+"bin_hi", - mlrval.MlrvalFromFloat64((tr.lo+float64(i+1))/tr.mul), + mlrval.FromFloat((tr.lo+float64(i+1))/tr.mul), ) for _, valueFieldName := range tr.valueFieldNames { outrec.PutReference( countFieldNames[valueFieldName], - mlrval.MlrvalFromInt(tr.countsByField[valueFieldName][i]), + mlrval.FromInt(tr.countsByField[valueFieldName][i]), ) } @@ -368,17 +368,17 @@ func (tr *TransformerHistogram) emitAuto( outrec.PutReference( tr.outputPrefix+"bin_lo", - mlrval.MlrvalFromFloat64((lo+float64(i))/mul), + mlrval.FromFloat((lo+float64(i))/mul), ) outrec.PutReference( tr.outputPrefix+"bin_hi", - mlrval.MlrvalFromFloat64((lo+float64(i+1))/mul), + mlrval.FromFloat((lo+float64(i+1))/mul), ) for _, valueFieldName := range tr.valueFieldNames { outrec.PutReference( countFieldNames[valueFieldName], - mlrval.MlrvalFromInt(tr.countsByField[valueFieldName][i]), + mlrval.FromInt(tr.countsByField[valueFieldName][i]), ) } diff --git a/internal/pkg/transformers/most-or-least-frequent.go b/internal/pkg/transformers/most-or-least-frequent.go index f5b75b5cf..7ab896124 100644 --- a/internal/pkg/transformers/most-or-least-frequent.go +++ b/internal/pkg/transformers/most-or-least-frequent.go @@ -285,7 +285,7 @@ func (tr *TransformerMostOrLeastFrequent) Transform( } if tr.showCounts { - outrec.PutReference(tr.outputFieldName, mlrval.MlrvalFromInt(sortPairs[i].count)) + outrec.PutReference(tr.outputFieldName, mlrval.FromInt(sortPairs[i].count)) } outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } diff --git a/internal/pkg/transformers/put-or-filter.go b/internal/pkg/transformers/put-or-filter.go index 17f09937f..c7a84901f 100644 --- a/internal/pkg/transformers/put-or-filter.go +++ b/internal/pkg/transformers/put-or-filter.go @@ -456,7 +456,7 @@ func NewTransformerPut( } key := pair[0] svalue := pair[1] - mvalue := mlrval.MlrvalFromInferredType(svalue) + mvalue := mlrval.FromInferredType(svalue) runtimeState.Oosvars.PutCopy(key, mvalue) } } diff --git a/internal/pkg/transformers/seqgen.go b/internal/pkg/transformers/seqgen.go index aeab63af1..495f1bb81 100644 --- a/internal/pkg/transformers/seqgen.go +++ b/internal/pkg/transformers/seqgen.go @@ -132,9 +132,9 @@ func NewTransformerSeqgen( stopString string, stepString string, ) (*TransformerSeqgen, error) { - start := mlrval.MlrvalFromInferredType(startString) - stop := mlrval.MlrvalFromInferredType(stopString) - step := mlrval.MlrvalFromInferredType(stepString) + start := mlrval.FromInferredType(startString) + stop := mlrval.FromInferredType(stopString) + step := mlrval.FromInferredType(stepString) var doneComparator types.BinaryFunc = nil fstart, startIsNumeric := start.GetNumericToFloatValue() @@ -187,7 +187,7 @@ func NewTransformerSeqgen( stop: stop, step: step, doneComparator: doneComparator, - mdone: types.MLRVAL_FALSE, + mdone: mlrval.FALSE, }, nil } diff --git a/internal/pkg/transformers/step.go b/internal/pkg/transformers/step.go index c2238f810..30d595901 100644 --- a/internal/pkg/transformers/step.go +++ b/internal/pkg/transformers/step.go @@ -379,11 +379,11 @@ func (stepper *tStepperDelta) process( inrec *mlrval.Mlrmap, ) { if valueFieldValue.IsEmpty() { - inrec.PutCopy(stepper.outputFieldName, types.MLRVAL_VOID) + inrec.PutCopy(stepper.outputFieldName, mlrval.VOID) return } - delta := mlrval.MlrvalFromInt(0) + delta := mlrval.FromInt(0) if stepper.previous != nil { delta = bifs.BIF_minus_binary(valueFieldValue, stepper.previous) } @@ -414,7 +414,7 @@ func (stepper *tStepperShift) process( inrec *mlrval.Mlrmap, ) { if stepper.previous == nil { - shift := types.MLRVAL_VOID + shift := mlrval.VOID inrec.PutCopy(stepper.outputFieldName, shift) } else { inrec.PutCopy(stepper.outputFieldName, stepper.previous) @@ -444,7 +444,7 @@ func (stepper *tStepperFromFirst) process( valueFieldValue *mlrval.Mlrval, inrec *mlrval.Mlrmap, ) { - fromFirst := mlrval.MlrvalFromInt(0) + fromFirst := mlrval.FromInt(0) if stepper.first == nil { stepper.first = valueFieldValue.Copy() } else { @@ -475,11 +475,11 @@ func (stepper *tStepperRatio) process( inrec *mlrval.Mlrmap, ) { if valueFieldValue.IsEmpty() { - inrec.PutCopy(stepper.outputFieldName, types.MLRVAL_VOID) + inrec.PutCopy(stepper.outputFieldName, mlrval.VOID) return } - ratio := mlrval.MlrvalFromInt(1) + ratio := mlrval.FromInt(1) if stepper.previous != nil { ratio = bifs.BIF_divide(valueFieldValue, stepper.previous) } @@ -500,7 +500,7 @@ func stepperRsumAlloc( _unused2 []string, ) tStepper { return &tStepperRsum{ - rsum: mlrval.MlrvalFromInt(0), + rsum: mlrval.FromInt(0), outputFieldName: inputFieldName + "_rsum", } } @@ -510,7 +510,7 @@ func (stepper *tStepperRsum) process( inrec *mlrval.Mlrmap, ) { if valueFieldValue.IsEmpty() { - inrec.PutCopy(stepper.outputFieldName, types.MLRVAL_VOID) + inrec.PutCopy(stepper.outputFieldName, mlrval.VOID) } else { stepper.rsum = bifs.BIF_plus_binary(valueFieldValue, stepper.rsum) inrec.PutCopy(stepper.outputFieldName, stepper.rsum) @@ -530,8 +530,8 @@ func stepperCounterAlloc( _unused2 []string, ) tStepper { return &tStepperCounter{ - counter: mlrval.MlrvalFromInt(0), - one: mlrval.MlrvalFromInt(1), + counter: mlrval.FromInt(0), + one: mlrval.FromInt(1), outputFieldName: inputFieldName + "_counter", } } @@ -541,7 +541,7 @@ func (stepper *tStepperCounter) process( inrec *mlrval.Mlrmap, ) { if valueFieldValue.IsEmpty() { - inrec.PutCopy(stepper.outputFieldName, types.MLRVAL_VOID) + inrec.PutCopy(stepper.outputFieldName, mlrval.VOID) } else { stepper.counter = bifs.BIF_plus_binary(stepper.counter, stepper.one) inrec.PutCopy(stepper.outputFieldName, stepper.counter) @@ -592,9 +592,9 @@ func stepperEWMAAlloc( ) os.Exit(1) } - alphas[i] = mlrval.MlrvalFromFloat64(dalpha) - oneMinusAlphas[i] = mlrval.MlrvalFromFloat64(1.0 - dalpha) - prevs[i] = mlrval.MlrvalFromFloat64(0.0) + alphas[i] = mlrval.FromFloat(dalpha) + oneMinusAlphas[i] = mlrval.FromFloat(1.0 - dalpha) + prevs[i] = mlrval.FromFloat(0.0) outputFieldNames[i] = inputFieldName + "_ewma_" + suffix } diff --git a/internal/pkg/transformers/top.go b/internal/pkg/transformers/top.go index 5a8199d19..721768ee8 100644 --- a/internal/pkg/transformers/top.go +++ b/internal/pkg/transformers/top.go @@ -274,11 +274,11 @@ func (tr *TransformerTop) emit( topKeeper := pb.Value.(*utils.TopKeeper) key := valueFieldName + "_top" if i < topKeeper.GetSize() { - newrec.PutReference(tr.outputFieldName, mlrval.MlrvalFromInt(i+1)) + newrec.PutReference(tr.outputFieldName, mlrval.FromInt(i+1)) newrec.PutReference(key, topKeeper.TopValues[i].Copy()) } else { - newrec.PutReference(tr.outputFieldName, mlrval.MlrvalFromInt(i+1)) - newrec.PutCopy(key, types.MLRVAL_VOID) + newrec.PutReference(tr.outputFieldName, mlrval.FromInt(i+1)) + newrec.PutCopy(key, mlrval.VOID) } } diff --git a/internal/pkg/transformers/uniq.go b/internal/pkg/transformers/uniq.go index 3be2de7ea..8e8a22ad7 100644 --- a/internal/pkg/transformers/uniq.go +++ b/internal/pkg/transformers/uniq.go @@ -373,7 +373,7 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowCounts( for pe := tr.uniqifiedRecords.Head; pe != nil; pe = pe.Next { outrecAndContext := pe.Value.(*types.RecordAndContext) icount := tr.uniqifiedRecordCounts.Get(pe.Key) - mcount := mlrval.MlrvalFromInt(icount.(int)) + mcount := mlrval.FromInt(icount.(int)) outrecAndContext.Record.PrependReference(tr.outputFieldName, mcount) outputRecordsAndContexts.PushBack(outrecAndContext) } @@ -403,7 +403,7 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowNumDistinctOnly( outrec := mlrval.NewMlrmapAsRecord() outrec.PutReference( tr.outputFieldName, - mlrval.MlrvalFromInt(tr.uniqifiedRecordCounts.FieldCount), + mlrval.FromInt(tr.uniqifiedRecordCounts.FieldCount), ) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) @@ -480,7 +480,7 @@ func (tr *TransformerUniq) transformUnlashed( "value", tr.unlashedCountValues.Get(fieldName).(*lib.OrderedMap).Get(fieldValueString).(*mlrval.Mlrval), ) - outrec.PutReference("count", mlrval.MlrvalFromInt(pf.Value.(int))) + outrec.PutReference("count", mlrval.FromInt(pf.Value.(int))) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } } @@ -513,7 +513,7 @@ func (tr *TransformerUniq) transformNumDistinctOnly( outrec := mlrval.NewMlrmapAsRecord() outrec.PutReference( "count", - mlrval.MlrvalFromInt(tr.countsByGroup.FieldCount), + mlrval.FromInt(tr.countsByGroup.FieldCount), ) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) @@ -556,7 +556,7 @@ func (tr *TransformerUniq) transformWithCounts( if tr.showCounts { outrec.PutReference( tr.outputFieldName, - mlrval.MlrvalFromInt(pa.Value.(int)), + mlrval.FromInt(pa.Value.(int)), ) } outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) diff --git a/internal/pkg/transformers/utils/join-bucket.go b/internal/pkg/transformers/utils/join-bucket.go index 8cf9ef6e8..92b169541 100644 --- a/internal/pkg/transformers/utils/join-bucket.go +++ b/internal/pkg/transformers/utils/join-bucket.go @@ -8,7 +8,6 @@ import ( "container/list" "github.com/johnkerl/miller/internal/pkg/mlrval" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- diff --git a/internal/pkg/transformers/utils/percentile-keeper.go b/internal/pkg/transformers/utils/percentile-keeper.go index a13127fb4..8200826c7 100644 --- a/internal/pkg/transformers/utils/percentile-keeper.go +++ b/internal/pkg/transformers/utils/percentile-keeper.go @@ -9,8 +9,8 @@ import ( "math" "sort" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/mlrval" - "github.com/johnkerl/miller/internal/pkg/types" ) type PercentileKeeper struct { @@ -247,10 +247,10 @@ func getPercentileLinearlyInterpolated(array []*mlrval.Mlrval, n int, p float64) } else { // array[iindex] + frac * (array[iindex+1] - array[iindex]) // TODO: just do this in float64. - frac := mlrval.MlrvalFromFloat64(findex - float64(iindex)) - diff := types.BIF_minus_binary(array[iindex+1], array[iindex]) - prod := types.BIF_times(frac, diff) - return *types.BIF_plus_binary(array[iindex], prod) + frac := mlrval.FromFloat(findex - float64(iindex)) + diff := bifs.BIF_minus_binary(array[iindex+1], array[iindex]) + prod := bifs.BIF_times(frac, diff) + return *bifs.BIF_plus_binary(array[iindex], prod) } } @@ -258,7 +258,7 @@ func getPercentileLinearlyInterpolated(array []*mlrval.Mlrval, n int, p float64) func (keeper *PercentileKeeper) sortIfNecessary() { if !keeper.sorted { sort.Slice(keeper.data, func(i, j int) bool { - return mlrval.BIF_less_than_as_bool(keeper.data[i], keeper.data[j]) + return mlrval.LessThan(keeper.data[i], keeper.data[j]) }) keeper.sorted = true } @@ -275,7 +275,7 @@ func (keeper *PercentileKeeper) Emit(percentile float64) *mlrval.Mlrval { func (keeper *PercentileKeeper) EmitNonInterpolated(percentile float64) *mlrval.Mlrval { if len(keeper.data) == 0 { - return types.MLRVAL_VOID + return mlrval.VOID } keeper.sortIfNecessary() return keeper.data[computeIndexNoninterpolated(int(len(keeper.data)), percentile)].Copy() @@ -283,7 +283,7 @@ func (keeper *PercentileKeeper) EmitNonInterpolated(percentile float64) *mlrval. func (keeper *PercentileKeeper) EmitLinearlyInterpolated(percentile float64) *mlrval.Mlrval { if len(keeper.data) == 0 { - return types.MLRVAL_VOID + return mlrval.VOID } keeper.sortIfNecessary() output := getPercentileLinearlyInterpolated(keeper.data, int(len(keeper.data)), percentile) diff --git a/internal/pkg/transformers/utils/stats1-accumulators.go b/internal/pkg/transformers/utils/stats1-accumulators.go index 0bd4f9d07..51db05a1c 100644 --- a/internal/pkg/transformers/utils/stats1-accumulators.go +++ b/internal/pkg/transformers/utils/stats1-accumulators.go @@ -9,9 +9,9 @@ import ( "os" "strings" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -293,7 +293,7 @@ func (acc *Stats1CountAccumulator) Ingest(value *mlrval.Mlrval) { acc.count++ } func (acc *Stats1CountAccumulator) Emit() *mlrval.Mlrval { - return mlrval.MlrvalFromInt(acc.count) + return mlrval.FromInt(acc.count) } func (acc *Stats1CountAccumulator) Reset() { acc.count = 0 @@ -322,7 +322,7 @@ func (acc *Stats1ModeAccumulator) Ingest(value *mlrval.Mlrval) { } func (acc *Stats1ModeAccumulator) Emit() *mlrval.Mlrval { if acc.countsByValue.IsEmpty() { - return types.MLRVAL_VOID + return mlrval.VOID } maxValue := "" var maxCount = int(0) @@ -334,7 +334,7 @@ func (acc *Stats1ModeAccumulator) Emit() *mlrval.Mlrval { maxCount = count } } - return mlrval.MlrvalFromString(maxValue) + return mlrval.FromString(maxValue) } func (acc *Stats1ModeAccumulator) Reset() { acc.countsByValue = lib.NewOrderedMap() @@ -363,7 +363,7 @@ func (acc *Stats1AntimodeAccumulator) Ingest(value *mlrval.Mlrval) { } func (acc *Stats1AntimodeAccumulator) Emit() *mlrval.Mlrval { if acc.countsByValue.IsEmpty() { - return types.MLRVAL_VOID + return mlrval.VOID } minValue := "" var minCount = int(0) @@ -375,7 +375,7 @@ func (acc *Stats1AntimodeAccumulator) Emit() *mlrval.Mlrval { minCount = count } } - return mlrval.MlrvalFromString(minValue) + return mlrval.FromString(minValue) } func (acc *Stats1AntimodeAccumulator) Reset() { acc.countsByValue = lib.NewOrderedMap() @@ -388,17 +388,17 @@ type Stats1SumAccumulator struct { func NewStats1SumAccumulator() IStats1Accumulator { return &Stats1SumAccumulator{ - sum: mlrval.MlrvalFromInt(0), + sum: mlrval.FromInt(0), } } func (acc *Stats1SumAccumulator) Ingest(value *mlrval.Mlrval) { - acc.sum = types.BIF_plus_binary(acc.sum, value) + acc.sum = bifs.BIF_plus_binary(acc.sum, value) } func (acc *Stats1SumAccumulator) Emit() *mlrval.Mlrval { return acc.sum.Copy() } func (acc *Stats1SumAccumulator) Reset() { - acc.sum = mlrval.MlrvalFromInt(0) + acc.sum = mlrval.FromInt(0) } // ---------------------------------------------------------------- @@ -409,23 +409,23 @@ type Stats1MeanAccumulator struct { func NewStats1MeanAccumulator() IStats1Accumulator { return &Stats1MeanAccumulator{ - sum: mlrval.MlrvalFromInt(0), + sum: mlrval.FromInt(0), count: 0, } } func (acc *Stats1MeanAccumulator) Ingest(value *mlrval.Mlrval) { - acc.sum = types.BIF_plus_binary(acc.sum, value) + acc.sum = bifs.BIF_plus_binary(acc.sum, value) acc.count++ } func (acc *Stats1MeanAccumulator) Emit() *mlrval.Mlrval { if acc.count == 0 { - return types.MLRVAL_VOID + return mlrval.VOID } else { - return types.BIF_divide(acc.sum, mlrval.MlrvalFromInt(acc.count)) + return bifs.BIF_divide(acc.sum, mlrval.FromInt(acc.count)) } } func (acc *Stats1MeanAccumulator) Reset() { - acc.sum = mlrval.MlrvalFromInt(0) + acc.sum = mlrval.FromInt(0) acc.count = 0 } @@ -436,21 +436,21 @@ type Stats1MinAccumulator struct { func NewStats1MinAccumulator() IStats1Accumulator { return &Stats1MinAccumulator{ - min: types.MLRVAL_ABSENT, + min: mlrval.ABSENT, } } func (acc *Stats1MinAccumulator) Ingest(value *mlrval.Mlrval) { - acc.min = mlrval.BIF_min_binary(acc.min, value) + acc.min = bifs.BIF_min_binary(acc.min, value) } func (acc *Stats1MinAccumulator) Emit() *mlrval.Mlrval { if acc.min.IsAbsent() { - return types.MLRVAL_VOID + return mlrval.VOID } else { return acc.min.Copy() } } func (acc *Stats1MinAccumulator) Reset() { - acc.min = types.MLRVAL_ABSENT + acc.min = mlrval.ABSENT } // ---------------------------------------------------------------- @@ -460,21 +460,21 @@ type Stats1MaxAccumulator struct { func NewStats1MaxAccumulator() IStats1Accumulator { return &Stats1MaxAccumulator{ - max: types.MLRVAL_ABSENT, + max: mlrval.ABSENT, } } func (acc *Stats1MaxAccumulator) Ingest(value *mlrval.Mlrval) { - acc.max = mlrval.BIF_max_binary(acc.max, value) + acc.max = bifs.BIF_max_binary(acc.max, value) } func (acc *Stats1MaxAccumulator) Emit() *mlrval.Mlrval { if acc.max.IsAbsent() { - return types.MLRVAL_VOID + return mlrval.VOID } else { return acc.max.Copy() } } func (acc *Stats1MaxAccumulator) Reset() { - acc.max = types.MLRVAL_ABSENT + acc.max = mlrval.ABSENT } // ---------------------------------------------------------------- @@ -487,23 +487,23 @@ type Stats1VarAccumulator struct { func NewStats1VarAccumulator() IStats1Accumulator { return &Stats1VarAccumulator{ count: 0, - sum: mlrval.MlrvalFromInt(0), - sum2: mlrval.MlrvalFromInt(0), + sum: mlrval.FromInt(0), + sum2: mlrval.FromInt(0), } } func (acc *Stats1VarAccumulator) Ingest(value *mlrval.Mlrval) { - value2 := types.BIF_times(value, value) + value2 := bifs.BIF_times(value, value) acc.count++ - acc.sum = types.BIF_plus_binary(acc.sum, value) - acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) + acc.sum = bifs.BIF_plus_binary(acc.sum, value) + acc.sum2 = bifs.BIF_plus_binary(acc.sum2, value2) } func (acc *Stats1VarAccumulator) Emit() *mlrval.Mlrval { - return mlrval.BIF_get_var(mlrval.MlrvalFromInt(acc.count), acc.sum, acc.sum2) + return bifs.BIF_get_var(mlrval.FromInt(acc.count), acc.sum, acc.sum2) } func (acc *Stats1VarAccumulator) Reset() { acc.count = 0 - acc.sum = mlrval.MlrvalFromInt(0) - acc.sum2 = mlrval.MlrvalFromInt(0) + acc.sum = mlrval.FromInt(0) + acc.sum2 = mlrval.FromInt(0) } // ---------------------------------------------------------------- @@ -516,23 +516,23 @@ type Stats1StddevAccumulator struct { func NewStats1StddevAccumulator() IStats1Accumulator { return &Stats1StddevAccumulator{ count: 0, - sum: mlrval.MlrvalFromInt(0), - sum2: mlrval.MlrvalFromInt(0), + sum: mlrval.FromInt(0), + sum2: mlrval.FromInt(0), } } func (acc *Stats1StddevAccumulator) Ingest(value *mlrval.Mlrval) { - value2 := types.BIF_times(value, value) + value2 := bifs.BIF_times(value, value) acc.count++ - acc.sum = types.BIF_plus_binary(acc.sum, value) - acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) + acc.sum = bifs.BIF_plus_binary(acc.sum, value) + acc.sum2 = bifs.BIF_plus_binary(acc.sum2, value2) } func (acc *Stats1StddevAccumulator) Emit() *mlrval.Mlrval { - return mlrval.BIF_get_stddev(mlrval.MlrvalFromInt(acc.count), acc.sum, acc.sum2) + return bifs.BIF_get_stddev(mlrval.FromInt(acc.count), acc.sum, acc.sum2) } func (acc *Stats1StddevAccumulator) Reset() { acc.count = 0 - acc.sum = mlrval.MlrvalFromInt(0) - acc.sum2 = mlrval.MlrvalFromInt(0) + acc.sum = mlrval.FromInt(0) + acc.sum2 = mlrval.FromInt(0) } // ---------------------------------------------------------------- @@ -545,24 +545,24 @@ type Stats1MeanEBAccumulator struct { func NewStats1MeanEBAccumulator() IStats1Accumulator { return &Stats1MeanEBAccumulator{ count: 0, - sum: mlrval.MlrvalFromInt(0), - sum2: mlrval.MlrvalFromInt(0), + sum: mlrval.FromInt(0), + sum2: mlrval.FromInt(0), } } func (acc *Stats1MeanEBAccumulator) Ingest(value *mlrval.Mlrval) { - value2 := types.BIF_times(value, value) + value2 := bifs.BIF_times(value, value) acc.count++ - acc.sum = types.BIF_plus_binary(acc.sum, value) - acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) + acc.sum = bifs.BIF_plus_binary(acc.sum, value) + acc.sum2 = bifs.BIF_plus_binary(acc.sum2, value2) } func (acc *Stats1MeanEBAccumulator) Emit() *mlrval.Mlrval { - mcount := mlrval.MlrvalFromInt(acc.count) - return mlrval.BIF_get_mean_EB(mcount, acc.sum, acc.sum2) + mcount := mlrval.FromInt(acc.count) + return bifs.BIF_get_mean_EB(mcount, acc.sum, acc.sum2) } func (acc *Stats1MeanEBAccumulator) Reset() { acc.count = 0 - acc.sum = mlrval.MlrvalFromInt(0) - acc.sum2 = mlrval.MlrvalFromInt(0) + acc.sum = mlrval.FromInt(0) + acc.sum2 = mlrval.FromInt(0) } // ---------------------------------------------------------------- @@ -576,28 +576,28 @@ type Stats1SkewnessAccumulator struct { func NewStats1SkewnessAccumulator() IStats1Accumulator { return &Stats1SkewnessAccumulator{ count: 0, - sum: mlrval.MlrvalFromInt(0), - sum2: mlrval.MlrvalFromInt(0), - sum3: mlrval.MlrvalFromInt(0), + sum: mlrval.FromInt(0), + sum2: mlrval.FromInt(0), + sum3: mlrval.FromInt(0), } } func (acc *Stats1SkewnessAccumulator) Ingest(value *mlrval.Mlrval) { - value2 := types.BIF_times(value, value) - value3 := types.BIF_times(value, value2) + value2 := bifs.BIF_times(value, value) + value3 := bifs.BIF_times(value, value2) acc.count++ - acc.sum = types.BIF_plus_binary(acc.sum, value) - acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) - acc.sum3 = types.BIF_plus_binary(acc.sum3, value3) + acc.sum = bifs.BIF_plus_binary(acc.sum, value) + acc.sum2 = bifs.BIF_plus_binary(acc.sum2, value2) + acc.sum3 = bifs.BIF_plus_binary(acc.sum3, value3) } func (acc *Stats1SkewnessAccumulator) Emit() *mlrval.Mlrval { - mcount := mlrval.MlrvalFromInt(acc.count) - return mlrval.BIF_get_skewness(mcount, acc.sum, acc.sum2, acc.sum3) + mcount := mlrval.FromInt(acc.count) + return bifs.BIF_get_skewness(mcount, acc.sum, acc.sum2, acc.sum3) } func (acc *Stats1SkewnessAccumulator) Reset() { acc.count = 0 - acc.sum = mlrval.MlrvalFromInt(0) - acc.sum2 = mlrval.MlrvalFromInt(0) - acc.sum3 = mlrval.MlrvalFromInt(0) + acc.sum = mlrval.FromInt(0) + acc.sum2 = mlrval.FromInt(0) + acc.sum3 = mlrval.FromInt(0) } // ---------------------------------------------------------------- @@ -612,32 +612,32 @@ type Stats1KurtosisAccumulator struct { func NewStats1KurtosisAccumulator() IStats1Accumulator { return &Stats1KurtosisAccumulator{ count: 0, - sum: mlrval.MlrvalFromInt(0), - sum2: mlrval.MlrvalFromInt(0), - sum3: mlrval.MlrvalFromInt(0), - sum4: mlrval.MlrvalFromInt(0), + sum: mlrval.FromInt(0), + sum2: mlrval.FromInt(0), + sum3: mlrval.FromInt(0), + sum4: mlrval.FromInt(0), } } func (acc *Stats1KurtosisAccumulator) Ingest(value *mlrval.Mlrval) { - value2 := types.BIF_times(value, value) - value3 := types.BIF_times(value, value2) - value4 := types.BIF_times(value, value3) + value2 := bifs.BIF_times(value, value) + value3 := bifs.BIF_times(value, value2) + value4 := bifs.BIF_times(value, value3) acc.count++ - acc.sum = types.BIF_plus_binary(acc.sum, value) - acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) - acc.sum3 = types.BIF_plus_binary(acc.sum3, value3) - acc.sum4 = types.BIF_plus_binary(acc.sum4, value4) + acc.sum = bifs.BIF_plus_binary(acc.sum, value) + acc.sum2 = bifs.BIF_plus_binary(acc.sum2, value2) + acc.sum3 = bifs.BIF_plus_binary(acc.sum3, value3) + acc.sum4 = bifs.BIF_plus_binary(acc.sum4, value4) } func (acc *Stats1KurtosisAccumulator) Emit() *mlrval.Mlrval { - mcount := mlrval.MlrvalFromInt(acc.count) - return mlrval.BIF_get_kurtosis(mcount, acc.sum, acc.sum2, acc.sum3, acc.sum4) + mcount := mlrval.FromInt(acc.count) + return bifs.BIF_get_kurtosis(mcount, acc.sum, acc.sum2, acc.sum3, acc.sum4) } func (acc *Stats1KurtosisAccumulator) Reset() { acc.count = 0 - acc.sum = mlrval.MlrvalFromInt(0) - acc.sum2 = mlrval.MlrvalFromInt(0) - acc.sum3 = mlrval.MlrvalFromInt(0) - acc.sum4 = mlrval.MlrvalFromInt(0) + acc.sum = mlrval.FromInt(0) + acc.sum2 = mlrval.FromInt(0) + acc.sum3 = mlrval.FromInt(0) + acc.sum4 = mlrval.FromInt(0) } // ---------------------------------------------------------------- diff --git a/internal/pkg/transformers/utils/stats2-accumulators.go b/internal/pkg/transformers/utils/stats2-accumulators.go index 458a411a2..9c360a861 100644 --- a/internal/pkg/transformers/utils/stats2-accumulators.go +++ b/internal/pkg/transformers/utils/stats2-accumulators.go @@ -11,7 +11,6 @@ import ( "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -181,16 +180,16 @@ func (acc *Stats2LinRegOLSAccumulator) Populate( outrec *mlrval.Mlrmap, ) { if acc.count < 2 { - outrec.PutCopy(acc.mOutputFieldName, types.MLRVAL_VOID) - outrec.PutCopy(acc.bOutputFieldName, types.MLRVAL_VOID) + outrec.PutCopy(acc.mOutputFieldName, mlrval.VOID) + outrec.PutCopy(acc.bOutputFieldName, mlrval.VOID) } else { m, b := lib.GetLinearRegressionOLS(acc.count, acc.sumx, acc.sumx2, acc.sumxy, acc.sumy) - outrec.PutReference(acc.mOutputFieldName, mlrval.MlrvalFromFloat64(m)) - outrec.PutReference(acc.bOutputFieldName, mlrval.MlrvalFromFloat64(b)) + outrec.PutReference(acc.mOutputFieldName, mlrval.FromFloat(m)) + outrec.PutReference(acc.bOutputFieldName, mlrval.FromFloat(b)) } - outrec.PutReference(acc.nOutputFieldName, mlrval.MlrvalFromInt(acc.count)) + outrec.PutReference(acc.nOutputFieldName, mlrval.FromInt(acc.count)) } func (acc *Stats2LinRegOLSAccumulator) Fit( @@ -210,10 +209,10 @@ func (acc *Stats2LinRegOLSAccumulator) Fit( } if acc.count < 2 { - outrec.PutCopy(acc.fitOutputFieldName, types.MLRVAL_VOID) + outrec.PutCopy(acc.fitOutputFieldName, mlrval.VOID) } else { yfit := acc.m*x + acc.b - outrec.PutReference(acc.fitOutputFieldName, mlrval.MlrvalFromFloat64(yfit)) + outrec.PutReference(acc.fitOutputFieldName, mlrval.FromFloat(yfit)) } } @@ -267,14 +266,14 @@ func (acc *Stats2LogiRegAccumulator) Populate( ) { if len(acc.xs) < 2 { - outrec.PutCopy(acc.mOutputFieldName, types.MLRVAL_VOID) - outrec.PutCopy(acc.bOutputFieldName, types.MLRVAL_VOID) + outrec.PutCopy(acc.mOutputFieldName, mlrval.VOID) + outrec.PutCopy(acc.bOutputFieldName, mlrval.VOID) } else { m, b := lib.LogisticRegression(acc.xs, acc.ys) - outrec.PutCopy(acc.mOutputFieldName, mlrval.MlrvalFromFloat64(m)) - outrec.PutCopy(acc.bOutputFieldName, mlrval.MlrvalFromFloat64(b)) + outrec.PutCopy(acc.mOutputFieldName, mlrval.FromFloat(m)) + outrec.PutCopy(acc.bOutputFieldName, mlrval.FromFloat(b)) } - outrec.PutReference(acc.nOutputFieldName, mlrval.MlrvalFromInt(len(acc.xs))) + outrec.PutReference(acc.nOutputFieldName, mlrval.FromInt(len(acc.xs))) } func (acc *Stats2LogiRegAccumulator) Fit( @@ -294,10 +293,10 @@ func (acc *Stats2LogiRegAccumulator) Fit( } if len(acc.xs) < 2 { - outrec.PutCopy(acc.fitOutputFieldName, types.MLRVAL_VOID) + outrec.PutCopy(acc.fitOutputFieldName, mlrval.VOID) } else { yfit := 1.0 / (1.0 + math.Exp(-acc.m*x-acc.b)) - outrec.PutReference(acc.fitOutputFieldName, mlrval.MlrvalFromFloat64(yfit)) + outrec.PutReference(acc.fitOutputFieldName, mlrval.FromFloat(yfit)) } } @@ -352,7 +351,7 @@ func (acc *Stats2R2Accumulator) Populate( ) { if acc.count < 2 { - outrec.PutCopy(acc.r2OutputFieldName, types.MLRVAL_VOID) + outrec.PutCopy(acc.r2OutputFieldName, mlrval.VOID) } else { n := float64(acc.count) sumx := acc.sumx @@ -364,7 +363,7 @@ func (acc *Stats2R2Accumulator) Populate( numerator = numerator * numerator denominator := (n*sumx2 - sumx*sumx) * (n*sumy2 - sumy*sumy) output := numerator / denominator - outrec.PutReference(acc.r2OutputFieldName, mlrval.MlrvalFromFloat64(output)) + outrec.PutReference(acc.r2OutputFieldName, mlrval.FromFloat(output)) } } @@ -498,10 +497,10 @@ func (acc *Stats2CorrCovAccumulator) Populate( key10 := acc.covx10OutputFieldName key11 := acc.covx11OutputFieldName if acc.count < 2 { - outrec.PutCopy(key00, types.MLRVAL_VOID) - outrec.PutCopy(key01, types.MLRVAL_VOID) - outrec.PutCopy(key10, types.MLRVAL_VOID) - outrec.PutCopy(key11, types.MLRVAL_VOID) + outrec.PutCopy(key00, mlrval.VOID) + outrec.PutCopy(key01, mlrval.VOID) + outrec.PutCopy(key10, mlrval.VOID) + outrec.PutCopy(key11, mlrval.VOID) } else { Q := lib.GetCovMatrix( acc.count, @@ -511,10 +510,10 @@ func (acc *Stats2CorrCovAccumulator) Populate( acc.sumy2, acc.sumxy, ) - outrec.PutReference(key00, mlrval.MlrvalFromFloat64(Q[0][0])) - outrec.PutReference(key01, mlrval.MlrvalFromFloat64(Q[0][1])) - outrec.PutReference(key10, mlrval.MlrvalFromFloat64(Q[1][0])) - outrec.PutReference(key11, mlrval.MlrvalFromFloat64(Q[1][1])) + outrec.PutReference(key00, mlrval.FromFloat(Q[0][0])) + outrec.PutReference(key01, mlrval.FromFloat(Q[0][1])) + outrec.PutReference(key10, mlrval.FromFloat(Q[1][0])) + outrec.PutReference(key11, mlrval.FromFloat(Q[1][1])) } } else if acc.doWhich == DO_LINREG_PCA { @@ -531,19 +530,19 @@ func (acc *Stats2CorrCovAccumulator) Populate( keyv22 := acc.pca_v22OutputFieldName if acc.count < 2 { - outrec.PutCopy(keym, types.MLRVAL_VOID) - outrec.PutCopy(keyb, types.MLRVAL_VOID) - outrec.PutCopy(keyn, types.MLRVAL_VOID) - outrec.PutCopy(keyq, types.MLRVAL_VOID) + outrec.PutCopy(keym, mlrval.VOID) + outrec.PutCopy(keyb, mlrval.VOID) + outrec.PutCopy(keyn, mlrval.VOID) + outrec.PutCopy(keyq, mlrval.VOID) if acc.doVerbose { - outrec.PutCopy(keyl1, types.MLRVAL_VOID) - outrec.PutCopy(keyl2, types.MLRVAL_VOID) - outrec.PutCopy(keyv11, types.MLRVAL_VOID) - outrec.PutCopy(keyv12, types.MLRVAL_VOID) - outrec.PutCopy(keyv21, types.MLRVAL_VOID) - outrec.PutCopy(keyv22, types.MLRVAL_VOID) + outrec.PutCopy(keyl1, mlrval.VOID) + outrec.PutCopy(keyl2, mlrval.VOID) + outrec.PutCopy(keyv11, mlrval.VOID) + outrec.PutCopy(keyv12, mlrval.VOID) + outrec.PutCopy(keyv21, mlrval.VOID) + outrec.PutCopy(keyv22, mlrval.VOID) } } else { Q := lib.GetCovMatrix( @@ -561,18 +560,18 @@ func (acc *Stats2CorrCovAccumulator) Populate( y_mean := acc.sumy / float64(acc.count) m, b, q := lib.GetLinearRegressionPCA(l1, l2, v1, v2, x_mean, y_mean) - outrec.PutReference(keym, mlrval.MlrvalFromFloat64(m)) - outrec.PutReference(keyb, mlrval.MlrvalFromFloat64(b)) - outrec.PutReference(keyn, mlrval.MlrvalFromInt(acc.count)) - outrec.PutReference(keyq, mlrval.MlrvalFromFloat64(q)) + outrec.PutReference(keym, mlrval.FromFloat(m)) + outrec.PutReference(keyb, mlrval.FromFloat(b)) + outrec.PutReference(keyn, mlrval.FromInt(acc.count)) + outrec.PutReference(keyq, mlrval.FromFloat(q)) if acc.doVerbose { - outrec.PutReference(keyl1, mlrval.MlrvalFromFloat64(l1)) - outrec.PutReference(keyl2, mlrval.MlrvalFromFloat64(l2)) - outrec.PutReference(keyv11, mlrval.MlrvalFromFloat64(v1[0])) - outrec.PutReference(keyv12, mlrval.MlrvalFromFloat64(v1[1])) - outrec.PutReference(keyv21, mlrval.MlrvalFromFloat64(v2[0])) - outrec.PutReference(keyv22, mlrval.MlrvalFromFloat64(v2[1])) + outrec.PutReference(keyl1, mlrval.FromFloat(l1)) + outrec.PutReference(keyl2, mlrval.FromFloat(l2)) + outrec.PutReference(keyv11, mlrval.FromFloat(v1[0])) + outrec.PutReference(keyv12, mlrval.FromFloat(v1[1])) + outrec.PutReference(keyv21, mlrval.FromFloat(v2[0])) + outrec.PutReference(keyv22, mlrval.FromFloat(v2[1])) } } } else { @@ -581,7 +580,7 @@ func (acc *Stats2CorrCovAccumulator) Populate( key = acc.covOutputFieldName } if acc.count < 2 { - outrec.PutCopy(key, types.MLRVAL_VOID) + outrec.PutCopy(key, mlrval.VOID) } else { output := lib.GetCov(acc.count, acc.sumx, acc.sumy, acc.sumxy) if acc.doWhich == DO_CORR { @@ -589,7 +588,7 @@ func (acc *Stats2CorrCovAccumulator) Populate( sigmay := math.Sqrt(lib.GetVar(acc.count, acc.sumy, acc.sumy2)) output = output / sigmax / sigmay } - outrec.PutReference(key, mlrval.MlrvalFromFloat64(output)) + outrec.PutReference(key, mlrval.FromFloat(output)) } } } @@ -620,10 +619,10 @@ func (acc *Stats2CorrCovAccumulator) Fit( acc.fitReady = true } if acc.count < 2 { - outrec.PutCopy(acc.pca_fitOutputFieldName, types.MLRVAL_VOID) + outrec.PutCopy(acc.pca_fitOutputFieldName, mlrval.VOID) } else { yfit := acc.m*x + acc.b - outrec.PutCopy(acc.pca_fitOutputFieldName, mlrval.MlrvalFromFloat64(yfit)) + outrec.PutCopy(acc.pca_fitOutputFieldName, mlrval.FromFloat(yfit)) } } diff --git a/internal/pkg/transformers/utils/top-keeper.go b/internal/pkg/transformers/utils/top-keeper.go index 073e99c47..6951b9c00 100644 --- a/internal/pkg/transformers/utils/top-keeper.go +++ b/internal/pkg/transformers/utils/top-keeper.go @@ -15,7 +15,7 @@ type TopKeeper struct { TopRecordsAndContexts []*types.RecordAndContext size int capacity int - bsearchFunc types.BsearchMlrvalArrayFunc + bsearchFunc mlrval.BsearchMlrvalArrayFunc } // ---------------------------------------------------------------- From cbe8ee84e49ea05a5a046fb2787a5e2e5a6e12c2 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Fri, 17 Dec 2021 16:48:01 -0500 Subject: [PATCH 12/19] first clean compile in a long while on this branch --- internal/pkg/auxents/help/entry.go | 2 +- internal/pkg/auxents/repl/dsl.go | 1 - internal/pkg/auxents/repl/verbs.go | 1 + internal/pkg/bifs/collections.go | 50 +++++------ internal/pkg/bifs/types.go | 2 +- internal/pkg/climain/mlrcli_parse.go | 4 +- internal/pkg/dsl/cst/builtin_functions.go | 16 ++-- internal/pkg/dsl/cst/collections.go | 24 +++--- internal/pkg/dsl/cst/for.go | 30 +++---- internal/pkg/dsl/cst/hofs.go | 17 ++-- internal/pkg/dsl/cst/leaves.go | 8 +- internal/pkg/dsl/cst/lvalues.go | 3 +- internal/pkg/lib/util.go | 2 +- .../mlrmap_flatten_unflatten.go | 28 +++++-- .../mlrval_accessors.go | 24 +----- internal/pkg/mlrval/mlrval_infer.go | 2 +- internal/pkg/mlrval/mlrval_new.go | 2 +- internal/pkg/mlrval/tbf.go | 84 ++++++++++++++----- internal/pkg/transformers/altkv.go | 1 + internal/pkg/transformers/bar.go | 4 +- internal/pkg/transformers/clean-whitespace.go | 5 +- internal/pkg/transformers/cut.go | 1 + internal/pkg/transformers/fill-down.go | 4 +- internal/pkg/transformers/fill-empty.go | 4 +- internal/pkg/transformers/format-values.go | 13 +-- internal/pkg/transformers/fraction.go | 3 +- internal/pkg/transformers/gap.go | 1 + internal/pkg/transformers/join.go | 1 + internal/pkg/transformers/json-stringify.go | 11 +-- internal/pkg/transformers/merge-fields.go | 6 +- internal/pkg/transformers/nest.go | 16 ++-- internal/pkg/transformers/regularize.go | 1 + .../pkg/transformers/remove-empty-columns.go | 3 +- internal/pkg/transformers/reorder.go | 1 + internal/pkg/transformers/reshape.go | 4 +- internal/pkg/transformers/sec2gmt.go | 2 +- internal/pkg/transformers/sec2gmtdate.go | 1 + internal/pkg/transformers/seqgen.go | 5 +- internal/pkg/transformers/sort.go | 25 +++--- internal/pkg/transformers/step.go | 11 +-- internal/pkg/transformers/template.go | 2 +- internal/pkg/transformers/uniq.go | 2 +- internal/pkg/transformers/unsparsify.go | 2 +- .../transformers/utils/join-bucket-keeper.go | 2 +- .../transformers/utils/stats1-accumulators.go | 2 +- internal/pkg/transformers/utils/top-keeper.go | 4 +- internal/pkg/types/context.go | 29 ++++--- 47 files changed, 259 insertions(+), 207 deletions(-) rename internal/pkg/{types-parked => mlrval}/mlrmap_flatten_unflatten.go (92%) rename internal/pkg/{types-parked => mlrval}/mlrval_accessors.go (85%) diff --git a/internal/pkg/auxents/help/entry.go b/internal/pkg/auxents/help/entry.go index 932f7dc12..c223279e8 100644 --- a/internal/pkg/auxents/help/entry.go +++ b/internal/pkg/auxents/help/entry.go @@ -10,12 +10,12 @@ import ( "github.com/mattn/go-isatty" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/dsl/cst" "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/transformers" - "github.com/johnkerl/miller/internal/pkg/types" ) // ================================================================ diff --git a/internal/pkg/auxents/repl/dsl.go b/internal/pkg/auxents/repl/dsl.go index 045341bfd..ce09f3636 100644 --- a/internal/pkg/auxents/repl/dsl.go +++ b/internal/pkg/auxents/repl/dsl.go @@ -26,7 +26,6 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/dsl/cst" "github.com/johnkerl/miller/internal/pkg/mlrval" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- diff --git a/internal/pkg/auxents/repl/verbs.go b/internal/pkg/auxents/repl/verbs.go index f889069ac..c03b0ec08 100644 --- a/internal/pkg/auxents/repl/verbs.go +++ b/internal/pkg/auxents/repl/verbs.go @@ -14,6 +14,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/dsl/cst" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) diff --git a/internal/pkg/bifs/collections.go b/internal/pkg/bifs/collections.go index 480697f8e..602b15c96 100644 --- a/internal/pkg/bifs/collections.go +++ b/internal/pkg/bifs/collections.go @@ -285,31 +285,31 @@ func BIF_mapsum(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { return mlrval.FromMap(newMap) } -//// ---------------------------------------------------------------- -//func BIF_mapdiff(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { -// if len(mlrvals) == 0 { -// return mlrval.FromEmptyMap() -// } -// if len(mlrvals) == 1 { -// return mlrvals[0] -// } -// if mlrvals[0].Type() != MT_MAP { -// return mlrval.ERROR -// } -// newMap := mlrvals[0].AcquireMapValue().Copy() -// -// for _, otherMapArg := range mlrvals[1:] { -// if otherMapArg.Type() != MT_MAP { -// return mlrval.ERROR -// } -// -// for pe := otherMapArg.AcquireMapValue().Head; pe != nil; pe = pe.Next { -// newMap.Remove(pe.Key) -// } -// } -// -// return mlrval.FromMap(newMap) -//} +// ---------------------------------------------------------------- +func BIF_mapdiff(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { + if len(mlrvals) == 0 { + return mlrval.FromEmptyMap() + } + if len(mlrvals) == 1 { + return mlrvals[0] + } + if !mlrvals[0].IsMap() { + return mlrval.ERROR + } + newMap := mlrvals[0].AcquireMapValue().Copy() + + for _, otherMapArg := range mlrvals[1:] { + if !otherMapArg.IsMap() { + return mlrval.ERROR + } + + for pe := otherMapArg.AcquireMapValue().Head; pe != nil; pe = pe.Next { + newMap.Remove(pe.Key) + } + } + + return mlrval.FromMap(newMap) +} // ================================================================ // joink([1,2,3], ",") -> "1,2,3" diff --git a/internal/pkg/bifs/types.go b/internal/pkg/bifs/types.go index e11021fa3..668248e02 100644 --- a/internal/pkg/bifs/types.go +++ b/internal/pkg/bifs/types.go @@ -56,7 +56,7 @@ func BIF_int(input1 *mlrval.Mlrval) *mlrval.Mlrval { // ---------------------------------------------------------------- func string_to_float(input1 *mlrval.Mlrval) *mlrval.Mlrval { - f, ok := lib.TryFloat64FromString(input1.AcquireStringValue()) + f, ok := lib.TryFloatFromString(input1.AcquireStringValue()) if ok { return mlrval.FromFloat(f) } else { diff --git a/internal/pkg/climain/mlrcli_parse.go b/internal/pkg/climain/mlrcli_parse.go index 8b61bbd5c..2a2cb3bb3 100644 --- a/internal/pkg/climain/mlrcli_parse.go +++ b/internal/pkg/climain/mlrcli_parse.go @@ -75,8 +75,8 @@ import ( "github.com/johnkerl/miller/internal/pkg/auxents/help" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/transformers" - "github.com/johnkerl/miller/internal/pkg/types" "github.com/johnkerl/miller/internal/pkg/version" ) @@ -280,7 +280,7 @@ func parseCommandLinePassTwo( // Set an optional global formatter for floating-point values if options.WriterOptions.FPOFMT != "" { - err = types.SetMlrvalFloatOutputFormat(options.WriterOptions.FPOFMT) + err = mlrval.SetFloatOutputFormat(options.WriterOptions.FPOFMT) if err != nil { return options, recordTransformers, err } diff --git a/internal/pkg/dsl/cst/builtin_functions.go b/internal/pkg/dsl/cst/builtin_functions.go index 2907bb9a1..bc110b406 100644 --- a/internal/pkg/dsl/cst/builtin_functions.go +++ b/internal/pkg/dsl/cst/builtin_functions.go @@ -778,7 +778,7 @@ func (node *LogicalANDOperatorNode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { aout := node.a.Evaluate(state) - atype := aout.GetType() + atype := aout.Type() if !(atype == mlrval.MT_ABSENT || atype == mlrval.MT_BOOL) { return mlrval.ERROR } @@ -794,7 +794,7 @@ func (node *LogicalANDOperatorNode) Evaluate( } bout := node.b.Evaluate(state) - btype := bout.GetType() + btype := bout.Type() if !(btype == mlrval.MT_ABSENT || btype == mlrval.MT_BOOL) { return mlrval.ERROR } @@ -802,7 +802,7 @@ func (node *LogicalANDOperatorNode) Evaluate( return mlrval.ABSENT } - return mlrval.BIF_logical_AND(aout, bout) + return bifs.BIF_logical_AND(aout, bout) } // ================================================================ @@ -826,7 +826,7 @@ func (node *LogicalOROperatorNode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { aout := node.a.Evaluate(state) - atype := aout.GetType() + atype := aout.Type() if !(atype == mlrval.MT_ABSENT || atype == mlrval.MT_BOOL) { return mlrval.ERROR } @@ -842,14 +842,14 @@ func (node *LogicalOROperatorNode) Evaluate( } bout := node.b.Evaluate(state) - btype := bout.GetType() + btype := bout.Type() if !(btype == mlrval.MT_ABSENT || btype == mlrval.MT_BOOL) { return mlrval.ERROR } if btype == mlrval.MT_ABSENT { return mlrval.ABSENT } - return mlrval.BIF_logical_OR(aout, bout) + return bifs.BIF_logical_OR(aout, bout) } // ================================================================ @@ -868,7 +868,7 @@ func (node *AbsentCoalesceOperatorNode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { aout := node.a.Evaluate(state) - if aout.GetType() != mlrval.MT_ABSENT { + if aout.Type() != mlrval.MT_ABSENT { return aout } @@ -891,7 +891,7 @@ func (node *EmptyCoalesceOperatorNode) Evaluate( state *runtime.State, ) *mlrval.Mlrval { aout := node.a.Evaluate(state) - atype := aout.GetType() + atype := aout.Type() if atype == mlrval.MT_ABSENT || atype == mlrval.MT_VOID || (atype == mlrval.MT_STRING && aout.String() == "") { return node.b.Evaluate(state) } else { diff --git a/internal/pkg/dsl/cst/collections.go b/internal/pkg/dsl/cst/collections.go index ef736d984..be2ca67e1 100644 --- a/internal/pkg/dsl/cst/collections.go +++ b/internal/pkg/dsl/cst/collections.go @@ -6,11 +6,11 @@ package cst import ( + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -48,7 +48,7 @@ func (node *ArrayLiteralNode) Evaluate( for i := range node.evaluables { mlrvals[i] = *node.evaluables[i].Evaluate(state) } - return mlrval.FromArrayReference(mlrvals) + return mlrval.FromArray(mlrvals) } // ---------------------------------------------------------------- @@ -102,7 +102,7 @@ func (node *CollectionIndexAccessNode) Evaluate( // Handle UTF-8 correctly: len(input1.printrep) will count bytes, not runes. runes := []rune(baseMlrval.String()) // Miller uses 1-up, and negatively aliased, indexing for strings and arrays. - zindex, inBounds := types.UnaliasArrayLengthIndex(len(runes), mindex) + zindex, inBounds := mlrval.UnaliasArrayLengthIndex(len(runes), mindex) if !inBounds { return mlrval.ERROR } @@ -182,7 +182,7 @@ func (node *ArraySliceAccessNode) Evaluate( lowerIndex, ok := lowerIndexMlrval.GetIntValue() if !ok { - if lowerIndexMlrval.IsEmpty() { + if lowerIndexMlrval.IsVoid() { lowerIndex = 1 } else { return mlrval.ERROR @@ -190,7 +190,7 @@ func (node *ArraySliceAccessNode) Evaluate( } upperIndex, ok := upperIndexMlrval.GetIntValue() if !ok { - if upperIndexMlrval.IsEmpty() { + if upperIndexMlrval.IsVoid() { upperIndex = n } else { return mlrval.ERROR @@ -200,11 +200,11 @@ func (node *ArraySliceAccessNode) Evaluate( // UnaliasArrayIndex returns a boolean second return value to indicate // whether the index is in range. But here, for the slicing operation, we // inspect the in-range-ness ourselves so we discard that 2nd return value. - lowerZindex, _ := types.UnaliasArrayIndex(&array, lowerIndex) - upperZindex, _ := types.UnaliasArrayIndex(&array, upperIndex) + lowerZindex, _ := mlrval.UnaliasArrayIndex(&array, lowerIndex) + upperZindex, _ := mlrval.UnaliasArrayIndex(&array, upperIndex) if lowerZindex > upperZindex { - return mlrval.FromArrayReference(make([]mlrval.Mlrval, 0)) + return mlrval.FromArray(make([]mlrval.Mlrval, 0)) } // Semantics: say x=[1,2,3,4,5]. Then x[3:10] is [3,4,5]. @@ -221,13 +221,13 @@ func (node *ArraySliceAccessNode) Evaluate( if lowerZindex < 0 { lowerZindex = 0 if lowerZindex > upperZindex { - return mlrval.FromArrayReference(make([]mlrval.Mlrval, 0)) + return mlrval.FromArray(make([]mlrval.Mlrval, 0)) } } if upperZindex > n-1 { upperZindex = n - 1 if lowerZindex > upperZindex { - return mlrval.FromArrayReference(make([]mlrval.Mlrval, 0)) + return mlrval.FromArray(make([]mlrval.Mlrval, 0)) } } @@ -243,7 +243,7 @@ func (node *ArraySliceAccessNode) Evaluate( di++ } - return mlrval.FromArrayReference(retval) + return mlrval.FromArray(retval) } // ================================================================ @@ -388,7 +388,7 @@ func (node *ArrayOrMapPositionalNameAccessNode) Evaluate( if baseMlrval.IsArray() { n, _ := baseMlrval.GetArrayLength() - zindex, ok := types.UnaliasArrayLengthIndex(int(n), index) + zindex, ok := mlrval.UnaliasArrayLengthIndex(int(n), index) if ok { return mlrval.FromInt(zindex + 1) // Miller user-space indices are 1-up } else { diff --git a/internal/pkg/dsl/cst/for.go b/internal/pkg/dsl/cst/for.go index 2653fc4fa..14008bf3f 100644 --- a/internal/pkg/dsl/cst/for.go +++ b/internal/pkg/dsl/cst/for.go @@ -524,17 +524,17 @@ func (node *ForLoopMultivariableNode) Execute(state *runtime.State) (*BlockExitP // ---------------------------------------------------------------- func (node *ForLoopMultivariableNode) executeOuter( - mlrval *mlrval.Mlrval, + mv *mlrval.Mlrval, keyIndexVariables []*runtime.StackVariable, state *runtime.State, ) (*BlockExitPayload, error) { if len(keyIndexVariables) == 1 { - return node.executeInner(mlrval, keyIndexVariables[0], state) + return node.executeInner(mv, keyIndexVariables[0], state) } // else, recurse - if mlrval.IsMap() { - mapval := mlrval.GetMap() + if mv.IsMap() { + mapval := mv.GetMap() for pe := mapval.Head; pe != nil; pe = pe.Next { mapkey := mlrval.FromString(pe.Key) @@ -563,8 +563,8 @@ func (node *ForLoopMultivariableNode) executeOuter( } } - } else if mlrval.IsArray() { - arrayval := mlrval.GetArray() + } else if mv.IsArray() { + arrayval := mv.GetArray() // Note: Miller user-space array indices ("mindex") are 1-up. Internal // Go storage ("zindex") is 0-up. @@ -597,7 +597,7 @@ func (node *ForLoopMultivariableNode) executeOuter( } } - } else if mlrval.IsAbsent() { + } else if mv.IsAbsent() { // Data-heterogeneity no-op } @@ -608,7 +608,7 @@ func (node *ForLoopMultivariableNode) executeOuter( // return nil, errors.New( // fmt.Sprintf( // "mlr: looped-over item is not a map or array; got %s", - // mlrval.GetTypeName(), + // mv.GetTypeName(), // ), // ) @@ -617,12 +617,12 @@ func (node *ForLoopMultivariableNode) executeOuter( // ---------------------------------------------------------------- func (node *ForLoopMultivariableNode) executeInner( - mlrval *mlrval.Mlrval, + mv *mlrval.Mlrval, keyIndexVariable *runtime.StackVariable, state *runtime.State, ) (*BlockExitPayload, error) { - if mlrval.IsMap() { - mapval := mlrval.GetMap() + if mv.IsMap() { + mapval := mv.GetMap() for pe := mapval.Head; pe != nil; pe = pe.Next { mapkey := mlrval.FromString(pe.Key) @@ -656,8 +656,8 @@ func (node *ForLoopMultivariableNode) executeInner( } } - } else if mlrval.IsArray() { - arrayval := mlrval.GetArray() + } else if mv.IsArray() { + arrayval := mv.GetArray() // Note: Miller user-space array indices ("mindex") are 1-up. Internal // Go storage ("zindex") is 0-up. @@ -694,7 +694,7 @@ func (node *ForLoopMultivariableNode) executeInner( } } - } else if mlrval.IsAbsent() { + } else if mv.IsAbsent() { // Data-heterogeneity no-op } @@ -705,7 +705,7 @@ func (node *ForLoopMultivariableNode) executeInner( // return nil, errors.New( // fmt.Sprintf( // "mlr: looped-over item is not a map or array; got %s", - // mlrval.GetTypeName(), + // mv.GetTypeName(), // ), // ) diff --git a/internal/pkg/dsl/cst/hofs.go b/internal/pkg/dsl/cst/hofs.go index b5960cb13..0f4a60f11 100644 --- a/internal/pkg/dsl/cst/hofs.go +++ b/internal/pkg/dsl/cst/hofs.go @@ -15,7 +15,6 @@ import ( "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // Most function types are in the github.com/johnkerl/miller/internal/pkg/types package. These types, though, @@ -243,7 +242,7 @@ func selectArray( outputArray = append(outputArray, *inputArray[i].Copy()) } } - return mlrval.FromArrayReference(outputArray) + return mlrval.FromArray(outputArray) } func selectMap( @@ -324,7 +323,7 @@ func applyArray( isNonAbsentOrDie(&retval, "apply") outputArray[i] = retval } - return mlrval.FromArrayReference(outputArray) + return mlrval.FromArray(outputArray) } func applyMap( @@ -616,11 +615,11 @@ func sortA( func sortANumerical(array []mlrval.Mlrval, reverse bool) { if !reverse { sort.Slice(array, func(i, j int) bool { - return mlrval.BIF_less_than_as_bool(&array[i], &array[j]) + return mlrval.LessThan(&array[i], &array[j]) }) } else { sort.Slice(array, func(i, j int) bool { - return mlrval.BIF_greater_than_as_bool(&array[i], &array[j]) + return mlrval.GreaterThan(&array[i], &array[j]) }) } } @@ -689,7 +688,7 @@ func sortMK( outmap.PutCopy(key, inmap.Get(key)) } - return mlrval.FromMapReferenced(outmap) + return mlrval.FromMap(outmap) } func sortMKNumerical(array []string, reverse bool) { @@ -759,7 +758,7 @@ func sortAF( udfCallsite := hofSpace.udfCallsite argsArray := hofSpace.argsArray - outputArray := types.CopyMlrvalArray(inputArray) + outputArray := mlrval.CopyMlrvalArray(inputArray) sort.Slice(outputArray, func(i, j int) bool { argsArray[0] = &outputArray[i] @@ -781,7 +780,7 @@ func sortAF( // Go sort-callback conventions: true if a < b, false otherwise. return nret < 0 }) - return mlrval.FromArrayReference(outputArray) + return mlrval.FromArray(outputArray) } // sortAF implements sort on arrays with callback UDF. @@ -829,7 +828,7 @@ func sortMF( }) sortedMap := mlrval.MlrmapFromPairsArray(pairsArray) - return mlrval.FromMapReferenced(sortedMap) + return mlrval.FromMap(sortedMap) } // ================================================================ diff --git a/internal/pkg/dsl/cst/leaves.go b/internal/pkg/dsl/cst/leaves.go index 8e802ea92..7726493f8 100644 --- a/internal/pkg/dsl/cst/leaves.go +++ b/internal/pkg/dsl/cst/leaves.go @@ -332,8 +332,10 @@ type IntLiteralNode struct { } func (root *RootNode) BuildIntLiteralNode(literal string) *IntLiteralNode { + ival, ok := lib.TryIntFromString(literal) + lib.InternalCodingErrorIf(!ok) return &IntLiteralNode{ - literal: mlrval.FromPrevalidatedIntString(literal), + literal: mlrval.FromPrevalidatedIntString(literal, ival), } } func (node *IntLiteralNode) Evaluate( @@ -348,8 +350,10 @@ type FloatLiteralNode struct { } func (root *RootNode) BuildFloatLiteralNode(literal string) *FloatLiteralNode { + fval, ok := lib.TryFloatFromString(literal) + lib.InternalCodingErrorIf(!ok) return &FloatLiteralNode{ - literal: mlrval.FromPrevalidatedFloat64String(literal), + literal: mlrval.FromPrevalidatedFloatString(literal, fval), } } func (node *FloatLiteralNode) Evaluate( diff --git a/internal/pkg/dsl/cst/lvalues.go b/internal/pkg/dsl/cst/lvalues.go index 55338b669..34dae1532 100644 --- a/internal/pkg/dsl/cst/lvalues.go +++ b/internal/pkg/dsl/cst/lvalues.go @@ -14,7 +14,6 @@ import ( "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -1022,7 +1021,7 @@ func (node *IndexedLvalueNode) Assign( // This lets the user do '$y[ ["a", "b", "c"] ] = $x' in lieu of // '$y["a"]["b"]["c"] = $x'. if len(indices) == 1 && indices[0].IsArray() { - indices = types.MakePointerArray(indices[0].GetArray()) + indices = mlrval.MakePointerArray(indices[0].GetArray()) } return node.baseLvalue.AssignIndexed(rvalue, indices, state) diff --git a/internal/pkg/lib/util.go b/internal/pkg/lib/util.go index c25b25597..232329180 100644 --- a/internal/pkg/lib/util.go +++ b/internal/pkg/lib/util.go @@ -114,7 +114,7 @@ func TryIntFromString(input string) (int, bool) { return 0, false } -func TryFloat64FromString(input string) (float64, bool) { +func TryFloatFromString(input string) (float64, bool) { fval, err := strconv.ParseFloat(input, 64) if err == nil { return fval, true diff --git a/internal/pkg/types-parked/mlrmap_flatten_unflatten.go b/internal/pkg/mlrval/mlrmap_flatten_unflatten.go similarity index 92% rename from internal/pkg/types-parked/mlrmap_flatten_unflatten.go rename to internal/pkg/mlrval/mlrmap_flatten_unflatten.go index bfea723a7..fbaeffc40 100644 --- a/internal/pkg/types-parked/mlrmap_flatten_unflatten.go +++ b/internal/pkg/mlrval/mlrmap_flatten_unflatten.go @@ -20,7 +20,7 @@ // otherwise) when we convert to/from JSON. // ================================================================ -package types +package mlrval import ( "strings" @@ -136,7 +136,7 @@ func (mlrmap *Mlrmap) CopyUnflattened( for pe := mlrmap.Head; pe != nil; pe = pe.Next { // Is the field name something dot something? if strings.Contains(pe.Key, separator) { - arrayOfIndices := mlrvalSplitAXHelper(pe.Key, separator) + arrayOfIndices := SplitAXHelper(pe.Key, separator) lib.InternalCodingErrorIf(len(arrayOfIndices.arrayval) < 1) // If the input field name was "x.a" then remember the "x". baseIndex := arrayOfIndices.arrayval[0].String() @@ -157,7 +157,7 @@ func (mlrmap *Mlrmap) CopyUnflattened( for baseIndex := range affectedBaseIndices { oldValue := other.Get(baseIndex) lib.InternalCodingErrorIf(oldValue == nil) - newValue := BIF_arrayify(oldValue) + newValue := oldValue.Arrayify() other.PutReference(baseIndex, newValue) } @@ -186,7 +186,7 @@ func (mlrmap *Mlrmap) CopyUnflattenFields( for pe := mlrmap.Head; pe != nil; pe = pe.Next { // Is the field name something dot something? if strings.Contains(pe.Key, separator) { - arrayOfIndices := mlrvalSplitAXHelper(pe.Key, separator) + arrayOfIndices := SplitAXHelper(pe.Key, separator) lib.InternalCodingErrorIf(len(arrayOfIndices.arrayval) < 1) // If the input field name was "x.a" then remember the "x". baseIndex := arrayOfIndices.arrayval[0].String() @@ -211,7 +211,7 @@ func (mlrmap *Mlrmap) CopyUnflattenFields( for baseIndex := range affectedBaseIndices { oldValue := other.Get(baseIndex) lib.InternalCodingErrorIf(oldValue == nil) - newValue := BIF_arrayify(oldValue) + newValue := oldValue.Arrayify() other.PutReference(baseIndex, newValue) } @@ -231,10 +231,24 @@ func unflattenTerminal(input *Mlrval) *Mlrval { return input } if input.printrep == "{}" { - return MlrvalFromMapReferenced(NewMlrmap()) + return FromMap(NewMlrmap()) } if input.printrep == "[]" { - return MlrvalFromArrayReference(make([]Mlrval, 0)) + return FromArray(make([]Mlrval, 0)) } return input } + +// SplitAXHelper is split out for the benefit of BIF_splitax and +// BIF_unflatten. +func SplitAXHelper(input string, separator string) *Mlrval { + fields := lib.SplitString(input, separator) + + output := FromArray(make([]Mlrval, len(fields))) + + for i, field := range fields { + output.arrayval[i] = *FromString(field) + } + + return output +} diff --git a/internal/pkg/types-parked/mlrval_accessors.go b/internal/pkg/mlrval/mlrval_accessors.go similarity index 85% rename from internal/pkg/types-parked/mlrval_accessors.go rename to internal/pkg/mlrval/mlrval_accessors.go index b4af1bf5e..366485957 100644 --- a/internal/pkg/types-parked/mlrval_accessors.go +++ b/internal/pkg/mlrval/mlrval_accessors.go @@ -1,17 +1,11 @@ -package types +package mlrval import ( - "fmt" - "os" "strconv" "github.com/johnkerl/miller/internal/pkg/lib" ) -func GetTypeName(mvtype MVType) string { - return TYPE_NAMES[mvtype] -} - func (mv *Mlrval) GetArrayLength() (int, bool) { if mv.IsArray() { return len(mv.arrayval), true @@ -20,16 +14,6 @@ func (mv *Mlrval) GetArrayLength() (int, bool) { } } -func (mv *Mlrval) Copy() *Mlrval { - other := *mv - if mv.IsMap() { - other.mapval = mv.mapval.Copy() - } else if mv.IsArray() { - other.arrayval = CopyMlrvalArray(mv.arrayval) - } - return &other -} - func CopyMlrvalArray(input []Mlrval) []Mlrval { output := make([]Mlrval, len(input)) for i, element := range input { @@ -57,7 +41,7 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { // values would disappear entirely in a JSON-to-CSV conversion. if mv.mapval.IsEmpty() { if prefix != "" { - retval.PutCopy(prefix, mlrval.FromString("{}")) + retval.PutCopy(prefix, FromString("{}")) } } @@ -82,7 +66,7 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { // values would disappear entirely in a JSON-to-CSV conversion. if len(mv.arrayval) == 0 { if prefix != "" { - retval.PutCopy(prefix, mlrval.FromString("[]")) + retval.PutCopy(prefix, FromString("[]")) } } @@ -106,7 +90,7 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { retval.PutCopy(prefix, mv.Copy()) } - return *MlrvalFromMapReferenced(retval) + return *FromMap(retval) } // ---------------------------------------------------------------- diff --git a/internal/pkg/mlrval/mlrval_infer.go b/internal/pkg/mlrval/mlrval_infer.go index 682ad3171..2c123fe17 100644 --- a/internal/pkg/mlrval/mlrval_infer.go +++ b/internal/pkg/mlrval/mlrval_infer.go @@ -57,7 +57,7 @@ func inferNormally(mv *Mlrval, input string, inferBool bool) *Mlrval { } if downcasedFloatNamesToNotInfer[strings.ToLower(input)] == false { - floatval, fok := lib.TryFloat64FromString(input) + floatval, fok := lib.TryFloatFromString(input) if fok { return mv.SetFromPrevalidatedFloatString(input, floatval) } diff --git a/internal/pkg/mlrval/mlrval_new.go b/internal/pkg/mlrval/mlrval_new.go index 77e89e87f..2e37c1dd0 100644 --- a/internal/pkg/mlrval/mlrval_new.go +++ b/internal/pkg/mlrval/mlrval_new.go @@ -139,7 +139,7 @@ func FromFloat(input float64) *Mlrval { // then its printrep will be "[[3.1400]]" and it will be MT_STRING. This // function supports that. func TryFromFloatString(input string) *Mlrval { - floatval, ok := lib.TryFloat64FromString(input) + floatval, ok := lib.TryFloatFromString(input) if ok { return FromPrevalidatedFloatString(input, floatval) } else { diff --git a/internal/pkg/mlrval/tbf.go b/internal/pkg/mlrval/tbf.go index 21b1c34c4..b73b03f9a 100644 --- a/internal/pkg/mlrval/tbf.go +++ b/internal/pkg/mlrval/tbf.go @@ -2,6 +2,9 @@ package mlrval import ( "errors" + "fmt" + "os" + "strconv" "github.com/johnkerl/miller/internal/pkg/lib" ) @@ -27,6 +30,49 @@ func NewMlrvalForAutoDeepen(mvtype MVType) (*Mlrval, error) { } } +func (mv *Mlrval) Arrayify() *Mlrval { + if mv.IsMap() { + if mv.mapval.IsEmpty() { + return mv + } + + convertible := true + i := 0 + for pe := mv.mapval.Head; pe != nil; pe = pe.Next { + sval := strconv.Itoa(i + 1) // Miller user-space indices are 1-up + i++ + if pe.Key != sval { + convertible = false + } + pe.Value = pe.Value.Arrayify() + } + + if convertible { + arrayval := make([]Mlrval, mv.mapval.FieldCount) + i := 0 + for pe := mv.mapval.Head; pe != nil; pe = pe.Next { + arrayval[i] = *pe.Value.Copy() + i++ + } + return FromArray(arrayval) + + } else { + return mv + } + + } else if mv.IsArray() { + // TODO: comment (or rethink) that this modifies its inputs!! + output := mv.Copy() + for i := range mv.arrayval { + output.arrayval[i] = *output.arrayval[i].Arrayify() + } + return output + + } else { + return mv + } +} + //func MlrvalFromEmptyMap() *Mlrval { // return &Mlrval{ // mvtype: MT_MAP, @@ -171,27 +217,19 @@ func LengthenMlrvalArray(array *[]Mlrval, newLength64 int) { // return inferrer(input, true) //} -//func (mv *Mlrval) GetNumericToFloatValueOrDie() (floatValue float64) { -// floatValue, ok := mv.GetNumericToFloatValue() -// if !ok { -// fmt.Fprintf( -// os.Stderr, -// "%s: couldn't parse \"%s\" as number.", -// "mlr", mv.String(), -// ) -// os.Exit(1) -// } -// return floatValue -//} +func (mv *Mlrval) GetNumericToFloatValueOrDie() (floatValue float64) { + floatValue, ok := mv.GetNumericToFloatValue() + if !ok { + fmt.Fprintf( + os.Stderr, + "%s: couldn't parse \"%s\" as number.", + "mlr", mv.String(), + ) + os.Exit(1) + } + return floatValue +} -//func (mv *Mlrval) AssertNumeric() { -// _ = mv.GetNumericToFloatValueOrDie() -//} - -//func (mv *Mlrval) GetArrayLength() (int, bool) { -// if mv.IsArray() { -// return len(mv.arrayval), true -// } else { -// return -999, false -// } -//} +func (mv *Mlrval) AssertNumeric() { + _ = mv.GetNumericToFloatValueOrDie() +} diff --git a/internal/pkg/transformers/altkv.go b/internal/pkg/transformers/altkv.go index 88bbba1f8..e7fddb3c9 100644 --- a/internal/pkg/transformers/altkv.go +++ b/internal/pkg/transformers/altkv.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) diff --git a/internal/pkg/transformers/bar.go b/internal/pkg/transformers/bar.go index ed425b75c..6172e6a73 100644 --- a/internal/pkg/transformers/bar.go +++ b/internal/pkg/transformers/bar.go @@ -252,7 +252,7 @@ func (tr *TransformerBar) processNoAuto( if idx > tr.width { idx = tr.width } - inrec.PutReference(fieldName, mlrval.mlrval.FromString(tr.bars[idx])) + inrec.PutReference(fieldName, mlrval.FromString(tr.bars[idx])) } outputRecordsAndContexts.PushBack(inrecAndContext) @@ -340,7 +340,7 @@ func (tr *TransformerBar) processAuto( buffer.WriteString("[") buffer.WriteString(shi) buffer.WriteString("]") - record.PutReference(fieldName, mlrval.mlrval.FromString(buffer.String())) + record.PutReference(fieldName, mlrval.FromString(buffer.String())) } } diff --git a/internal/pkg/transformers/clean-whitespace.go b/internal/pkg/transformers/clean-whitespace.go index 1f2d4bdb9..efb0a5db7 100644 --- a/internal/pkg/transformers/clean-whitespace.go +++ b/internal/pkg/transformers/clean-whitespace.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" @@ -153,7 +154,7 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeysAndValues( newrec := mlrval.NewMlrmapAsRecord() for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { - oldKey := mlrval.mlrval.FromString(pe.Key) + oldKey := mlrval.FromString(pe.Key) // xxx temp newKey := bifs.BIF_clean_whitespace(oldKey) newValue := bifs.BIF_clean_whitespace(pe.Value) @@ -178,7 +179,7 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeys( newrec := mlrval.NewMlrmapAsRecord() for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { - oldKey := mlrval.mlrval.FromString(pe.Key) + oldKey := mlrval.FromString(pe.Key) newKey := bifs.BIF_clean_whitespace(oldKey) // Transferring ownership from old record to new record; no copy needed newrec.PutReference(newKey.String(), pe.Value) diff --git a/internal/pkg/transformers/cut.go b/internal/pkg/transformers/cut.go index 51b038c1d..aff700163 100644 --- a/internal/pkg/transformers/cut.go +++ b/internal/pkg/transformers/cut.go @@ -9,6 +9,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) diff --git a/internal/pkg/transformers/fill-down.go b/internal/pkg/transformers/fill-down.go index fc3b16789..60dc26a5a 100644 --- a/internal/pkg/transformers/fill-down.go +++ b/internal/pkg/transformers/fill-down.go @@ -176,7 +176,7 @@ func (tr *TransformerFillDown) transformSpecified( if tr.onlyIfAbsent { present = value != nil } else { - present = value != nil && !value.IsEmpty() + present = value != nil && !value.IsVoid() } if present { @@ -215,7 +215,7 @@ func (tr *TransformerFillDown) transformAll( if tr.onlyIfAbsent { present = value != nil } else { - present = value != nil && !value.IsEmpty() + present = value != nil && !value.IsVoid() } if present { diff --git a/internal/pkg/transformers/fill-empty.go b/internal/pkg/transformers/fill-empty.go index 2b57de4e1..840b75ab1 100644 --- a/internal/pkg/transformers/fill-empty.go +++ b/internal/pkg/transformers/fill-empty.go @@ -105,7 +105,7 @@ func NewTransformerFillEmpty( if inferType { tr.fillValue = mlrval.FromInferredType(fillString) } else { - tr.fillValue = mlrval.mlrval.FromString(fillString) + tr.fillValue = mlrval.FromString(fillString) } return tr, nil } @@ -123,7 +123,7 @@ func (tr *TransformerFillEmpty) Transform( inrec := inrecAndContext.Record for pe := inrec.Head; pe != nil; pe = pe.Next { - if pe.Value.IsEmpty() { + if pe.Value.IsVoid() { pe.Value = tr.fillValue } } diff --git a/internal/pkg/transformers/format-values.go b/internal/pkg/transformers/format-values.go index 1a74ed14c..bcde9511f 100644 --- a/internal/pkg/transformers/format-values.go +++ b/internal/pkg/transformers/format-values.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -131,9 +132,9 @@ func transformerFormatValuesParseCLI( // ---------------------------------------------------------------- type TransformerFormatValues struct { - stringFormatter types.IMlrvalFormatter - intFormatter types.IMlrvalFormatter - floatFormatter types.IMlrvalFormatter + stringFormatter mlrval.IFormatter + intFormatter mlrval.IFormatter + floatFormatter mlrval.IFormatter coerceIntToFloat bool } @@ -143,17 +144,17 @@ func NewTransformerFormatValues( floatFormat string, coerceIntToFloat bool, ) (*TransformerFormatValues, error) { - stringFormatter, err := types.GetMlrvalFormatter(stringFormat) + stringFormatter, err := mlrval.GetFormatter(stringFormat) if err != nil { return nil, err } - intFormatter, err := types.GetMlrvalFormatter(intFormat) + intFormatter, err := mlrval.GetFormatter(intFormat) if err != nil { return nil, err } - floatFormatter, err := types.GetMlrvalFormatter(floatFormat) + floatFormatter, err := mlrval.GetFormatter(floatFormat) if err != nil { return nil, err } diff --git a/internal/pkg/transformers/fraction.go b/internal/pkg/transformers/fraction.go index 5178511d1..462b139fb 100644 --- a/internal/pkg/transformers/fraction.go +++ b/internal/pkg/transformers/fraction.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" @@ -267,7 +268,7 @@ func (tr *TransformerFraction) Transform( } denominator := sumsForGroup[fractionFieldName] - if !mlrval.BIF_equals_as_bool(value, tr.zero) { + if !mlrval.Equals(value, tr.zero) { outputValue = bifs.BIF_divide(numerator, denominator) outputValue = bifs.BIF_times(outputValue, tr.multiplier) } else { diff --git a/internal/pkg/transformers/gap.go b/internal/pkg/transformers/gap.go index d7ec544fd..881406c1e 100644 --- a/internal/pkg/transformers/gap.go +++ b/internal/pkg/transformers/gap.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/types" ) diff --git a/internal/pkg/transformers/join.go b/internal/pkg/transformers/join.go index 794a15a47..3b6147fa5 100644 --- a/internal/pkg/transformers/join.go +++ b/internal/pkg/transformers/join.go @@ -9,6 +9,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/input" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/transformers/utils" "github.com/johnkerl/miller/internal/pkg/types" ) diff --git a/internal/pkg/transformers/json-stringify.go b/internal/pkg/transformers/json-stringify.go index 68154e18f..045bc134a 100644 --- a/internal/pkg/transformers/json-stringify.go +++ b/internal/pkg/transformers/json-stringify.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -84,11 +85,11 @@ func transformerJSONStringifyParseCLI( } } - var jsonFormatting types.TJSONFormatting = types.JSON_SINGLE_LINE + var jsonFormatting mlrval.TJSONFormatting = mlrval.JSON_SINGLE_LINE if jvStack { - jsonFormatting = types.JSON_MULTILINE + jsonFormatting = mlrval.JSON_MULTILINE } else { - jsonFormatting = types.JSON_SINGLE_LINE + jsonFormatting = mlrval.JSON_SINGLE_LINE } *pargi = argi @@ -111,7 +112,7 @@ func transformerJSONStringifyParseCLI( // ---------------------------------------------------------------- type TransformerJSONStringify struct { // input - jsonFormatting types.TJSONFormatting + jsonFormatting mlrval.TJSONFormatting fieldNameSet map[string]bool // state @@ -119,7 +120,7 @@ type TransformerJSONStringify struct { } func NewTransformerJSONStringify( - jsonFormatting types.TJSONFormatting, + jsonFormatting mlrval.TJSONFormatting, fieldNames []string, ) (*TransformerJSONStringify, error) { var fieldNameSet map[string]bool = nil diff --git a/internal/pkg/transformers/merge-fields.go b/internal/pkg/transformers/merge-fields.go index bfdef6e9e..84f421b32 100644 --- a/internal/pkg/transformers/merge-fields.go +++ b/internal/pkg/transformers/merge-fields.go @@ -351,7 +351,7 @@ func (tr *TransformerMergeFields) transformByNameList( continue } - if mvalue.IsEmpty() { // key present with empty value + if mvalue.IsVoid() { // key present with empty value if !tr.keepInputFields { inrec.Remove(valueFieldName) } @@ -418,7 +418,7 @@ func (tr *TransformerMergeFields) transformByNameRegex( continue } - if mvalue.IsEmpty() { // Key present with empty value + if mvalue.IsVoid() { // Key present with empty value if !tr.keepInputFields { // We are modifying the record while iterating over it. next := pe.Next inrec.Unlink(pe) @@ -521,7 +521,7 @@ func (tr *TransformerMergeFields) transformByCollapsing( // The accumulator has been initialized with default values; continue // here. (If we were to continue before namedAccumulators.Put(...) we // would be failing to construct the accumulator.) - if mvalue.IsEmpty() { // key present with empty value + if mvalue.IsVoid() { // key present with empty value if !tr.keepInputFields { // We are modifying the record while iterating over it. next := pe.Next inrec.Unlink(pe) diff --git a/internal/pkg/transformers/nest.go b/internal/pkg/transformers/nest.go index 220ec6917..e3cc86260 100644 --- a/internal/pkg/transformers/nest.go +++ b/internal/pkg/transformers/nest.go @@ -336,7 +336,7 @@ func (tr *TransformerNest) explodeValuesAcrossFields( i := 1 for _, piece := range pieces { key := tr.fieldName + "_" + strconv.Itoa(i) - value := mlrval.mlrval.FromString(piece) + value := mlrval.FromString(piece) recordEntry = inrec.PutReferenceAfter(recordEntry, key, value) i++ } @@ -369,7 +369,7 @@ func (tr *TransformerNest) explodeValuesAcrossRecords( pieces := strings.Split(svalue, tr.nestedFS) for _, piece := range pieces { outrec := inrec.Copy() - outrec.PutReference(tr.fieldName, mlrval.mlrval.FromString(piece)) + outrec.PutReference(tr.fieldName, mlrval.FromString(piece)) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } @@ -405,13 +405,13 @@ func (tr *TransformerNest) explodePairsAcrossFields( recordEntry = inrec.PutReferenceAfter( recordEntry, pair[0], - mlrval.mlrval.FromString(pair[1]), + mlrval.FromString(pair[1]), ) } else { // there is not a pair recordEntry = inrec.PutReferenceAfter( recordEntry, tr.fieldName, - mlrval.mlrval.FromString(piece), + mlrval.FromString(piece), ) } } @@ -450,9 +450,9 @@ func (tr *TransformerNest) explodePairsAcrossRecords( // name, in which case replace its value. pair := strings.SplitN(piece, tr.nestedPS, 2) if len(pair) == 2 { // there is a pair - outrec.PutReferenceAfter(originalEntry, pair[0], mlrval.mlrval.FromString(pair[1])) + outrec.PutReferenceAfter(originalEntry, pair[0], mlrval.FromString(pair[1])) } else { // there is not a pair - outrec.PutReferenceAfter(originalEntry, tr.fieldName, mlrval.mlrval.FromString(piece)) + outrec.PutReferenceAfter(originalEntry, tr.fieldName, mlrval.FromString(piece)) } outrec.Unlink(originalEntry) @@ -498,7 +498,7 @@ func (tr *TransformerNest) implodeValuesAcrossFields( } if fieldCount > 0 { - newValue := mlrval.mlrval.FromString(buffer.String()) + newValue := mlrval.FromString(buffer.String()) if previousEntry == nil { // No record before the unlinked one, i.e. list-head. inrec.PrependReference(tr.fieldName, newValue) } else { @@ -577,7 +577,7 @@ func (tr *TransformerNest) implodeValueAcrossRecords( } // tr.fieldName was already present so we'll overwrite it in-place here. - outrec.PutReference(tr.fieldName, mlrval.mlrval.FromString(buffer.String())) + outrec.PutReference(tr.fieldName, mlrval.FromString(buffer.String())) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } } diff --git a/internal/pkg/transformers/regularize.go b/internal/pkg/transformers/regularize.go index 1e20a7d5b..f50c5c439 100644 --- a/internal/pkg/transformers/regularize.go +++ b/internal/pkg/transformers/regularize.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) diff --git a/internal/pkg/transformers/remove-empty-columns.go b/internal/pkg/transformers/remove-empty-columns.go index 411e635a9..12fcd03e2 100644 --- a/internal/pkg/transformers/remove-empty-columns.go +++ b/internal/pkg/transformers/remove-empty-columns.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -107,7 +108,7 @@ func (tr *TransformerRemoveEmptyColumns) Transform( tr.recordsAndContexts.PushBack(inrecAndContext) for pe := inrec.Head; pe != nil; pe = pe.Next { - if !pe.Value.IsEmpty() { + if !pe.Value.IsVoid() { tr.namesWithNonEmptyValues[pe.Key] = true } } diff --git a/internal/pkg/transformers/reorder.go b/internal/pkg/transformers/reorder.go index 9c75bcf4e..39ac531be 100644 --- a/internal/pkg/transformers/reorder.go +++ b/internal/pkg/transformers/reorder.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) diff --git a/internal/pkg/transformers/reshape.go b/internal/pkg/transformers/reshape.go index 99ce0e0c7..b2838874b 100644 --- a/internal/pkg/transformers/reshape.go +++ b/internal/pkg/transformers/reshape.go @@ -321,7 +321,7 @@ func (tr *TransformerReshape) wideToLongNoRegex( } else { for pf := pairs.Head; pf != nil; pf = pf.Next { outrec := inrec.Copy() - outrec.PutReference(tr.outputKeyFieldName, mlrval.mlrval.FromString(pf.Key)) + outrec.PutReference(tr.outputKeyFieldName, mlrval.FromString(pf.Key)) outrec.PutReference(tr.outputValueFieldName, pf.Value) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } @@ -363,7 +363,7 @@ func (tr *TransformerReshape) wideToLongRegex( } else { for pf := pairs.Head; pf != nil; pf = pf.Next { outrec := inrec.Copy() - outrec.PutReference(tr.outputKeyFieldName, mlrval.mlrval.FromString(pf.Key)) + outrec.PutReference(tr.outputKeyFieldName, mlrval.FromString(pf.Key)) outrec.PutReference(tr.outputValueFieldName, pf.Value) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } diff --git a/internal/pkg/transformers/sec2gmt.go b/internal/pkg/transformers/sec2gmt.go index 9bb064474..6973d2f02 100644 --- a/internal/pkg/transformers/sec2gmt.go +++ b/internal/pkg/transformers/sec2gmt.go @@ -164,7 +164,7 @@ func (tr *TransformerSec2GMT) Transform( if value != nil { floatval, ok := value.GetNumericToFloatValue() if ok { - newValue := mlrval.mlrval.FromString(lib.Sec2GMT( + newValue := mlrval.FromString(lib.Sec2GMT( floatval/tr.preDivide, tr.numDecimalPlaces, )) diff --git a/internal/pkg/transformers/sec2gmtdate.go b/internal/pkg/transformers/sec2gmtdate.go index ea5cc3940..1f886eba5 100644 --- a/internal/pkg/transformers/sec2gmtdate.go +++ b/internal/pkg/transformers/sec2gmtdate.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/types" diff --git a/internal/pkg/transformers/seqgen.go b/internal/pkg/transformers/seqgen.go index 495f1bb81..427fe1a36 100644 --- a/internal/pkg/transformers/seqgen.go +++ b/internal/pkg/transformers/seqgen.go @@ -7,6 +7,7 @@ import ( "os" "strings" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" @@ -121,7 +122,7 @@ type TransformerSeqgen struct { start *mlrval.Mlrval stop *mlrval.Mlrval step *mlrval.Mlrval - doneComparator types.BinaryFunc + doneComparator bifs.BinaryFunc mdone *mlrval.Mlrval } @@ -135,7 +136,7 @@ func NewTransformerSeqgen( start := mlrval.FromInferredType(startString) stop := mlrval.FromInferredType(stopString) step := mlrval.FromInferredType(stepString) - var doneComparator types.BinaryFunc = nil + var doneComparator bifs.BinaryFunc = nil fstart, startIsNumeric := start.GetNumericToFloatValue() if !startIsNumeric { diff --git a/internal/pkg/transformers/sort.go b/internal/pkg/transformers/sort.go index 90d8d5767..4365f9bf9 100644 --- a/internal/pkg/transformers/sort.go +++ b/internal/pkg/transformers/sort.go @@ -48,6 +48,7 @@ import ( "sort" "strings" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" @@ -110,7 +111,7 @@ func transformerSortParseCLI( argi++ groupByFieldNames := make([]string, 0) - comparatorFuncs := make([]types.ComparatorFunc, 0) + comparatorFuncs := make([]bifs.ComparatorFunc, 0) for argi < argc /* variable increment: 1 or 2 depending on flag */ { opt := args[argi] @@ -129,7 +130,7 @@ func transformerSortParseCLI( subList := cli.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.LexicalAscendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.LexicalAscendingComparator) } } else if opt == "-c" { @@ -140,14 +141,14 @@ func transformerSortParseCLI( subList := cli.VerbGetStringArrayArgOrDie(verb, "-nr", args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.CaseFoldDescendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.CaseFoldDescendingComparator) } } else { subList := cli.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.CaseFoldAscendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.CaseFoldAscendingComparator) } } @@ -155,7 +156,7 @@ func transformerSortParseCLI( subList := cli.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.LexicalDescendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.LexicalDescendingComparator) } } else if opt == "-n" { @@ -188,7 +189,7 @@ func transformerSortParseCLI( subList := cli.VerbGetStringArrayArgOrDie(verb, "-nf", args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.NumericAscendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.NumericAscendingComparator) } } else if args[argi] == "-r" { @@ -197,7 +198,7 @@ func transformerSortParseCLI( subList := cli.VerbGetStringArrayArgOrDie(verb, "-nr", args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.NumericDescendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.NumericDescendingComparator) } } else { @@ -205,7 +206,7 @@ func transformerSortParseCLI( subList := cli.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.NumericAscendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.NumericAscendingComparator) } } @@ -213,14 +214,14 @@ func transformerSortParseCLI( subList := cli.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.NumericAscendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.NumericAscendingComparator) } } else if opt == "-nr" { subList := cli.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.NumericDescendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.NumericDescendingComparator) } } else { @@ -269,7 +270,7 @@ func transformerSortParseCLI( type TransformerSort struct { // -- Input groupByFieldNames []string - comparatorFuncs []types.ComparatorFunc + comparatorFuncs []bifs.ComparatorFunc // -- State // Map from string to *list.List: @@ -281,7 +282,7 @@ type TransformerSort struct { func NewTransformerSort( groupByFieldNames []string, - comparatorFuncs []types.ComparatorFunc, + comparatorFuncs []bifs.ComparatorFunc, ) (*TransformerSort, error) { tr := &TransformerSort{ diff --git a/internal/pkg/transformers/step.go b/internal/pkg/transformers/step.go index 30d595901..3f763eb45 100644 --- a/internal/pkg/transformers/step.go +++ b/internal/pkg/transformers/step.go @@ -7,6 +7,7 @@ import ( "os" "strings" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/mlrval" @@ -378,7 +379,7 @@ func (stepper *tStepperDelta) process( valueFieldValue *mlrval.Mlrval, inrec *mlrval.Mlrmap, ) { - if valueFieldValue.IsEmpty() { + if valueFieldValue.IsVoid() { inrec.PutCopy(stepper.outputFieldName, mlrval.VOID) return } @@ -474,7 +475,7 @@ func (stepper *tStepperRatio) process( valueFieldValue *mlrval.Mlrval, inrec *mlrval.Mlrmap, ) { - if valueFieldValue.IsEmpty() { + if valueFieldValue.IsVoid() { inrec.PutCopy(stepper.outputFieldName, mlrval.VOID) return } @@ -509,7 +510,7 @@ func (stepper *tStepperRsum) process( valueFieldValue *mlrval.Mlrval, inrec *mlrval.Mlrmap, ) { - if valueFieldValue.IsEmpty() { + if valueFieldValue.IsVoid() { inrec.PutCopy(stepper.outputFieldName, mlrval.VOID) } else { stepper.rsum = bifs.BIF_plus_binary(valueFieldValue, stepper.rsum) @@ -540,7 +541,7 @@ func (stepper *tStepperCounter) process( valueFieldValue *mlrval.Mlrval, inrec *mlrval.Mlrmap, ) { - if valueFieldValue.IsEmpty() { + if valueFieldValue.IsVoid() { inrec.PutCopy(stepper.outputFieldName, mlrval.VOID) } else { stepper.counter = bifs.BIF_plus_binary(stepper.counter, stepper.one) @@ -583,7 +584,7 @@ func stepperEWMAAlloc( for i, stringAlpha := range stringAlphas { suffix := suffixes[i] - dalpha, ok := lib.TryFloat64FromString(stringAlpha) + dalpha, ok := lib.TryFloatFromString(stringAlpha) if !ok { fmt.Fprintf( os.Stderr, diff --git a/internal/pkg/transformers/template.go b/internal/pkg/transformers/template.go index 285ad307e..605d09cbb 100644 --- a/internal/pkg/transformers/template.go +++ b/internal/pkg/transformers/template.go @@ -130,7 +130,7 @@ func NewTransformerTemplate( return &TransformerTemplate{ fieldNameList: fieldNames, fieldNameSet: lib.StringListToSet(fieldNames), - fillWith: mlrval.mlrval.FromString(fillWith), + fillWith: mlrval.FromString(fillWith), }, nil } diff --git a/internal/pkg/transformers/uniq.go b/internal/pkg/transformers/uniq.go index 8e8a22ad7..5cafd37f0 100644 --- a/internal/pkg/transformers/uniq.go +++ b/internal/pkg/transformers/uniq.go @@ -475,7 +475,7 @@ func (tr *TransformerUniq) transformUnlashed( for pf := countsForFieldName.Head; pf != nil; pf = pf.Next { fieldValueString := pf.Key outrec := mlrval.NewMlrmapAsRecord() - outrec.PutReference("field", mlrval.mlrval.FromString(fieldName)) + outrec.PutReference("field", mlrval.FromString(fieldName)) outrec.PutCopy( "value", tr.unlashedCountValues.Get(fieldName).(*lib.OrderedMap).Get(fieldValueString).(*mlrval.Mlrval), diff --git a/internal/pkg/transformers/unsparsify.go b/internal/pkg/transformers/unsparsify.go index b8a3850c4..365d86689 100644 --- a/internal/pkg/transformers/unsparsify.go +++ b/internal/pkg/transformers/unsparsify.go @@ -128,7 +128,7 @@ func NewTransformerUnsparsify( } tr := &TransformerUnsparsify{ - fillerMlrval: mlrval.mlrval.FromString(fillerString), + fillerMlrval: mlrval.FromString(fillerString), recordsAndContexts: list.New(), fieldNamesSeen: fieldNamesSeen, } diff --git a/internal/pkg/transformers/utils/join-bucket-keeper.go b/internal/pkg/transformers/utils/join-bucket-keeper.go index 6f14eebd8..ea8b87e5c 100644 --- a/internal/pkg/transformers/utils/join-bucket-keeper.go +++ b/internal/pkg/transformers/utils/join-bucket-keeper.go @@ -466,7 +466,7 @@ func (keeper *JoinBucketKeeper) fillNextJoinBucket() { os.Exit(1) } - keeper.JoinBucket.leftFieldValues = types.CopyMlrvalPointerArray(peekFieldValues) + keeper.JoinBucket.leftFieldValues = mlrval.CopyMlrvalPointerArray(peekFieldValues) keeper.JoinBucket.RecordsAndContexts.PushBack(keeper.peekRecordAndContext) keeper.JoinBucket.WasPaired = false diff --git a/internal/pkg/transformers/utils/stats1-accumulators.go b/internal/pkg/transformers/utils/stats1-accumulators.go index 51db05a1c..08f21a02e 100644 --- a/internal/pkg/transformers/utils/stats1-accumulators.go +++ b/internal/pkg/transformers/utils/stats1-accumulators.go @@ -187,7 +187,7 @@ func tryPercentileFromName(accumulatorName string) (float64, bool) { return 50.0, true } if strings.HasPrefix(accumulatorName, "p") { - percentile, ok := lib.TryFloat64FromString(accumulatorName[1:]) + percentile, ok := lib.TryFloatFromString(accumulatorName[1:]) if !ok { return 0.0, false } diff --git a/internal/pkg/transformers/utils/top-keeper.go b/internal/pkg/transformers/utils/top-keeper.go index 6951b9c00..a8ecef371 100644 --- a/internal/pkg/transformers/utils/top-keeper.go +++ b/internal/pkg/transformers/utils/top-keeper.go @@ -27,9 +27,9 @@ func NewTopKeeper(capacity int, doMax bool) *TopKeeper { capacity: capacity, } if doMax { - keeper.bsearchFunc = types.BsearchMlrvalArrayForDescendingInsert + keeper.bsearchFunc = mlrval.BsearchMlrvalArrayForDescendingInsert } else { - keeper.bsearchFunc = types.BsearchMlrvalArrayForAscendingInsert + keeper.bsearchFunc = mlrval.BsearchMlrvalArrayForAscendingInsert } return keeper } diff --git a/internal/pkg/types/context.go b/internal/pkg/types/context.go index 91a82540b..b64476116 100644 --- a/internal/pkg/types/context.go +++ b/internal/pkg/types/context.go @@ -40,21 +40,20 @@ func NewRecordAndContext( } } -// TODO: uncomment and integrate -//// For the record-readers to update their initial context as each new record is read. -//func (rac *RecordAndContext) Copy() *RecordAndContext { -// if rac == nil { -// return nil -// } -// recordCopy := rac.Record.Copy() -// contextCopy := rac.Context -// return &RecordAndContext{ -// Record: recordCopy, -// Context: contextCopy, -// OutputString: "", -// EndOfStream: false, -// } -//} +// For the record-readers to update their initial context as each new record is read. +func (rac *RecordAndContext) Copy() *RecordAndContext { + if rac == nil { + return nil + } + recordCopy := rac.Record.Copy() + contextCopy := rac.Context + return &RecordAndContext{ + Record: recordCopy, + Context: contextCopy, + OutputString: "", + EndOfStream: false, + } +} // For print/dump/etc to insert strings sequenced into the record-output // stream. This avoids race conditions between different goroutines printing From 9a0b959d480f45a537b01b98e0722a030efeba1e Mon Sep 17 00:00:00 2001 From: John Kerl Date: Fri, 17 Dec 2021 17:35:43 -0500 Subject: [PATCH 13/19] results of first post-compile profiling --- internal/pkg/input/pseudo_reader_gen.go | 2 +- internal/pkg/input/record_reader_dkvp_nidx.go | 6 +++--- internal/pkg/mlrval/mlrval_copy.go | 9 ++++----- internal/pkg/mlrval/mlrval_is.go | 16 +++++++++++++--- internal/pkg/mlrval/mlrval_output.go | 5 ++++- scripts/chain-cmps.sh | 14 +++++++------- scripts/chain-lengths.sh | 12 ++++++------ scripts/time-big-files | 6 ++++++ todo.txt | 10 ++++++---- 9 files changed, 50 insertions(+), 30 deletions(-) diff --git a/internal/pkg/input/pseudo_reader_gen.go b/internal/pkg/input/pseudo_reader_gen.go index 5aad82399..21172583d 100644 --- a/internal/pkg/input/pseudo_reader_gen.go +++ b/internal/pkg/input/pseudo_reader_gen.go @@ -122,7 +122,7 @@ func (reader *PseudoReaderGen) tryParse( name string, svalue string, ) (*mlrval.Mlrval, error) { - mvalue := mlrval.FromInferredType(svalue) + mvalue := mlrval.FromDeferredType(svalue) if mvalue == nil || !mvalue.IsNumeric() { return nil, errors.New( fmt.Sprintf("mlr: gen: %s \"%s\" is not parseable as number", name, svalue), diff --git a/internal/pkg/input/record_reader_dkvp_nidx.go b/internal/pkg/input/record_reader_dkvp_nidx.go index 0c65e7f6d..6c6039f9f 100644 --- a/internal/pkg/input/record_reader_dkvp_nidx.go +++ b/internal/pkg/input/record_reader_dkvp_nidx.go @@ -180,11 +180,11 @@ func recordFromDKVPLine(reader *RecordReaderDKVPNIDX, line string) *mlrval.Mlrma // "a=". Here we use the positional index as the key. This way // DKVP is a generalization of NIDX. key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := mlrval.FromInferredType(kv[0]) + value := mlrval.FromDeferredType(kv[0]) record.PutReference(key, value) } else { key := kv[0] - value := mlrval.FromInferredType(kv[1]) + value := mlrval.FromDeferredType(kv[1]) record.PutReference(key, value) } } @@ -209,7 +209,7 @@ func recordFromNIDXLine(reader *RecordReaderDKVPNIDX, line string) *mlrval.Mlrma for _, value := range values { i++ key := strconv.Itoa(i) - mval := mlrval.FromInferredType(value) + mval := mlrval.FromDeferredType(value) record.PutReference(key, mval) } return record diff --git a/internal/pkg/mlrval/mlrval_copy.go b/internal/pkg/mlrval/mlrval_copy.go index 8d3775c9f..115882c4e 100644 --- a/internal/pkg/mlrval/mlrval_copy.go +++ b/internal/pkg/mlrval/mlrval_copy.go @@ -2,12 +2,11 @@ package mlrval // TODO: comment about mvtype; deferrence; copying of deferrence. func (mv *Mlrval) Copy() *Mlrval { + other := *mv if mv.mvtype == MT_MAP { - panic("mlrval map-valued copy unimplemented, pending refactor") + other.mapval = mv.mapval.Copy() } else if mv.mvtype == MT_ARRAY { - panic("mlrval array-valued copy unimplemented, pending refactor") - } else { - other := *mv - return &other + other.arrayval = CopyMlrvalArray(mv.arrayval) } + return &other } diff --git a/internal/pkg/mlrval/mlrval_is.go b/internal/pkg/mlrval/mlrval_is.go index b154ec957..e8317e08f 100644 --- a/internal/pkg/mlrval/mlrval_is.go +++ b/internal/pkg/mlrval/mlrval_is.go @@ -88,13 +88,23 @@ func (mv *Mlrval) IsFalse() bool { } func (mv *Mlrval) IsArray() bool { - return mv.Type() == MT_ARRAY + // TODO: comment non-deferrable type -- don't force a (potentially + // expensive in bulk) JIT-infer of other types + // return mv.Type() == MT_ARRAY + return mv.mvtype == MT_ARRAY } func (mv *Mlrval) IsMap() bool { - return mv.Type() == MT_MAP + // TODO: comment non-deferrable type -- don't force a (potentially + // expensive in bulk) JIT-infer of other types + // return mv.Type() == MT_ARRAY + return mv.mvtype == MT_MAP } func (mv *Mlrval) IsArrayOrMap() bool { - t := mv.Type() + // TODO: comment why not + // In flatten we don't want to type-infer things that don't need to be jitted. + // Arrays & maps are never from deferred type. + // t := mv.Type() + t := mv.mvtype return t == MT_ARRAY || t == MT_MAP } diff --git a/internal/pkg/mlrval/mlrval_output.go b/internal/pkg/mlrval/mlrval_output.go index 0adcc69dd..5d74dfd25 100644 --- a/internal/pkg/mlrval/mlrval_output.go +++ b/internal/pkg/mlrval/mlrval_output.go @@ -7,7 +7,10 @@ import ( // Must have non-pointer receiver in order to implement the fmt.Stringer // interface to make this printable via fmt.Println et al. func (mv Mlrval) String() string { - if mv.IsFloat() && floatOutputFormatter != nil { + // TODO: comment re deferral -- important perf effect! + // if mv.IsFloat() && floatOutputFormatter != nil + // if mv.mvtype == MT_FLOAT && floatOutputFormatter != nil { + if floatOutputFormatter != nil && (mv.mvtype == MT_FLOAT || mv.mvtype == MT_PENDING) { // Use the format string from global --ofmt, if supplied return floatOutputFormatter.FormatFloat(mv.floatval) } else { diff --git a/scripts/chain-cmps.sh b/scripts/chain-cmps.sh index d818c8897..5b54e79fa 100755 --- a/scripts/chain-cmps.sh +++ b/scripts/chain-cmps.sh @@ -1,40 +1,40 @@ -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "./mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ check \ | md5sum done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "./mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ cat \ | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "./mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ head \ | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "./mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ tail \ | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "./mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ tac \ | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "./mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ sort -f shape \ | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "./mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ sort -n quantity \ | md5sum; diff --git a/scripts/chain-lengths.sh b/scripts/chain-lengths.sh index b6d2b6122..f059bec06 100755 --- a/scripts/chain-lengths.sh +++ b/scripts/chain-lengths.sh @@ -1,17 +1,17 @@ -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ then put -f scripts/chain-1.mlr \ | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ then put -f scripts/chain-1.mlr \ then put -f scripts/chain-1.mlr \ | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ then put -f scripts/chain-1.mlr \ then put -f scripts/chain-1.mlr \ @@ -19,7 +19,7 @@ echo; for m in mlr5 "./mlr -S"; do | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ then put -f scripts/chain-1.mlr \ then put -f scripts/chain-1.mlr \ @@ -28,7 +28,7 @@ echo; for m in mlr5 "./mlr -S"; do | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ then put -f scripts/chain-1.mlr \ then put -f scripts/chain-1.mlr \ @@ -38,7 +38,7 @@ echo; for m in mlr5 "./mlr -S"; do | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ then put -f scripts/chain-1.mlr \ then put -f scripts/chain-1.mlr \ diff --git a/scripts/time-big-files b/scripts/time-big-files index c334e233c..6c5ba45b0 100755 --- a/scripts/time-big-files +++ b/scripts/time-big-files @@ -3,30 +3,36 @@ ourdir=$(dirname $0) $ourdir/time-big-file dkvp mlr5 +$ourdir/time-big-file dkvp ~/tmp/miller/mlr $ourdir/time-big-file dkvp mlr $ourdir/time-big-file dkvp 'mlr -S' echo $ourdir/time-big-file nidx mlr5 +$ourdir/time-big-file nidx ~/tmp/miller/mlr $ourdir/time-big-file nidx mlr $ourdir/time-big-file nidx 'mlr -S' echo $ourdir/time-big-file xtab mlr5 +$ourdir/time-big-file xtab ~/tmp/miller/mlr $ourdir/time-big-file xtab mlr $ourdir/time-big-file xtab 'mlr -S' echo $ourdir/time-big-file csv mlr5 +$ourdir/time-big-file csv ~/tmp/miller/mlr $ourdir/time-big-file csv mlr $ourdir/time-big-file csv 'mlr -S' echo $ourdir/time-big-file csvlite mlr5 +$ourdir/time-big-file csvlite ~/tmp/miller/mlr $ourdir/time-big-file csvlite mlr $ourdir/time-big-file csvlite 'mlr -S' echo $ourdir/time-big-file json mlr5 +$ourdir/time-big-file json ~/tmp/miller/mlr $ourdir/time-big-file json mlr $ourdir/time-big-file json 'mlr -S' diff --git a/todo.txt b/todo.txt index c23183bd0..1f3bdfce7 100644 --- a/todo.txt +++ b/todo.txt @@ -3,12 +3,11 @@ PUNCHDOWN LIST * --ifs-regex & --ips-regex -- guessing is not safe as evidence by '.' and '|' * JIT next: - o mv.Equals & mv.Copy - o mlrmap copy - o move all but wrappers from $bifs/booleans.go to mlrval; also copy-reduce for sort -- ? o JSON I/O UT o krepl miller6 new doclink-comments - ? []Mlrval -> []*Mlrval ? + o IsArrayOrMap infers all -- it should not & it should say why not + o fix: + mlr --from test/input/abixy --opprint put -v -f test/cases/dsl-for-srec-loops/0005/mlr * perf next: o batchify source-quench @@ -74,6 +73,9 @@ PUNCHDOWN LIST - fix record/line sequencing regressions - maybe increase records-per-batch in join-bucket-keeper; and/or refactor - maybe increase records-per-batch in repl; and/or refactor + - lots of reading/neatening on types, bifs, mlrval split-up x all source files + - follow-ons for lots of UT per se on types, bifs, mlrval + ? []Mlrval -> []*Mlrval ? o goals: - keep goroutines -- including per-transformer -- for parallelism - look for flex ideas on how to structure that parallelism From 4f9eae7dfa9112ccf2375a1189770d59f3895537 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sun, 19 Dec 2021 14:04:27 -0500 Subject: [PATCH 14/19] testing --- internal/pkg/bifs/math.go | 25 ------------- internal/pkg/bifs/mathlib.go | 21 +++++++++++ internal/pkg/mlrval/mlrval_cmp.go | 14 ++++---- internal/pkg/mlrval/mlrval_collections.go | 6 ++-- internal/pkg/mlrval/mlrval_constants.go | 2 +- internal/pkg/mlrval/mlrval_infer.go | 2 +- internal/pkg/mlrval/mlrval_is.go | 43 ++++++++++++++++------- internal/pkg/mlrval/mlrval_json.go | 2 +- internal/pkg/mlrval/mlrval_new.go | 14 ++++---- internal/pkg/mlrval/mlrval_output.go | 36 +++++++++---------- internal/pkg/mlrval/mlrval_type.go | 6 ++-- internal/pkg/mlrval/tbf.go | 2 +- scripts/time-big-files | 12 +++---- todo.txt | 11 ++++++ 14 files changed, 110 insertions(+), 86 deletions(-) delete mode 100644 internal/pkg/bifs/math.go diff --git a/internal/pkg/bifs/math.go b/internal/pkg/bifs/math.go deleted file mode 100644 index e92712c4f..000000000 --- a/internal/pkg/bifs/math.go +++ /dev/null @@ -1,25 +0,0 @@ -package bifs - -import ( - "github.com/johnkerl/miller/internal/pkg/mlrval" -) - -// Return error (unary math-library func) -func _math_unary_erro1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { - return mlrval.ERROR -} - -// Return absent (unary math-library func) -func _math_unary_absn1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { - return mlrval.ABSENT -} - -// Return null (unary math-library func) -func _math_unary_null1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { - return mlrval.NULL -} - -// Return void (unary math-library func) -func _math_unary_void1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { - return mlrval.VOID -} diff --git a/internal/pkg/bifs/mathlib.go b/internal/pkg/bifs/mathlib.go index f98fd7b51..33bc36798 100644 --- a/internal/pkg/bifs/mathlib.go +++ b/internal/pkg/bifs/mathlib.go @@ -11,6 +11,27 @@ import ( "github.com/johnkerl/miller/internal/pkg/mlrval" ) +// ---------------------------------------------------------------- +// Return error (unary math-library func) +func _math_unary_erro1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.ERROR +} + +// Return absent (unary math-library func) +func _math_unary_absn1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.ABSENT +} + +// Return null (unary math-library func) +func _math_unary_null1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.NULL +} + +// Return void (unary math-library func) +func _math_unary_void1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.VOID +} + // ---------------------------------------------------------------- func math_unary_f_i(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { return mlrval.FromFloat(f(float64(input1.AcquireIntValue()))) diff --git a/internal/pkg/mlrval/mlrval_cmp.go b/internal/pkg/mlrval/mlrval_cmp.go index 940577ddb..6b0907b83 100644 --- a/internal/pkg/mlrval/mlrval_cmp.go +++ b/internal/pkg/mlrval/mlrval_cmp.go @@ -17,25 +17,25 @@ type CmpFuncInt func(input1, input2 *Mlrval) int // -1, 0, 1 for <=> // Exported methods func Equals(input1, input2 *Mlrval) bool { - return eq_dispositions[input1.mvtype][input2.mvtype](input1, input2) + return eq_dispositions[input1.Type()][input2.Type()](input1, input2) } func NotEquals(input1, input2 *Mlrval) bool { - return ne_dispositions[input1.mvtype][input2.mvtype](input1, input2) + return ne_dispositions[input1.Type()][input2.Type()](input1, input2) } func GreaterThan(input1, input2 *Mlrval) bool { - return gt_dispositions[input1.mvtype][input2.mvtype](input1, input2) + return gt_dispositions[input1.Type()][input2.Type()](input1, input2) } func GreaterThanOrEquals(input1, input2 *Mlrval) bool { - return ge_dispositions[input1.mvtype][input2.mvtype](input1, input2) + return ge_dispositions[input1.Type()][input2.Type()](input1, input2) } func LessThan(input1, input2 *Mlrval) bool { - return lt_dispositions[input1.mvtype][input2.mvtype](input1, input2) + return lt_dispositions[input1.Type()][input2.Type()](input1, input2) } func LessThanOrEquals(input1, input2 *Mlrval) bool { - return le_dispositions[input1.mvtype][input2.mvtype](input1, input2) + return le_dispositions[input1.Type()][input2.Type()](input1, input2) } func Cmp(input1, input2 *Mlrval) int { - return cmp_dispositions[input1.mvtype][input2.mvtype](input1, input2) + return cmp_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- diff --git a/internal/pkg/mlrval/mlrval_collections.go b/internal/pkg/mlrval/mlrval_collections.go index 9adf56450..52287460d 100644 --- a/internal/pkg/mlrval/mlrval_collections.go +++ b/internal/pkg/mlrval/mlrval_collections.go @@ -353,7 +353,7 @@ func putIndexedOnMap(baseMap *Mlrmap, indices []*Mlrval, rvalue *Mlrval) error { nextIndex := indices[1] var err error = nil - baseValue, err = NewMlrvalForAutoDeepen(nextIndex.mvtype) + baseValue, err = NewMlrvalForAutoDeepen(nextIndex.Type()) if err != nil { return err } @@ -407,11 +407,11 @@ func putIndexedOnArray( // Overwrite what's in this slot if it's the wrong type if nextIndex.IsString() { - if (*baseArray)[zindex].mvtype != MT_MAP { + if !(*baseArray)[zindex].IsMap() { (*baseArray)[zindex] = *FromEmptyMap() } } else if nextIndex.IsInt() { - if (*baseArray)[zindex].mvtype != MT_ARRAY { + if !(*baseArray)[zindex].IsArray() { (*baseArray)[zindex] = *FromEmptyArray() } } else { diff --git a/internal/pkg/mlrval/mlrval_constants.go b/internal/pkg/mlrval/mlrval_constants.go index bbc6dbe1d..bfaacf77f 100644 --- a/internal/pkg/mlrval/mlrval_constants.go +++ b/internal/pkg/mlrval/mlrval_constants.go @@ -12,7 +12,7 @@ package mlrval func MlrvalFromPending() Mlrval { return Mlrval{ mvtype: MT_PENDING, - printrep: INVALID_PRINTREP, + printrep: "(bug-if-you-see-this:case-3)", // INVALID_PRINTREP, } } diff --git a/internal/pkg/mlrval/mlrval_infer.go b/internal/pkg/mlrval/mlrval_infer.go index 2c123fe17..c168cf639 100644 --- a/internal/pkg/mlrval/mlrval_infer.go +++ b/internal/pkg/mlrval/mlrval_infer.go @@ -6,7 +6,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/lib" ) -// TODO: no infer bool. Always false in this path. +// TODO: no infer-bool from data files. Always false in this path. // It's essential that we use mv.Type() not mv.mvtype since types are // JIT-computed on first access for most data-file values. See type.go for more diff --git a/internal/pkg/mlrval/mlrval_is.go b/internal/pkg/mlrval/mlrval_is.go index e8317e08f..abaa315c9 100644 --- a/internal/pkg/mlrval/mlrval_is.go +++ b/internal/pkg/mlrval/mlrval_is.go @@ -1,5 +1,9 @@ package mlrval +import ( + "github.com/johnkerl/miller/internal/pkg/lib" +) + // It's essential that we use mv.Type() not mv.mvtype since types are // JIT-computed on first access for most data-file values. See type.go for more // information. @@ -8,8 +12,9 @@ func (mv *Mlrval) IsLegit() bool { return mv.Type() >= MT_VOID } +// TODO: comment no JIT-infer here -- absent is non-inferrable and we needn't take the expense of JIT. func (mv *Mlrval) IsErrorOrAbsent() bool { - t := mv.Type() + t := mv.mvtype return t == MT_ERROR || t == MT_ABSENT } @@ -17,21 +22,30 @@ func (mv *Mlrval) IsError() bool { return mv.Type() == MT_ERROR } +// TODO: comment no JIT-infer here -- absent is non-inferrable and we needn't take the expense of JIT. func (mv *Mlrval) IsAbsent() bool { - return mv.Type() == MT_ABSENT + return mv.mvtype == MT_ABSENT } +// TODO: comment no JIT-infer here -- NULL is non-inferrable and we needn't take the expense of JIT. +// This is a literal in JSON files, or else explicitly set to NULL. func (mv *Mlrval) IsNull() bool { - return mv.Type() == MT_NULL + return mv.mvtype == MT_NULL } func (mv *Mlrval) IsVoid() bool { - return mv.Type() == MT_VOID + if mv.mvtype == MT_VOID { + return true + } + if mv.mvtype == MT_PENDING && mv.printrep == "" { + lib.InternalCodingErrorIf(!mv.printrepValid) + return true + } + return false } func (mv *Mlrval) IsErrorOrVoid() bool { - t := mv.Type() - return t == MT_ERROR || t == MT_VOID + return mv.IsError() || mv.IsVoid() } // * Error is non-empty @@ -40,14 +54,17 @@ func (mv *Mlrval) IsErrorOrVoid() bool { // * Empty string is empty // * Int/float/bool/array/map are all non-empty func (mv *Mlrval) IsEmptyString() bool { - t := mv.Type() - if t == MT_VOID { + if mv.mvtype == MT_VOID { return true - } else if t == MT_STRING { - return mv.printrep == "" - } else { - return false } + if mv.mvtype == MT_STRING && mv.printrep == "" { + return true + } + if mv.mvtype == MT_PENDING && mv.printrep == "" { + lib.InternalCodingErrorIf(!mv.printrepValid) + return true + } + return false } func (mv *Mlrval) IsString() bool { @@ -109,5 +126,5 @@ func (mv *Mlrval) IsArrayOrMap() bool { } func (mv *Mlrval) IsFunction() bool { - return mv.Type() == MT_FUNC + return mv.mvtype == MT_FUNC } diff --git a/internal/pkg/mlrval/mlrval_json.go b/internal/pkg/mlrval/mlrval_json.go index 9e17b49d8..3da3368aa 100644 --- a/internal/pkg/mlrval/mlrval_json.go +++ b/internal/pkg/mlrval/mlrval_json.go @@ -271,7 +271,7 @@ func (mv *Mlrval) marshalJSONAux( elementNestingDepth int, outputIsStdout bool, ) (string, error) { - switch mv.mvtype { + switch mv.Type() { case MT_PENDING: return mv.marshalJSONPending(outputIsStdout) break diff --git a/internal/pkg/mlrval/mlrval_new.go b/internal/pkg/mlrval/mlrval_new.go index 2e37c1dd0..507f5ad53 100644 --- a/internal/pkg/mlrval/mlrval_new.go +++ b/internal/pkg/mlrval/mlrval_new.go @@ -14,7 +14,7 @@ import ( func FromPending() *Mlrval { return &Mlrval{ mvtype: MT_PENDING, - printrep: "(bug-if-you-see-this-pending)", + printrep: "(bug-if-you-see-this:case-1)", printrepValid: false, } } @@ -210,9 +210,9 @@ func FromFunction(funcval interface{}, name string) *Mlrval { func FromArray(arrayval []Mlrval) *Mlrval { return &Mlrval{ mvtype: MT_ARRAY, - printrep: INVALID_PRINTREP, - printrepValid: true, - arrayval: arrayval, + printrep: "(bug-if-you-see-this:case-4)", // INVALID_PRINTREP, + printrepValid: false, + arrayval: CopyMlrvalArray(arrayval), } } @@ -223,9 +223,9 @@ func FromEmptyArray() *Mlrval { func FromMap(mapval *Mlrmap) *Mlrval { return &Mlrval{ mvtype: MT_MAP, - printrep: INVALID_PRINTREP, - printrepValid: true, - mapval: mapval, + printrep: "(bug-if-you-see-this:case-5)", // INVALID_PRINTREP, + printrepValid: false, + mapval: mapval.Copy(), } } diff --git a/internal/pkg/mlrval/mlrval_output.go b/internal/pkg/mlrval/mlrval_output.go index 5d74dfd25..5f0e27a32 100644 --- a/internal/pkg/mlrval/mlrval_output.go +++ b/internal/pkg/mlrval/mlrval_output.go @@ -1,6 +1,8 @@ package mlrval import ( + "fmt" + "os" "strconv" ) @@ -28,7 +30,7 @@ func (mv *Mlrval) setPrintRep() { // Should not have gotten outside of the JSON decoder, so flag this // clearly visually if it should (buggily) slip through to // user-level visibility. - mv.printrep = "(bug-if-you-see-this)" // xxx constdef at top of file + mv.printrep = "(bug-if-you-see-this:case=3)" // xxx constdef at top of file break case MT_ERROR: @@ -39,7 +41,7 @@ func (mv *Mlrval) setPrintRep() { // Callsites should be using absence to do non-assigns, so flag // this clearly visually if it should (buggily) slip through to // user-level visibility. - mv.printrep = "(bug-if-you-see-this)" // xxx constdef at top of file + mv.printrep = "(bug-if-you-see-this:case=4)" // xxx constdef at top of file break case MT_VOID: @@ -66,26 +68,24 @@ func (mv *Mlrval) setPrintRep() { break case MT_ARRAY: - panic("mlrval array-to-string unimplemented, pending refactor") - //bytes, err := mv.MarshalJSON(JSON_MULTILINE, false) - //// maybe just InternalCodingErrorIf(err != nil) - //if err != nil { - // fmt.Fprintln(os.Stderr, err) - // os.Exit(1) - //} - //mv.printrep = string(bytes) + bytes, err := mv.MarshalJSON(JSON_MULTILINE, false) + // maybe just InternalCodingErrorIf(err != nil) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + mv.printrep = string(bytes) break case MT_MAP: - panic("mlrval map-to-string unimplemented, pending refactor") - //bytes, err := mv.MarshalJSON(JSON_MULTILINE, false) - //// maybe just InternalCodingErrorIf(err != nil) - //if err != nil { - // fmt.Fprintln(os.Stderr, err) - // os.Exit(1) - //} - //mv.printrep = string(bytes) + bytes, err := mv.MarshalJSON(JSON_MULTILINE, false) + // maybe just InternalCodingErrorIf(err != nil) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + mv.printrep = string(bytes) break } diff --git a/internal/pkg/mlrval/mlrval_type.go b/internal/pkg/mlrval/mlrval_type.go index 3b46aa610..4b5d9bef6 100644 --- a/internal/pkg/mlrval/mlrval_type.go +++ b/internal/pkg/mlrval/mlrval_type.go @@ -69,12 +69,12 @@ type Mlrval struct { mapval *Mlrmap // First-class-function literals from internal/pkg/dsl/cst. // Interfaced here to avoid package-dependency cycles. - funcval interface{} + funcval interface{} } -const INVALID_PRINTREP = "(bug-if-you-see-this)" +const INVALID_PRINTREP = "(bug-if-you-see-this:case-2)" const ERROR_PRINTREP = "(error)" -const ABSENT_PRINTREP = "(error)" +const ABSENT_PRINTREP = "(absent)" // Enumeration for mlrval types // diff --git a/internal/pkg/mlrval/tbf.go b/internal/pkg/mlrval/tbf.go index b73b03f9a..09063757d 100644 --- a/internal/pkg/mlrval/tbf.go +++ b/internal/pkg/mlrval/tbf.go @@ -82,7 +82,7 @@ func (mv *Mlrval) Arrayify() *Mlrval { // } //} // -//func MlrvalFromMap(mlrmap *Mlrmap) *Mlrval { +//func FromMap(mlrmap *Mlrmap) *Mlrval { // mv := MlrvalFromEmptyMap() // if mlrmap == nil { // // TODO maybe return 2nd-arg error in the API diff --git a/scripts/time-big-files b/scripts/time-big-files index 6c5ba45b0..e74f06922 100755 --- a/scripts/time-big-files +++ b/scripts/time-big-files @@ -4,35 +4,35 @@ ourdir=$(dirname $0) $ourdir/time-big-file dkvp mlr5 $ourdir/time-big-file dkvp ~/tmp/miller/mlr -$ourdir/time-big-file dkvp mlr $ourdir/time-big-file dkvp 'mlr -S' +$ourdir/time-big-file dkvp mlr echo $ourdir/time-big-file nidx mlr5 $ourdir/time-big-file nidx ~/tmp/miller/mlr -$ourdir/time-big-file nidx mlr $ourdir/time-big-file nidx 'mlr -S' +$ourdir/time-big-file nidx mlr echo $ourdir/time-big-file xtab mlr5 $ourdir/time-big-file xtab ~/tmp/miller/mlr -$ourdir/time-big-file xtab mlr $ourdir/time-big-file xtab 'mlr -S' +$ourdir/time-big-file xtab mlr echo $ourdir/time-big-file csv mlr5 $ourdir/time-big-file csv ~/tmp/miller/mlr -$ourdir/time-big-file csv mlr $ourdir/time-big-file csv 'mlr -S' +$ourdir/time-big-file csv mlr echo $ourdir/time-big-file csvlite mlr5 $ourdir/time-big-file csvlite ~/tmp/miller/mlr -$ourdir/time-big-file csvlite mlr $ourdir/time-big-file csvlite 'mlr -S' +$ourdir/time-big-file csvlite mlr echo $ourdir/time-big-file json mlr5 $ourdir/time-big-file json ~/tmp/miller/mlr -$ourdir/time-big-file json mlr $ourdir/time-big-file json 'mlr -S' +$ourdir/time-big-file json mlr diff --git a/todo.txt b/todo.txt index 1f3bdfce7..927316543 100644 --- a/todo.txt +++ b/todo.txt @@ -8,6 +8,16 @@ PUNCHDOWN LIST o IsArrayOrMap infers all -- it should not & it should say why not o fix: mlr --from test/input/abixy --opprint put -v -f test/cases/dsl-for-srec-loops/0005/mlr + -> for(k,v in $*) {...} while modifying $* in-loop: $* isn't being copied + o fix: + mlr --from data/small put 'dump $*' + o look at pre/post FromMap; also check PutReference/PutCopy callsites + o UTs for gd-2 + - mlrval_cmp.go + - JSON I/O + - mv from-array/from-map w/ copy & mutate orig & check new -- & vice versa + - dash-O and octal infer + o grep tmiller map put ref vs map put copy * perf next: o batchify source-quench @@ -47,6 +57,7 @@ PUNCHDOWN LIST o mods: ! need cmps for array/map -- no absent/error anymore here ! moreover, proofread all mlrval_cmp.go dispo mxes + ! array/map fields: marshal as JSON_SINGLE_LINE - replace rac-list with batch-array -- ? profile to see what value. - unaliasArrayIndex in multiple packages for cycdep break -- ? - randsel for arrays From 5ba8d494f841764ba917d60bd003adde4c527914 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 20 Dec 2021 18:19:12 -0500 Subject: [PATCH 15/19] bugfix in ofmt formatting --- internal/pkg/mlrval/mlrval_output.go | 3 ++- todo.txt | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/pkg/mlrval/mlrval_output.go b/internal/pkg/mlrval/mlrval_output.go index 5f0e27a32..b3cd4d10c 100644 --- a/internal/pkg/mlrval/mlrval_output.go +++ b/internal/pkg/mlrval/mlrval_output.go @@ -12,7 +12,8 @@ func (mv Mlrval) String() string { // TODO: comment re deferral -- important perf effect! // if mv.IsFloat() && floatOutputFormatter != nil // if mv.mvtype == MT_FLOAT && floatOutputFormatter != nil { - if floatOutputFormatter != nil && (mv.mvtype == MT_FLOAT || mv.mvtype == MT_PENDING) { + //if floatOutputFormatter != nil && (mv.mvtype == MT_FLOAT || mv.mvtype == MT_PENDING) { + if floatOutputFormatter != nil && mv.Type() == MT_FLOAT { // Use the format string from global --ofmt, if supplied return floatOutputFormatter.FormatFloat(mv.floatval) } else { diff --git a/todo.txt b/todo.txt index 927316543..10b6dc53d 100644 --- a/todo.txt +++ b/todo.txt @@ -11,6 +11,8 @@ PUNCHDOWN LIST -> for(k,v in $*) {...} while modifying $* in-loop: $* isn't being copied o fix: mlr --from data/small put 'dump $*' + ! fix + echo x=0xff | mlr --oxtab --ofmt '%f' cat o look at pre/post FromMap; also check PutReference/PutCopy callsites o UTs for gd-2 - mlrval_cmp.go From 6274e611262746e53ce1dd70c050e21a79b67adc Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 20 Dec 2021 22:12:39 -0500 Subject: [PATCH 16/19] bugfix in octal-supporess --- internal/pkg/mlrval/mlrval_infer.go | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/internal/pkg/mlrval/mlrval_infer.go b/internal/pkg/mlrval/mlrval_infer.go index c168cf639..e445d04ad 100644 --- a/internal/pkg/mlrval/mlrval_infer.go +++ b/internal/pkg/mlrval/mlrval_infer.go @@ -1,6 +1,7 @@ package mlrval import ( + "regexp" "strings" "github.com/johnkerl/miller/internal/pkg/lib" @@ -72,28 +73,24 @@ func inferNormally(mv *Mlrval, input string, inferBool bool) *Mlrval { return mv.SetFromString(input) } +var octalDetector = regexp.MustCompile("^-?0[0-9]+") + func inferWithOctalSuppress(mv *Mlrval, input string, inferBool bool) *Mlrval { inferNormally(mv, input, inferBool) - if mv.mvtype == MT_INT { - if input[0] == '0' && len(input) > 1 { - c := input[1] - if c != 'x' && c != 'X' && c != 'b' && c != 'B' { - return mv.SetFromString(input) - } - } - if strings.HasPrefix(input, "-0") && len(input) > 2 { - c := input[2] - if c != 'x' && c != 'X' && c != 'b' && c != 'B' { - return mv.SetFromString(input) - } - } + if mv.mvtype != MT_INT && mv.mvtype != MT_FLOAT { + return mv + } + + if octalDetector.MatchString(mv.printrep) { + return mv.SetFromString(input) + } else { + return mv } - return mv } func inferWithIntAsFloat(mv *Mlrval, input string, inferBool bool) *Mlrval { inferNormally(mv, input, inferBool) - if mv.mvtype == MT_INT { + if mv.Type() == MT_INT { mv.floatval = float64(mv.intval) mv.mvtype = MT_FLOAT } From fc665c5b0d1cc6ad2cb00526c1dafa59a27d3045 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 20 Dec 2021 22:17:21 -0500 Subject: [PATCH 17/19] go fmt --- Makefile | 5 +- internal/pkg/bifs/collections.go | 2 +- internal/pkg/bifs/mathlib.go | 78 ++++++++++++++++--------- internal/pkg/dsl/cst/cond.go | 2 +- internal/pkg/mlrval/mlrmap_accessors.go | 2 +- internal/pkg/mlrval/mlrval_format.go | 1 - internal/pkg/mlrval/mlrval_get.go | 1 - internal/pkg/runtime/state.go | 1 - internal/pkg/transformers/gap.go | 2 +- 9 files changed, 58 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index 310e562c7..187182701 100644 --- a/Makefile +++ b/Makefile @@ -161,8 +161,11 @@ tests-in-order: mlrval-tests mlrmap-tests input-tests bifs-tests regression-test: go test -v regression_test.go +# go fmt ./... finds experimental C files which we want to ignore. fmt: - -go fmt ./... + -go fmt ./cmd/... + -go fmt ./internal/pkg/... + -go fmt ./regression_test.go # ---------------------------------------------------------------- # For developers before pushing to GitHub. diff --git a/internal/pkg/bifs/collections.go b/internal/pkg/bifs/collections.go index 602b15c96..a5ecb028d 100644 --- a/internal/pkg/bifs/collections.go +++ b/internal/pkg/bifs/collections.go @@ -640,7 +640,7 @@ func BIF_get_values(input1 *mlrval.Mlrval) *mlrval.Mlrval { for i, value := range inputarrayval { arrayval[i] = *value.Copy() } - return mlrval.FromArray(arrayval) + return mlrval.FromArray(arrayval) } else { return mlrval.ERROR diff --git a/internal/pkg/bifs/mathlib.go b/internal/pkg/bifs/mathlib.go index 33bc36798..698791907 100644 --- a/internal/pkg/bifs/mathlib.go +++ b/internal/pkg/bifs/mathlib.go @@ -58,29 +58,45 @@ var mudispo = [mlrval.MT_DIM]mathLibUnaryFuncWrapper{ /*FUNC */ _math_unary_erro1, } -func BIF_acos(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Acos) } -func BIF_acosh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Acosh) } -func BIF_asin(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Asin) } -func BIF_asinh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Asinh) } -func BIF_atan(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Atan) } -func BIF_atanh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Atanh) } -func BIF_cbrt(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Cbrt) } -func BIF_cos(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Cos) } -func BIF_cosh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Cosh) } -func BIF_erf(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Erf) } -func BIF_erfc(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Erfc) } -func BIF_exp(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Exp) } -func BIF_expm1(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Expm1) } -func BIF_invqnorm(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, lib.Invqnorm) } -func BIF_log(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Log) } -func BIF_log10(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Log10) } -func BIF_log1p(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Log1p) } -func BIF_qnorm(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, lib.Qnorm) } -func BIF_sin(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Sin) } -func BIF_sinh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Sinh) } -func BIF_sqrt(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Sqrt) } -func BIF_tan(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Tan) } -func BIF_tanh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Tanh) } +func BIF_acos(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Acos) } +func BIF_acosh(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, math.Acosh) +} +func BIF_asin(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Asin) } +func BIF_asinh(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, math.Asinh) +} +func BIF_atan(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Atan) } +func BIF_atanh(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, math.Atanh) +} +func BIF_cbrt(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Cbrt) } +func BIF_cos(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Cos) } +func BIF_cosh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Cosh) } +func BIF_erf(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Erf) } +func BIF_erfc(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Erfc) } +func BIF_exp(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Exp) } +func BIF_expm1(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, math.Expm1) +} +func BIF_invqnorm(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, lib.Invqnorm) +} +func BIF_log(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Log) } +func BIF_log10(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, math.Log10) +} +func BIF_log1p(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, math.Log1p) +} +func BIF_qnorm(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, lib.Qnorm) +} +func BIF_sin(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Sin) } +func BIF_sinh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Sinh) } +func BIF_sqrt(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Sqrt) } +func BIF_tan(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Tan) } +func BIF_tanh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Tanh) } // Disposition vector for unary mathlib functions which are int-preserving var imudispo = [mlrval.MT_DIM]mathLibUnaryFuncWrapper{ @@ -98,11 +114,17 @@ var imudispo = [mlrval.MT_DIM]mathLibUnaryFuncWrapper{ } // Int-preserving -func BIF_abs(input1 *mlrval.Mlrval) *mlrval.Mlrval { return imudispo[input1.Type()](input1, math.Abs) } // xxx -func BIF_ceil(input1 *mlrval.Mlrval) *mlrval.Mlrval { return imudispo[input1.Type()](input1, math.Ceil) } // xxx -func BIF_floor(input1 *mlrval.Mlrval) *mlrval.Mlrval { return imudispo[input1.Type()](input1, math.Floor) } // xxx -func BIF_round(input1 *mlrval.Mlrval) *mlrval.Mlrval { return imudispo[input1.Type()](input1, math.Round) } // xxx -func BIF_sgn(input1 *mlrval.Mlrval) *mlrval.Mlrval { return imudispo[input1.Type()](input1, lib.Sgn) } // xxx +func BIF_abs(input1 *mlrval.Mlrval) *mlrval.Mlrval { return imudispo[input1.Type()](input1, math.Abs) } // xxx +func BIF_ceil(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return imudispo[input1.Type()](input1, math.Ceil) +} // xxx +func BIF_floor(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return imudispo[input1.Type()](input1, math.Floor) +} // xxx +func BIF_round(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return imudispo[input1.Type()](input1, math.Round) +} // xxx +func BIF_sgn(input1 *mlrval.Mlrval) *mlrval.Mlrval { return imudispo[input1.Type()](input1, lib.Sgn) } // xxx // ================================================================ // Exponentiation: DSL operator '**'. See also diff --git a/internal/pkg/dsl/cst/cond.go b/internal/pkg/dsl/cst/cond.go index fb79bbcfd..28e6cf336 100644 --- a/internal/pkg/dsl/cst/cond.go +++ b/internal/pkg/dsl/cst/cond.go @@ -10,8 +10,8 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" - "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/mlrval" + "github.com/johnkerl/miller/internal/pkg/runtime" ) type CondBlockNode struct { diff --git a/internal/pkg/mlrval/mlrmap_accessors.go b/internal/pkg/mlrval/mlrmap_accessors.go index 33592f33c..47e0634de 100644 --- a/internal/pkg/mlrval/mlrmap_accessors.go +++ b/internal/pkg/mlrval/mlrmap_accessors.go @@ -770,7 +770,7 @@ func (mlrmap *Mlrmap) IsNested() bool { if mlrmap.Head == nil { return false } else if mlrmap.Head.Value.GetMap() == nil { - //TODO: check IsArrayOrMap() + //TODO: check IsArrayOrMap() return false } else { return true diff --git a/internal/pkg/mlrval/mlrval_format.go b/internal/pkg/mlrval/mlrval_format.go index ae3bf2810..8ecc8df93 100644 --- a/internal/pkg/mlrval/mlrval_format.go +++ b/internal/pkg/mlrval/mlrval_format.go @@ -195,7 +195,6 @@ func (formatter *formatterToInt) FormatFloat(floatValue float64) string { return fmt.Sprintf(formatter.goFormatString, int(floatValue)) } - // ---------------------------------------------------------------- type formatterToString struct { diff --git a/internal/pkg/mlrval/mlrval_get.go b/internal/pkg/mlrval/mlrval_get.go index 88706199b..3c411088c 100644 --- a/internal/pkg/mlrval/mlrval_get.go +++ b/internal/pkg/mlrval/mlrval_get.go @@ -92,7 +92,6 @@ func GetTypeName(mvtype MVType) string { return TYPE_NAMES[mvtype] } - // These are for built-in functions operating within type-keyed // disposition-vector/disposition-matrix context. They've already computed // mv.Type() -- it's a fatal error if they haven't -- and they need the typed diff --git a/internal/pkg/runtime/state.go b/internal/pkg/runtime/state.go index aca8fcee2..7df865705 100644 --- a/internal/pkg/runtime/state.go +++ b/internal/pkg/runtime/state.go @@ -16,7 +16,6 @@ import ( ) type State struct { - Inrec *mlrval.Mlrmap Context *types.Context Oosvars *mlrval.Mlrmap diff --git a/internal/pkg/transformers/gap.go b/internal/pkg/transformers/gap.go index 881406c1e..c0f6de7a0 100644 --- a/internal/pkg/transformers/gap.go +++ b/internal/pkg/transformers/gap.go @@ -6,8 +6,8 @@ import ( "os" "strings" - "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) From 30bcb69e2e0419d99b7a081c5494b87d590f15c5 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 20 Dec 2021 22:44:44 -0500 Subject: [PATCH 18/19] neaten --- internal/pkg/mlrval/mlrmap_accessors.go | 5 +---- internal/pkg/mlrval/mlrval_is.go | 5 +++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/pkg/mlrval/mlrmap_accessors.go b/internal/pkg/mlrval/mlrmap_accessors.go index 47e0634de..5fccacbae 100644 --- a/internal/pkg/mlrval/mlrmap_accessors.go +++ b/internal/pkg/mlrval/mlrmap_accessors.go @@ -209,10 +209,7 @@ func (mlrmap *Mlrmap) findEntryByPositionalIndex(position int) *MlrmapEntry { } func (mlrmap *Mlrmap) PutCopyWithMlrvalIndex(key *Mlrval, value *Mlrval) error { - if key.IsString() { - mlrmap.PutCopy(key.printrep, value) - return nil - } else if key.IsInt() { + if key.IsStringOrInt() { mlrmap.PutCopy(key.String(), value) return nil } else { diff --git a/internal/pkg/mlrval/mlrval_is.go b/internal/pkg/mlrval/mlrval_is.go index abaa315c9..6fbb51f9b 100644 --- a/internal/pkg/mlrval/mlrval_is.go +++ b/internal/pkg/mlrval/mlrval_is.go @@ -76,6 +76,11 @@ func (mv *Mlrval) IsStringOrVoid() bool { return t == MT_STRING || t == MT_VOID } +func (mv *Mlrval) IsStringOrInt() bool { + t := mv.Type() + return t == MT_STRING || t == MT_VOID || t == MT_INT +} + func (mv *Mlrval) IsInt() bool { return mv.Type() == MT_INT } From 79b5bc1180d040f059bd13650f336122dd1f17ee Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 20 Dec 2021 23:25:36 -0500 Subject: [PATCH 19/19] regression tests all passing --- internal/pkg/mlrval/mlrval_accessors.go | 4 +- internal/pkg/mlrval/mlrval_collections.go | 92 ++++++++- internal/pkg/mlrval/mlrval_get.go | 20 ++ internal/pkg/mlrval/tbf.go | 235 ---------------------- 4 files changed, 111 insertions(+), 240 deletions(-) delete mode 100644 internal/pkg/mlrval/tbf.go diff --git a/internal/pkg/mlrval/mlrval_accessors.go b/internal/pkg/mlrval/mlrval_accessors.go index 366485957..90a6d73fc 100644 --- a/internal/pkg/mlrval/mlrval_accessors.go +++ b/internal/pkg/mlrval/mlrval_accessors.go @@ -93,9 +93,7 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { return *FromMap(retval) } -// ---------------------------------------------------------------- -// Used by stats1. - +// Increment is used by stats1. func (mv *Mlrval) Increment() { if mv.mvtype == MT_INT { mv.intval++ diff --git a/internal/pkg/mlrval/mlrval_collections.go b/internal/pkg/mlrval/mlrval_collections.go index 52287460d..fdf225d8b 100644 --- a/internal/pkg/mlrval/mlrval_collections.go +++ b/internal/pkg/mlrval/mlrval_collections.go @@ -72,6 +72,7 @@ import ( "errors" "fmt" "os" + "strconv" "github.com/johnkerl/miller/internal/pkg/lib" ) @@ -299,10 +300,10 @@ func (mv *Mlrval) PutIndexed(indices []*Mlrval, rvalue *Mlrval) error { } else { baseIndex := indices[0] if baseIndex.IsString() { - mv = FromEmptyMap() + *mv = *FromEmptyMap() return putIndexedOnMap(mv.mapval, indices, rvalue) } else if baseIndex.IsInt() { - mv = FromEmptyArray() + *mv = *FromEmptyArray() return putIndexedOnArray(&mv.arrayval, indices, rvalue) } else { return errors.New( @@ -654,3 +655,90 @@ func BsearchMlrvalArrayForAscendingInsert( return lo } + +// NewMlrvalForAutoDeepen is for auto-deepen of nested maps in things like +// +// $foo[1]["a"][2]["b"] = 3 +// +// Autocreated levels are maps. Array levels can be explicitly created e.g. +// +// $foo[1]["a"] ??= [] +// $foo[1]["a"][2]["b"] = 3 +func NewMlrvalForAutoDeepen(mvtype MVType) (*Mlrval, error) { + if mvtype == MT_STRING || mvtype == MT_INT { + empty := FromEmptyMap() + return empty, nil + } else { + return nil, errors.New( + "mlr: indices must be string, int, or array thereof; got " + GetTypeName(mvtype), + ) + } +} + +func (mv *Mlrval) Arrayify() *Mlrval { + if mv.IsMap() { + if mv.mapval.IsEmpty() { + return mv + } + + convertible := true + i := 0 + for pe := mv.mapval.Head; pe != nil; pe = pe.Next { + sval := strconv.Itoa(i + 1) // Miller user-space indices are 1-up + i++ + if pe.Key != sval { + convertible = false + } + pe.Value = pe.Value.Arrayify() + } + + if convertible { + arrayval := make([]Mlrval, mv.mapval.FieldCount) + i := 0 + for pe := mv.mapval.Head; pe != nil; pe = pe.Next { + arrayval[i] = *pe.Value.Copy() + i++ + } + return FromArray(arrayval) + + } else { + return mv + } + + } else if mv.IsArray() { + // TODO: comment (or rethink) that this modifies its inputs!! + output := mv.Copy() + for i := range mv.arrayval { + output.arrayval[i] = *output.arrayval[i].Arrayify() + } + return output + + } else { + return mv + } +} + +func LengthenMlrvalArray(array *[]Mlrval, newLength64 int) { + newLength := int(newLength64) + lib.InternalCodingErrorIf(newLength <= len(*array)) + + if newLength <= cap(*array) { + newArray := (*array)[:newLength] + for zindex := len(*array); zindex < newLength; zindex++ { + // TODO: comment why not MT_ABSENT or MT_VOID + newArray[zindex] = *NULL + } + *array = newArray + } else { + newArray := make([]Mlrval, newLength, 2*newLength) + zindex := 0 + for zindex = 0; zindex < len(*array); zindex++ { + newArray[zindex] = (*array)[zindex] + } + for zindex = len(*array); zindex < newLength; zindex++ { + // TODO: comment why not MT_ABSENT or MT_VOID + newArray[zindex] = *NULL + } + *array = newArray + } +} diff --git a/internal/pkg/mlrval/mlrval_get.go b/internal/pkg/mlrval/mlrval_get.go index 3c411088c..072e75595 100644 --- a/internal/pkg/mlrval/mlrval_get.go +++ b/internal/pkg/mlrval/mlrval_get.go @@ -1,6 +1,9 @@ package mlrval import ( + "fmt" + "os" + "github.com/johnkerl/miller/internal/pkg/lib" ) @@ -126,3 +129,20 @@ func (mv *Mlrval) AcquireMapValue() *Mlrmap { lib.InternalCodingErrorIf(mv.mvtype != MT_MAP) return mv.mapval } + +func (mv *Mlrval) GetNumericToFloatValueOrDie() (floatValue float64) { + floatValue, ok := mv.GetNumericToFloatValue() + if !ok { + fmt.Fprintf( + os.Stderr, + "%s: couldn't parse \"%s\" as number.", + "mlr", mv.String(), + ) + os.Exit(1) + } + return floatValue +} + +func (mv *Mlrval) AssertNumeric() { + _ = mv.GetNumericToFloatValueOrDie() +} diff --git a/internal/pkg/mlrval/tbf.go b/internal/pkg/mlrval/tbf.go deleted file mode 100644 index 09063757d..000000000 --- a/internal/pkg/mlrval/tbf.go +++ /dev/null @@ -1,235 +0,0 @@ -package mlrval - -import ( - "errors" - "fmt" - "os" - "strconv" - - "github.com/johnkerl/miller/internal/pkg/lib" -) - -// things to be filed - -// NewMlrvalForAutoDeepen is for auto-deepen of nested maps in things like -// -// $foo[1]["a"][2]["b"] = 3 -// -// Autocreated levels are maps. Array levels can be explicitly created e.g. -// -// $foo[1]["a"] ??= [] -// $foo[1]["a"][2]["b"] = 3 -func NewMlrvalForAutoDeepen(mvtype MVType) (*Mlrval, error) { - if mvtype == MT_STRING || mvtype == MT_INT { - empty := FromEmptyMap() - return empty, nil - } else { - return nil, errors.New( - "mlr: indices must be string, int, or array thereof; got " + GetTypeName(mvtype), - ) - } -} - -func (mv *Mlrval) Arrayify() *Mlrval { - if mv.IsMap() { - if mv.mapval.IsEmpty() { - return mv - } - - convertible := true - i := 0 - for pe := mv.mapval.Head; pe != nil; pe = pe.Next { - sval := strconv.Itoa(i + 1) // Miller user-space indices are 1-up - i++ - if pe.Key != sval { - convertible = false - } - pe.Value = pe.Value.Arrayify() - } - - if convertible { - arrayval := make([]Mlrval, mv.mapval.FieldCount) - i := 0 - for pe := mv.mapval.Head; pe != nil; pe = pe.Next { - arrayval[i] = *pe.Value.Copy() - i++ - } - return FromArray(arrayval) - - } else { - return mv - } - - } else if mv.IsArray() { - // TODO: comment (or rethink) that this modifies its inputs!! - output := mv.Copy() - for i := range mv.arrayval { - output.arrayval[i] = *output.arrayval[i].Arrayify() - } - return output - - } else { - return mv - } -} - -//func MlrvalFromEmptyMap() *Mlrval { -// return &Mlrval{ -// mvtype: MT_MAP, -// printrep: "(bug-if-you-see-this-map-type)", -// printrepValid: false, -// mapval: NewMlrmap(), -// } -//} -// -//func FromMap(mlrmap *Mlrmap) *Mlrval { -// mv := MlrvalFromEmptyMap() -// if mlrmap == nil { -// // TODO maybe return 2nd-arg error in the API -// return ERROR -// } -// -// for pe := mlrmap.Head; pe != nil; pe = pe.Next { -// mv.mapval.PutCopy(pe.Key, pe.Value) -// } -// return mv -//} -// -//// Like previous but doesn't copy. Only safe when the argument's sole purpose -//// is to be passed into here. -//func MlrvalFromMapReferenced(mlrmap *Mlrmap) *Mlrval { -// mv := MlrvalFromEmptyMap() -// if mlrmap == nil { -// // xxx maybe return 2nd-arg error in the API -// return ERROR -// } -// -// for pe := mlrmap.Head; pe != nil; pe = pe.Next { -// mv.mapval.PutReference(pe.Key, pe.Value) -// } -// return mv -//} -// -//// Does not copy the data. We can make a MlrvalFromArrayLiteralCopy if needed, -//// using values.CopyMlrvalArray(). -//func MlrvalEmptyArray() Mlrval { -// return Mlrval{ -// mvtype: MT_ARRAY, -// printrep: "(bug-if-you-see-this-array-type)", -// printrepValid: false, -// intval: 0, -// floatval: 0.0, -// boolval: false, -// arrayval: make([]Mlrval, 0, 10), -// mapval: nil, -// } -//} -// -//// Users can do things like '$new[1][2][3] = 4' even if '$new' isn't already -//// allocated. This function supports that. -//func NewSizedMlrvalArray(length int) *Mlrval { -// arrayval := make([]Mlrval, length, 2*length) -// -// for i := 0; i < int(length); i++ { -// arrayval[i] = *VOID -// } -// -// return &Mlrval{ -// mvtype: MT_ARRAY, -// printrep: "(bug-if-you-see-this-array-type)", -// printrepValid: false, -// intval: 0, -// floatval: 0.0, -// boolval: false, -// arrayval: arrayval, -// mapval: nil, -// } -//} -// -//// Does not copy the data. We can make a SetFromArrayLiteralCopy if needed -//// using values.CopyMlrvalArray(). -//func MlrvalFromArrayReference(input []Mlrval) *Mlrval { -// return &Mlrval{ -// mvtype: MT_ARRAY, -// printrepValid: false, -// arrayval: input, -// } -//} - -func LengthenMlrvalArray(array *[]Mlrval, newLength64 int) { - newLength := int(newLength64) - lib.InternalCodingErrorIf(newLength <= len(*array)) - - if newLength <= cap(*array) { - newArray := (*array)[:newLength] - for zindex := len(*array); zindex < newLength; zindex++ { - // TODO: comment why not MT_ABSENT or MT_VOID - newArray[zindex] = *NULL - } - *array = newArray - } else { - newArray := make([]Mlrval, newLength, 2*newLength) - zindex := 0 - for zindex = 0; zindex < len(*array); zindex++ { - newArray[zindex] = (*array)[zindex] - } - for zindex = len(*array); zindex < newLength; zindex++ { - // TODO: comment why not MT_ABSENT or MT_VOID - newArray[zindex] = *NULL - } - *array = newArray - } -} - -//// NewMlrvalForAutoDeepen is for auto-deepen of nested maps in things like -//// -//// $foo[1]["a"][2]["b"] = 3 -//// -//// Autocreated levels are maps. Array levels can be explicitly created e.g. -//// -//// $foo[1]["a"] ??= [] -//// $foo[1]["a"][2]["b"] = 3 -//func NewMlrvalForAutoDeepen(mvtype MVType) (*Mlrval, error) { -// if mvtype == MT_STRING || mvtype == MT_INT { -// empty := MlrvalFromEmptyMap() -// return empty, nil -// } else { -// return nil, errors.New( -// "mlr: indices must be string, int, or array thereof; got " + GetTypeName(mvtype), -// ) -// } -//} - -// TODO: FILE -// -//// MlrvalFromInferredTypeForDataFiles is for parsing field values directly from -//// data files (except JSON, which is typed -- "true" and true are distinct). -//// Mostly the same as MlrvalFromInferredType, except it doesn't auto-infer -//// true/false to bool; don't auto-infer NaN/Inf to float; etc. -//func MlrvalFromInferredTypeForDataFiles(input string) *Mlrval { -// return inferrer(input, false) -//} -// -//// MlrvalFromInferredType is for parsing field values not directly from data -//// files. Mostly the same as MlrvalFromInferredTypeForDataFiles, except it -//// auto-infers true/false to bool; don't auto-infer NaN/Inf to float; etc. -//func MlrvalFromInferredType(input string) *Mlrval { -// return inferrer(input, true) -//} - -func (mv *Mlrval) GetNumericToFloatValueOrDie() (floatValue float64) { - floatValue, ok := mv.GetNumericToFloatValue() - if !ok { - fmt.Fprintf( - os.Stderr, - "%s: couldn't parse \"%s\" as number.", - "mlr", mv.String(), - ) - os.Exit(1) - } - return floatValue -} - -func (mv *Mlrval) AssertNumeric() { - _ = mv.GetNumericToFloatValueOrDie() -}