diff --git a/.vimrc b/.vimrc index 18c0048c3..bef8ce9ef 100644 --- a/.vimrc +++ b/.vimrc @@ -1,2 +1,8 @@ map \d :w:!clear;echo Building ...; echo; make mlr -map \f :w:!clear;echo Building ...; echo; make mall +map \f :w:!clear;echo Building ...; echo; make tests-in-order +"map \r :w:!clear;echo Building ...; echo; make mlrval-tests +"map \r :w:!clear;echo Building ...; echo; make mlrmap-tests +"map \r :w:!clear;echo Building ...; echo; make input-tests +"map \r :w:!clear;echo Building ...; echo; make mlrval-format-test +"map \r :w:!clear;echo Building ...; echo; make bifs-tests +map \r :w:!clear;echo Building ...; echo; make bifs-collections-test diff --git a/Makefile b/Makefile index 276dcae21..187182701 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ PREFIX=/usr/local INSTALLDIR=$(PREFIX)/bin +# ================================================================ +# General-use targets + # This must remain the first target in this file, which is what 'make' with no # arguments will run. build: @@ -14,38 +17,157 @@ check: unit_test regression_test @echo "by './configure --prefix=/your/install/path' if you wish to install to" @echo "somewhere other than /usr/local/bin -- the default prefix is /usr/local." -# Unit tests (small number) -unit_test: - go test github.com/johnkerl/miller/internal/pkg/... - -# Keystroke-savers -unbackslash_test: - go test $(ls internal/pkg/lib/*.go|grep -v test) internal/pkg/lib/unbackslash_test.go -mlrval_functions_test: - go test internal/pkg/types/mlrval_functions_test.go $(ls internal/pkg/types/*.go | grep -v test) -mlrval_format_test: - go test internal/pkg/types/mlrval_format_test.go $(ls internal/pkg/types/*.go|grep -v test) -regex_test: - go test internal/pkg/lib/regex_test.go internal/pkg/lib/regex.go - -# Regression tests (large number) -# -# See ./regression_test.go for information on how to get more details -# for debugging. TL;DR is for CI jobs, we have 'go test -v'; for -# interactive use, instead of 'go test -v' simply use 'mlr regtest -# -vvv' or 'mlr regtest -s 20'. See also internal/pkg/auxents/regtest. -regression_test: - go test -v regression_test.go - # DESTDIR is for package installs; nominally blank when this is run interactively. # See also https://www.gnu.org/prep/standards/html_node/DESTDIR.html install: build cp mlr $(DESTDIR)/$(INSTALLDIR) make -C man install -fmt: - -go fmt ./... +# ================================================================ +# Dev targets +# ---------------------------------------------------------------- +# Unit tests (small number) +unit-test ut: + go test github.com/johnkerl/miller/internal/pkg/... + +# Keystroke-savers +lib-unbackslash-test: + go test internal/pkg/lib/unbackslash_test.go internal/pkg/lib/unbackslash.go +lib_regex_test: + go test internal/pkg/lib/regex_test.go internal/pkg/lib/regex.go +lib-tests: + go test github.com/johnkerl/miller/internal/pkg/lib/... + +mlrval-new-test: + go test internal/pkg/mlrval/new_test.go \ + internal/pkg/mlrval/mlrval_type.go \ + internal/pkg/mlrval/mlrval_constants.go \ + internal/pkg/mlrval/mlrval_new.go \ + internal/pkg/mlrval/mlrval_infer.go +mlrval-is-test: + go test internal/pkg/mlrval/is_test.go \ + internal/pkg/mlrval/mlrval_type.go \ + internal/pkg/mlrval/mlrval_constants.go \ + internal/pkg/mlrval/mlrval_new.go \ + internal/pkg/mlrval/mlrval_infer.go \ + internal/pkg/mlrval/mlrval_is.go +mlrval-get-test: + go test internal/pkg/mlrval/get_test.go \ + internal/pkg/mlrval/mlrval_type.go \ + internal/pkg/mlrval/mlrval_constants.go \ + internal/pkg/mlrval/mlrval_new.go \ + internal/pkg/mlrval/mlrval_infer.go \ + internal/pkg/mlrval/mlrval_is.go \ + internal/pkg/mlrval/mlrval_get.go +mlrval-output-test: + go test internal/pkg/mlrval/output_test.go \ + internal/pkg/mlrval/mlrval_type.go \ + internal/pkg/mlrval/mlrval_constants.go \ + internal/pkg/mlrval/mlrval_new.go \ + internal/pkg/mlrval/mlrval_infer.go \ + internal/pkg/mlrval/mlrval_is.go \ + internal/pkg/mlrval/mlrval_get.go \ + internal/pkg/mlrval/mlrval_output.go \ + internal/pkg/mlrval/mlrval_format.go +mlrval-format-test: + go test internal/pkg/mlrval/format_test.go \ + internal/pkg/mlrval/mlrval_type.go \ + internal/pkg/mlrval/mlrval_constants.go \ + internal/pkg/mlrval/mlrval_new.go \ + internal/pkg/mlrval/mlrval_infer.go \ + internal/pkg/mlrval/mlrval_is.go \ + internal/pkg/mlrval/mlrval_get.go \ + internal/pkg/mlrval/mlrval_output.go \ + internal/pkg/mlrval/mlrval_format.go +mlrval-tests: + go test github.com/johnkerl/miller/internal/pkg/mlrval/... + +mlrmap-new-test: + go test internal/pkg/mlrval/mlrmap_new_test.go \ + internal/pkg/mlrval/mlrmap.go \ + internal/pkg/mlrval/mlrval_type.go \ + internal/pkg/mlrval/mlrval_constants.go \ + internal/pkg/mlrval/mlrval_new.go \ + internal/pkg/mlrval/mlrval_infer.go \ + internal/pkg/mlrval/mlrval_is.go \ + internal/pkg/mlrval/mlrval_get.go \ + internal/pkg/mlrval/mlrval_output.go \ + internal/pkg/mlrval/mlrval_format.go +mlrmap-accessors-test: + go test internal/pkg/mlrval/mlrmap_accessors_test.go \ + internal/pkg/mlrval/mlrmap.go \ + internal/pkg/mlrval/mlrmap_accessors.go \ + internal/pkg/mlrval/mlrval_type.go \ + internal/pkg/mlrval/mlrval_constants.go \ + internal/pkg/mlrval/mlrval_new.go \ + internal/pkg/mlrval/mlrval_cmp.go \ + internal/pkg/mlrval/mlrval_copy.go \ + internal/pkg/mlrval/mlrval_infer.go \ + internal/pkg/mlrval/mlrval_is.go \ + internal/pkg/mlrval/mlrval_get.go \ + internal/pkg/mlrval/mlrval_output.go \ + internal/pkg/mlrval/mlrval_format.go + +mlrmap-tests: mlrmap-new-test mlrmap-accessors-test + +input-dkvp-test: + go test internal/pkg/input/record_reader_dkvp_test.go \ + internal/pkg/input/record_reader.go \ + internal/pkg/input/record_reader_dkvp_nidx.go +input-tests: input-dkvp-test + +bifs-arithmetic-test: + go test internal/pkg/bifs/arithmetic_test.go \ + internal/pkg/bifs/base.go \ + internal/pkg/bifs/arithmetic.go +bifs-bits-test: + go test internal/pkg/bifs/bits_test.go \ + internal/pkg/bifs/base.go \ + internal/pkg/bifs/arithmetic.go \ + internal/pkg/bifs/bits.go +bifs-collections-test: + go test internal/pkg/bifs/collections_test.go \ + internal/pkg/bifs/base.go \ + internal/pkg/bifs/arithmetic.go \ + internal/pkg/bifs/collections.go +bifs-hashing-test: + go test internal/pkg/bifs/hashing_test.go \ + internal/pkg/bifs/base.go \ + internal/pkg/bifs/arithmetic.go \ + internal/pkg/bifs/hashing.go +bifs-sort-test: + go test internal/pkg/bifs/sort_test.go \ + internal/pkg/bifs/base.go \ + internal/pkg/bifs/arithmetic.go \ + internal/pkg/bifs/sort.go + +bifs-tests: bifs-arithmetic-test bifs-bits-test bifs-collections-test bifs-hashing-test bifs-sort-test + +#mlrval_functions_test: +# go test internal/pkg/mlrval/mlrval_functions_test.go $(ls internal/pkg/types/*.go | grep -v test) +#mlrval_format_test: +# go test internal/pkg/mlrval/mlrval_format_test.go $(ls internal/pkg/types/*.go|grep -v test) + +tests-in-order: mlrval-tests mlrmap-tests input-tests bifs-tests + +# ---------------------------------------------------------------- +# Regression tests (large number) +# +# See ./regression_test.go for information on how to get more details +# for debugging. TL;DR is for CI jobs, we have 'go test -v'; for +# interactive use, instead of 'go test -v' simply use 'mlr regtest +# -vvv' or 'mlr regtest -s 20'. See also internal/pkg/auxents/regtest. +regression-test: + go test -v regression_test.go + +# go fmt ./... finds experimental C files which we want to ignore. +fmt: + -go fmt ./cmd/... + -go fmt ./internal/pkg/... + -go fmt ./regression_test.go + +# ---------------------------------------------------------------- # For developers before pushing to GitHub. # # These steps are done in a particular order: @@ -67,6 +189,7 @@ dev: make -C docs @echo DONE +# ---------------------------------------------------------------- # Keystroke-savers it: build check so: install @@ -85,10 +208,12 @@ mprof5: go build github.com/johnkerl/miller/cmd/mprof5 mall: mprof5 mprof4 mprof3 mprof2 mprof mlr +# ---------------------------------------------------------------- # Please see comments in ./create-release-tarball as well as # https://miller.readthedocs.io/en/latest/build/#creating-a-new-release-for-developers release_tarball: build check ./create-release-tarball +# ================================================================ # Go does its own dependency management, outside of make. .PHONY: build mlr mprof mprof2 mprof3 mprof4 mprof5 check unit_test regression_test fmt dev diff --git a/cmd/mprof/main.go b/cmd/mprof/main.go index 1bd60daab..e7fa4c082 100644 --- a/cmd/mprof/main.go +++ b/cmd/mprof/main.go @@ -243,8 +243,8 @@ func (reader *RecordReaderDKVPNonPipelined) processHandle( func (reader *RecordReaderDKVPNonPipelined) recordFromDKVPLine( line string, -) *types.Mlrmap { - record := types.NewMlrmap() +) *mlrval.Mlrmap { + record := mlrval.NewMlrmap() var pairs []string if reader.readerOptions.IFSRegex == nil { // e.g. --no-ifs-regex @@ -268,11 +268,11 @@ func (reader *RecordReaderDKVPNonPipelined) recordFromDKVPLine( // "a=". Here we use the positional index as the key. This way // DKVP is a generalization of NIDX. key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := types.MlrvalFromInferredTypeForDataFiles(kv[0]) + value := mlrval.FromInferredTypeForDataFiles(kv[0]) record.PutReference(key, value) } else { key := kv[0] - value := types.MlrvalFromInferredTypeForDataFiles(kv[1]) + value := mlrval.FromInferredTypeForDataFiles(kv[1]) record.PutReference(key, value) } } @@ -379,8 +379,8 @@ func (reader *RecordReaderDKVPListPipelined) processHandle( func (reader *RecordReaderDKVPListPipelined) recordFromDKVPLine( line string, -) *types.Mlrmap { - record := types.NewMlrmap() +) *mlrval.Mlrmap { + record := mlrval.NewMlrmap() var pairs []string if reader.readerOptions.IFSRegex == nil { // e.g. --no-ifs-regex @@ -404,11 +404,11 @@ func (reader *RecordReaderDKVPListPipelined) recordFromDKVPLine( // "a=". Here we use the positional index as the key. This way // DKVP is a generalization of NIDX. key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := types.MlrvalFromInferredTypeForDataFiles(kv[0]) + value := mlrval.FromInferredTypeForDataFiles(kv[0]) record.PutReference(key, value) } else { key := kv[0] - value := types.MlrvalFromInferredTypeForDataFiles(kv[1]) + value := mlrval.FromInferredTypeForDataFiles(kv[1]) record.PutReference(key, value) } } @@ -531,8 +531,8 @@ func (reader *RecordReaderDKVPChanPipelined) processHandle( func (reader *RecordReaderDKVPChanPipelined) recordFromDKVPLine( line string, -) *types.Mlrmap { - record := types.NewMlrmap() +) *mlrval.Mlrmap { + record := mlrval.NewMlrmap() var pairs []string if reader.readerOptions.IFSRegex == nil { // e.g. --no-ifs-regex @@ -556,11 +556,11 @@ func (reader *RecordReaderDKVPChanPipelined) recordFromDKVPLine( // "a=". Here we use the positional index as the key. This way // DKVP is a generalization of NIDX. key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := types.MlrvalFromInferredTypeForDataFiles(kv[0]) + value := mlrval.FromInferredTypeForDataFiles(kv[0]) record.PutReference(key, value) } else { key := kv[0] - value := types.MlrvalFromInferredTypeForDataFiles(kv[1]) + value := mlrval.FromInferredTypeForDataFiles(kv[1]) record.PutReference(key, value) } } @@ -643,7 +643,7 @@ func NewRecordWriterDKVP2(writerOptions *cli.TWriterOptions) (*RecordWriterDKVP2 } func (writer *RecordWriterDKVP2) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ostream *bufio.Writer, ) { // End of record stream: nothing special for this output format diff --git a/cmd/mprof2/main.go b/cmd/mprof2/main.go index b680e48a9..6619e2b41 100644 --- a/cmd/mprof2/main.go +++ b/cmd/mprof2/main.go @@ -255,8 +255,8 @@ func (reader *RecordReaderDKVPChanPipelined) processHandle( func (reader *RecordReaderDKVPChanPipelined) recordFromDKVPLine( line string, -) *types.Mlrmap { - record := types.NewMlrmap() +) *mlrval.Mlrmap { + record := mlrval.NewMlrmap() var pairs []string if reader.readerOptions.IFSRegex == nil { // e.g. --no-ifs-regex @@ -280,11 +280,11 @@ func (reader *RecordReaderDKVPChanPipelined) recordFromDKVPLine( // "a=". Here we use the positional index as the key. This way // DKVP is a generalization of NIDX. key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := types.MlrvalFromInferredTypeForDataFiles(kv[0]) + value := mlrval.FromInferredTypeForDataFiles(kv[0]) record.PutReference(key, value) } else { key := kv[0] - value := types.MlrvalFromInferredTypeForDataFiles(kv[1]) + value := mlrval.FromInferredTypeForDataFiles(kv[1]) record.PutReference(key, value) } } diff --git a/cmd/mprof3/main.go b/cmd/mprof3/main.go index fe30e8c52..a30bc0d65 100644 --- a/cmd/mprof3/main.go +++ b/cmd/mprof3/main.go @@ -269,8 +269,8 @@ func (reader *RecordReaderDKVPChanPipelined) getRecordBatch( func (reader *RecordReaderDKVPChanPipelined) recordFromDKVPLine( line string, -) *types.Mlrmap { - record := types.NewMlrmapAsRecord() +) *mlrval.Mlrmap { + record := mlrval.NewMlrmapAsRecord() var pairs []string if reader.readerOptions.IFSRegex == nil { // e.g. --no-ifs-regex @@ -294,11 +294,11 @@ func (reader *RecordReaderDKVPChanPipelined) recordFromDKVPLine( // "a=". Here we use the positional index as the key. This way // DKVP is a generalization of NIDX. key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := types.MlrvalFromInferredTypeForDataFiles(kv[0]) + value := mlrval.FromInferredTypeForDataFiles(kv[0]) record.PutReference(key, value) } else { key := kv[0] - value := types.MlrvalFromInferredTypeForDataFiles(kv[1]) + value := mlrval.FromInferredTypeForDataFiles(kv[1]) record.PutReference(key, value) } } diff --git a/go.mod b/go.mod index 6348401c6..65acb8d29 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/lestrrat-go/strftime v1.0.4 github.com/mattn/go-isatty v0.0.12 github.com/pkg/profile v1.6.0 // indirect + github.com/stretchr/testify v1.7.0 // indirect golang.org/x/sys v0.0.0-20210326220804-49726bf1d181 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf ) diff --git a/go.sum b/go.sum index 04746f2e1..268f50836 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -49,3 +51,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/pkg/auxents/help/entry.go b/internal/pkg/auxents/help/entry.go index 7e8c14597..c223279e8 100644 --- a/internal/pkg/auxents/help/entry.go +++ b/internal/pkg/auxents/help/entry.go @@ -10,11 +10,12 @@ import ( "github.com/mattn/go-isatty" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/dsl/cst" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/transformers" - "github.com/johnkerl/miller/internal/pkg/types" ) // ================================================================ @@ -447,11 +448,11 @@ func helpOutputColorization() { // ---------------------------------------------------------------- func helpTypeArithmeticInfo() { - mlrvals := []*types.Mlrval{ - types.MlrvalFromInt(1), - types.MlrvalFromFloat64(2.5), - types.MLRVAL_ABSENT, - types.MLRVAL_ERROR, + mlrvals := []*mlrval.Mlrval{ + mlrval.FromInt(1), + mlrval.FromFloat(2.5), + mlrval.ABSENT, + mlrval.ERROR, } n := len(mlrvals) @@ -470,7 +471,7 @@ func helpTypeArithmeticInfo() { } else if i == -1 { fmt.Printf(" %-10s", "------") } else { - sum := types.BIF_plus_binary(mlrvals[i], mlrvals[j]) + sum := bifs.BIF_plus_binary(mlrvals[i], mlrvals[j]) fmt.Printf(" %-10s", sum.String()) } } diff --git a/internal/pkg/auxents/repl/dsl.go b/internal/pkg/auxents/repl/dsl.go index ae98538d8..ce09f3636 100644 --- a/internal/pkg/auxents/repl/dsl.go +++ b/internal/pkg/auxents/repl/dsl.go @@ -25,7 +25,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/dsl/cst" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ---------------------------------------------------------------- @@ -83,8 +83,8 @@ func (repl *Repl) handleDSLStringAux( // The filter expression for the main Miller DSL is any non-assignment // statment like 'true' or '$x > 0.5' etc. For the REPL, we re-use this for // interactive expressions to be printed to the terminal. For the main DSL, - // the default is types.MlrvalFromTrue(); for the REPL, the default is - // types.MLRVAL_VOID. + // the default is mlrval.FromTrue(); for the REPL, the default is + // mlrval.VOID. filterExpression := repl.runtimeState.FilterExpression if filterExpression.IsVoid() { // nothing to print @@ -93,7 +93,7 @@ func (repl *Repl) handleDSLStringAux( } else { fmt.Println(filterExpression.String()) } - repl.runtimeState.FilterExpression = types.MLRVAL_VOID + repl.runtimeState.FilterExpression = mlrval.VOID } return nil diff --git a/internal/pkg/auxents/repl/session.go b/internal/pkg/auxents/repl/session.go index 04dfd3015..20f8dfe10 100644 --- a/internal/pkg/auxents/repl/session.go +++ b/internal/pkg/auxents/repl/session.go @@ -29,6 +29,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl/cst" "github.com/johnkerl/miller/internal/pkg/input" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/output" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" @@ -58,7 +59,7 @@ func NewRepl( } // $* is the empty map {} until/unless the user opens a file and reads records from it. - inrec := types.NewMlrmapAsRecord() + inrec := mlrval.NewMlrmapAsRecord() // NR is 0, etc until/unless the user opens a file and reads records from it. context := types.NewContext() @@ -67,9 +68,9 @@ func NewRepl( // The filter expression for the main Miller DSL is any non-assignment // statment like 'true' or '$x > 0.5' etc. For the REPL, we re-use this for // interactive expressions to be printed to the terminal. For the main DSL, - // the default is types.MlrvalFromTrue(); for the REPL, the default is - // types.MLRVAL_VOID. - runtimeState.FilterExpression = types.MLRVAL_VOID + // the default is mlrval.FromTrue(); for the REPL, the default is + // mlrval.VOID. + runtimeState.FilterExpression = mlrval.VOID // For control-C handling sysToSignalHandlerChannel := make(chan os.Signal, 1) // Our signal handler reads system notification here diff --git a/internal/pkg/auxents/repl/verbs.go b/internal/pkg/auxents/repl/verbs.go index 5197dc8b6..c03b0ec08 100644 --- a/internal/pkg/auxents/repl/verbs.go +++ b/internal/pkg/auxents/repl/verbs.go @@ -14,6 +14,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/dsl/cst" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -608,7 +609,7 @@ func handleWrite(repl *Repl, args []string) bool { return true } -func writeRecord(repl *Repl, outrec *types.Mlrmap) { +func writeRecord(repl *Repl, outrec *mlrval.Mlrmap) { if outrec != nil { // E.g. '{"req": {"method": "GET", "path": "/api/check"}}' becomes // req.method=GET,req.path=/api/check. diff --git a/internal/pkg/types/mlrval_functions_arithmetic.go b/internal/pkg/bifs/arithmetic.go similarity index 62% rename from internal/pkg/types/mlrval_functions_arithmetic.go rename to internal/pkg/bifs/arithmetic.go index 239e18934..03aaacdc6 100644 --- a/internal/pkg/types/mlrval_functions_arithmetic.go +++ b/internal/pkg/bifs/arithmetic.go @@ -1,13 +1,15 @@ -package types +package bifs import ( "math" + + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ================================================================ // Unary plus operator -var upos_dispositions = [MT_DIM]UnaryFunc{ +var upos_dispositions = [mlrval.MT_DIM]UnaryFunc{ /*ERROR */ _erro1, /*ABSENT */ _absn1, /*NULL */ _null1, @@ -21,22 +23,22 @@ var upos_dispositions = [MT_DIM]UnaryFunc{ /*FUNC */ _erro1, } -func BIF_plus_unary(input1 *Mlrval) *Mlrval { - return upos_dispositions[input1.mvtype](input1) +func BIF_plus_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return upos_dispositions[input1.Type()](input1) } // ================================================================ // Unary minus operator -func uneg_i_i(input1 *Mlrval) *Mlrval { - return MlrvalFromInt(-input1.intval) +func uneg_i_i(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(-input1.AcquireIntValue()) } -func uneg_f_f(input1 *Mlrval) *Mlrval { - return MlrvalFromFloat64(-input1.floatval) +func uneg_f_f(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(-input1.AcquireFloatValue()) } -var uneg_dispositions = [MT_DIM]UnaryFunc{ +var uneg_dispositions = [mlrval.MT_DIM]UnaryFunc{ /*ERROR */ _erro1, /*ABSENT */ _absn1, /*NULL */ _null1, @@ -50,30 +52,19 @@ var uneg_dispositions = [MT_DIM]UnaryFunc{ /*FUNC */ _erro1, } -func BIF_minus_unary(input1 *Mlrval) *Mlrval { - return uneg_dispositions[input1.mvtype](input1) -} - -// ================================================================ -// Logical NOT operator - -func BIF_logicalnot(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_BOOL { - return MlrvalFromBool(!input1.boolval) - } else { - return MLRVAL_ERROR - } +func BIF_minus_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return uneg_dispositions[input1.Type()](input1) } // ================================================================ // Addition with auto-overflow from int to float when necessary. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic // Auto-overflows up to float. Additions & subtractions overflow by at most // one bit so it suffices to check sign-changes. -func plus_n_ii(input1, input2 *Mlrval) *Mlrval { - a := input1.intval - b := input2.intval +func plus_n_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() c := a + b overflowed := false @@ -88,23 +79,23 @@ func plus_n_ii(input1, input2 *Mlrval) *Mlrval { } if overflowed { - return MlrvalFromFloat64(float64(a) + float64(b)) + return mlrval.FromFloat(float64(a) + float64(b)) } else { - return MlrvalFromInt(c) + return mlrval.FromInt(c) } } -func plus_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) + input2.floatval) +func plus_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) + input2.AcquireFloatValue()) } -func plus_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval + float64(input2.intval)) +func plus_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() + float64(input2.AcquireIntValue())) } -func plus_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval + input2.floatval) +func plus_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() + input2.AcquireFloatValue()) } -var plus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var plus_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _2___, _erro, _absn, _absn, _erro}, @@ -119,19 +110,19 @@ var plus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_plus_binary(input1, input2 *Mlrval) *Mlrval { - return plus_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_plus_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return plus_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ // Subtraction with auto-overflow from int to float when necessary. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic // Adds & subtracts overflow by at most one bit so it suffices to check // sign-changes. -func minus_n_ii(input1, input2 *Mlrval) *Mlrval { - a := input1.intval - b := input2.intval +func minus_n_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() c := a - b overflowed := false @@ -146,23 +137,23 @@ func minus_n_ii(input1, input2 *Mlrval) *Mlrval { } if overflowed { - return MlrvalFromFloat64(float64(a) - float64(b)) + return mlrval.FromFloat(float64(a) - float64(b)) } else { - return MlrvalFromInt(c) + return mlrval.FromInt(c) } } -func minus_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) - input2.floatval) +func minus_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) - input2.AcquireFloatValue()) } -func minus_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval - float64(input2.intval)) +func minus_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() - float64(input2.AcquireIntValue())) } -func minus_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval - input2.floatval) +func minus_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() - input2.AcquireFloatValue()) } -var minus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var minus_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _2___, _erro, _absn, _absn, _erro}, @@ -177,13 +168,13 @@ var minus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_minus_binary(input1, input2 *Mlrval) *Mlrval { - return minus_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_minus_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return minus_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ // Multiplication with auto-overflow from int to float when necessary. See -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic // Auto-overflows up to float. // @@ -214,29 +205,29 @@ func BIF_minus_binary(input1, input2 *Mlrval) *Mlrval { // double less than 2**63. (An alterative would be to do all integer multiplies // using handcrafted multi-word 128-bit arithmetic). -func times_n_ii(input1, input2 *Mlrval) *Mlrval { - a := input1.intval - b := input2.intval +func times_n_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() c := float64(a) * float64(b) if math.Abs(c) > 9223372036854774784.0 { - return MlrvalFromFloat64(c) + return mlrval.FromFloat(c) } else { - return MlrvalFromInt(a * b) + return mlrval.FromInt(a * b) } } -func times_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) * input2.floatval) +func times_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) * input2.AcquireFloatValue()) } -func times_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval * float64(input2.intval)) +func times_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() * float64(input2.AcquireIntValue())) } -func times_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval * input2.floatval) +func times_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() * input2.AcquireFloatValue()) } -var times_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var times_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _2___, _erro, _absn, _absn, _erro}, @@ -251,13 +242,13 @@ var times_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_times(input1, input2 *Mlrval) *Mlrval { - return times_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_times(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return times_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ // Pythonic division. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic // // Int/int pairings don't produce overflow. // @@ -272,42 +263,42 @@ func BIF_times(input1, input2 *Mlrval) *Mlrval { // $ echo 'x=1e-300,y=1e300' | mlr put '$z=$y/$x' // x=1e-300,y=1e300,z=+Inf -func divide_n_ii(input1, input2 *Mlrval) *Mlrval { - a := input1.intval - b := input2.intval +func divide_n_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() if b == 0 { // Compute inf/nan as with floats rather than fatal runtime FPE on integer divide by zero - return MlrvalFromFloat64(float64(a) / float64(b)) + return mlrval.FromFloat(float64(a) / float64(b)) } // Pythonic division, not C division. if a%b == 0 { - return MlrvalFromInt(a / b) + return mlrval.FromInt(a / b) } else { - return MlrvalFromFloat64(float64(a) / float64(b)) + return mlrval.FromFloat(float64(a) / float64(b)) } c := float64(a) * float64(b) if math.Abs(c) > 9223372036854774784.0 { - return MlrvalFromFloat64(c) + return mlrval.FromFloat(c) } else { - return MlrvalFromInt(a * b) + return mlrval.FromInt(a * b) } } -func divide_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) / input2.floatval) +func divide_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) / input2.AcquireFloatValue()) } -func divide_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval / float64(input2.intval)) +func divide_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() / float64(input2.AcquireIntValue())) } -func divide_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval / input2.floatval) +func divide_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() / input2.AcquireFloatValue()) } -var divide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var divide_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _i0__, _f0__, _erro, _absn, _absn, _erro}, @@ -322,21 +313,21 @@ var divide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_divide(input1, input2 *Mlrval) *Mlrval { - return divide_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return divide_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ // Integer division: DSL operator '//' as in Python. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic -func int_divide_n_ii(input1, input2 *Mlrval) *Mlrval { - a := input1.intval - b := input2.intval +func int_divide_n_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() if b == 0 { // Compute inf/nan as with floats rather than fatal runtime FPE on integer divide by zero - return MlrvalFromFloat64(float64(a) / float64(b)) + return mlrval.FromFloat(float64(a) / float64(b)) } // Pythonic division, not C division. @@ -355,20 +346,20 @@ func int_divide_n_ii(input1, input2 *Mlrval) *Mlrval { } } } - return MlrvalFromInt(q) + return mlrval.FromInt(q) } -func int_divide_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Floor(float64(input1.intval) / input2.floatval)) +func int_divide_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Floor(float64(input1.AcquireIntValue()) / input2.AcquireFloatValue())) } -func int_divide_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Floor(input1.floatval / float64(input2.intval))) +func int_divide_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Floor(input1.AcquireFloatValue() / float64(input2.AcquireIntValue()))) } -func int_divide_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Floor(input1.floatval / input2.floatval)) +func int_divide_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Floor(input1.AcquireFloatValue() / input2.AcquireFloatValue())) } -var int_divide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var int_divide_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _i0__, _f0__, _erro, _absn, _absn, _erro}, @@ -383,28 +374,28 @@ var int_divide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_int_divide(input1, input2 *Mlrval) *Mlrval { - return int_divide_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_int_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return int_divide_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ // Non-auto-overflowing addition: DSL operator '.+'. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic -func dotplus_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval + input2.intval) +func dotplus_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() + input2.AcquireIntValue()) } -func dotplus_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) + input2.floatval) +func dotplus_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) + input2.AcquireFloatValue()) } -func dotplus_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval + float64(input2.intval)) +func dotplus_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() + float64(input2.AcquireIntValue())) } -func dotplus_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval + input2.floatval) +func dotplus_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() + input2.AcquireFloatValue()) } -var dot_plus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var dot_plus_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _2___, _erro, _absn, _absn, _erro}, @@ -419,28 +410,28 @@ var dot_plus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_dot_plus(input1, input2 *Mlrval) *Mlrval { - return dot_plus_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_dot_plus(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return dot_plus_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ // Non-auto-overflowing subtraction: DSL operator '.-'. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic -func dotminus_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval - input2.intval) +func dotminus_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() - input2.AcquireIntValue()) } -func dotminus_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) - input2.floatval) +func dotminus_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) - input2.AcquireFloatValue()) } -func dotminus_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval - float64(input2.intval)) +func dotminus_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() - float64(input2.AcquireIntValue())) } -func dotminus_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval - input2.floatval) +func dotminus_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() - input2.AcquireFloatValue()) } -var dotminus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var dotminus_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _n2__, _n2__, _erro, _absn, _absn, _erro}, @@ -455,28 +446,28 @@ var dotminus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_dot_minus(input1, input2 *Mlrval) *Mlrval { - return dotminus_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_dot_minus(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return dotminus_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // Non-auto-overflowing multiplication: DSL operator '.*'. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic -func dottimes_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval * input2.intval) +func dottimes_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() * input2.AcquireIntValue()) } -func dottimes_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) * input2.floatval) +func dottimes_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) * input2.AcquireFloatValue()) } -func dottimes_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval * float64(input2.intval)) +func dottimes_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() * float64(input2.AcquireIntValue())) } -func dottimes_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval * input2.floatval) +func dottimes_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() * input2.AcquireFloatValue()) } -var dottimes_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var dottimes_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _2___, _erro, _absn, _absn, _erro}, @@ -491,28 +482,28 @@ var dottimes_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_dot_times(input1, input2 *Mlrval) *Mlrval { - return dottimes_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_dot_times(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return dottimes_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // 64-bit integer division: DSL operator './'. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic -func dotdivide_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval / input2.intval) +func dotdivide_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() / input2.AcquireIntValue()) } -func dotdivide_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval) / input2.floatval) +func dotdivide_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue()) / input2.AcquireFloatValue()) } -func dotdivide_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval / float64(input2.intval)) +func dotdivide_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() / float64(input2.AcquireIntValue())) } -func dotdivide_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(input1.floatval / input2.floatval) +func dotdivide_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(input1.AcquireFloatValue() / input2.AcquireFloatValue()) } -var dotdivide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var dotdivide_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _2___, _erro, _absn, _absn, _erro}, @@ -527,21 +518,21 @@ var dotdivide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_dot_divide(input1, input2 *Mlrval) *Mlrval { - return dotdivide_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_dot_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return dotdivide_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // 64-bit integer division: DSL operator './/'. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html +// https://miller.readthedocs.io/en/latest/reference-main-arithmetic -func dotidivide_i_ii(input1, input2 *Mlrval) *Mlrval { - a := input1.intval - b := input2.intval +func dotidivide_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() if b == 0 { // Compute inf/nan as with floats rather than fatal runtime FPE on integer divide by zero - return MlrvalFromFloat64(float64(a) / float64(b)) + return mlrval.FromFloat(float64(a) / float64(b)) } // Pythonic division, not C division. @@ -560,20 +551,20 @@ func dotidivide_i_ii(input1, input2 *Mlrval) *Mlrval { } } } - return MlrvalFromInt(q) + return mlrval.FromInt(q) } -func dotidivide_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Floor(float64(input1.intval) / input2.floatval)) +func dotidivide_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Floor(float64(input1.AcquireIntValue()) / input2.AcquireFloatValue())) } -func dotidivide_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Floor(input1.floatval / float64(input2.intval))) +func dotidivide_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Floor(input1.AcquireFloatValue() / float64(input2.AcquireIntValue()))) } -func dotidivide_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Floor(input1.floatval / input2.floatval)) +func dotidivide_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Floor(input1.AcquireFloatValue() / input2.AcquireFloatValue())) } -var dotidivide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var dotidivide_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _erro, _absn, _erro, _2___, _2___, _erro, _absn, _absn, _erro}, @@ -588,20 +579,20 @@ var dotidivide_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func MlrvalDotIntDivide(input1, input2 *Mlrval) *Mlrval { - return dotidivide_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_dot_int_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return dotidivide_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // Modulus -func modulus_i_ii(input1, input2 *Mlrval) *Mlrval { - a := input1.intval - b := input2.intval +func modulus_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() if b == 0 { // Compute inf/nan as with floats rather than fatal runtime FPE on integer divide by zero - return MlrvalFromFloat64(float64(a) / float64(b)) + return mlrval.FromFloat(float64(a) / float64(b)) } // Pythonic division, not C division. @@ -616,28 +607,28 @@ func modulus_i_ii(input1, input2 *Mlrval) *Mlrval { } } - return MlrvalFromInt(m) + return mlrval.FromInt(m) } -func modulus_f_fi(input1, input2 *Mlrval) *Mlrval { - a := input1.floatval - b := float64(input2.intval) - return MlrvalFromFloat64(a - b*math.Floor(a/b)) +func modulus_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireFloatValue() + b := float64(input2.AcquireIntValue()) + return mlrval.FromFloat(a - b*math.Floor(a/b)) } -func modulus_f_if(input1, input2 *Mlrval) *Mlrval { - a := float64(input1.intval) - b := input2.floatval - return MlrvalFromFloat64(a - b*math.Floor(a/b)) +func modulus_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := float64(input1.AcquireIntValue()) + b := input2.AcquireFloatValue() + return mlrval.FromFloat(a - b*math.Floor(a/b)) } -func modulus_f_ff(input1, input2 *Mlrval) *Mlrval { - a := input1.floatval - b := input2.floatval - return MlrvalFromFloat64(a - b*math.Floor(a/b)) +func modulus_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireFloatValue() + b := input2.AcquireFloatValue() + return mlrval.FromFloat(a - b*math.Floor(a/b)) } -var modulus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var modulus_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _i0__, _f0__, _erro, _absn, _absn, _erro}, @@ -652,8 +643,8 @@ var modulus_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_modulus(input1, input2 *Mlrval) *Mlrval { - return modulus_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_modulus(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return modulus_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ @@ -701,7 +692,7 @@ func imodexp(a, e, m int) int { return c } -func imodop(input1, input2, input3 *Mlrval, iop i_iii_func) *Mlrval { +func imodop(input1, input2, input3 *mlrval.Mlrval, iop i_iii_func) *mlrval.Mlrval { if !input1.IsLegit() { return input1 } @@ -712,34 +703,34 @@ func imodop(input1, input2, input3 *Mlrval, iop i_iii_func) *Mlrval { return input3 } if !input1.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input3.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromInt(iop(input1.intval, input2.intval, input3.intval)) + return mlrval.FromInt(iop(input1.AcquireIntValue(), input2.AcquireIntValue(), input3.AcquireIntValue())) } -func BIF_mod_add(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_mod_add(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { return imodop(input1, input2, input3, imodadd) } -func BIF_mod_sub(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_mod_sub(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { return imodop(input1, input2, input3, imodsub) } -func BIF_mod_mul(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_mod_mul(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { return imodop(input1, input2, input3, imodmul) } -func BIF_mod_exp(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_mod_exp(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { // Pre-check for negative exponent - if input2.mvtype == MT_INT && input2.intval < 0 { - return MLRVAL_ERROR + if input2.IsInt() && input2.AcquireIntValue() < 0 { + return mlrval.ERROR } return imodop(input1, input2, input3, imodexp) } @@ -759,27 +750,27 @@ func BIF_mod_exp(input1, input2, input3 *Mlrval) *Mlrval { // * empty-null always loses against numbers // ---------------------------------------------------------------- -func min_f_ff(input1, input2 *Mlrval) *Mlrval { - var a float64 = input1.floatval - var b float64 = input2.floatval - return MlrvalFromFloat64(math.Min(a, b)) +func min_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a float64 = input1.AcquireFloatValue() + var b float64 = input2.AcquireFloatValue() + return mlrval.FromFloat(math.Min(a, b)) } -func min_f_fi(input1, input2 *Mlrval) *Mlrval { - var a float64 = input1.floatval - var b float64 = float64(input2.intval) - return MlrvalFromFloat64(math.Min(a, b)) +func min_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a float64 = input1.AcquireFloatValue() + var b float64 = float64(input2.AcquireIntValue()) + return mlrval.FromFloat(math.Min(a, b)) } -func min_f_if(input1, input2 *Mlrval) *Mlrval { - var a float64 = float64(input1.intval) - var b float64 = input2.floatval - return MlrvalFromFloat64(math.Min(a, b)) +func min_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a float64 = float64(input1.AcquireIntValue()) + var b float64 = input2.AcquireFloatValue() + return mlrval.FromFloat(math.Min(a, b)) } -func min_i_ii(input1, input2 *Mlrval) *Mlrval { - var a int = input1.intval - var b int = input2.intval +func min_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a int = input1.AcquireIntValue() + var b int = input2.AcquireIntValue() if a < b { return input1 } else { @@ -791,17 +782,17 @@ func min_i_ii(input1, input2 *Mlrval) *Mlrval { // --- + ----- ----- // a=F | min=a min=a // a=T | min=b min=b -func min_b_bb(input1, input2 *Mlrval) *Mlrval { - if input1.boolval == false { +func min_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.AcquireBoolValue() == false { return input1 } else { return input2 } } -func min_s_ss(input1, input2 *Mlrval) *Mlrval { - var a string = input1.printrep - var b string = input2.printrep +func min_s_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a string = input1.AcquireStringValue() + var b string = input2.AcquireStringValue() if a < b { return input1 } else { @@ -809,7 +800,7 @@ func min_s_ss(input1, input2 *Mlrval) *Mlrval { } } -var min_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var min_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _null, _2___, _2___, _2___, _2___, _2___, _absn, _absn, _erro}, @@ -824,18 +815,18 @@ var min_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func MlrvalBinaryMin(input1, input2 *Mlrval) *Mlrval { - return (min_dispositions[input1.mvtype][input2.mvtype])(input1, input2) +func BIF_min_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return (min_dispositions[input1.Type()][input2.Type()])(input1, input2) } -func BIF_min_variadic(mlrvals []*Mlrval) *Mlrval { +func BIF_min_variadic(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { if len(mlrvals) == 0 { - return MLRVAL_VOID + return mlrval.VOID } else { retval := mlrvals[0] for i := range mlrvals { if i > 0 { - retval = MlrvalBinaryMin(retval, mlrvals[i]) + retval = BIF_min_binary(retval, mlrvals[i]) } } return retval @@ -843,27 +834,27 @@ func BIF_min_variadic(mlrvals []*Mlrval) *Mlrval { } // ---------------------------------------------------------------- -func max_f_ff(input1, input2 *Mlrval) *Mlrval { - var a float64 = input1.floatval - var b float64 = input2.floatval - return MlrvalFromFloat64(math.Max(a, b)) +func max_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a float64 = input1.AcquireFloatValue() + var b float64 = input2.AcquireFloatValue() + return mlrval.FromFloat(math.Max(a, b)) } -func max_f_fi(input1, input2 *Mlrval) *Mlrval { - var a float64 = input1.floatval - var b float64 = float64(input2.intval) - return MlrvalFromFloat64(math.Max(a, b)) +func max_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a float64 = input1.AcquireFloatValue() + var b float64 = float64(input2.AcquireIntValue()) + return mlrval.FromFloat(math.Max(a, b)) } -func max_f_if(input1, input2 *Mlrval) *Mlrval { - var a float64 = float64(input1.intval) - var b float64 = input2.floatval - return MlrvalFromFloat64(math.Max(a, b)) +func max_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a float64 = float64(input1.AcquireIntValue()) + var b float64 = input2.AcquireFloatValue() + return mlrval.FromFloat(math.Max(a, b)) } -func max_i_ii(input1, input2 *Mlrval) *Mlrval { - var a int = input1.intval - var b int = input2.intval +func max_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a int = input1.AcquireIntValue() + var b int = input2.AcquireIntValue() if a > b { return input1 } else { @@ -875,17 +866,17 @@ func max_i_ii(input1, input2 *Mlrval) *Mlrval { // --- + ----- ----- // a=F | max=a max=b // a=T | max=a max=b -func max_b_bb(input1, input2 *Mlrval) *Mlrval { - if input2.boolval == false { +func max_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input2.AcquireBoolValue() == false { return input1 } else { return input2 } } -func max_s_ss(input1, input2 *Mlrval) *Mlrval { - var a string = input1.printrep - var b string = input2.printrep +func max_s_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var a string = input1.AcquireStringValue() + var b string = input2.AcquireStringValue() if a > b { return input1 } else { @@ -893,7 +884,7 @@ func max_s_ss(input1, input2 *Mlrval) *Mlrval { } } -var max_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var max_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _null, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _2___, _2___, _2___, _2___, _2___, _absn, _absn, _erro}, @@ -908,18 +899,18 @@ var max_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func MlrvalBinaryMax(input1, input2 *Mlrval) *Mlrval { - return (max_dispositions[input1.mvtype][input2.mvtype])(input1, input2) +func BIF_max_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return (max_dispositions[input1.Type()][input2.Type()])(input1, input2) } -func BIF_max_variadic(mlrvals []*Mlrval) *Mlrval { +func BIF_max_variadic(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { if len(mlrvals) == 0 { - return MLRVAL_VOID + return mlrval.VOID } else { retval := mlrvals[0] for i := range mlrvals { if i > 0 { - retval = MlrvalBinaryMax(retval, mlrvals[i]) + retval = BIF_max_binary(retval, mlrvals[i]) } } return retval diff --git a/internal/pkg/bifs/arithmetic_test.go b/internal/pkg/bifs/arithmetic_test.go new file mode 100644 index 000000000..5935854ac --- /dev/null +++ b/internal/pkg/bifs/arithmetic_test.go @@ -0,0 +1,90 @@ +package bifs + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + +func TestBIF_plus_unary(t *testing.T) { + input := mlrval.FromDeferredType("123") + output := BIF_plus_unary(input) + intval, ok := output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 123, intval) + + input = mlrval.FromDeferredType("-123.5") + output = BIF_plus_unary(input) + floatval, ok := output.GetFloatValue() + assert.True(t, ok) + assert.Equal(t, -123.5, floatval) +} + +func TestBIF_minus_unary(t *testing.T) { + input := mlrval.FromDeferredType("123") + output := BIF_minus_unary(input) + intval, ok := output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, -123, intval) + + input = mlrval.FromDeferredType("-123.5") + output = BIF_minus_unary(input) + floatval, ok := output.GetFloatValue() + assert.True(t, ok) + assert.Equal(t, 123.5, floatval) +} + +func TestBIF_plus_binary(t *testing.T) { + input1 := mlrval.FromDeferredType("123") + input2 := mlrval.FromDeferredType("456") + output := BIF_plus_binary(input1, input2) + intval, ok := output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 579, intval) + + input1 = mlrval.FromDeferredType("123.5") + input2 = mlrval.FromDeferredType("456") + output = BIF_plus_binary(input1, input2) + floatval, ok := output.GetFloatValue() + assert.True(t, ok) + assert.Equal(t, 579.5, floatval) +} + +func TestBIF_plus_binary_overflow(t *testing.T) { + input1 := mlrval.FromInt(0x07ffffffffffffff) + input2 := mlrval.FromInt(0x07fffffffffffffe) + output := BIF_plus_binary(input1, input2) + intval, ok := output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 0x0ffffffffffffffd, intval) + + input1 = mlrval.FromInt(0x7fffffffffffffff) + input2 = mlrval.FromInt(0x7ffffffffffffffe) + output = BIF_plus_binary(input1, input2) + floatval, ok := output.GetFloatValue() + assert.True(t, ok) + assert.Equal(t, 18446744073709552000.0, floatval) +} + +// TODO: copy in more unit-test cases from existing regression-test data + +//func BIF_minus_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_times(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_int_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_dot_plus(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_dot_minus(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_dot_times(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_dot_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_dot_int_divide(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_modulus(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_mod_add(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_mod_sub(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_mod_mul(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_mod_exp(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_min_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_min_variadic(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval +//func BIF_max_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +//func BIF_max_variadic(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval diff --git a/internal/pkg/types/mlrval_functions_base.go b/internal/pkg/bifs/base.go similarity index 62% rename from internal/pkg/types/mlrval_functions_base.go rename to internal/pkg/bifs/base.go index 68559ab6d..cd7c04160 100644 --- a/internal/pkg/types/mlrval_functions_base.go +++ b/internal/pkg/bifs/base.go @@ -12,7 +12,7 @@ // example: adding two ints, or an int and a float, or int and boolean (the // latter being an error). // -// The next-past-highest mlrval type enum is called MT_DIM and that is the +// The next-past-highest mlrval type enum is called mlrval.MT_DIM and that is the // dimension of the binary-operator disposition matrices and unary-operator // disposition vectors. // @@ -45,38 +45,43 @@ // ('s') to anything else ('x'). // ================================================================ -package types +package bifs + +import ( + "github.com/johnkerl/miller/internal/pkg/mlrval" + "github.com/johnkerl/miller/internal/pkg/types" +) // Function-pointer type for zary functions. -type ZaryFunc func() *Mlrval +type ZaryFunc func() *mlrval.Mlrval // Function-pointer type for unary-operator disposition vectors. -type UnaryFunc func(input1 *Mlrval) *Mlrval +type UnaryFunc func(input1 *mlrval.Mlrval) *mlrval.Mlrval // The asserting_{type} need access to the context to say things like 'Assertion ... failed // at filename {FILENAME} record number {NR}'. -type UnaryFuncWithContext func(input1 *Mlrval, context *Context) *Mlrval +type UnaryFuncWithContext func(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval // Returns nil, or one-up captures array as array slots 1..9 of 10-element // array for "\1".."\9". -type RegexCaptureBinaryFunc func(input *Mlrval, sregex *Mlrval) (*Mlrval, []string) +type RegexCaptureBinaryFunc func(input *mlrval.Mlrval, sregex *mlrval.Mlrval) (*mlrval.Mlrval, []string) // Helps keystroke-saving for wrapping Go math-library functions // Examples: cos, sin, etc. type mathLibUnaryFunc func(float64) float64 -type mathLibUnaryFuncWrapper func(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval +type mathLibUnaryFuncWrapper func(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval // Function-pointer type for binary-operator disposition matrices. -type BinaryFunc func(input1, input2 *Mlrval) *Mlrval +type BinaryFunc func(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval // Function-pointer type for ternary functions -type TernaryFunc func(input1, input2, input3 *Mlrval) *Mlrval +type TernaryFunc func(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval // Function-pointer type for variadic functions. -type VariadicFunc func(inputs []*Mlrval) *Mlrval +type VariadicFunc func(inputs []*mlrval.Mlrval) *mlrval.Mlrval // Function-pointer type for sorting. Returns < 0 if a < b, 0 if a == b, > 0 if a > b. -type ComparatorFunc func(*Mlrval, *Mlrval) int +type ComparatorFunc func(*mlrval.Mlrval, *mlrval.Mlrval) int // ================================================================ // The following are frequently used in disposition matrices for various @@ -86,102 +91,102 @@ type ComparatorFunc func(*Mlrval, *Mlrval) int // ---------------------------------------------------------------- // Return error (unary) -func _erro1(input1 *Mlrval) *Mlrval { - return MLRVAL_ERROR +func _erro1(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.ERROR } // Return absent (unary) -func _absn1(input1 *Mlrval) *Mlrval { - return MLRVAL_ABSENT +func _absn1(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.ABSENT } // Return zero (unary) -func _zero1(input1 *Mlrval) *Mlrval { - return MlrvalFromInt(0) +func _zero1(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(0) } // Return null (unary) -func _null1(input1 *Mlrval) *Mlrval { - return MLRVAL_NULL +func _null1(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.NULL } // Return void (unary) -func _void1(input1 *Mlrval) *Mlrval { - return MLRVAL_VOID +func _void1(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.VOID } // Return argument (unary) -func _1u___(input1 *Mlrval) *Mlrval { +func _1u___(input1 *mlrval.Mlrval) *mlrval.Mlrval { return input1 } // ---------------------------------------------------------------- // Return error (binary) -func _erro(input1, input2 *Mlrval) *Mlrval { - return MLRVAL_ERROR +func _erro(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.ERROR } // Return absent (binary) -func _absn(input1, input2 *Mlrval) *Mlrval { - return MLRVAL_ABSENT +func _absn(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.ABSENT } // Return null (binary) -func _null(input1, input2 *Mlrval) *Mlrval { - return MLRVAL_NULL +func _null(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.NULL } // Return void (binary) -func _void(input1, input2 *Mlrval) *Mlrval { - return MLRVAL_VOID +func _void(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.VOID } // Return 0 (binary) -func _zero2(input1 *Mlrval) *Mlrval { - return MlrvalFromInt(0) +func _zero2(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(0) } // Return first argument (binary) -func _1___(input1, input2 *Mlrval) *Mlrval { +func _1___(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return input1 } // Return second argument (binary) -func _2___(input1, input2 *Mlrval) *Mlrval { +func _2___(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return input2 } // Return negative second argument (binary) -func _n2__(input1, input2 *Mlrval) *Mlrval { +func _n2__(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return BIF_minus_unary(input2) } // Return first argument, as string (binary) -func _s1__(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromString(input1.String()) +func _s1__(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromString(input1.String()) } // Return second argument, as string (binary) -func _s2__(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromString(input2.String()) +func _s2__(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromString(input2.String()) } // Return integer zero (binary) -func _i0__(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(0) +func _i0__(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(0) } // Return float zero (binary) -func _f0__(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(0) +func _f0__(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(0) } // Return boolean true (binary) -func _true(input1, input2 *Mlrval) *Mlrval { - return MLRVAL_TRUE +func _true(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.TRUE } // Return boolean false (binary) -func _fals(input1, input2 *Mlrval) *Mlrval { - return MLRVAL_FALSE +func _fals(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FALSE } diff --git a/internal/pkg/types/mlrval_functions_bits.go b/internal/pkg/bifs/bits.go similarity index 75% rename from internal/pkg/types/mlrval_functions_bits.go rename to internal/pkg/bifs/bits.go index f722f8391..71be7c9a6 100644 --- a/internal/pkg/types/mlrval_functions_bits.go +++ b/internal/pkg/bifs/bits.go @@ -1,13 +1,17 @@ -package types +package bifs + +import ( + "github.com/johnkerl/miller/internal/pkg/mlrval" +) // ================================================================ // Bitwise NOT -func bitwise_not_i_i(input1 *Mlrval) *Mlrval { - return MlrvalFromInt(^input1.intval) +func bitwise_not_i_i(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(^input1.AcquireIntValue()) } -var bitwise_not_dispositions = [MT_DIM]UnaryFunc{ +var bitwise_not_dispositions = [mlrval.MT_DIM]UnaryFunc{ /*ERROR */ _erro1, /*ABSENT */ _absn1, /*NULL */ _null1, @@ -21,8 +25,8 @@ var bitwise_not_dispositions = [MT_DIM]UnaryFunc{ /*FUNC */ _erro1, } -func BIF_bitwise_not(input1 *Mlrval) *Mlrval { - return bitwise_not_dispositions[input1.mvtype](input1) +func BIF_bitwise_not(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return bitwise_not_dispositions[input1.Type()](input1) } // ================================================================ @@ -36,18 +40,18 @@ const _m08 uint64 = 0x00ff00ff00ff00ff const _m16 uint64 = 0x0000ffff0000ffff const _m32 uint64 = 0x00000000ffffffff -func bitcount_i_i(input1 *Mlrval) *Mlrval { - a := uint64(input1.intval) +func bitcount_i_i(input1 *mlrval.Mlrval) *mlrval.Mlrval { + a := uint64(input1.AcquireIntValue()) a = (a & _m01) + ((a >> 1) & _m01) a = (a & _m02) + ((a >> 2) & _m02) a = (a & _m04) + ((a >> 4) & _m04) a = (a & _m08) + ((a >> 8) & _m08) a = (a & _m16) + ((a >> 16) & _m16) a = (a & _m32) + ((a >> 32) & _m32) - return MlrvalFromInt(int(a)) + return mlrval.FromInt(int(a)) } -var bitcount_dispositions = [MT_DIM]UnaryFunc{ +var bitcount_dispositions = [mlrval.MT_DIM]UnaryFunc{ /*ERROR */ _erro1, /*ABSENT */ _absn1, /*NULL */ _zero1, @@ -61,18 +65,18 @@ var bitcount_dispositions = [MT_DIM]UnaryFunc{ /*FUNC */ _erro1, } -func BIF_bitcount(input1 *Mlrval) *Mlrval { - return bitcount_dispositions[input1.mvtype](input1) +func BIF_bitcount(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return bitcount_dispositions[input1.Type()](input1) } // ================================================================ // Bitwise AND -func bitwise_and_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval & input2.intval) +func bitwise_and_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() & input2.AcquireIntValue()) } -var bitwise_and_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var bitwise_and_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _erro, _erro, _absn, _absn, _erro}, @@ -87,18 +91,18 @@ var bitwise_and_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_bitwise_and(input1, input2 *Mlrval) *Mlrval { - return bitwise_and_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_bitwise_and(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return bitwise_and_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // Bitwise OR -func bitwise_or_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval | input2.intval) +func bitwise_or_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() | input2.AcquireIntValue()) } -var bitwise_or_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var bitwise_or_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _erro, _erro, _absn, _absn, _erro}, @@ -113,18 +117,18 @@ var bitwise_or_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_bitwise_or(input1, input2 *Mlrval) *Mlrval { - return bitwise_or_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_bitwise_or(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return bitwise_or_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // Bitwise XOR -func bitwise_xor_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval ^ input2.intval) +func bitwise_xor_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() ^ input2.AcquireIntValue()) } -var bitwise_xor_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var bitwise_xor_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _erro, _erro, _absn, _absn, _erro}, @@ -139,18 +143,18 @@ var bitwise_xor_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_bitwise_xor(input1, input2 *Mlrval) *Mlrval { - return bitwise_xor_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_bitwise_xor(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return bitwise_xor_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // Left shift -func lsh_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval << uint64(input2.intval)) +func lsh_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() << uint64(input2.AcquireIntValue())) } -var left_shift_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var left_shift_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _erro, _erro, _absn, _absn, _erro}, @@ -165,18 +169,18 @@ var left_shift_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_left_shift(input1, input2 *Mlrval) *Mlrval { - return left_shift_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_left_shift(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return left_shift_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // Signed right shift -func srsh_i_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(input1.intval >> uint64(input2.intval)) +func srsh_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(input1.AcquireIntValue() >> uint64(input2.AcquireIntValue())) } -var signed_right_shift_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var signed_right_shift_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _erro, _erro, _absn, _absn, _erro}, @@ -191,21 +195,21 @@ var signed_right_shift_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_signed_right_shift(input1, input2 *Mlrval) *Mlrval { - return signed_right_shift_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_signed_right_shift(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return signed_right_shift_dispositions[input1.Type()][input2.Type()](input1, input2) } // ---------------------------------------------------------------- // Unsigned right shift -func ursh_i_ii(input1, input2 *Mlrval) *Mlrval { - var ua uint64 = uint64(input1.intval) - var ub uint64 = uint64(input2.intval) +func ursh_i_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var ua uint64 = uint64(input1.AcquireIntValue()) + var ub uint64 = uint64(input2.AcquireIntValue()) var uc = ua >> ub - return MlrvalFromInt(int(uc)) + return mlrval.FromInt(int(uc)) } -var unsigned_right_shift_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var unsigned_right_shift_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _2___, _erro, _erro, _absn, _absn, _erro}, @@ -220,6 +224,6 @@ var unsigned_right_shift_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_unsigned_right_shift(input1, input2 *Mlrval) *Mlrval { - return unsigned_right_shift_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_unsigned_right_shift(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return unsigned_right_shift_dispositions[input1.Type()][input2.Type()](input1, input2) } diff --git a/internal/pkg/bifs/bits_test.go b/internal/pkg/bifs/bits_test.go new file mode 100644 index 000000000..1c5bec2a4 --- /dev/null +++ b/internal/pkg/bifs/bits_test.go @@ -0,0 +1,19 @@ +package bifs + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + +func TestBIF_bitcount(t *testing.T) { + input1 := mlrval.FromDeferredType("0xcafe") + output := BIF_bitcount(input1) + intval, ok := output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 11, intval) +} + +// TODO: copy in more unit-test cases from existing regression-test data diff --git a/internal/pkg/bifs/booleans.go b/internal/pkg/bifs/booleans.go new file mode 100644 index 000000000..ce7f513db --- /dev/null +++ b/internal/pkg/bifs/booleans.go @@ -0,0 +1,480 @@ +// ================================================================ +// Boolean expressions for ==, !=, >, >=, <, <= +// ================================================================ + +package bifs + +import ( + "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// string_cmp implements the spaceship operator for strings. +func string_cmp(a, b string) int { + if a < b { + return -1 + } + if a > b { + return 1 + } + return 0 +} + +// int_cmp implements the spaceship operator for ints. +func int_cmp(a, b int) int { + if a < b { + return -1 + } + if a > b { + return 1 + } + return 0 +} + +// float_cmp implements the spaceship operator for floats. +func float_cmp(a, b float64) int { + if a < b { + return -1 + } + if a > b { + return 1 + } + return 0 +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +func eq_b_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() == input2.AcquireStringValue()) +} +func ne_b_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() != input2.AcquireStringValue()) +} +func gt_b_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() > input2.AcquireStringValue()) +} +func ge_b_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() >= input2.AcquireStringValue()) +} +func lt_b_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() < input2.AcquireStringValue()) +} +func le_b_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() <= input2.AcquireStringValue()) +} +func cmp_b_ss(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(string_cmp(input1.AcquireStringValue(), input2.AcquireStringValue())) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +func eq_b_xs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.String() == input2.AcquireStringValue()) +} +func ne_b_xs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.String() != input2.AcquireStringValue()) +} +func gt_b_xs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.String() > input2.AcquireStringValue()) +} +func ge_b_xs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.String() >= input2.AcquireStringValue()) +} +func lt_b_xs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.String() < input2.AcquireStringValue()) +} +func le_b_xs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.String() <= input2.AcquireStringValue()) +} +func cmp_b_xs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(string_cmp(input1.String(), input2.AcquireStringValue())) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +func eq_b_sx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() == input2.String()) +} +func ne_b_sx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() != input2.String()) +} +func gt_b_sx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() > input2.String()) +} +func ge_b_sx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() >= input2.String()) +} +func lt_b_sx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() < input2.String()) +} +func le_b_sx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireStringValue() <= input2.String()) +} +func cmp_b_sx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(string_cmp(input1.AcquireStringValue(), input2.String())) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +func eq_b_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireIntValue() == input2.AcquireIntValue()) +} +func ne_b_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireIntValue() != input2.AcquireIntValue()) +} +func gt_b_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireIntValue() > input2.AcquireIntValue()) +} +func ge_b_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireIntValue() >= input2.AcquireIntValue()) +} +func lt_b_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireIntValue() < input2.AcquireIntValue()) +} +func le_b_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireIntValue() <= input2.AcquireIntValue()) +} +func cmp_b_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(int_cmp(input1.AcquireIntValue(), input2.AcquireIntValue())) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +func eq_b_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(float64(input1.AcquireIntValue()) == input2.AcquireFloatValue()) +} +func ne_b_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(float64(input1.AcquireIntValue()) != input2.AcquireFloatValue()) +} +func gt_b_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(float64(input1.AcquireIntValue()) > input2.AcquireFloatValue()) +} +func ge_b_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(float64(input1.AcquireIntValue()) >= input2.AcquireFloatValue()) +} +func lt_b_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(float64(input1.AcquireIntValue()) < input2.AcquireFloatValue()) +} +func le_b_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(float64(input1.AcquireIntValue()) <= input2.AcquireFloatValue()) +} +func cmp_b_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(float_cmp(float64(input1.AcquireIntValue()), input2.AcquireFloatValue())) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +func eq_b_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() == float64(input2.AcquireIntValue())) +} +func ne_b_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() != float64(input2.AcquireIntValue())) +} +func gt_b_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() > float64(input2.AcquireIntValue())) +} +func ge_b_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() >= float64(input2.AcquireIntValue())) +} +func lt_b_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() < float64(input2.AcquireIntValue())) +} +func le_b_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() <= float64(input2.AcquireIntValue())) +} +func cmp_b_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(float_cmp(input1.AcquireFloatValue(), float64(input2.AcquireIntValue()))) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +func eq_b_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() == input2.AcquireFloatValue()) +} +func ne_b_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() != input2.AcquireFloatValue()) +} +func gt_b_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() > input2.AcquireFloatValue()) +} +func ge_b_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() >= input2.AcquireFloatValue()) +} +func lt_b_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() < input2.AcquireFloatValue()) +} +func le_b_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() <= input2.AcquireFloatValue()) +} +func cmp_b_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(float_cmp(input1.AcquireFloatValue(), input2.AcquireFloatValue())) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +func eq_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireBoolValue() == input2.AcquireBoolValue()) +} +func ne_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireBoolValue() != input2.AcquireBoolValue()) +} + +// We could say ordering on bool is error, but, Miller allows +// sorting on bool so it should allow ordering on bool. + +func gt_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(lib.BoolToInt(input1.AcquireBoolValue()) > lib.BoolToInt(input2.AcquireBoolValue())) +} +func ge_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(lib.BoolToInt(input1.AcquireBoolValue()) >= lib.BoolToInt(input2.AcquireBoolValue())) +} +func lt_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(lib.BoolToInt(input1.AcquireBoolValue()) < lib.BoolToInt(input2.AcquireBoolValue())) +} +func le_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(lib.BoolToInt(input1.AcquireBoolValue()) <= lib.BoolToInt(input2.AcquireBoolValue())) +} +func cmp_b_bb(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(int_cmp(lib.BoolToInt(input1.AcquireBoolValue()), lib.BoolToInt(input2.AcquireBoolValue()))) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +func eq_b_aa(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + a := input1.AcquireArrayValue() + b := input2.AcquireArrayValue() + + // Different-length arrays are not equal + if len(a) != len(b) { + return mlrval.FALSE + } + + // Same-length arrays: return false if any slot is not equal, else true. + for i := range a { + eq := BIF_equals(&a[i], &b[i]) + lib.InternalCodingErrorIf(eq.Type() != mlrval.MT_BOOL) + if eq.AcquireBoolValue() == false { + return mlrval.FALSE + } + } + + return mlrval.TRUE +} +func ne_b_aa(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + output := eq_b_aa(input1, input2) + return mlrval.FromBool(!output.AcquireBoolValue()) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +func eq_b_mm(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireMapValue().Equals(input2.AcquireMapValue())) +} +func ne_b_mm(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(!input1.AcquireMapValue().Equals(input2.AcquireMapValue())) +} + +// We get a Golang "initialization loop" due to recursive depth computation +// if this is defined statically. So, we use a "package init" function. +var eq_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{} + +func init() { + eq_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, + /*ABSENT */ {_erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, + /*NULL */ {_erro, _absn, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _erro}, + /*VOID */ {_erro, _absn, _fals, eq_b_ss, eq_b_ss, eq_b_sx, eq_b_sx, _fals, _fals, _fals, _erro}, + /*STRING */ {_erro, _absn, _fals, eq_b_ss, eq_b_ss, eq_b_sx, eq_b_sx, _fals, _fals, _fals, _erro}, + /*INT */ {_erro, _absn, _fals, eq_b_xs, eq_b_xs, eq_b_ii, eq_b_if, _fals, _fals, _fals, _erro}, + /*FLOAT */ {_erro, _absn, _fals, eq_b_xs, eq_b_xs, eq_b_fi, eq_b_ff, _fals, _fals, _fals, _erro}, + /*BOOL */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, eq_b_bb, _fals, _fals, _erro}, + /*ARRAY */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, _fals, eq_b_aa, _fals, _erro}, + /*MAP */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _fals, eq_b_mm, _erro}, + /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, + } +} + +var ne_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, + /*ABSENT */ {_erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, + /*NULL */ {_erro, _absn, _fals, _true, _true, _true, _true, _true, _true, _true, _erro}, + /*VOID */ {_erro, _absn, _true, ne_b_ss, ne_b_ss, ne_b_sx, ne_b_sx, _true, _true, _true, _erro}, + /*STRING */ {_erro, _absn, _true, ne_b_ss, ne_b_ss, ne_b_sx, ne_b_sx, _true, _true, _true, _erro}, + /*INT */ {_erro, _absn, _true, ne_b_xs, ne_b_xs, ne_b_ii, ne_b_if, _true, _true, _true, _erro}, + /*FLOAT */ {_erro, _absn, _true, ne_b_xs, ne_b_xs, ne_b_fi, ne_b_ff, _true, _true, _true, _erro}, + /*BOOL */ {_erro, _absn, _true, _true, _true, _true, _true, ne_b_bb, _true, _true, _erro}, + /*ARRAY */ {_erro, _absn, _true, _true, _true, _true, _true, _true, ne_b_aa, _true, _erro}, + /*MAP */ {_erro, _absn, _true, _true, _true, _true, _true, _true, _true, ne_b_mm, _erro}, + /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, +} + +var gt_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_erro, _erro, _fals, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, + /*ABSENT */ {_erro, _absn, _true, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, + /*NULL */ {_true, _fals, _fals, _true, _true, _true, _true, _true, _absn, _absn, _erro}, + /*VOID */ {_erro, _absn, _fals, gt_b_ss, gt_b_ss, gt_b_sx, gt_b_sx, _fals, _fals, _fals, _erro}, + /*STRING */ {_erro, _absn, _fals, gt_b_ss, gt_b_ss, gt_b_sx, gt_b_sx, _fals, _fals, _fals, _erro}, + /*INT */ {_erro, _absn, _fals, gt_b_xs, gt_b_xs, gt_b_ii, gt_b_if, _fals, _fals, _fals, _erro}, + /*FLOAT */ {_erro, _absn, _fals, gt_b_xs, gt_b_xs, gt_b_fi, gt_b_ff, _fals, _fals, _fals, _erro}, + /*BOOL */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, gt_b_bb, _fals, _fals, _erro}, + /*ARRAY */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _erro, _fals, _erro}, + /*MAP */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _erro, _erro}, + /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, +} + +var ge_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_erro, _erro, _fals, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, + /*ABSENT */ {_erro, _absn, _true, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, + /*NULL */ {_true, _fals, _true, _true, _true, _true, _true, _true, _absn, _absn, _erro}, + /*VOID */ {_erro, _absn, _fals, ge_b_ss, ge_b_ss, ge_b_sx, ge_b_sx, _fals, _fals, _fals, _erro}, + /*STRING */ {_erro, _absn, _fals, ge_b_ss, ge_b_ss, ge_b_sx, ge_b_sx, _fals, _fals, _fals, _erro}, + /*INT */ {_erro, _absn, _fals, ge_b_xs, ge_b_xs, ge_b_ii, ge_b_if, _fals, _fals, _fals, _erro}, + /*FLOAT */ {_erro, _absn, _fals, ge_b_xs, ge_b_xs, ge_b_fi, ge_b_ff, _fals, _fals, _fals, _erro}, + /*BOOL */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, ge_b_bb, _fals, _fals, _erro}, + /*ARRAY */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _erro, _fals, _erro}, + /*MAP */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _erro, _erro}, + /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, +} + +var lt_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_erro, _erro, _true, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, + /*ABSENT */ {_erro, _absn, _fals, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, + /*NULL */ {_fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _absn, _absn, _erro}, + /*VOID */ {_erro, _absn, _true, lt_b_ss, lt_b_ss, lt_b_sx, lt_b_sx, _fals, _fals, _fals, _erro}, + /*STRING */ {_erro, _absn, _true, lt_b_ss, lt_b_ss, lt_b_sx, lt_b_sx, _fals, _fals, _fals, _erro}, + /*INT */ {_erro, _absn, _true, lt_b_xs, lt_b_xs, lt_b_ii, lt_b_if, _fals, _fals, _fals, _erro}, + /*FLOAT */ {_erro, _absn, _true, lt_b_xs, lt_b_xs, lt_b_fi, lt_b_ff, _fals, _fals, _fals, _erro}, + /*BOOL */ {_erro, _absn, _true, _fals, _fals, _fals, _fals, lt_b_bb, _fals, _fals, _erro}, + /*ARRAY */ {_erro, _absn, _absn, _fals, _fals, _fals, _fals, _fals, _erro, _fals, _erro}, + /*MAP */ {_erro, _absn, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _erro, _erro}, + /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, +} + +var le_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_erro, _erro, _true, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, + /*ABSENT */ {_erro, _absn, _fals, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, + /*NULL */ {_fals, _true, _true, _fals, _fals, _fals, _fals, _fals, _absn, _absn, _erro}, + /*VOID */ {_erro, _absn, _true, le_b_ss, le_b_ss, le_b_sx, le_b_sx, _fals, _fals, _fals, _erro}, + /*STRING */ {_erro, _absn, _true, le_b_ss, le_b_ss, le_b_sx, le_b_sx, _fals, _fals, _fals, _erro}, + /*INT */ {_erro, _absn, _true, le_b_xs, le_b_xs, le_b_ii, le_b_if, _fals, _fals, _fals, _erro}, + /*FLOAT */ {_erro, _absn, _true, le_b_xs, le_b_xs, le_b_fi, le_b_ff, _fals, _fals, _fals, _erro}, + /*BOOL */ {_erro, _absn, _true, _fals, _fals, _fals, _fals, le_b_bb, _fals, _fals, _erro}, + /*ARRAY */ {_erro, _absn, _absn, _fals, _fals, _fals, _fals, _fals, _erro, _fals, _erro}, + /*MAP */ {_erro, _absn, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _erro, _erro}, + /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, +} + +var cmp_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_erro, _erro, _true, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, + /*ABSENT */ {_erro, _absn, _fals, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, + /*NULL */ {_fals, _true, _true, _fals, _fals, _fals, _fals, _fals, _absn, _absn, _erro}, + /*VOID */ {_erro, _absn, _true, cmp_b_ss, cmp_b_ss, cmp_b_sx, cmp_b_sx, _fals, _fals, _fals, _erro}, + /*STRING */ {_erro, _absn, _true, cmp_b_ss, cmp_b_ss, cmp_b_sx, cmp_b_sx, _fals, _fals, _fals, _erro}, + /*INT */ {_erro, _absn, _true, cmp_b_xs, cmp_b_xs, cmp_b_ii, cmp_b_if, _fals, _fals, _fals, _erro}, + /*FLOAT */ {_erro, _absn, _true, cmp_b_xs, cmp_b_xs, cmp_b_fi, cmp_b_ff, _fals, _fals, _fals, _erro}, + /*BOOL */ {_erro, _absn, _true, _fals, _fals, _fals, _fals, cmp_b_bb, _fals, _fals, _erro}, + /*ARRAY */ {_erro, _absn, _absn, _fals, _fals, _fals, _fals, _fals, _erro, _fals, _erro}, + /*MAP */ {_erro, _absn, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _erro, _erro}, + /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, +} + +func BIF_equals(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return eq_dispositions[input1.Type()][input2.Type()](input1, input2) +} +func BIF_not_equals(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return ne_dispositions[input1.Type()][input2.Type()](input1, input2) +} +func BIF_greater_than(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return gt_dispositions[input1.Type()][input2.Type()](input1, input2) +} +func BIF_greater_than_or_equals(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return ge_dispositions[input1.Type()][input2.Type()](input1, input2) +} +func BIF_less_than(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return lt_dispositions[input1.Type()][input2.Type()](input1, input2) +} +func BIF_less_than_or_equals(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return le_dispositions[input1.Type()][input2.Type()](input1, input2) +} +func BIF_cmp(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return cmp_dispositions[input1.Type()][input2.Type()](input1, input2) +} + +// For Go's sort.Slice. +func BIF_less_than_as_bool(input1, input2 *mlrval.Mlrval) bool { + // TODO refactor to avoid copy + // This is a hot path for sort GC and is worth significant hand-optimization + mretval := lt_dispositions[input1.Type()][input2.Type()](input1, input2) + retval, ok := mretval.GetBoolValue() + lib.InternalCodingErrorIf(!ok) + return retval +} + +// For Go's sort.Slice. +func BIF_less_than_or_equals_as_bool(input1, input2 *mlrval.Mlrval) bool { + // TODO refactor to avoid copy + // This is a hot path for sort GC and is worth significant hand-optimization + mretval := le_dispositions[input1.Type()][input2.Type()](input1, input2) + retval, ok := mretval.GetBoolValue() + lib.InternalCodingErrorIf(!ok) + return retval +} + +// For top-keeper +func BIF_greater_than_as_bool(input1, input2 *mlrval.Mlrval) bool { + // TODO refactor to avoid copy + // This is a hot path for sort GC and is worth significant hand-optimization + mretval := gt_dispositions[input1.Type()][input2.Type()](input1, input2) + retval, ok := mretval.GetBoolValue() + lib.InternalCodingErrorIf(!ok) + return retval +} + +// For top-keeper +func BIF_greater_than_or_equals_as_bool(input1, input2 *mlrval.Mlrval) bool { + // TODO refactor to avoid copy + // This is a hot path for sort GC and is worth significant hand-optimization + mretval := ge_dispositions[input1.Type()][input2.Type()](input1, input2) + retval, ok := mretval.GetBoolValue() + lib.InternalCodingErrorIf(!ok) + return retval +} + +// Convenience wrapper for non-DSL callsites that just want a bool +func BIF_equals_as_bool(input1, input2 *mlrval.Mlrval) bool { + mretval := eq_dispositions[input1.Type()][input2.Type()](input1, input2) + retval, ok := mretval.GetBoolValue() + lib.InternalCodingErrorIf(!ok) + return retval +} + +// ---------------------------------------------------------------- +func BIF_logical_NOT(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsBool() { + return mlrval.FromBool(!input1.AcquireBoolValue()) + } else { + return mlrval.ERROR + } +} + +func BIF_logical_AND(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsBool() && input2.IsBool() { + return mlrval.FromBool(input1.AcquireBoolValue() && input2.AcquireBoolValue()) + } else { + return mlrval.ERROR + } +} + +func BIF_logical_OR(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsBool() && input2.IsBool() { + return mlrval.FromBool(input1.AcquireBoolValue() || input2.AcquireBoolValue()) + } else { + return mlrval.ERROR + } +} + +func BIF_logical_XOR(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsBool() && input2.IsBool() { + return mlrval.FromBool(input1.AcquireBoolValue() != input2.AcquireBoolValue()) + } else { + return mlrval.ERROR + } +} diff --git a/internal/pkg/bifs/collections.go b/internal/pkg/bifs/collections.go new file mode 100644 index 000000000..a5ecb028d --- /dev/null +++ b/internal/pkg/bifs/collections.go @@ -0,0 +1,852 @@ +package bifs + +import ( + "bytes" + "strconv" + "strings" + + "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + +// ================================================================ +// Map/array count. Scalars (including strings) have length 1; strlen is for string length. +func BIF_length(input1 *mlrval.Mlrval) *mlrval.Mlrval { + switch input1.Type() { + case mlrval.MT_ERROR: + return mlrval.FromInt(0) + break + case mlrval.MT_ABSENT: + return mlrval.FromInt(0) + break + case mlrval.MT_ARRAY: + arrayval := input1.AcquireArrayValue() + return mlrval.FromInt(int(len(arrayval))) + break + case mlrval.MT_MAP: + mapval := input1.AcquireMapValue() + return mlrval.FromInt(int(mapval.FieldCount)) + break + } + return mlrval.FromInt(1) +} + +// ================================================================ +func depth_from_array(input1 *mlrval.Mlrval) *mlrval.Mlrval { + maxChildDepth := 0 + arrayval := input1.AcquireArrayValue() + for _, child := range arrayval { + childDepth := BIF_depth(&child) + lib.InternalCodingErrorIf(!childDepth.IsInt()) + iChildDepth := int(childDepth.AcquireIntValue()) + if iChildDepth > maxChildDepth { + maxChildDepth = iChildDepth + } + } + return mlrval.FromInt(int(1 + maxChildDepth)) +} + +func depth_from_map(input1 *mlrval.Mlrval) *mlrval.Mlrval { + maxChildDepth := 0 + mapval := input1.AcquireMapValue() + for pe := mapval.Head; pe != nil; pe = pe.Next { + child := pe.Value + childDepth := BIF_depth(child) + lib.InternalCodingErrorIf(!childDepth.IsInt()) + iChildDepth := int(childDepth.AcquireIntValue()) + if iChildDepth > maxChildDepth { + maxChildDepth = iChildDepth + } + } + return mlrval.FromInt(int(1 + maxChildDepth)) +} + +func depth_from_scalar(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(0) +} + +// We get a Golang "initialization loop" due to recursive depth computation +// if this is defined statically. So, we use a "package init" function. +var depth_dispositions = [mlrval.MT_DIM]UnaryFunc{} + +func init() { + depth_dispositions = [mlrval.MT_DIM]UnaryFunc{ + /*ERROR */ _erro1, + /*ABSENT */ _absn1, + /*NULL */ _zero1, + /*VOID */ depth_from_scalar, + /*STRING */ depth_from_scalar, + /*INT */ depth_from_scalar, + /*FLOAT */ depth_from_scalar, + /*BOOL */ depth_from_scalar, + /*ARRAY */ depth_from_array, + /*MAP */ depth_from_map, + /*FUNC */ _erro1, + } +} + +func BIF_depth(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return depth_dispositions[input1.Type()](input1) +} + +// ================================================================ +func leafcount_from_array(input1 *mlrval.Mlrval) *mlrval.Mlrval { + sumChildLeafCount := 0 + arrayval := input1.AcquireArrayValue() + for _, child := range arrayval { + // Golang initialization loop if we do this :( + // childLeafCount := BIF_leafcount(&child) + + childLeafCount := mlrval.FromInt(1) + if child.IsArray() { + childLeafCount = leafcount_from_array(&child) + } else if child.IsMap() { + childLeafCount = leafcount_from_map(&child) + } + + lib.InternalCodingErrorIf(!childLeafCount.IsInt()) + iChildLeafCount := int(childLeafCount.AcquireIntValue()) + sumChildLeafCount += iChildLeafCount + } + return mlrval.FromInt(int(sumChildLeafCount)) +} + +func leafcount_from_map(input1 *mlrval.Mlrval) *mlrval.Mlrval { + sumChildLeafCount := 0 + mapval := input1.AcquireMapValue() + for pe := mapval.Head; pe != nil; pe = pe.Next { + child := pe.Value + + // Golang initialization loop if we do this :( + // childLeafCount := BIF_leafcount(child) + + childLeafCount := mlrval.FromInt(1) + if child.IsArray() { + childLeafCount = leafcount_from_array(child) + } else if child.IsMap() { + childLeafCount = leafcount_from_map(child) + } + + lib.InternalCodingErrorIf(!childLeafCount.IsInt()) + iChildLeafCount := int(childLeafCount.AcquireIntValue()) + sumChildLeafCount += iChildLeafCount + } + return mlrval.FromInt(int(sumChildLeafCount)) +} + +func leafcount_from_scalar(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(1) +} + +var leafcount_dispositions = [mlrval.MT_DIM]UnaryFunc{ + /*ERROR */ _erro1, + /*ABSENT */ _absn1, + /*NULL */ _zero1, + /*VOID */ leafcount_from_scalar, + /*STRING */ leafcount_from_scalar, + /*INT */ leafcount_from_scalar, + /*FLOAT */ leafcount_from_scalar, + /*BOOL */ leafcount_from_scalar, + /*ARRAY */ leafcount_from_array, + /*MAP */ leafcount_from_map, + /*FUNC */ _erro1, +} + +func BIF_leafcount(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return leafcount_dispositions[input1.Type()](input1) +} + +// ---------------------------------------------------------------- +func has_key_in_array(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input2.IsString() { + return mlrval.FALSE + } + if !input2.IsInt() { + return mlrval.ERROR + } + arrayval := input1.AcquireArrayValue() + _, ok := unaliasArrayIndex(&arrayval, input2.AcquireIntValue()) + return mlrval.FromBool(ok) +} + +func has_key_in_map(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input2.IsString() || input2.IsInt() { + return mlrval.FromBool(input1.AcquireMapValue().Has(input2.String())) + } else { + return mlrval.ERROR + } +} + +func BIF_haskey(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsArray() { + return has_key_in_array(input1, input2) + } else if input1.IsMap() { + return has_key_in_map(input1, input2) + } else { + return mlrval.ERROR + } +} + +// ================================================================ +func BIF_mapselect(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { + if len(mlrvals) < 1 { + return mlrval.ERROR + } + if !mlrvals[0].IsMap() { + return mlrval.ERROR + } + oldmap := mlrvals[0].AcquireMapValue() + newMap := mlrval.NewMlrmap() + + newKeys := make(map[string]bool) + for _, selectArg := range mlrvals[1:] { + if selectArg.IsString() { + newKeys[selectArg.AcquireStringValue()] = true + } else if selectArg.IsInt() { + newKeys[selectArg.String()] = true + } else if selectArg.IsArray() { + for _, element := range selectArg.AcquireArrayValue() { + if element.IsString() { + newKeys[element.AcquireStringValue()] = true + } else { + return mlrval.ERROR + } + } + } else { + return mlrval.ERROR + } + } + + for pe := oldmap.Head; pe != nil; pe = pe.Next { + oldKey := pe.Key + _, present := newKeys[oldKey] + if present { + newMap.PutCopy(oldKey, oldmap.Get(oldKey)) + } + } + + return mlrval.FromMap(newMap) +} + +// ---------------------------------------------------------------- +func BIF_mapexcept(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { + if len(mlrvals) < 1 { + return mlrval.ERROR + } + if !mlrvals[0].IsMap() { + return mlrval.ERROR + } + newMap := mlrvals[0].AcquireMapValue().Copy() + + for _, exceptArg := range mlrvals[1:] { + if exceptArg.IsString() { + newMap.Remove(exceptArg.AcquireStringValue()) + } else if exceptArg.IsInt() { + newMap.Remove(exceptArg.String()) + } else if exceptArg.IsArray() { + for _, element := range exceptArg.AcquireArrayValue() { + if element.IsString() { + newMap.Remove(element.AcquireStringValue()) + } else { + return mlrval.ERROR + } + } + } else { + return mlrval.ERROR + } + } + + return mlrval.FromMap(newMap) +} + +// ---------------------------------------------------------------- +func BIF_mapsum(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { + if len(mlrvals) == 0 { + return mlrval.FromEmptyMap() + } + if len(mlrvals) == 1 { + return mlrvals[0] + } + if mlrvals[0].Type() != mlrval.MT_MAP { + return mlrval.ERROR + } + newMap := mlrvals[0].AcquireMapValue().Copy() + + for _, otherMapArg := range mlrvals[1:] { + if otherMapArg.Type() != mlrval.MT_MAP { + return mlrval.ERROR + } + + for pe := otherMapArg.AcquireMapValue().Head; pe != nil; pe = pe.Next { + newMap.PutCopy(pe.Key, pe.Value) + } + } + + return mlrval.FromMap(newMap) +} + +// ---------------------------------------------------------------- +func BIF_mapdiff(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval { + if len(mlrvals) == 0 { + return mlrval.FromEmptyMap() + } + if len(mlrvals) == 1 { + return mlrvals[0] + } + if !mlrvals[0].IsMap() { + return mlrval.ERROR + } + newMap := mlrvals[0].AcquireMapValue().Copy() + + for _, otherMapArg := range mlrvals[1:] { + if !otherMapArg.IsMap() { + return mlrval.ERROR + } + + for pe := otherMapArg.AcquireMapValue().Head; pe != nil; pe = pe.Next { + newMap.Remove(pe.Key) + } + } + + return mlrval.FromMap(newMap) +} + +// ================================================================ +// joink([1,2,3], ",") -> "1,2,3" +// joink({"a":3,"b":4,"c":5}, ",") -> "a,b,c" +func BIF_joink(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if !input2.IsString() { + return mlrval.ERROR + } + fieldSeparator := input2.AcquireStringValue() + if input1.IsMap() { + var buffer bytes.Buffer + + for pe := input1.AcquireMapValue().Head; pe != nil; pe = pe.Next { + buffer.WriteString(pe.Key) + if pe.Next != nil { + buffer.WriteString(fieldSeparator) + } + } + + return mlrval.FromString(buffer.String()) + } else if input1.IsArray() { + var buffer bytes.Buffer + + for i := range input1.AcquireArrayValue() { + if i > 0 { + buffer.WriteString(fieldSeparator) + } + // Miller userspace array indices are 1-up + buffer.WriteString(strconv.Itoa(i + 1)) + } + + return mlrval.FromString(buffer.String()) + } else { + return mlrval.ERROR + } +} + +// ---------------------------------------------------------------- +// joinv([3,4,5], ",") -> "3,4,5" +// joinv({"a":3,"b":4,"c":5}, ",") -> "3,4,5" +func BIF_joinv(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if !input2.IsString() { + return mlrval.ERROR + } + fieldSeparator := input2.AcquireStringValue() + + if input1.IsMap() { + var buffer bytes.Buffer + + for pe := input1.AcquireMapValue().Head; pe != nil; pe = pe.Next { + buffer.WriteString(pe.Value.String()) + if pe.Next != nil { + buffer.WriteString(fieldSeparator) + } + } + + return mlrval.FromString(buffer.String()) + } else if input1.IsArray() { + var buffer bytes.Buffer + + for i, element := range input1.AcquireArrayValue() { + if i > 0 { + buffer.WriteString(fieldSeparator) + } + buffer.WriteString(element.String()) + } + + return mlrval.FromString(buffer.String()) + } else { + return mlrval.ERROR + } +} + +// ---------------------------------------------------------------- +// joinkv([3,4,5], "=", ",") -> "1=3,2=4,3=5" +// joinkv({"a":3,"b":4,"c":5}, "=", ",") -> "a=3,b=4,c=5" +func BIF_joinkv(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { + if !input2.IsString() { + return mlrval.ERROR + } + pairSeparator := input2.AcquireStringValue() + if !input3.IsString() { + return mlrval.ERROR + } + fieldSeparator := input3.AcquireStringValue() + + if input1.IsMap() { + var buffer bytes.Buffer + + for pe := input1.AcquireMapValue().Head; pe != nil; pe = pe.Next { + buffer.WriteString(pe.Key) + buffer.WriteString(pairSeparator) + buffer.WriteString(pe.Value.String()) + if pe.Next != nil { + buffer.WriteString(fieldSeparator) + } + } + + return mlrval.FromString(buffer.String()) + } else if input1.IsArray() { + var buffer bytes.Buffer + + for i, element := range input1.AcquireArrayValue() { + if i > 0 { + buffer.WriteString(fieldSeparator) + } + // Miller userspace array indices are 1-up + buffer.WriteString(strconv.Itoa(i + 1)) + buffer.WriteString(pairSeparator) + buffer.WriteString(element.String()) + } + + return mlrval.FromString(buffer.String()) + } else { + return mlrval.ERROR + } +} + +// ================================================================ +// splitkv("a=3,b=4,c=5", "=", ",") -> {"a":3,"b":4,"c":5} +func BIF_splitkv(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsStringOrVoid() { + return mlrval.ERROR + } + if !input2.IsString() { + return mlrval.ERROR + } + pairSeparator := input2.AcquireStringValue() + if !input3.IsString() { + return mlrval.ERROR + } + fieldSeparator := input3.AcquireStringValue() + + output := mlrval.FromMap(mlrval.NewMlrmap()) + + fields := lib.SplitString(input1.AcquireStringValue(), fieldSeparator) + for i, field := range fields { + pair := strings.SplitN(field, pairSeparator, 2) + if len(pair) == 1 { + key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up + value := mlrval.FromInferredType(pair[0]) + output.AcquireMapValue().PutReference(key, value) + } else if len(pair) == 2 { + key := pair[0] + value := mlrval.FromInferredType(pair[1]) + output.AcquireMapValue().PutReference(key, value) + } else { + lib.InternalCodingErrorIf(true) + } + } + return output +} + +// ---------------------------------------------------------------- +// splitkvx("a=3,b=4,c=5", "=", ",") -> {"a":"3","b":"4","c":"5"} +func BIF_splitkvx(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsStringOrVoid() { + return mlrval.ERROR + } + if !input2.IsString() { + return mlrval.ERROR + } + pairSeparator := input2.AcquireStringValue() + if !input3.IsString() { + return mlrval.ERROR + } + fieldSeparator := input3.AcquireStringValue() + + output := mlrval.FromMap(mlrval.NewMlrmap()) + + fields := lib.SplitString(input1.AcquireStringValue(), fieldSeparator) + for i, field := range fields { + pair := strings.SplitN(field, pairSeparator, 2) + if len(pair) == 1 { + key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up + value := mlrval.FromString(pair[0]) + output.AcquireMapValue().PutReference(key, value) + } else if len(pair) == 2 { + key := pair[0] + value := mlrval.FromString(pair[1]) + output.AcquireMapValue().PutReference(key, value) + } else { + lib.InternalCodingErrorIf(true) + } + } + + return output +} + +// ---------------------------------------------------------------- +// splitnv("a,b,c", ",") -> {"1":"a","2":"b","3":"c"} +func BIF_splitnv(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsStringOrVoid() { + return mlrval.ERROR + } + if !input2.IsString() { + return mlrval.ERROR + } + + output := mlrval.FromMap(mlrval.NewMlrmap()) + + fields := lib.SplitString(input1.AcquireStringValue(), input2.AcquireStringValue()) + for i, field := range fields { + key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up + value := mlrval.FromInferredType(field) + output.AcquireMapValue().PutReference(key, value) + } + + return output +} + +// ---------------------------------------------------------------- +// splitnvx("3,4,5", ",") -> {"1":"3","2":"4","3":"5"} +func BIF_splitnvx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsStringOrVoid() { + return mlrval.ERROR + } + if !input2.IsString() { + return mlrval.ERROR + } + + output := mlrval.FromMap(mlrval.NewMlrmap()) + + fields := lib.SplitString(input1.AcquireStringValue(), input2.AcquireStringValue()) + for i, field := range fields { + key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up + value := mlrval.FromString(field) + output.AcquireMapValue().PutReference(key, value) + } + + return output +} + +// ---------------------------------------------------------------- +// splita("3,4,5", ",") -> [3,4,5] +func BIF_splita(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsStringOrVoid() { + return mlrval.ERROR + } + if !input2.IsString() { + return mlrval.ERROR + } + fieldSeparator := input2.AcquireStringValue() + + fields := lib.SplitString(input1.AcquireStringValue(), fieldSeparator) + + arrayval := make([]mlrval.Mlrval, len(fields)) + + for i, field := range fields { + value := mlrval.FromInferredType(field) + arrayval[i] = *value + } + + return mlrval.FromArray(arrayval) +} + +// ---------------------------------------------------------------- +// BIF_splitax splits a string to an array, without type-inference: +// e.g. splitax("3,4,5", ",") -> ["3","4","5"] +func BIF_splitax(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsStringOrVoid() { + return mlrval.ERROR + } + if !input2.IsString() { + return mlrval.ERROR + } + input := input1.AcquireStringValue() + fieldSeparator := input2.AcquireStringValue() + + return bif_splitax_helper(input, fieldSeparator) +} + +// bif_splitax_helper is split out for the benefit of BIF_splitax and +// BIF_unflatten. +func bif_splitax_helper(input string, separator string) *mlrval.Mlrval { + fields := lib.SplitString(input, separator) + + arrayval := make([]mlrval.Mlrval, len(fields)) + + for i, field := range fields { + arrayval[i] = *mlrval.FromString(field) + } + + return mlrval.FromArray(arrayval) +} + +// ---------------------------------------------------------------- +func BIF_get_keys(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsMap() { + // TODO: make a ReferenceFrom with comments + mapval := input1.AcquireMapValue() + arrayval := make([]mlrval.Mlrval, mapval.FieldCount) + i := 0 + for pe := mapval.Head; pe != nil; pe = pe.Next { + arrayval[i] = *mlrval.FromString(pe.Key) + i++ + } + return mlrval.FromArray(arrayval) + + } else if input1.IsArray() { + inputarrayval := input1.AcquireArrayValue() + arrayval := make([]mlrval.Mlrval, len(inputarrayval)) + for i := range inputarrayval { + arrayval[i] = *mlrval.FromInt(int(i + 1)) // Miller user-space indices are 1-up + } + return mlrval.FromArray(arrayval) + + } else { + return mlrval.ERROR + } +} + +// ---------------------------------------------------------------- +func BIF_get_values(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsMap() { + mapval := input1.AcquireMapValue() + arrayval := make([]mlrval.Mlrval, mapval.FieldCount) + i := 0 + for pe := mapval.Head; pe != nil; pe = pe.Next { + arrayval[i] = *pe.Value.Copy() + i++ + } + return mlrval.FromArray(arrayval) + + } else if input1.IsArray() { + inputarrayval := input1.AcquireArrayValue() + arrayval := make([]mlrval.Mlrval, len(inputarrayval)) + for i, value := range inputarrayval { + arrayval[i] = *value.Copy() + } + return mlrval.FromArray(arrayval) + + } else { + return mlrval.ERROR + } +} + +// ---------------------------------------------------------------- +func BIF_append(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsArray() { + return mlrval.ERROR + } + + output := input1.Copy() + output.ArrayAppend(input2.Copy()) + return output +} + +// ---------------------------------------------------------------- +// First argumemnt is prefix. +// Second argument is delimiter. +// Third argument is map or array. +// flatten("a", ".", {"b": { "c": 4 }}) is {"a.b.c" : 4}. +// flatten("", ".", {"a": { "b": 3 }}) is {"a.b" : 3}. +func BIF_flatten(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { + if input3.IsMap() || input3.IsArray() { + if !input1.IsString() && input1.Type() != mlrval.MT_VOID { + return mlrval.ERROR + } + prefix := input1.AcquireStringValue() + if !input2.IsString() { + return mlrval.ERROR + } + delimiter := input2.AcquireStringValue() + + retval := input3.FlattenToMap(prefix, delimiter) + return &retval + } else { + return input3 + } +} + +// flatten($*, ".") is the same as flatten("", ".", $*) +func BIF_flatten_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return BIF_flatten(mlrval.VOID, input2, input1) +} + +// ---------------------------------------------------------------- +// First argument is a map. +// Second argument is a delimiter string. +// unflatten({"a.b.c", ".") is {"a": { "b": { "c": 4}}}. +func BIF_unflatten(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if !input2.IsString() { + return mlrval.ERROR + } + if input1.Type() != mlrval.MT_MAP { + return input1 + } + oldmap := input1.AcquireMapValue() + separator := input2.AcquireStringValue() + newmap := oldmap.CopyUnflattened(separator) + return mlrval.FromMap(newmap) +} + +// ---------------------------------------------------------------- +// Converts maps with "1", "2", ... keys into arrays. Recurses nested data structures. +func BIF_arrayify(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsMap() { + if input1.AcquireMapValue().IsEmpty() { + return input1 + } + + convertible := true + i := 0 + for pe := input1.AcquireMapValue().Head; pe != nil; pe = pe.Next { + sval := strconv.Itoa(i + 1) // Miller user-space indices are 1-up + i++ + if pe.Key != sval { + convertible = false + } + pe.Value = BIF_arrayify(pe.Value) + } + + if convertible { + mapval := input1.AcquireMapValue() + arrayval := make([]mlrval.Mlrval, input1.AcquireMapValue().FieldCount) + i := 0 + for pe := mapval.Head; pe != nil; pe = pe.Next { + arrayval[i] = *pe.Value.Copy() + i++ + } + return mlrval.FromArray(arrayval) + + } else { + return input1 + } + + } else if input1.IsArray() { + // TODO: comment (or rethink) that this modifies its inputs!! + output := input1.Copy() + for i := range input1.AcquireArrayValue() { + output.AcquireArrayValue()[i] = *BIF_arrayify(&output.AcquireArrayValue()[i]) + } + return output + + } else { + return input1 + } +} + +// ---------------------------------------------------------------- +func BIF_json_parse(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsVoid() { + return input1 + } else if !input1.IsString() { + return mlrval.ERROR + } else { + output := mlrval.FromPending() + err := output.UnmarshalJSON([]byte(input1.AcquireStringValue())) + if err != nil { + return mlrval.ERROR + } + return output + } +} + +func BIF_json_stringify_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { + outputBytes, err := input1.MarshalJSON(mlrval.JSON_SINGLE_LINE, false) + if err != nil { + return mlrval.ERROR + } else { + return mlrval.FromString(string(outputBytes)) + } +} + +func BIF_json_stringify_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + var jsonFormatting mlrval.TJSONFormatting = mlrval.JSON_SINGLE_LINE + useMultiline, ok := input2.GetBoolValue() + if !ok { + return mlrval.ERROR + } + if useMultiline { + jsonFormatting = mlrval.JSON_MULTILINE + } + + outputBytes, err := input1.MarshalJSON(jsonFormatting, false) + if err != nil { + return mlrval.ERROR + } else { + return mlrval.FromString(string(outputBytes)) + } +} + +func unaliasArrayIndex(array *[]mlrval.Mlrval, mindex int) (int, bool) { + n := int(len(*array)) + return unaliasArrayLengthIndex(n, mindex) +} + +// Input "mindex" is a Miller DSL array index. These are 1-up, so 1..n where n +// is the length of the array. Also, -n..-1 are aliases to 1..n. 0 is never a +// valid index. +// +// Output "zindex" is a Golang array index. These are 0-up, so 0..(n-1). +// +// The second return value indicates whether the Miller index is in-bounds. +// Even if it's out of bounds, while the second return value is false, the +// first return is correctly de-aliased. E.g. if the array has length 5 and the +// mindex is 8, zindex is 7 and valid=false. This is so in array-slice +// operations like 'v = myarray[2:8]' the callsite can hand back slots 2-5 of +// the array (which is the same way Python handles beyond-the-end indexing). + +// Examples with n = 5: +// +// mindex zindex ok +// -7 -2 false +// -6 -1 false +// -5 0 true +// -4 1 true +// -3 2 true +// -2 3 true +// -1 4 true +// 0 -1 false +// 1 0 true +// 2 1 true +// 3 2 true +// 4 3 true +// 5 4 true +// 6 5 false +// 7 6 false + +func unaliasArrayLengthIndex(n int, mindex int) (int, bool) { + if 1 <= mindex { + zindex := mindex - 1 + if mindex <= n { // in bounds + return zindex, true + } else { // out of bounds + return zindex, false + } + } else if mindex <= -1 { + zindex := mindex + n + if -n <= mindex { // in bounds + return zindex, true + } else { // out of bounds + return zindex, false + } + } else { + // mindex is 0 + return -1, false + } +} diff --git a/internal/pkg/bifs/collections_test.go b/internal/pkg/bifs/collections_test.go new file mode 100644 index 000000000..08b25061b --- /dev/null +++ b/internal/pkg/bifs/collections_test.go @@ -0,0 +1,75 @@ +package bifs + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + +func TestBIF_length(t *testing.T) { + input1 := mlrval.FromInt(123) + output := BIF_length(input1) + intval, ok := output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 1, intval) +} + +func TestBIF_depth(t *testing.T) { + input1 := mlrval.FromInt(123) + output := BIF_depth(input1) + intval, ok := output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 0, intval) + + mapval := mlrval.NewMlrmap() + mapval.PutReference("key", mlrval.FromString("value")) + input1 = mlrval.FromMap(mapval) + output = BIF_depth(input1) + intval, ok = output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 1, intval) + + arrayval := make([]mlrval.Mlrval, 1) + arrayval[0] = *mlrval.FromString("value") + input1 = mlrval.FromArray(arrayval) + output = BIF_depth(input1) + intval, ok = output.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 1, intval) +} + +// TODO: copy in more unit-test cases from existing regression-test data + +// func leafcount_from_array(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func leafcount_from_map(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func leafcount_from_scalar(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_leafcount(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func has_key_in_array(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func has_key_in_map(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_haskey(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_mapselect(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval +// func BIF_mapexcept(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval +// func BIF_mapsum(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval +// func BIF_mapdiff(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval +// func BIF_joink(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_joinv(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_joinkv(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_splitkv(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_splitkvx(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_splitnv(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_splitnvx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_splita(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_splitax(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func mlrvalSplitAXHelper(input string, separator string) *mlrval.Mlrval +// func BIF_get_keys(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_get_values(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_append(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_flatten(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_flatten_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_unflatten(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_arrayify(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_json_parse(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_json_stringify_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval +// func BIF_json_stringify_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval diff --git a/internal/pkg/types/mlrval_functions_datetime.go b/internal/pkg/bifs/datetime.go similarity index 61% rename from internal/pkg/types/mlrval_functions_datetime.go rename to internal/pkg/bifs/datetime.go index 9ae6e244c..aaac7555b 100644 --- a/internal/pkg/types/mlrval_functions_datetime.go +++ b/internal/pkg/bifs/datetime.go @@ -1,4 +1,4 @@ -package types +package bifs import ( "fmt" @@ -9,22 +9,23 @@ import ( "github.com/lestrrat-go/strftime" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) const ISO8601_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" -var ptr_ISO8601_TIME_FORMAT = MlrvalFromString("%Y-%m-%dT%H:%M:%SZ") -var ptr_ISO8601_LOCAL_TIME_FORMAT = MlrvalFromString("%Y-%m-%d %H:%M:%S") -var ptr_YMD_FORMAT = MlrvalFromString("%Y-%m-%d") +var ptr_ISO8601_TIME_FORMAT = mlrval.FromString("%Y-%m-%dT%H:%M:%SZ") +var ptr_ISO8601_LOCAL_TIME_FORMAT = mlrval.FromString("%Y-%m-%d %H:%M:%S") +var ptr_YMD_FORMAT = mlrval.FromString("%Y-%m-%d") // ================================================================ -func BIF_systime() *Mlrval { - return MlrvalFromFloat64( +func BIF_systime() *mlrval.Mlrval { + return mlrval.FromFloat( float64(time.Now().UnixNano()) / 1.0e9, ) } -func BIF_systimeint() *Mlrval { - return MlrvalFromInt(int(time.Now().Unix())) +func BIF_systimeint() *mlrval.Mlrval { + return mlrval.FromInt(int(time.Now().Unix())) } var startTime float64 @@ -32,15 +33,15 @@ var startTime float64 func init() { startTime = float64(time.Now().UnixNano()) / 1.0e9 } -func BIF_uptime() *Mlrval { - return MlrvalFromFloat64( +func BIF_uptime() *mlrval.Mlrval { + return mlrval.FromFloat( float64(time.Now().UnixNano())/1.0e9 - startTime, ) } // ================================================================ -func BIF_sec2gmt_unary(input1 *Mlrval) *Mlrval { +func BIF_sec2gmt_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { floatValue, isNumeric := input1.GetNumericToFloatValue() if !isNumeric { return input1 @@ -48,10 +49,10 @@ func BIF_sec2gmt_unary(input1 *Mlrval) *Mlrval { numDecimalPlaces := 0 - return MlrvalFromString(lib.Sec2GMT(floatValue, numDecimalPlaces)) + return mlrval.FromString(lib.Sec2GMT(floatValue, numDecimalPlaces)) } -func BIF_sec2gmt_binary(input1, input2 *Mlrval) *Mlrval { +func BIF_sec2gmt_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { floatValue, isNumeric := input1.GetNumericToFloatValue() if !isNumeric { return input1 @@ -59,13 +60,13 @@ func BIF_sec2gmt_binary(input1, input2 *Mlrval) *Mlrval { numDecimalPlaces, isInt := input2.GetIntValue() if !isInt { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromString(lib.Sec2GMT(floatValue, numDecimalPlaces)) + return mlrval.FromString(lib.Sec2GMT(floatValue, numDecimalPlaces)) } -func BIF_sec2localtime_unary(input1 *Mlrval) *Mlrval { +func BIF_sec2localtime_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { floatValue, isNumeric := input1.GetNumericToFloatValue() if !isNumeric { return input1 @@ -73,10 +74,10 @@ func BIF_sec2localtime_unary(input1 *Mlrval) *Mlrval { numDecimalPlaces := 0 - return MlrvalFromString(lib.Sec2LocalTime(floatValue, numDecimalPlaces)) + return mlrval.FromString(lib.Sec2LocalTime(floatValue, numDecimalPlaces)) } -func BIF_sec2localtime_binary(input1, input2 *Mlrval) *Mlrval { +func BIF_sec2localtime_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { floatValue, isNumeric := input1.GetNumericToFloatValue() if !isNumeric { return input1 @@ -84,13 +85,13 @@ func BIF_sec2localtime_binary(input1, input2 *Mlrval) *Mlrval { numDecimalPlaces, isInt := input2.GetIntValue() if !isInt { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromString(lib.Sec2LocalTime(floatValue, numDecimalPlaces)) + return mlrval.FromString(lib.Sec2LocalTime(floatValue, numDecimalPlaces)) } -func BIF_sec2localtime_ternary(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_sec2localtime_ternary(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { floatValue, isNumeric := input1.GetNumericToFloatValue() if !isNumeric { return input1 @@ -98,37 +99,37 @@ func BIF_sec2localtime_ternary(input1, input2, input3 *Mlrval) *Mlrval { numDecimalPlaces, isInt := input2.GetIntValue() if !isInt { - return MLRVAL_ERROR + return mlrval.ERROR } - locationString, isString := input3.GetString() + locationString, isString := input3.GetStringValue() if !isString { - return MLRVAL_ERROR + return mlrval.ERROR } location, err := time.LoadLocation(locationString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromString(lib.Sec2LocationTime(floatValue, numDecimalPlaces, location)) + return mlrval.FromString(lib.Sec2LocationTime(floatValue, numDecimalPlaces, location)) } -func BIF_sec2gmtdate(input1 *Mlrval) *Mlrval { +func BIF_sec2gmtdate(input1 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsNumeric() { return input1 } return BIF_strftime(input1, ptr_YMD_FORMAT) } -func BIF_sec2localdate_unary(input1 *Mlrval) *Mlrval { +func BIF_sec2localdate_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsNumeric() { return input1 } return BIF_strftime_local_binary(input1, ptr_YMD_FORMAT) } -func BIF_sec2localdate_binary(input1, input2 *Mlrval) *Mlrval { +func BIF_sec2localdate_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsNumeric() { return input1 } @@ -137,32 +138,32 @@ func BIF_sec2localdate_binary(input1, input2 *Mlrval) *Mlrval { // ---------------------------------------------------------------- -func BIF_localtime2gmt_unary(input1 *Mlrval) *Mlrval { +func BIF_localtime2gmt_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } return BIF_sec2gmt_unary(BIF_localtime2sec_unary(input1)) } -func BIF_localtime2gmt_binary(input1, input2 *Mlrval) *Mlrval { +func BIF_localtime2gmt_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } return BIF_sec2gmt_unary(BIF_localtime2sec_binary(input1, input2)) } -func BIF_gmt2localtime_unary(input1 *Mlrval) *Mlrval { +func BIF_gmt2localtime_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } return BIF_sec2localtime_unary(BIF_gmt2sec(input1)) } -func BIF_gmt2localtime_binary(input1, input2 *Mlrval) *Mlrval { +func BIF_gmt2localtime_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } - return BIF_sec2localtime_ternary(BIF_gmt2sec(input1), MlrvalFromInt(0), input2) + return BIF_sec2localtime_ternary(BIF_gmt2sec(input1), mlrval.FromInt(0), input2) } // ================================================================ @@ -171,38 +172,38 @@ func BIF_gmt2localtime_binary(input1, input2 *Mlrval) *Mlrval { var extensionRegex = regexp.MustCompile("([1-9])S") -func BIF_strftime(input1, input2 *Mlrval) *Mlrval { +func BIF_strftime(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return strftimeHelper(input1, input2, false, nil) } -func BIF_strftime_local_binary(input1, input2 *Mlrval) *Mlrval { +func BIF_strftime_local_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return strftimeHelper(input1, input2, true, nil) } -func BIF_strftime_local_ternary(input1, input2, input3 *Mlrval) *Mlrval { - locationString, isString := input3.GetString() +func BIF_strftime_local_ternary(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { + locationString, isString := input3.GetStringValue() if !isString { - return MLRVAL_ERROR + return mlrval.ERROR } location, err := time.LoadLocation(locationString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } return strftimeHelper(input1, input2, true, location) } -func strftimeHelper(input1, input2 *Mlrval, doLocal bool, location *time.Location) *Mlrval { - if input1.mvtype == MT_VOID { +func strftimeHelper(input1, input2 *mlrval.Mlrval, doLocal bool, location *time.Location) *mlrval.Mlrval { + if input1.IsVoid() { return input1 } epochSeconds, ok := input1.GetNumericToFloatValue() if !ok { - return MLRVAL_ERROR + return mlrval.ERROR } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR + if !input2.IsString() { + return mlrval.ERROR } // Convert argument1 from float seconds since the epoch to a Go time. @@ -223,16 +224,16 @@ func strftimeHelper(input1, input2 *Mlrval, doLocal bool, location *time.Locatio // implementation. However, in the strftime package we're using in the Go // port, extension-formats are only a single byte so we need to rewrite // them to "%6". - formatString := extensionRegex.ReplaceAllString(input2.printrep, "$1") + formatString := extensionRegex.ReplaceAllString(input2.AcquireStringValue(), "$1") formatter, err := strftime.New(formatString, strftimeExtensions) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } outputString := formatter.FormatString(inputTime) - return MlrvalFromString(outputString) + return mlrval.FromString(outputString) } // ---------------------------------------------------------------- @@ -299,82 +300,82 @@ func init() { // ================================================================ // Argument 1 is formatted date string like "2021-03-04 02:59:50". // Argument 2 is format string like "%Y-%m-%d %H:%M:%S". -func BIF_strptime(input1, input2 *Mlrval) *Mlrval { - if input1.mvtype != MT_STRING { - return MLRVAL_ERROR +func BIF_strptime(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsString() { + return mlrval.ERROR } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR + if !input2.IsString() { + return mlrval.ERROR } - timeString := input1.printrep - formatString := input2.printrep + timeString := input1.AcquireStringValue() + formatString := input2.AcquireStringValue() t, err := strptime.Parse(timeString, formatString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromFloat64(float64(t.UnixNano()) / 1.0e9) + return mlrval.FromFloat(float64(t.UnixNano()) / 1.0e9) } -func BIF_strptime_local_binary(input1, input2 *Mlrval) *Mlrval { - if input1.mvtype != MT_STRING { - return MLRVAL_ERROR +func BIF_strptime_local_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsString() { + return mlrval.ERROR } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR + if !input2.IsString() { + return mlrval.ERROR } - timeString := input1.printrep - formatString := input2.printrep + timeString := input1.AcquireStringValue() + formatString := input2.AcquireStringValue() t, err := strptime.ParseLocal(timeString, formatString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromFloat64(float64(t.UnixNano()) / 1.0e9) + return mlrval.FromFloat(float64(t.UnixNano()) / 1.0e9) } -func BIF_strptime_local_ternary(input1, input2, input3 *Mlrval) *Mlrval { - if input1.mvtype != MT_STRING { - return MLRVAL_ERROR +func BIF_strptime_local_ternary(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsString() { + return mlrval.ERROR } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR + if !input2.IsString() { + return mlrval.ERROR } - if input3.mvtype != MT_STRING { - return MLRVAL_ERROR + if !input3.IsString() { + return mlrval.ERROR } - timeString := input1.printrep - formatString := input2.printrep - locationString := input3.printrep + timeString := input1.AcquireStringValue() + formatString := input2.AcquireStringValue() + locationString := input3.AcquireStringValue() location, err := time.LoadLocation(locationString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } // TODO: use location t, err := strptime.ParseLocation(timeString, formatString, location) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromFloat64(float64(t.UnixNano()) / 1.0e9) + return mlrval.FromFloat(float64(t.UnixNano()) / 1.0e9) } // ================================================================ // Argument 1 is formatted date string like "2021-03-04T02:59:50Z". -func BIF_gmt2sec(input1 *Mlrval) *Mlrval { +func BIF_gmt2sec(input1 *mlrval.Mlrval) *mlrval.Mlrval { return BIF_strptime(input1, ptr_ISO8601_TIME_FORMAT) } -func BIF_localtime2sec_unary(input1 *Mlrval) *Mlrval { +func BIF_localtime2sec_unary(input1 *mlrval.Mlrval) *mlrval.Mlrval { return BIF_strptime_local_binary(input1, ptr_ISO8601_LOCAL_TIME_FORMAT) } -func BIF_localtime2sec_binary(input1, input2 *Mlrval) *Mlrval { +func BIF_localtime2sec_binary(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { return BIF_strptime_local_ternary(input1, ptr_ISO8601_LOCAL_TIME_FORMAT, input2) } diff --git a/internal/pkg/bifs/hashing.go b/internal/pkg/bifs/hashing.go new file mode 100644 index 000000000..6c6c0ce39 --- /dev/null +++ b/internal/pkg/bifs/hashing.go @@ -0,0 +1,63 @@ +package bifs + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "fmt" + + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + +func BIF_md5(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsStringOrVoid() { + return mlrval.ERROR + } else { + return mlrval.FromString( + fmt.Sprintf( + "%x", + md5.Sum([]byte(input1.AcquireStringValue())), + ), + ) + } +} + +func BIF_sha1(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsStringOrVoid() { + return mlrval.ERROR + } else { + return mlrval.FromString( + fmt.Sprintf( + "%x", + sha1.Sum([]byte(input1.AcquireStringValue())), + ), + ) + } +} + +func BIF_sha256(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsStringOrVoid() { + return mlrval.ERROR + } else { + return mlrval.FromString( + fmt.Sprintf( + "%x", + sha256.Sum256([]byte(input1.AcquireStringValue())), + ), + ) + } +} + +func BIF_sha512(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsStringOrVoid() { + return mlrval.ERROR + } else { + return mlrval.FromString( + fmt.Sprintf( + "%x", + sha512.Sum512([]byte(input1.AcquireStringValue())), + ), + ) + } +} diff --git a/internal/pkg/bifs/hashing_test.go b/internal/pkg/bifs/hashing_test.go new file mode 100644 index 000000000..3e736c0a0 --- /dev/null +++ b/internal/pkg/bifs/hashing_test.go @@ -0,0 +1,25 @@ +package bifs + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + +func TestBIF_md5(t *testing.T) { + input1 := mlrval.FromDeferredType("") + output := BIF_md5(input1) + stringval, ok := output.GetStringValue() + assert.True(t, ok) + assert.Equal(t, "d41d8cd98f00b204e9800998ecf8427e", stringval) + + input1 = mlrval.FromDeferredType("miller") + output = BIF_md5(input1) + stringval, ok = output.GetStringValue() + assert.True(t, ok) + assert.Equal(t, "f0af962ddbc82430e947390b2f3f6e49", stringval) +} + +// TODO: copy in more unit-test cases from existing regression-test data diff --git a/internal/pkg/bifs/mathlib.go b/internal/pkg/bifs/mathlib.go new file mode 100644 index 000000000..698791907 --- /dev/null +++ b/internal/pkg/bifs/mathlib.go @@ -0,0 +1,269 @@ +// ================================================================ +// Go math-library functions +// ================================================================ + +package bifs + +import ( + "math" + + "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + +// ---------------------------------------------------------------- +// Return error (unary math-library func) +func _math_unary_erro1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.ERROR +} + +// Return absent (unary math-library func) +func _math_unary_absn1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.ABSENT +} + +// Return null (unary math-library func) +func _math_unary_null1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.NULL +} + +// Return void (unary math-library func) +func _math_unary_void1(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.VOID +} + +// ---------------------------------------------------------------- +func math_unary_f_i(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.FromFloat(f(float64(input1.AcquireIntValue()))) +} +func math_unary_i_i(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.FromInt(int(f(float64(input1.AcquireIntValue())))) +} +func math_unary_f_f(input1 *mlrval.Mlrval, f mathLibUnaryFunc) *mlrval.Mlrval { + return mlrval.FromFloat(f(input1.AcquireFloatValue())) +} + +// Disposition vector for unary mathlib functions +var mudispo = [mlrval.MT_DIM]mathLibUnaryFuncWrapper{ + /*ERROR */ _math_unary_erro1, + /*ABSENT */ _math_unary_absn1, + /*NULL */ _math_unary_null1, + /*VOID */ _math_unary_void1, + /*STRING */ _math_unary_erro1, + /*INT */ math_unary_f_i, + /*FLOAT */ math_unary_f_f, + /*BOOL */ _math_unary_erro1, + /*ARRAY */ _math_unary_absn1, + /*MAP */ _math_unary_absn1, + /*FUNC */ _math_unary_erro1, +} + +func BIF_acos(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Acos) } +func BIF_acosh(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, math.Acosh) +} +func BIF_asin(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Asin) } +func BIF_asinh(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, math.Asinh) +} +func BIF_atan(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Atan) } +func BIF_atanh(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, math.Atanh) +} +func BIF_cbrt(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Cbrt) } +func BIF_cos(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Cos) } +func BIF_cosh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Cosh) } +func BIF_erf(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Erf) } +func BIF_erfc(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Erfc) } +func BIF_exp(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Exp) } +func BIF_expm1(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, math.Expm1) +} +func BIF_invqnorm(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, lib.Invqnorm) +} +func BIF_log(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Log) } +func BIF_log10(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, math.Log10) +} +func BIF_log1p(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, math.Log1p) +} +func BIF_qnorm(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mudispo[input1.Type()](input1, lib.Qnorm) +} +func BIF_sin(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Sin) } +func BIF_sinh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Sinh) } +func BIF_sqrt(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Sqrt) } +func BIF_tan(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Tan) } +func BIF_tanh(input1 *mlrval.Mlrval) *mlrval.Mlrval { return mudispo[input1.Type()](input1, math.Tanh) } + +// Disposition vector for unary mathlib functions which are int-preserving +var imudispo = [mlrval.MT_DIM]mathLibUnaryFuncWrapper{ + /*ERROR */ _math_unary_erro1, + /*ABSENT */ _math_unary_absn1, + /*NULL */ _math_unary_null1, + /*VOID */ _math_unary_void1, + /*STRING */ _math_unary_erro1, + /*INT */ math_unary_i_i, + /*FLOAT */ math_unary_f_f, + /*BOOL */ _math_unary_erro1, + /*ARRAY */ _math_unary_absn1, + /*MAP */ _math_unary_absn1, + /*FUNC */ _math_unary_erro1, +} + +// Int-preserving +func BIF_abs(input1 *mlrval.Mlrval) *mlrval.Mlrval { return imudispo[input1.Type()](input1, math.Abs) } // xxx +func BIF_ceil(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return imudispo[input1.Type()](input1, math.Ceil) +} // xxx +func BIF_floor(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return imudispo[input1.Type()](input1, math.Floor) +} // xxx +func BIF_round(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return imudispo[input1.Type()](input1, math.Round) +} // xxx +func BIF_sgn(input1 *mlrval.Mlrval) *mlrval.Mlrval { return imudispo[input1.Type()](input1, lib.Sgn) } // xxx + +// ================================================================ +// Exponentiation: DSL operator '**'. See also +// https://johnkerl.org/miller6/reference-main-arithmetic.html + +func pow_f_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + foutput := math.Pow(float64(input1.AcquireIntValue()), float64(input2.AcquireIntValue())) + ioutput := int(foutput) + // Int raised to int power should be float if it can be (i.e. unless overflow) + if float64(ioutput) == foutput { + return mlrval.FromInt(ioutput) + } else { + return mlrval.FromFloat(foutput) + } +} +func pow_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Pow(float64(input1.AcquireIntValue()), input2.AcquireFloatValue())) +} +func pow_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Pow(input1.AcquireFloatValue(), float64(input2.AcquireIntValue()))) +} +func pow_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Pow(input1.AcquireFloatValue(), input2.AcquireFloatValue())) +} + +var pow_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, + /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _i0__, _f0__, _erro, _absn, _absn, _erro}, + /*NULL */ {_erro, _absn, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, + /*VOID */ {_erro, _absn, _erro, _void, _erro, _void, _void, _erro, _absn, _absn, _erro}, + /*STRING */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, + /*INT */ {_erro, _1___, _erro, _void, _erro, pow_f_ii, pow_f_if, _erro, _absn, _absn, _erro}, + /*FLOAT */ {_erro, _1___, _erro, _void, _erro, pow_f_fi, pow_f_ff, _erro, _absn, _absn, _erro}, + /*BOOL */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, + /*ARRAY */ {_absn, _absn, _erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, + /*MAP */ {_absn, _absn, _erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, + /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, +} + +func BIF_pow(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return pow_dispositions[input1.Type()][input2.Type()](input1, input2) +} + +// ================================================================ +func atan2_f_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Atan2(float64(input1.AcquireIntValue()), float64(input2.AcquireIntValue()))) +} +func atan2_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Atan2(float64(input1.AcquireIntValue()), input2.AcquireFloatValue())) +} +func atan2_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Atan2(input1.AcquireFloatValue(), float64(input2.AcquireIntValue()))) +} +func atan2_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(math.Atan2(input1.AcquireFloatValue(), input2.AcquireFloatValue())) +} + +var atan2_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, + /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _i0__, _f0__, _erro, _absn, _absn, _erro}, + /*NULL */ {_erro, _absn, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, + /*VOID */ {_erro, _absn, _erro, _void, _erro, _void, _void, _erro, _absn, _absn, _erro}, + /*STRING */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, + /*INT */ {_erro, _1___, _erro, _void, _erro, atan2_f_ii, atan2_f_if, _erro, _absn, _absn, _erro}, + /*FLOAT */ {_erro, _1___, _erro, _void, _erro, atan2_f_fi, atan2_f_ff, _erro, _absn, _absn, _erro}, + /*BOOL */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, + /*ARRAY */ {_absn, _absn, _erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, + /*MAP */ {_absn, _absn, _erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, + /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, +} + +func BIF_atan2(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return atan2_dispositions[input1.Type()][input2.Type()](input1, input2) +} + +// ================================================================ +func mlr_roundm(x, m float64) float64 { + return math.Round(x/m) * m +} + +func roundm_f_ii(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(int(mlr_roundm(float64(input1.AcquireIntValue()), float64(input2.AcquireIntValue())))) +} +func roundm_f_if(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(mlr_roundm(float64(input1.AcquireIntValue()), input2.AcquireFloatValue())) +} +func roundm_f_fi(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(mlr_roundm(input1.AcquireFloatValue(), float64(input2.AcquireIntValue()))) +} +func roundm_f_ff(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(mlr_roundm(input1.AcquireFloatValue(), input2.AcquireFloatValue())) +} + +var roundm_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, + /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _i0__, _f0__, _erro, _absn, _absn, _erro}, + /*NULL */ {_erro, _absn, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, + /*VOID */ {_erro, _absn, _erro, _void, _erro, _void, _void, _erro, _absn, _absn, _erro}, + /*STRING */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, + /*INT */ {_erro, _1___, _erro, _void, _erro, roundm_f_ii, roundm_f_if, _erro, _absn, _absn, _erro}, + /*FLOAT */ {_erro, _1___, _erro, _void, _erro, roundm_f_fi, roundm_f_ff, _erro, _absn, _absn, _erro}, + /*BOOL */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, + /*ARRAY */ {_absn, _absn, _erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, + /*MAP */ {_absn, _absn, _erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, + /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, +} + +func BIF_roundm(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return roundm_dispositions[input1.Type()][input2.Type()](input1, input2) +} + +// ================================================================ +func BIF_logifit(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsLegit() { + return input1 + } + if !input2.IsLegit() { + return input2 + } + if !input3.IsLegit() { + return input3 + } + + // int/float OK; rest not + x, xok := input1.GetNumericToFloatValue() + if !xok { + return mlrval.ERROR + } + m, mok := input2.GetNumericToFloatValue() + if !mok { + return mlrval.ERROR + } + b, bok := input3.GetNumericToFloatValue() + if !bok { + return mlrval.ERROR + } + + return mlrval.FromFloat(1.0 / (1.0 + math.Exp(-m*x-b))) +} diff --git a/internal/pkg/types/mlrval_functions_random.go b/internal/pkg/bifs/random.go similarity index 58% rename from internal/pkg/types/mlrval_functions_random.go rename to internal/pkg/bifs/random.go index 6e5a4b9e3..b345e6d65 100644 --- a/internal/pkg/types/mlrval_functions_random.go +++ b/internal/pkg/bifs/random.go @@ -1,19 +1,20 @@ -package types +package bifs import ( "math" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) -func BIF_urand() *Mlrval { - return MlrvalFromFloat64( +func BIF_urand() *mlrval.Mlrval { + return mlrval.FromFloat( lib.RandFloat64(), ) } -func BIF_urand32() *Mlrval { - return MlrvalFromInt( +func BIF_urand32() *mlrval.Mlrval { + return mlrval.FromInt( int( lib.RandUint32(), ), @@ -21,7 +22,7 @@ func BIF_urand32() *Mlrval { } // TODO: use a disposition matrix -func BIF_urandint(input1, input2 *Mlrval) *Mlrval { +func BIF_urandint(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsLegit() { return input1 } @@ -29,14 +30,14 @@ func BIF_urandint(input1, input2 *Mlrval) *Mlrval { return input2 } if !input1.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } - a := input1.intval - b := input2.intval + a := input1.AcquireIntValue() + b := input2.AcquireIntValue() var lo int = 0 var hi int = 0 @@ -48,10 +49,10 @@ func BIF_urandint(input1, input2 *Mlrval) *Mlrval { hi = a + 1 } u := int(math.Floor(float64(lo) + float64((hi-lo))*lib.RandFloat64())) - return MlrvalFromInt(u) + return mlrval.FromInt(u) } -func BIF_urandrange(input1, input2 *Mlrval) *Mlrval { +func BIF_urandrange(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsLegit() { return input1 } @@ -61,12 +62,12 @@ func BIF_urandrange(input1, input2 *Mlrval) *Mlrval { a, aok := input1.GetNumericToFloatValue() b, bok := input2.GetNumericToFloatValue() if !aok { - return MLRVAL_ERROR + return mlrval.ERROR } if !bok { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromFloat64( + return mlrval.FromFloat( a + (b-a)*lib.RandFloat64(), ) } diff --git a/internal/pkg/types/mlrval_functions_regex.go b/internal/pkg/bifs/regex.go similarity index 52% rename from internal/pkg/types/mlrval_functions_regex.go rename to internal/pkg/bifs/regex.go index 72c9af669..448a62278 100644 --- a/internal/pkg/types/mlrval_functions_regex.go +++ b/internal/pkg/bifs/regex.go @@ -1,14 +1,15 @@ -package types +package bifs import ( "strings" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // BIF_ssub implements the ssub function -- no-frills string-replace, no // regexes, no escape sequences. -func BIF_ssub(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_ssub(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsErrorOrAbsent() { return input1 } @@ -19,16 +20,16 @@ func BIF_ssub(input1, input2, input3 *Mlrval) *Mlrval { return input3 } if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input3.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - return MlrvalFromString( - strings.Replace(input1.printrep, input2.printrep, input3.printrep, 1), + return mlrval.FromString( + strings.Replace(input1.AcquireStringValue(), input2.AcquireStringValue(), input3.AcquireStringValue(), 1), ) } @@ -39,7 +40,7 @@ func BIF_ssub(input1, input2, input3 *Mlrval) *Mlrval { // on each record. Likewise for other regex-using functions in this file. But // first, do a profiling run to see how much time would be saved, and if this // precomputing+caching would be worthwhile. -func BIF_sub(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_sub(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsErrorOrAbsent() { return input1 } @@ -50,26 +51,26 @@ func BIF_sub(input1, input2, input3 *Mlrval) *Mlrval { return input3 } if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input3.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - input := input1.printrep - sregex := input2.printrep - replacement := input3.printrep + input := input1.AcquireStringValue() + sregex := input2.AcquireStringValue() + replacement := input3.AcquireStringValue() stringOutput := lib.RegexSub(input, sregex, replacement) - return MlrvalFromString(stringOutput) + return mlrval.FromString(stringOutput) } // BIF_gsub implements the gsub function, with support for regexes and regex captures // of the form "\1" .. "\9". -func BIF_gsub(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_gsub(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsErrorOrAbsent() { return input1 } @@ -80,26 +81,26 @@ func BIF_gsub(input1, input2, input3 *Mlrval) *Mlrval { return input3 } if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input3.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - input := input1.printrep - sregex := input2.printrep - replacement := input3.printrep + input := input1.AcquireStringValue() + sregex := input2.AcquireStringValue() + replacement := input3.AcquireStringValue() stringOutput := lib.RegexGsub(input, sregex, replacement) - return MlrvalFromString(stringOutput) + return mlrval.FromString(stringOutput) } // BIF_string_matches_regexp implements the =~ operator, with support for // setting regex-captures for later expressions to access using "\1" .. "\9". -func BIF_string_matches_regexp(input1, input2 *Mlrval) (retval *Mlrval, captures []string) { +func BIF_string_matches_regexp(input1, input2 *mlrval.Mlrval) (retval *mlrval.Mlrval, captures []string) { if !input1.IsLegit() { return input1, nil } @@ -108,51 +109,51 @@ func BIF_string_matches_regexp(input1, input2 *Mlrval) (retval *Mlrval, captures } input1string := input1.String() if !input2.IsStringOrVoid() { - return MLRVAL_ERROR, nil + return mlrval.ERROR, nil } - boolOutput, captures := lib.RegexMatches(input1string, input2.printrep) - return MlrvalFromBool(boolOutput), captures + boolOutput, captures := lib.RegexMatches(input1string, input2.AcquireStringValue()) + return mlrval.FromBool(boolOutput), captures } // BIF_string_matches_regexp implements the !=~ operator. -func BIF_string_does_not_match_regexp(input1, input2 *Mlrval) (retval *Mlrval, captures []string) { +func BIF_string_does_not_match_regexp(input1, input2 *mlrval.Mlrval) (retval *mlrval.Mlrval, captures []string) { output, captures := BIF_string_matches_regexp(input1, input2) - if output.mvtype == MT_BOOL { - return MlrvalFromBool(!output.boolval), captures + if output.IsBool() { + return mlrval.FromBool(!output.AcquireBoolValue()), captures } else { // else leave it as error, absent, etc. return output, captures } } -func BIF_regextract(input1, input2 *Mlrval) *Mlrval { +func BIF_regextract(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } - regex := lib.CompileMillerRegexOrDie(input2.printrep) - match := regex.FindStringIndex(input1.printrep) + regex := lib.CompileMillerRegexOrDie(input2.AcquireStringValue()) + match := regex.FindStringIndex(input1.AcquireStringValue()) if match != nil { - return MlrvalFromString(input1.printrep[match[0]:match[1]]) + return mlrval.FromString(input1.AcquireStringValue()[match[0]:match[1]]) } else { - return MLRVAL_ABSENT + return mlrval.ABSENT } } -func BIF_regextract_or_else(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_regextract_or_else(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } - regex := lib.CompileMillerRegexOrDie(input2.printrep) - match := regex.FindStringIndex(input1.printrep) + regex := lib.CompileMillerRegexOrDie(input2.AcquireStringValue()) + match := regex.FindStringIndex(input1.AcquireStringValue()) if match != nil { - return MlrvalFromString(input1.printrep[match[0]:match[1]]) + return mlrval.FromString(input1.AcquireStringValue()[match[0]:match[1]]) } else { return input3 } diff --git a/internal/pkg/bifs/relative_time.go b/internal/pkg/bifs/relative_time.go new file mode 100644 index 000000000..4f09130d1 --- /dev/null +++ b/internal/pkg/bifs/relative_time.go @@ -0,0 +1,326 @@ +package bifs + +import ( + "fmt" + "math" + "strings" + + "github.com/johnkerl/miller/internal/pkg/mlrval" +) + +func BIF_dhms2sec(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsString() { + return mlrval.ERROR + } + var d, h, m, s int + + if strings.HasPrefix(input1.AcquireStringValue(), "-") { + + n, err := fmt.Sscanf(input1.AcquireStringValue(), "-%dd%dh%dm%ds", &d, &h, &m, &s) + if n == 4 && err == nil { + return mlrval.FromInt(-(s + m*60 + h*60*60 + d*60*60*24)) + } + n, err = fmt.Sscanf(input1.AcquireStringValue(), "-%dh%dm%ds", &h, &m, &s) + if n == 3 && err == nil { + return mlrval.FromInt(-(s + m*60 + h*60*60)) + } + n, err = fmt.Sscanf(input1.AcquireStringValue(), "-%dm%ds", &m, &s) + if n == 2 && err == nil { + return mlrval.FromInt(-(s + m*60)) + } + n, err = fmt.Sscanf(input1.AcquireStringValue(), "-%ds", &s) + if n == 1 && err == nil { + return mlrval.FromInt(-(s)) + } + + } else { + + n, err := fmt.Sscanf(input1.AcquireStringValue(), "%dd%dh%dm%ds", &d, &h, &m, &s) + if n == 4 && err == nil { + return mlrval.FromInt(s + m*60 + h*60*60 + d*60*60*24) + } + n, err = fmt.Sscanf(input1.AcquireStringValue(), "%dh%dm%ds", &h, &m, &s) + if n == 3 && err == nil { + return mlrval.FromInt(s + m*60 + h*60*60) + } + n, err = fmt.Sscanf(input1.AcquireStringValue(), "%dm%ds", &m, &s) + if n == 2 && err == nil { + return mlrval.FromInt(s + m*60) + } + n, err = fmt.Sscanf(input1.AcquireStringValue(), "%ds", &s) + if n == 1 && err == nil { + return mlrval.FromInt(s) + } + + } + return mlrval.ERROR +} + +func BIF_dhms2fsec(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsString() { + return mlrval.ERROR + } + + var d, h, m int + var s float64 + + if strings.HasPrefix(input1.AcquireStringValue(), "-") { + + n, err := fmt.Sscanf(input1.AcquireStringValue(), "-%dd%dh%dm%fs", &d, &h, &m, &s) + if n == 4 && err == nil { + return mlrval.FromFloat(-(s + float64(m*60+h*60*60+d*60*60*24))) + } + n, err = fmt.Sscanf(input1.AcquireStringValue(), "-%dh%dm%fs", &h, &m, &s) + if n == 3 && err == nil { + return mlrval.FromFloat(-(s + float64(m*60+h*60*60))) + } + n, err = fmt.Sscanf(input1.AcquireStringValue(), "-%dm%fs", &m, &s) + if n == 2 && err == nil { + return mlrval.FromFloat(-(s + float64(m*60))) + } + n, err = fmt.Sscanf(input1.AcquireStringValue(), "-%fs", &s) + if n == 1 && err == nil { + return mlrval.FromFloat(-(s)) + } + + } else { + + n, err := fmt.Sscanf(input1.AcquireStringValue(), "%dd%dh%dm%fs", &d, &h, &m, &s) + if n == 4 && err == nil { + return mlrval.FromFloat(s + float64(m*60+h*60*60+d*60*60*24)) + } + n, err = fmt.Sscanf(input1.AcquireStringValue(), "%dh%dm%fs", &h, &m, &s) + if n == 3 && err == nil { + return mlrval.FromFloat(s + float64(m*60+h*60*60)) + } + n, err = fmt.Sscanf(input1.AcquireStringValue(), "%dm%fs", &m, &s) + if n == 2 && err == nil { + return mlrval.FromFloat(s + float64(m*60)) + } + n, err = fmt.Sscanf(input1.AcquireStringValue(), "%fs", &s) + if n == 1 && err == nil { + return mlrval.FromFloat(s) + } + + } + return mlrval.ERROR +} + +func BIF_hms2sec(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsString() { + return mlrval.ERROR + } + if input1.AcquireStringValue() == "" { + return mlrval.ERROR + } + var h, m, s int + + if strings.HasPrefix(input1.AcquireStringValue(), "-") { + n, err := fmt.Sscanf(input1.AcquireStringValue(), "-%d:%d:%d", &h, &m, &s) + if n == 3 && err == nil { + return mlrval.FromInt(-(s + m*60 + h*60*60)) + } + } else { + n, err := fmt.Sscanf(input1.AcquireStringValue(), "%d:%d:%d", &h, &m, &s) + if n == 3 && err == nil { + return mlrval.FromInt(s + m*60 + h*60*60) + } + } + + return mlrval.ERROR +} + +func BIF_hms2fsec(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsString() { + return mlrval.ERROR + } + + var h, m int + var s float64 + + if strings.HasPrefix(input1.AcquireStringValue(), "-") { + n, err := fmt.Sscanf(input1.AcquireStringValue(), "-%d:%d:%f", &h, &m, &s) + if n == 3 && err == nil { + return mlrval.FromFloat(-(s + float64(m*60+h*60*60))) + } + } else { + n, err := fmt.Sscanf(input1.AcquireStringValue(), "%d:%d:%f", &h, &m, &s) + if n == 3 && err == nil { + return mlrval.FromFloat(s + float64(m*60+h*60*60)) + } + } + + return mlrval.ERROR +} + +func BIF_sec2dhms(input1 *mlrval.Mlrval) *mlrval.Mlrval { + isec, ok := input1.GetIntValue() + if !ok { + return mlrval.ERROR + } + + var d, h, m, s int + + splitIntToDHMS(isec, &d, &h, &m, &s) + if d != 0 { + return mlrval.FromString( + fmt.Sprintf("%dd%02dh%02dm%02ds", d, h, m, s), + ) + } else if h != 0 { + return mlrval.FromString( + fmt.Sprintf("%dh%02dm%02ds", h, m, s), + ) + } else if m != 0 { + return mlrval.FromString( + fmt.Sprintf("%dm%02ds", m, s), + ) + } else { + return mlrval.FromString( + fmt.Sprintf("%ds", s), + ) + } + + return mlrval.ERROR +} + +func BIF_sec2hms(input1 *mlrval.Mlrval) *mlrval.Mlrval { + isec, ok := input1.GetIntValue() + if !ok { + return mlrval.ERROR + } + sign := "" + if isec < 0 { + sign = "-" + isec = -isec + } + + var d, h, m, s int + + splitIntToDHMS(isec, &d, &h, &m, &s) + h += d * 24 + + return mlrval.FromString( + fmt.Sprintf("%s%02d:%02d:%02d", sign, h, m, s), + ) + + return mlrval.ERROR +} + +func BIF_fsec2dhms(input1 *mlrval.Mlrval) *mlrval.Mlrval { + fsec, ok := input1.GetNumericToFloatValue() + if !ok { + return mlrval.ERROR + } + + sign := 1 + if fsec < 0 { + sign = -1 + fsec = -fsec + } + isec := int(math.Trunc(fsec)) + fractional := fsec - float64(isec) + + var d, h, m, s int + + splitIntToDHMS(isec, &d, &h, &m, &s) + + if d != 0 { + d = sign * d + return mlrval.FromString( + fmt.Sprintf( + "%dd%02dh%02dm%09.6fs", + d, h, m, float64(s)+fractional), + ) + } else if h != 0 { + h = sign * h + return mlrval.FromString( + fmt.Sprintf( + "%dh%02dm%09.6fs", + h, m, float64(s)+fractional), + ) + } else if m != 0 { + m = sign * m + return mlrval.FromString( + fmt.Sprintf( + "%dm%09.6fs", + m, float64(s)+fractional), + ) + } else { + s = sign * s + fractional = float64(sign) * fractional + return mlrval.FromString( + fmt.Sprintf( + "%.6fs", + float64(s)+fractional), + ) + } +} + +func BIF_fsec2hms(input1 *mlrval.Mlrval) *mlrval.Mlrval { + fsec, ok := input1.GetNumericToFloatValue() + if !ok { + return mlrval.ERROR + } + + sign := "" + if fsec < 0 { + sign = "-" + fsec = -fsec + } + isec := int(math.Trunc(fsec)) + fractional := fsec - float64(isec) + + var d, h, m, s int + + splitIntToDHMS(isec, &d, &h, &m, &s) + h += d * 24 + + // "%02.6f" does not exist so we have to do our own zero-pad + if s < 10 { + return mlrval.FromString( + fmt.Sprintf("%s%02d:%02d:0%.6f", sign, h, m, float64(s)+fractional), + ) + } else { + return mlrval.FromString( + fmt.Sprintf("%s%02d:%02d:%.6f", sign, h, m, float64(s)+fractional), + ) + } + + return mlrval.ERROR +} + +// Helper function +func splitIntToDHMS(u int, pd, ph, pm, ps *int) { + d := 0 + h := 0 + m := 0 + s := 0 + sign := 1 + if u < 0 { + u = -u + sign = -1 + } + s = u % 60 + u = u / 60 + if u == 0 { + s = s * sign + } else { + m = u % 60 + u = u / 60 + if u == 0 { + m = m * sign + } else { + h = u % 24 + u = u / 24 + if u == 0 { + h = h * sign + } else { + d = u * sign + } + } + } + *pd = d + *ph = h + *pm = m + *ps = s +} diff --git a/internal/pkg/types/mlrval_functions_sort.go b/internal/pkg/bifs/sort.go similarity index 69% rename from internal/pkg/types/mlrval_functions_sort.go rename to internal/pkg/bifs/sort.go index d452bed44..d7d0ddafe 100644 --- a/internal/pkg/types/mlrval_functions_sort.go +++ b/internal/pkg/bifs/sort.go @@ -2,17 +2,19 @@ // For sorting // ================================================================ -package types +package bifs import ( "fmt" "os" "strings" + + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // LexicalAscendingComparator is for lexical sort: it stringifies // everything. -func LexicalAscendingComparator(input1 *Mlrval, input2 *Mlrval) int { +func LexicalAscendingComparator(input1 *mlrval.Mlrval, input2 *mlrval.Mlrval) int { sa := input1.String() sb := input2.String() if sa < sb { @@ -26,19 +28,19 @@ func LexicalAscendingComparator(input1 *Mlrval, input2 *Mlrval) int { // LexicalDescendingComparator is for reverse-lexical sort: it stringifies // everything. -func LexicalDescendingComparator(input1 *Mlrval, input2 *Mlrval) int { +func LexicalDescendingComparator(input1 *mlrval.Mlrval, input2 *mlrval.Mlrval) int { return LexicalAscendingComparator(input2, input1) } // CaseFoldAscendingComparator is for case-folded lexical sort: it stringifies // everything. -func CaseFoldAscendingComparator(input1 *Mlrval, input2 *Mlrval) int { +func CaseFoldAscendingComparator(input1 *mlrval.Mlrval, input2 *mlrval.Mlrval) int { sa := input1.String() sb := input2.String() - if input1.mvtype == MT_STRING { + if input1.IsString() { sa = strings.ToLower(sa) } - if input2.mvtype == MT_STRING { + if input2.IsString() { sb = strings.ToLower(sb) } if sa < sb { @@ -52,7 +54,7 @@ func CaseFoldAscendingComparator(input1 *Mlrval, input2 *Mlrval) int { // CaseFoldDescendingComparator is for case-folded lexical sort: it stringifies // everything. -func CaseFoldDescendingComparator(input1 *Mlrval, input2 *Mlrval) int { +func CaseFoldDescendingComparator(input1 *mlrval.Mlrval, input2 *mlrval.Mlrval) int { return CaseFoldAscendingComparator(input2, input1) } @@ -65,29 +67,29 @@ func CaseFoldDescendingComparator(input1 *Mlrval, input2 *Mlrval) int { // * numeric compares on numbers // * false < true -func _neg1(input1, input2 *Mlrval) int { +func _neg1(input1, input2 *mlrval.Mlrval) int { return -1 } -func _zero(input1, input2 *Mlrval) int { +func _zero(input1, input2 *mlrval.Mlrval) int { return 0 } -func _pos1(input1, input2 *Mlrval) int { +func _pos1(input1, input2 *mlrval.Mlrval) int { return 1 } -func _scmp(input1, input2 *Mlrval) int { - if input1.printrep < input2.printrep { +func _scmp(input1, input2 *mlrval.Mlrval) int { + if input1.AcquireStringValue() < input2.AcquireStringValue() { return -1 - } else if input1.printrep > input2.printrep { + } else if input1.AcquireStringValue() > input2.AcquireStringValue() { return 1 } else { return 0 } } -func iicmp(input1, input2 *Mlrval) int { - ca := input1.intval - cb := input2.intval +func iicmp(input1, input2 *mlrval.Mlrval) int { + ca := input1.AcquireIntValue() + cb := input2.AcquireIntValue() if ca < cb { return -1 } else if ca > cb { @@ -96,9 +98,9 @@ func iicmp(input1, input2 *Mlrval) int { return 0 } } -func ifcmp(input1, input2 *Mlrval) int { - ca := float64(input1.intval) - cb := input2.floatval +func ifcmp(input1, input2 *mlrval.Mlrval) int { + ca := float64(input1.AcquireIntValue()) + cb := input2.AcquireFloatValue() if ca < cb { return -1 } else if ca > cb { @@ -107,9 +109,9 @@ func ifcmp(input1, input2 *Mlrval) int { return 0 } } -func ficmp(input1, input2 *Mlrval) int { - ca := input1.floatval - cb := float64(input2.intval) +func ficmp(input1, input2 *mlrval.Mlrval) int { + ca := input1.AcquireFloatValue() + cb := float64(input2.AcquireIntValue()) if ca < cb { return -1 } else if ca > cb { @@ -118,9 +120,9 @@ func ficmp(input1, input2 *Mlrval) int { return 0 } } -func ffcmp(input1, input2 *Mlrval) int { - ca := input1.floatval - cb := input2.floatval +func ffcmp(input1, input2 *mlrval.Mlrval) int { + ca := input1.AcquireFloatValue() + cb := input2.AcquireFloatValue() if ca < cb { return -1 } else if ca > cb { @@ -130,9 +132,9 @@ func ffcmp(input1, input2 *Mlrval) int { } } -func bbcmp(input1, input2 *Mlrval) int { - a := input1.boolval - b := input2.boolval +func bbcmp(input1, input2 *mlrval.Mlrval) int { + a := input1.AcquireBoolValue() + b := input2.AcquireBoolValue() if a == false { if b == false { return 0 @@ -148,7 +150,7 @@ func bbcmp(input1, input2 *Mlrval) int { } } -func _xcmp(input1, input2 *Mlrval) int { +func _xcmp(input1, input2 *mlrval.Mlrval) int { fmt.Fprintf(os.Stderr, "mlr: functions cannot be sorted.\n") os.Exit(1) return 0 @@ -165,7 +167,7 @@ func _xcmp(input1, input2 *Mlrval) int { // * false < true // typed_cmp_dispositions is the disposition matrix for numerical sorting of Mlrvals. -var typed_cmp_typedpositions = [MT_DIM][MT_DIM]ComparatorFunc{ +var typed_cmp_typedpositions = [mlrval.MT_DIM][mlrval.MT_DIM]ComparatorFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_zero, _neg1, _neg1, _pos1, _pos1, _pos1, _pos1, _pos1, _zero, _zero, _xcmp}, /*ABSENT */ {_pos1, _zero, _pos1, _pos1, _pos1, _pos1, _pos1, _pos1, _zero, _zero, _xcmp}, @@ -182,12 +184,12 @@ var typed_cmp_typedpositions = [MT_DIM][MT_DIM]ComparatorFunc{ // NumericAscendingComparator is for "numerical" sort: it uses Mlrval sorting // rules by type, including numeric sort for numeric types. -func NumericAscendingComparator(input1 *Mlrval, input2 *Mlrval) int { - return typed_cmp_typedpositions[input1.mvtype][input2.mvtype](input1, input2) +func NumericAscendingComparator(input1 *mlrval.Mlrval, input2 *mlrval.Mlrval) int { + return typed_cmp_typedpositions[input1.Type()][input2.Type()](input1, input2) } // NumericDescendingComparator is for "numerical" sort: it uses Mlrval sorting // rules by type, including numeric sort for numeric types. -func NumericDescendingComparator(input1 *Mlrval, input2 *Mlrval) int { +func NumericDescendingComparator(input1 *mlrval.Mlrval, input2 *mlrval.Mlrval) int { return NumericAscendingComparator(input2, input1) } diff --git a/internal/pkg/types/mlrval_functions_test.go b/internal/pkg/bifs/sort_test.go similarity index 84% rename from internal/pkg/types/mlrval_functions_test.go rename to internal/pkg/bifs/sort_test.go index 26b69a898..a47574c80 100644 --- a/internal/pkg/types/mlrval_functions_test.go +++ b/internal/pkg/bifs/sort_test.go @@ -1,12 +1,9 @@ -// ================================================================ -// Most Miller tests (thousands of them) are command-line-driven via -// mlr regtest. Here are some cases needing special focus. -// ================================================================ - -package types +package bifs import ( "testing" + + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ---------------------------------------------------------------- @@ -25,18 +22,18 @@ import ( func TestComparators(t *testing.T) { - i10 := MlrvalFromInt(10) - i2 := MlrvalFromInt(2) + i10 := mlrval.FromInt(10) + i2 := mlrval.FromInt(2) - bfalse := MlrvalFromBool(false) - btrue := MlrvalFromBool(true) + bfalse := mlrval.FromBool(false) + btrue := mlrval.FromBool(true) - sabc := MlrvalFromString("abc") - sdef := MlrvalFromString("def") + sabc := mlrval.FromString("abc") + sdef := mlrval.FromString("def") - e := *MLRVAL_ERROR + e := *mlrval.ERROR - a := *MLRVAL_ABSENT + a := *mlrval.ABSENT // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // Within-type lexical comparisons diff --git a/internal/pkg/types/mlrval_functions_stats.go b/internal/pkg/bifs/stats.go similarity index 84% rename from internal/pkg/types/mlrval_functions_stats.go rename to internal/pkg/bifs/stats.go index 3ef436c53..efcabec76 100644 --- a/internal/pkg/types/mlrval_functions_stats.go +++ b/internal/pkg/bifs/stats.go @@ -1,9 +1,10 @@ -package types +package bifs import ( "math" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ---------------------------------------------------------------- @@ -23,7 +24,7 @@ import ( // output = [m, b, math.sqrt(var_m), math.sqrt(var_b)] // ---------------------------------------------------------------- -func MlrvalGetVar(mn, msum, msum2 *Mlrval) *Mlrval { +func BIF_get_var(mn, msum, msum2 *mlrval.Mlrval) *mlrval.Mlrval { n, isInt := mn.GetIntValue() lib.InternalCodingErrorIf(!isInt) sum, isNumber := msum.GetNumericToFloatValue() @@ -32,7 +33,7 @@ func MlrvalGetVar(mn, msum, msum2 *Mlrval) *Mlrval { lib.InternalCodingErrorIf(!isNumber) if n < 2 { - return MLRVAL_VOID + return mlrval.VOID } mean := float64(sum) / float64(n) @@ -41,12 +42,12 @@ func MlrvalGetVar(mn, msum, msum2 *Mlrval) *Mlrval { numerator = 0.0 } denominator := float64(n - 1) - return MlrvalFromFloat64(numerator / denominator) + return mlrval.FromFloat(numerator / denominator) } // ---------------------------------------------------------------- -func MlrvalGetStddev(mn, msum, msum2 *Mlrval) *Mlrval { - mvar := MlrvalGetVar(mn, msum, msum2) +func BIF_get_stddev(mn, msum, msum2 *mlrval.Mlrval) *mlrval.Mlrval { + mvar := BIF_get_var(mn, msum, msum2) if mvar.IsVoid() { return mvar } @@ -54,8 +55,8 @@ func MlrvalGetStddev(mn, msum, msum2 *Mlrval) *Mlrval { } // ---------------------------------------------------------------- -func MlrvalGetMeanEB(mn, msum, msum2 *Mlrval) *Mlrval { - mvar := MlrvalGetVar(mn, msum, msum2) +func BIF_get_mean_EB(mn, msum, msum2 *mlrval.Mlrval) *mlrval.Mlrval { + mvar := BIF_get_var(mn, msum, msum2) if mvar.IsVoid() { return mvar } @@ -86,11 +87,11 @@ func MlrvalGetMeanEB(mn, msum, msum2 *Mlrval) *Mlrval { // = sumx2 - n mean^2 // ---------------------------------------------------------------- -func MlrvalGetSkewness(mn, msum, msum2, msum3 *Mlrval) *Mlrval { +func BIF_get_skewness(mn, msum, msum2, msum3 *mlrval.Mlrval) *mlrval.Mlrval { n, isInt := mn.GetIntValue() lib.InternalCodingErrorIf(!isInt) if n < 2 { - return MLRVAL_VOID + return mlrval.VOID } fn := float64(n) sum, isNumber := msum.GetNumericToFloatValue() @@ -105,7 +106,7 @@ func MlrvalGetSkewness(mn, msum, msum2, msum3 *Mlrval) *Mlrval { numerator = numerator / fn denominator := (sum2 - fn*mean*mean) / (fn - 1.0) denominator = math.Pow(denominator, 1.5) - return MlrvalFromFloat64(numerator / denominator) + return mlrval.FromFloat(numerator / denominator) } // Unbiased: @@ -123,11 +124,11 @@ func MlrvalGetSkewness(mn, msum, msum2, msum3 *Mlrval) *Mlrval { // = sumx4 - mean*(4 sumx3 - mean*(6 sumx2 - 3 n mean^2)) // ---------------------------------------------------------------- -func MlrvalGetKurtosis(mn, msum, msum2, msum3, msum4 *Mlrval) *Mlrval { +func BIF_get_kurtosis(mn, msum, msum2, msum3, msum4 *mlrval.Mlrval) *mlrval.Mlrval { n, isInt := mn.GetIntValue() lib.InternalCodingErrorIf(!isInt) if n < 2 { - return MLRVAL_VOID + return mlrval.VOID } fn := float64(n) sum, isNumber := msum.GetNumericToFloatValue() @@ -145,6 +146,6 @@ func MlrvalGetKurtosis(mn, msum, msum2, msum3, msum4 *Mlrval) *Mlrval { numerator = numerator / fn denominator := (sum2 - fn*mean*mean) / fn denominator = denominator * denominator - return MlrvalFromFloat64(numerator/denominator - 3.0) + return mlrval.FromFloat(numerator/denominator - 3.0) } diff --git a/internal/pkg/types/mlrval_functions_strings.go b/internal/pkg/bifs/strings.go similarity index 58% rename from internal/pkg/types/mlrval_functions_strings.go rename to internal/pkg/bifs/strings.go index 81dd4e548..5f4dea02c 100644 --- a/internal/pkg/types/mlrval_functions_strings.go +++ b/internal/pkg/bifs/strings.go @@ -1,4 +1,4 @@ -package types +package bifs import ( "regexp" @@ -7,20 +7,21 @@ import ( "unicode/utf8" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ================================================================ -func BIF_strlen(input1 *Mlrval) *Mlrval { +func BIF_strlen(input1 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } else { - return MlrvalFromInt(int(utf8.RuneCountInString(input1.printrep))) + return mlrval.FromInt(int(utf8.RuneCountInString(input1.AcquireStringValue()))) } } // ================================================================ -func BIF_string(input1 *Mlrval) *Mlrval { - return MlrvalFromString(input1.String()) +func BIF_string(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromString(input1.String()) } // ================================================================ @@ -37,11 +38,11 @@ func BIF_string(input1 *Mlrval) *Mlrval { // should be: always the string concatenation of the string representations of // the two arguments. So, we do the string-cast for the user. -func dot_s_xx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromString(input1.String() + input2.String()) +func dot_s_xx(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromString(input1.String() + input2.String()) } -var dot_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var dot_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, /*ABSENT */ {_erro, _absn, _null, _void, _2___, _s2__, _s2__, _s2__, _absn, _absn, _erro}, @@ -56,53 +57,53 @@ var dot_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_dot(input1, input2 *Mlrval) *Mlrval { - return dot_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_dot(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return dot_dispositions[input1.Type()][input2.Type()](input1, input2) } // ================================================================ // substr1(s,m,n) gives substring of s from 1-up position m to n inclusive. // Negative indices -len .. -1 alias to 0 .. len-1. -func BIF_substr_1_up(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_substr_1_up(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - // Handle UTF-8 correctly: len(input1.printrep) will count bytes, not runes. - runes := []rune(input1.printrep) + // Handle UTF-8 correctly: len(input1.AcquireStringValue()) will count bytes, not runes. + runes := []rune(input1.AcquireStringValue()) strlen := int(len(runes)) // For array slices like s[1:2], s[:2], s[1:], when the lower index is // empty in the DSL expression it comes in here as a 1. But when the upper // index is empty in the DSL expression it comes in here as "". if !input2.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } - lowerMindex := input2.intval + lowerMindex := input2.AcquireIntValue() upperMindex := strlen - if input3.IsEmpty() { + if input3.IsVoid() { // Keep strlen } else if !input3.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } else { - upperMindex = input3.intval + upperMindex = input3.AcquireIntValue() } // Convert from negative-aliased 1-up to positive-only 0-up - m, mok := UnaliasArrayLengthIndex(strlen, lowerMindex) - n, nok := UnaliasArrayLengthIndex(strlen, upperMindex) + m, mok := unaliasArrayLengthIndex(strlen, lowerMindex) + n, nok := unaliasArrayLengthIndex(strlen, upperMindex) if !mok || !nok { - return MLRVAL_VOID + return mlrval.VOID } else if m > n { - return MLRVAL_VOID + return mlrval.VOID } else { // Note Golang slice indices are 0-up, and the 1st index is inclusive // while the 2nd is exclusive. For Miller, indices are 1-up and both // are inclusive. - return MlrvalFromString(string(runes[m : n+1])) + return mlrval.FromString(string(runes[m : n+1])) } } @@ -110,34 +111,34 @@ func BIF_substr_1_up(input1, input2, input3 *Mlrval) *Mlrval { // substr0(s,m,n) gives substring of s from 0-up position m to n inclusive. // Negative indices -len .. -1 alias to 0 .. len-1. -func BIF_substr_0_up(input1, input2, input3 *Mlrval) *Mlrval { +func BIF_substr_0_up(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } - // Handle UTF-8 correctly: len(input1.printrep) will count bytes, not runes. - runes := []rune(input1.printrep) + // Handle UTF-8 correctly: len(input1.AcquireStringValue()) will count bytes, not runes. + runes := []rune(input1.AcquireStringValue()) strlen := int(len(runes)) // For array slices like s[1:2], s[:2], s[1:], when the lower index is // empty in the DSL expression it comes in here as a 1. But when the upper // index is empty in the DSL expression it comes in here as "". if !input2.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } - lowerMindex := input2.intval + lowerMindex := input2.AcquireIntValue() if lowerMindex >= 0 { // Make 1-up lowerMindex += 1 } upperMindex := strlen - if input3.IsEmpty() { + if input3.IsVoid() { // Keep strlen } else if !input3.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } else { - upperMindex = input3.intval + upperMindex = input3.AcquireIntValue() if upperMindex >= 0 { // Make 1-up upperMindex += 1 @@ -145,23 +146,23 @@ func BIF_substr_0_up(input1, input2, input3 *Mlrval) *Mlrval { } // Convert from negative-aliased 1-up to positive-only 0-up - m, mok := UnaliasArrayLengthIndex(strlen, lowerMindex) - n, nok := UnaliasArrayLengthIndex(strlen, upperMindex) + m, mok := unaliasArrayLengthIndex(strlen, lowerMindex) + n, nok := unaliasArrayLengthIndex(strlen, upperMindex) if !mok || !nok { - return MLRVAL_VOID + return mlrval.VOID } else if m > n { - return MLRVAL_VOID + return mlrval.VOID } else { // Note Golang slice indices are 0-up, and the 1st index is inclusive // while the 2nd is exclusive. For Miller, indices are 1-up and both // are inclusive. - return MlrvalFromString(string(runes[m : n+1])) + return mlrval.FromString(string(runes[m : n+1])) } } // ================================================================ -func BIF_truncate(input1, input2 *Mlrval) *Mlrval { +func BIF_truncate(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if input1.IsErrorOrAbsent() { return input1 } @@ -169,59 +170,59 @@ func BIF_truncate(input1, input2 *Mlrval) *Mlrval { return input2 } if !input1.IsStringOrVoid() { - return MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsInt() { - return MLRVAL_ERROR + return mlrval.ERROR } - if input2.intval < 0 { - return MLRVAL_ERROR + if input2.AcquireIntValue() < 0 { + return mlrval.ERROR } - // Handle UTF-8 correctly: len(input1.printrep) will count bytes, not runes. - runes := []rune(input1.printrep) + // Handle UTF-8 correctly: len(input1.AcquireStringValue()) will count bytes, not runes. + runes := []rune(input1.AcquireStringValue()) oldLength := int(len(runes)) - maxLength := input2.intval + maxLength := input2.AcquireIntValue() if oldLength <= maxLength { return input1 } else { - return MlrvalFromString(string(runes[0:maxLength])) + return mlrval.FromString(string(runes[0:maxLength])) } } // ================================================================ -func BIF_lstrip(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_STRING { - return MlrvalFromString(strings.TrimLeft(input1.printrep, " \t")) +func BIF_lstrip(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsString() { + return mlrval.FromString(strings.TrimLeft(input1.AcquireStringValue(), " \t")) } else { return input1 } } -func BIF_rstrip(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_STRING { - return MlrvalFromString(strings.TrimRight(input1.printrep, " \t")) +func BIF_rstrip(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsString() { + return mlrval.FromString(strings.TrimRight(input1.AcquireStringValue(), " \t")) } else { return input1 } } -func BIF_strip(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_STRING { - return MlrvalFromString(strings.Trim(input1.printrep, " \t")) +func BIF_strip(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsString() { + return mlrval.FromString(strings.Trim(input1.AcquireStringValue(), " \t")) } else { return input1 } } // ---------------------------------------------------------------- -func BIF_collapse_whitespace(input1 *Mlrval) *Mlrval { - return MlrvalCollapseWhitespaceRegexp(input1, WhitespaceRegexp()) +func BIF_collapse_whitespace(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return BIF_collapse_whitespace_regexp(input1, WhitespaceRegexp()) } -func MlrvalCollapseWhitespaceRegexp(input1 *Mlrval, whitespaceRegexp *regexp.Regexp) *Mlrval { - if input1.mvtype == MT_STRING { - return MlrvalFromString(whitespaceRegexp.ReplaceAllString(input1.printrep, " ")) +func BIF_collapse_whitespace_regexp(input1 *mlrval.Mlrval, whitespaceRegexp *regexp.Regexp) *mlrval.Mlrval { + if input1.IsString() { + return mlrval.FromString(whitespaceRegexp.ReplaceAllString(input1.AcquireStringValue(), " ")) } else { return input1 } @@ -232,37 +233,37 @@ func WhitespaceRegexp() *regexp.Regexp { } // ================================================================ -func BIF_toupper(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_STRING { - return MlrvalFromString(strings.ToUpper(input1.printrep)) - } else if input1.mvtype == MT_VOID { +func BIF_toupper(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsString() { + return mlrval.FromString(strings.ToUpper(input1.AcquireStringValue())) + } else if input1.IsVoid() { return input1 } else { return input1 } } -func BIF_tolower(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_STRING { - return MlrvalFromString(strings.ToLower(input1.printrep)) - } else if input1.mvtype == MT_VOID { +func BIF_tolower(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsString() { + return mlrval.FromString(strings.ToLower(input1.AcquireStringValue())) + } else if input1.IsVoid() { return input1 } else { return input1 } } -func BIF_capitalize(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_STRING { - if input1.printrep == "" { +func BIF_capitalize(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsString() { + if input1.AcquireStringValue() == "" { return input1 } else { - runes := []rune(input1.printrep) + runes := []rune(input1.AcquireStringValue()) rfirst := runes[0] rrest := runes[1:] sfirst := strings.ToUpper(string(rfirst)) srest := string(rrest) - return MlrvalFromString(sfirst + srest) + return mlrval.FromString(sfirst + srest) } } else { return input1 @@ -270,66 +271,66 @@ func BIF_capitalize(input1 *Mlrval) *Mlrval { } // ---------------------------------------------------------------- -func BIF_clean_whitespace(input1 *Mlrval) *Mlrval { +func BIF_clean_whitespace(input1 *mlrval.Mlrval) *mlrval.Mlrval { return BIF_strip( - MlrvalCollapseWhitespaceRegexp( + BIF_collapse_whitespace_regexp( input1, WhitespaceRegexp(), ), ) } // ================================================================ -func BIF_hexfmt(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_INT { - return MlrvalFromString("0x" + strconv.FormatUint(uint64(input1.intval), 16)) +func BIF_hexfmt(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsInt() { + return mlrval.FromString("0x" + strconv.FormatUint(uint64(input1.AcquireIntValue()), 16)) } else { return input1 } } // ---------------------------------------------------------------- -func fmtnum_is(input1, input2 *Mlrval) *Mlrval { +func fmtnum_is(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input2.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } - formatString := input2.printrep - formatter, err := GetMlrvalFormatter(formatString) + formatString := input2.AcquireStringValue() + formatter, err := mlrval.GetFormatter(formatString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } return formatter.Format(input1) } -func fmtnum_fs(input1, input2 *Mlrval) *Mlrval { +func fmtnum_fs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input2.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } - formatString := input2.printrep - formatter, err := GetMlrvalFormatter(formatString) + formatString := input2.AcquireStringValue() + formatter, err := mlrval.GetFormatter(formatString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } return formatter.Format(input1) } -func fmtnum_bs(input1, input2 *Mlrval) *Mlrval { +func fmtnum_bs(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { if !input2.IsString() { - return MLRVAL_ERROR + return mlrval.ERROR } - formatString := input2.printrep - formatter, err := GetMlrvalFormatter(formatString) + formatString := input2.AcquireStringValue() + formatter, err := mlrval.GetFormatter(formatString) if err != nil { - return MLRVAL_ERROR + return mlrval.ERROR } - intMv := MlrvalFromInt(lib.BoolToInt(input1.boolval)) + intMv := mlrval.FromInt(lib.BoolToInt(input1.AcquireBoolValue())) return formatter.Format(intMv) } -var fmtnum_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ +var fmtnum_dispositions = [mlrval.MT_DIM][mlrval.MT_DIM]BinaryFunc{ // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _absn, _absn, _absn, _erro, _erro, _erro, _erro}, @@ -344,6 +345,6 @@ var fmtnum_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, } -func BIF_fmtnum(input1, input2 *Mlrval) *Mlrval { - return fmtnum_dispositions[input1.mvtype][input2.mvtype](input1, input2) +func BIF_fmtnum(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { + return fmtnum_dispositions[input1.Type()][input2.Type()](input1, input2) } diff --git a/internal/pkg/bifs/system.go b/internal/pkg/bifs/system.go new file mode 100644 index 000000000..974325b8e --- /dev/null +++ b/internal/pkg/bifs/system.go @@ -0,0 +1,46 @@ +package bifs + +import ( + "os" + "os/exec" + "runtime" + "strings" + + "github.com/johnkerl/miller/internal/pkg/mlrval" + "github.com/johnkerl/miller/internal/pkg/platform" + "github.com/johnkerl/miller/internal/pkg/version" +) + +func BIF_version() *mlrval.Mlrval { + return mlrval.FromString(version.STRING) +} + +func BIF_os() *mlrval.Mlrval { + return mlrval.FromString(runtime.GOOS) +} + +func BIF_hostname() *mlrval.Mlrval { + hostname, err := os.Hostname() + if err != nil { + return mlrval.ERROR + } else { + return mlrval.FromString(hostname) + } +} + +func BIF_system(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if !input1.IsStringOrVoid() { + return mlrval.ERROR + } + commandString := input1.AcquireStringValue() + + shellRunArray := platform.GetShellRunArray(commandString) + + outputBytes, err := exec.Command(shellRunArray[0], shellRunArray[1:]...).Output() + if err != nil { + return mlrval.ERROR + } + outputString := strings.TrimRight(string(outputBytes), "\n") + + return mlrval.FromString(outputString) +} diff --git a/internal/pkg/bifs/types.go b/internal/pkg/bifs/types.go new file mode 100644 index 000000000..668248e02 --- /dev/null +++ b/internal/pkg/bifs/types.go @@ -0,0 +1,288 @@ +package bifs + +import ( + "fmt" + "os" + + "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" + "github.com/johnkerl/miller/internal/pkg/types" +) + +// ================================================================ +func BIF_typeof(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromString(input1.GetTypeName()) +} + +// ---------------------------------------------------------------- +func string_to_int(input1 *mlrval.Mlrval) *mlrval.Mlrval { + i, ok := lib.TryIntFromString(input1.AcquireStringValue()) + if ok { + return mlrval.FromInt(i) + } else { + return mlrval.ERROR + } +} + +func float_to_int(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromInt(int(input1.AcquireFloatValue())) +} + +func bool_to_int(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.AcquireBoolValue() == true { + return mlrval.FromInt(1) + } else { + return mlrval.FromInt(0) + } +} + +var to_int_dispositions = [mlrval.MT_DIM]UnaryFunc{ + /*ERROR */ _erro1, + /*ABSENT */ _absn1, + /*NULL */ _null1, + /*VOID */ _void1, + /*STRING */ string_to_int, + /*INT */ _1u___, + /*FLOAT */ float_to_int, + /*BOOL */ bool_to_int, + /*ARRAY */ _erro1, + /*MAP */ _erro1, + /*FUNC */ _erro1, +} + +func BIF_int(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return to_int_dispositions[input1.Type()](input1) +} + +// ---------------------------------------------------------------- +func string_to_float(input1 *mlrval.Mlrval) *mlrval.Mlrval { + f, ok := lib.TryFloatFromString(input1.AcquireStringValue()) + if ok { + return mlrval.FromFloat(f) + } else { + return mlrval.ERROR + } +} + +func int_to_float(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromFloat(float64(input1.AcquireIntValue())) +} + +func bool_to_float(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.AcquireBoolValue() == true { + return mlrval.FromFloat(1.0) + } else { + return mlrval.FromFloat(0.0) + } +} + +var to_float_dispositions = [mlrval.MT_DIM]UnaryFunc{ + /*ERROR */ _erro1, + /*ABSENT */ _absn1, + /*NULL */ _null1, + /*VOID */ _void1, + /*STRING */ string_to_float, + /*INT */ int_to_float, + /*FLOAT */ _1u___, + /*BOOL */ bool_to_float, + /*ARRAY */ _erro1, + /*MAP */ _erro1, + /*FUNC */ _erro1, +} + +func BIF_float(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return to_float_dispositions[input1.Type()](input1) +} + +// ---------------------------------------------------------------- +func string_to_boolean(input1 *mlrval.Mlrval) *mlrval.Mlrval { + b, ok := lib.TryBoolFromBoolString(input1.AcquireStringValue()) + if ok { + return mlrval.FromBool(b) + } else { + return mlrval.ERROR + } +} + +func int_to_bool(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireIntValue() != 0) +} + +func float_to_bool(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.AcquireFloatValue() != 0.0) +} + +var to_boolean_dispositions = [mlrval.MT_DIM]UnaryFunc{ + /*ERROR */ _erro1, + /*ABSENT */ _absn1, + /*NULL */ _null1, + /*VOID */ _void1, + /*STRING */ string_to_boolean, + /*INT */ int_to_bool, + /*FLOAT */ float_to_bool, + /*BOOL */ _1u___, + /*ARRAY */ _erro1, + /*MAP */ _erro1, + /*FUNC */ _erro1, +} + +func BIF_boolean(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return to_boolean_dispositions[input1.Type()](input1) +} + +// ---------------------------------------------------------------- +func BIF_is_absent(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsAbsent()) +} +func BIF_is_error(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsError()) +} +func BIF_is_bool(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsBool()) +} +func BIF_is_boolean(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsBool()) +} +func BIF_is_empty(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsVoid() { + return mlrval.TRUE + } else if input1.IsString() { + if input1.AcquireStringValue() == "" { + return mlrval.TRUE + } else { + return mlrval.FALSE + } + } else { + return mlrval.FALSE + } +} +func BIF_is_emptymap(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsMap() && input1.AcquireMapValue().IsEmpty()) +} +func BIF_is_float(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsFloat()) +} +func BIF_is_int(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsInt()) +} +func BIF_is_map(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsMap()) +} +func BIF_is_array(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsArray()) +} +func BIF_is_nonemptymap(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsMap() && input1.AcquireMapValue().FieldCount != 0) +} +func BIF_is_notempty(input1 *mlrval.Mlrval) *mlrval.Mlrval { + if input1.IsVoid() { + return mlrval.FALSE + } else if input1.IsString() { + if input1.AcquireStringValue() == "" { + return mlrval.FALSE + } else { + return mlrval.TRUE + } + } else { + return mlrval.TRUE + } +} +func BIF_is_notmap(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(!input1.IsMap()) +} +func BIF_is_notarray(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(!input1.IsArray()) +} +func BIF_is_notnull(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(!input1.IsAbsent() && !input1.IsVoid()) +} +func BIF_is_null(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsAbsent() || input1.IsVoid()) +} +func BIF_is_numeric(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsInt() || input1.IsFloat()) +} +func BIF_is_present(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(!input1.IsAbsent()) +} +func BIF_is_string(input1 *mlrval.Mlrval) *mlrval.Mlrval { + return mlrval.FromBool(input1.IsStringOrVoid()) +} + +// ---------------------------------------------------------------- +func assertingCommon(input1, check *mlrval.Mlrval, description string, context *types.Context) *mlrval.Mlrval { + if check.IsFalse() { + // TODO: get context as in the C impl + //fprintf(stderr, "%s: %s type-assertion failed at NR=%lld FNR=%lld FILENAME=%s\n", + //MLR_GLOBALS.bargv0, pstate->desc, pvars->pctx->nr, pvars->pctx->fnr, pvars->pctx->filename); + //exit(1); + fmt.Fprintf( + os.Stderr, + "mlr: %s type-assertion failed at NR=%d FNR=%d FILENAME=%s\n", + description, + context.NR, + context.FNR, + context.FILENAME, + ) + os.Exit(1) + } + return input1 +} + +func BIF_asserting_absent(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_absent(input1), "is_absent", context) +} +func BIF_asserting_error(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_error(input1), "is_error", context) +} +func BIF_asserting_bool(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_bool(input1), "is_bool", context) +} +func BIF_asserting_boolean(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_boolean(input1), "is_boolean", context) +} +func BIF_asserting_empty(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_empty(input1), "is_empty", context) +} +func BIF_asserting_emptyMap(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_emptymap(input1), "is_empty_map", context) +} +func BIF_asserting_float(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_float(input1), "is_float", context) +} +func BIF_asserting_int(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_int(input1), "is_int", context) +} +func BIF_asserting_map(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_map(input1), "is_map", context) +} +func BIF_asserting_array(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_array(input1), "is_array", context) +} +func BIF_asserting_nonempty_map(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_nonemptymap(input1), "is_non_empty_map", context) +} +func BIF_asserting_not_empty(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_notempty(input1), "is_not_empty", context) +} +func BIF_asserting_not_map(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_notmap(input1), "is_not_map", context) +} +func BIF_asserting_not_array(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_notarray(input1), "is_not_array", context) +} +func BIF_asserting_not_null(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_notnull(input1), "is_not_null", context) +} +func BIF_asserting_null(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_null(input1), "is_null", context) +} +func BIF_asserting_numeric(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_numeric(input1), "is_numeric", context) +} +func BIF_asserting_present(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_present(input1), "is_present", context) +} +func BIF_asserting_string(input1 *mlrval.Mlrval, context *types.Context) *mlrval.Mlrval { + return assertingCommon(input1, BIF_is_string(input1), "is_string", context) +} diff --git a/internal/pkg/cli/option_parse.go b/internal/pkg/cli/option_parse.go index 839e0282a..68a680dbc 100644 --- a/internal/pkg/cli/option_parse.go +++ b/internal/pkg/cli/option_parse.go @@ -15,7 +15,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/colorizer" "github.com/johnkerl/miller/internal/pkg/lib" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // FinalizeReaderOptions does a few things. @@ -2584,7 +2584,7 @@ In general --no-hash-records is faster, and is the default. For specific use-cas data having many fields, and many of them being processed during a given processing run, --hash-records might offer a slight performance benefit.`, parser: func(args []string, argc int, pargi *int, options *TOptions) { - types.HashRecords(true) + mlrval.HashRecords(true) *pargi += 1 }, }, @@ -2593,7 +2593,7 @@ data having many fields, and many of them being processed during a given process name: "--no-hash-records", help: `See --hash-records.`, parser: func(args []string, argc int, pargi *int, options *TOptions) { - types.HashRecords(false) + mlrval.HashRecords(false) *pargi += 1 }, }, @@ -2603,7 +2603,7 @@ data having many fields, and many of them being processed during a given process altNames: []string{"-S"}, help: `Don't treat values like 123 or 456.7 in data files as int/float; leave them as strings.`, parser: func(args []string, argc int, pargi *int, options *TOptions) { - types.SetInferrerStringOnly() + mlrval.SetInferrerStringOnly() *pargi += 1 }, }, @@ -2613,7 +2613,7 @@ data having many fields, and many of them being processed during a given process altNames: []string{"-A"}, help: `Cast all integers in data files to floats.`, parser: func(args []string, argc int, pargi *int, options *TOptions) { - types.SetInferrerIntAsFloat() + mlrval.SetInferrerIntAsFloat() *pargi += 1 }, }, @@ -2623,7 +2623,7 @@ data having many fields, and many of them being processed during a given process altNames: []string{"-O"}, help: `Treat numbers like 0123 in data files as string "0123", not octal for decimal 83 etc.`, parser: func(args []string, argc int, pargi *int, options *TOptions) { - types.SetInferrerNoOctal() + mlrval.SetInferrerNoOctal() *pargi += 1 }, }, diff --git a/internal/pkg/climain/mlrcli_parse.go b/internal/pkg/climain/mlrcli_parse.go index 8b61bbd5c..2a2cb3bb3 100644 --- a/internal/pkg/climain/mlrcli_parse.go +++ b/internal/pkg/climain/mlrcli_parse.go @@ -75,8 +75,8 @@ import ( "github.com/johnkerl/miller/internal/pkg/auxents/help" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/transformers" - "github.com/johnkerl/miller/internal/pkg/types" "github.com/johnkerl/miller/internal/pkg/version" ) @@ -280,7 +280,7 @@ func parseCommandLinePassTwo( // Set an optional global formatter for floating-point values if options.WriterOptions.FPOFMT != "" { - err = types.SetMlrvalFloatOutputFormat(options.WriterOptions.FPOFMT) + err = mlrval.SetFloatOutputFormat(options.WriterOptions.FPOFMT) if err != nil { return options, recordTransformers, err } diff --git a/internal/pkg/dsl/cst/builtin_function_manager.go b/internal/pkg/dsl/cst/builtin_function_manager.go index d90eec326..f1c05f6b9 100644 --- a/internal/pkg/dsl/cst/builtin_function_manager.go +++ b/internal/pkg/dsl/cst/builtin_function_manager.go @@ -19,9 +19,9 @@ import ( "sort" "strings" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/colorizer" "github.com/johnkerl/miller/internal/pkg/lib" - "github.com/johnkerl/miller/internal/pkg/types" ) type TFunctionClass string @@ -53,16 +53,16 @@ type BuiltinFunctionInfo struct { hasMultipleArities bool minimumVariadicArity int maximumVariadicArity int // 0 means no max - zaryFunc types.ZaryFunc - unaryFunc types.UnaryFunc - binaryFunc types.BinaryFunc - ternaryFunc types.TernaryFunc - variadicFunc types.VariadicFunc - unaryFuncWithContext types.UnaryFuncWithContext // asserting_{typename} - regexCaptureBinaryFunc types.RegexCaptureBinaryFunc // =~ and !=~ - binaryFuncWithState BinaryFuncWithState // select, apply, reduce - ternaryFuncWithState TernaryFuncWithState // fold - variadicFuncWithState VariadicFuncWithState // sort + zaryFunc bifs.ZaryFunc + unaryFunc bifs.UnaryFunc + binaryFunc bifs.BinaryFunc + ternaryFunc bifs.TernaryFunc + variadicFunc bifs.VariadicFunc + unaryFuncWithContext bifs.UnaryFuncWithContext // asserting_{typename} + regexCaptureBinaryFunc bifs.RegexCaptureBinaryFunc // =~ and !=~ + binaryFuncWithState BinaryFuncWithState // select, apply, reduce + ternaryFuncWithState TernaryFuncWithState // fold + variadicFuncWithState VariadicFuncWithState // sort } // ================================================================ @@ -88,8 +88,8 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { name: "+", class: FUNC_CLASS_ARITHMETIC, help: `Addition as binary operator; unary plus operator.`, - unaryFunc: types.BIF_plus_unary, - binaryFunc: types.BIF_plus_binary, + unaryFunc: bifs.BIF_plus_unary, + binaryFunc: bifs.BIF_plus_binary, hasMultipleArities: true, }, @@ -97,8 +97,8 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { name: "-", class: FUNC_CLASS_ARITHMETIC, help: `Subtraction as binary operator; unary negation operator.`, - unaryFunc: types.BIF_minus_unary, - binaryFunc: types.BIF_minus_binary, + unaryFunc: bifs.BIF_minus_unary, + binaryFunc: bifs.BIF_minus_binary, hasMultipleArities: true, }, @@ -106,154 +106,154 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { name: "*", class: FUNC_CLASS_ARITHMETIC, help: `Multiplication, with integer*integer overflow to float.`, - binaryFunc: types.BIF_times, + binaryFunc: bifs.BIF_times, }, { name: "/", class: FUNC_CLASS_ARITHMETIC, help: `Division. Integer / integer is floating-point.`, - binaryFunc: types.BIF_divide, + binaryFunc: bifs.BIF_divide, }, { name: "//", class: FUNC_CLASS_ARITHMETIC, help: `Pythonic integer division, rounding toward negative.`, - binaryFunc: types.BIF_int_divide, + binaryFunc: bifs.BIF_int_divide, }, { name: "**", class: FUNC_CLASS_ARITHMETIC, help: `Exponentiation. Same as pow, but as an infix operator.`, - binaryFunc: types.BIF_pow, + binaryFunc: bifs.BIF_pow, }, { name: "pow", class: FUNC_CLASS_ARITHMETIC, help: `Exponentiation. Same as **, but as a function.`, - binaryFunc: types.BIF_pow, + binaryFunc: bifs.BIF_pow, }, { name: ".+", class: FUNC_CLASS_ARITHMETIC, help: `Addition, with integer-to-integer overflow.`, - binaryFunc: types.BIF_dot_plus, + binaryFunc: bifs.BIF_dot_plus, }, { name: ".-", class: FUNC_CLASS_ARITHMETIC, help: `Subtraction, with integer-to-integer overflow.`, - binaryFunc: types.BIF_dot_minus, + binaryFunc: bifs.BIF_dot_minus, }, { name: ".*", class: FUNC_CLASS_ARITHMETIC, help: `Multiplication, with integer-to-integer overflow.`, - binaryFunc: types.BIF_dot_times, + binaryFunc: bifs.BIF_dot_times, }, { name: "./", class: FUNC_CLASS_ARITHMETIC, help: `Integer division; not pythonic.`, - binaryFunc: types.BIF_dot_divide, + binaryFunc: bifs.BIF_dot_divide, }, { name: "%", class: FUNC_CLASS_ARITHMETIC, help: `Remainder; never negative-valued (pythonic).`, - binaryFunc: types.BIF_modulus, + binaryFunc: bifs.BIF_modulus, }, { name: "~", class: FUNC_CLASS_ARITHMETIC, help: `Bitwise NOT. Beware '$y=~$x' since =~ is the regex-match operator: try '$y = ~$x'.`, - unaryFunc: types.BIF_bitwise_not, + unaryFunc: bifs.BIF_bitwise_not, }, { name: "&", class: FUNC_CLASS_ARITHMETIC, help: `Bitwise AND.`, - binaryFunc: types.BIF_bitwise_and, + binaryFunc: bifs.BIF_bitwise_and, }, { name: "|", class: FUNC_CLASS_ARITHMETIC, help: `Bitwise OR.`, - binaryFunc: types.BIF_bitwise_or, + binaryFunc: bifs.BIF_bitwise_or, }, { name: "^", help: `Bitwise XOR.`, class: FUNC_CLASS_ARITHMETIC, - binaryFunc: types.BIF_bitwise_xor, + binaryFunc: bifs.BIF_bitwise_xor, }, { name: "<<", class: FUNC_CLASS_ARITHMETIC, help: `Bitwise left-shift.`, - binaryFunc: types.BIF_left_shift, + binaryFunc: bifs.BIF_left_shift, }, { name: ">>", class: FUNC_CLASS_ARITHMETIC, help: `Bitwise signed right-shift.`, - binaryFunc: types.BIF_signed_right_shift, + binaryFunc: bifs.BIF_signed_right_shift, }, { name: ">>>", class: FUNC_CLASS_ARITHMETIC, help: `Bitwise unsigned right-shift.`, - binaryFunc: types.BIF_unsigned_right_shift, + binaryFunc: bifs.BIF_unsigned_right_shift, }, { name: "bitcount", class: FUNC_CLASS_ARITHMETIC, help: "Count of 1-bits.", - unaryFunc: types.BIF_bitcount, + unaryFunc: bifs.BIF_bitcount, }, { name: "madd", class: FUNC_CLASS_ARITHMETIC, help: `a + b mod m (integers)`, - ternaryFunc: types.BIF_mod_add, + ternaryFunc: bifs.BIF_mod_add, }, { name: "msub", class: FUNC_CLASS_ARITHMETIC, help: `a - b mod m (integers)`, - ternaryFunc: types.BIF_mod_sub, + ternaryFunc: bifs.BIF_mod_sub, }, { name: "mmul", class: FUNC_CLASS_ARITHMETIC, help: `a * b mod m (integers)`, - ternaryFunc: types.BIF_mod_mul, + ternaryFunc: bifs.BIF_mod_mul, }, { name: "mexp", class: FUNC_CLASS_ARITHMETIC, help: `a ** b mod m (integers)`, - ternaryFunc: types.BIF_mod_exp, + ternaryFunc: bifs.BIF_mod_exp, }, // ---------------------------------------------------------------- @@ -263,7 +263,7 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { name: "!", class: FUNC_CLASS_BOOLEAN, help: `Logical negation.`, - unaryFunc: types.BIF_logicalnot, + unaryFunc: bifs.BIF_logical_NOT, }, { @@ -271,63 +271,63 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { class: FUNC_CLASS_BOOLEAN, help: `String/numeric equality. Mixing number and string results in string compare.`, - binaryFunc: types.BIF_equals, + binaryFunc: bifs.BIF_equals, }, { name: "!=", class: FUNC_CLASS_BOOLEAN, help: `String/numeric inequality. Mixing number and string results in string compare.`, - binaryFunc: types.BIF_not_equals, + binaryFunc: bifs.BIF_not_equals, }, { name: ">", help: `String/numeric greater-than. Mixing number and string results in string compare.`, class: FUNC_CLASS_BOOLEAN, - binaryFunc: types.BIF_greater_than, + binaryFunc: bifs.BIF_greater_than, }, { name: ">=", help: `String/numeric greater-than-or-equals. Mixing number and string results in string compare.`, class: FUNC_CLASS_BOOLEAN, - binaryFunc: types.BIF_greater_than_or_equals, + binaryFunc: bifs.BIF_greater_than_or_equals, }, { name: "<=>", help: `Comparator, nominally for sorting. Given a <=> b, returns <0, 0, >0 as a < b, a == b, or a > b, respectively.`, class: FUNC_CLASS_BOOLEAN, - binaryFunc: types.BIF_cmp, + binaryFunc: bifs.BIF_cmp, }, { name: "<", class: FUNC_CLASS_BOOLEAN, help: `String/numeric less-than. Mixing number and string results in string compare.`, - binaryFunc: types.BIF_less_than, + binaryFunc: bifs.BIF_less_than, }, { name: "<=", class: FUNC_CLASS_BOOLEAN, help: `String/numeric less-than-or-equals. Mixing number and string results in string compare.`, - binaryFunc: types.BIF_less_than_or_equals, + binaryFunc: bifs.BIF_less_than_or_equals, }, { name: "=~", class: FUNC_CLASS_BOOLEAN, help: `String (left-hand side) matches regex (right-hand side), e.g. '$name =~ "^a.*b$"'.`, - regexCaptureBinaryFunc: types.BIF_string_matches_regexp, + regexCaptureBinaryFunc: bifs.BIF_string_matches_regexp, }, { name: "!=~", class: FUNC_CLASS_BOOLEAN, help: `String (left-hand side) does not match regex (right-hand side), e.g. '$name !=~ "^a.*b$"'.`, - regexCaptureBinaryFunc: types.BIF_string_does_not_match_regexp, + regexCaptureBinaryFunc: bifs.BIF_string_does_not_match_regexp, }, { @@ -348,7 +348,7 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { name: "^^", class: FUNC_CLASS_BOOLEAN, help: `Logical XOR.`, - binaryFunc: types.BIF_logicalxor, + binaryFunc: bifs.BIF_logical_XOR, }, { @@ -379,91 +379,91 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { name: ".", class: FUNC_CLASS_STRING, help: `String concatenation.`, - binaryFunc: types.BIF_dot, + binaryFunc: bifs.BIF_dot, }, { name: "capitalize", class: FUNC_CLASS_STRING, help: "Convert string's first character to uppercase.", - unaryFunc: types.BIF_capitalize, + unaryFunc: bifs.BIF_capitalize, }, { name: "clean_whitespace", class: FUNC_CLASS_STRING, help: "Same as collapse_whitespace and strip.", - unaryFunc: types.BIF_clean_whitespace, + unaryFunc: bifs.BIF_clean_whitespace, }, { name: "collapse_whitespace", class: FUNC_CLASS_STRING, help: "Strip repeated whitespace from string.", - unaryFunc: types.BIF_collapse_whitespace, + unaryFunc: bifs.BIF_collapse_whitespace, }, { name: "gsub", class: FUNC_CLASS_STRING, help: `'$name=gsub($name, "old", "new")' (replace all).`, - ternaryFunc: types.BIF_gsub, + ternaryFunc: bifs.BIF_gsub, }, { name: "lstrip", class: FUNC_CLASS_STRING, help: "Strip leading whitespace from string.", - unaryFunc: types.BIF_lstrip, + unaryFunc: bifs.BIF_lstrip, }, { name: "regextract", class: FUNC_CLASS_STRING, help: `'$name=regextract($name, "[A-Z]{3}[0-9]{2}")'`, - binaryFunc: types.BIF_regextract, + binaryFunc: bifs.BIF_regextract, }, { name: "regextract_or_else", class: FUNC_CLASS_STRING, help: `'$name=regextract_or_else($name, "[A-Z]{3}[0-9]{2}", "default")'`, - ternaryFunc: types.BIF_regextract_or_else, + ternaryFunc: bifs.BIF_regextract_or_else, }, { name: "rstrip", class: FUNC_CLASS_STRING, help: "Strip trailing whitespace from string.", - unaryFunc: types.BIF_rstrip, + unaryFunc: bifs.BIF_rstrip, }, { name: "strip", class: FUNC_CLASS_STRING, help: "Strip leading and trailing whitespace from string.", - unaryFunc: types.BIF_strip, + unaryFunc: bifs.BIF_strip, }, { name: "strlen", class: FUNC_CLASS_STRING, help: "String length.", - unaryFunc: types.BIF_strlen, + unaryFunc: bifs.BIF_strlen, }, { name: "ssub", class: FUNC_CLASS_STRING, help: `Like sub but does no regexing. No characters are special.`, - ternaryFunc: types.BIF_ssub, + ternaryFunc: bifs.BIF_ssub, }, { name: "sub", class: FUNC_CLASS_STRING, help: `'$name=sub($name, "old", "new")' (replace once).`, - ternaryFunc: types.BIF_sub, + ternaryFunc: bifs.BIF_sub, }, { @@ -471,14 +471,14 @@ func makeBuiltinFunctionLookupTable() []BuiltinFunctionInfo { class: FUNC_CLASS_STRING, help: `substr0(s,m,n) gives substring of s from 0-up position m to n inclusive. Negative indices -len .. -1 alias to 0 .. len-1. See also substr and substr1.`, - ternaryFunc: types.BIF_substr_0_up, + ternaryFunc: bifs.BIF_substr_0_up, }, { name: "substr1", class: FUNC_CLASS_STRING, help: `substr1(s,m,n) gives substring of s from 1-up position m to n inclusive. Negative indices -len .. -1 alias to 1 .. len. See also substr and substr0.`, - ternaryFunc: types.BIF_substr_1_up, + ternaryFunc: bifs.BIF_substr_1_up, }, { name: "substr", @@ -486,28 +486,28 @@ Negative indices -len .. -1 alias to 1 .. len. See also substr and substr0.`, help: `substr is an alias for substr0. See also substr1. Miller is generally 1-up with all array and string indices, but, this is a backward-compatibility issue with Miller 5 and below. Arrays are new in Miller 6; the substr function is older.`, - ternaryFunc: types.BIF_substr_0_up, + ternaryFunc: bifs.BIF_substr_0_up, }, { name: "tolower", class: FUNC_CLASS_STRING, help: "Convert string to lowercase.", - unaryFunc: types.BIF_tolower, + unaryFunc: bifs.BIF_tolower, }, { name: "toupper", class: FUNC_CLASS_STRING, help: "Convert string to uppercase.", - unaryFunc: types.BIF_toupper, + unaryFunc: bifs.BIF_toupper, }, { name: "truncate", class: FUNC_CLASS_STRING, help: `Truncates string first argument to max length of int second argument.`, - binaryFunc: types.BIF_truncate, + binaryFunc: bifs.BIF_truncate, }, // ---------------------------------------------------------------- @@ -517,25 +517,25 @@ Arrays are new in Miller 6; the substr function is older.`, name: "md5", class: FUNC_CLASS_HASHING, help: `MD5 hash.`, - unaryFunc: types.BIF_md5, + unaryFunc: bifs.BIF_md5, }, { name: "sha1", class: FUNC_CLASS_HASHING, help: `SHA1 hash.`, - unaryFunc: types.BIF_sha1, + unaryFunc: bifs.BIF_sha1, }, { name: "sha256", class: FUNC_CLASS_HASHING, help: `SHA256 hash.`, - unaryFunc: types.BIF_sha256, + unaryFunc: bifs.BIF_sha256, }, { name: "sha512", class: FUNC_CLASS_HASHING, help: `SHA512 hash.`, - unaryFunc: types.BIF_sha512, + unaryFunc: bifs.BIF_sha512, }, // ---------------------------------------------------------------- @@ -545,119 +545,119 @@ Arrays are new in Miller 6; the substr function is older.`, name: "abs", class: FUNC_CLASS_MATH, help: "Absolute value.", - unaryFunc: types.BIF_abs, + unaryFunc: bifs.BIF_abs, }, { name: "acos", class: FUNC_CLASS_MATH, help: "Inverse trigonometric cosine.", - unaryFunc: types.BIF_acos, + unaryFunc: bifs.BIF_acos, }, { name: "acosh", class: FUNC_CLASS_MATH, help: "Inverse hyperbolic cosine.", - unaryFunc: types.BIF_acosh, + unaryFunc: bifs.BIF_acosh, }, { name: "asin", class: FUNC_CLASS_MATH, help: "Inverse trigonometric sine.", - unaryFunc: types.BIF_asin, + unaryFunc: bifs.BIF_asin, }, { name: "asinh", class: FUNC_CLASS_MATH, help: "Inverse hyperbolic sine.", - unaryFunc: types.BIF_asinh, + unaryFunc: bifs.BIF_asinh, }, { name: "atan", class: FUNC_CLASS_MATH, help: "One-argument arctangent.", - unaryFunc: types.BIF_atan, + unaryFunc: bifs.BIF_atan, }, { name: "atan2", class: FUNC_CLASS_MATH, help: "Two-argument arctangent.", - binaryFunc: types.BIF_atan2, + binaryFunc: bifs.BIF_atan2, }, { name: "atanh", class: FUNC_CLASS_MATH, help: "Inverse hyperbolic tangent.", - unaryFunc: types.BIF_atanh, + unaryFunc: bifs.BIF_atanh, }, { name: "cbrt", class: FUNC_CLASS_MATH, help: "Cube root.", - unaryFunc: types.BIF_cbrt, + unaryFunc: bifs.BIF_cbrt, }, { name: "ceil", class: FUNC_CLASS_MATH, help: "Ceiling: nearest integer at or above.", - unaryFunc: types.BIF_ceil, + unaryFunc: bifs.BIF_ceil, }, { name: "cos", class: FUNC_CLASS_MATH, help: "Trigonometric cosine.", - unaryFunc: types.BIF_cos, + unaryFunc: bifs.BIF_cos, }, { name: "cosh", class: FUNC_CLASS_MATH, help: "Hyperbolic cosine.", - unaryFunc: types.BIF_cosh, + unaryFunc: bifs.BIF_cosh, }, { name: "erf", class: FUNC_CLASS_MATH, help: "Error function.", - unaryFunc: types.BIF_erf, + unaryFunc: bifs.BIF_erf, }, { name: "erfc", class: FUNC_CLASS_MATH, help: "Complementary error function.", - unaryFunc: types.BIF_erfc, + unaryFunc: bifs.BIF_erfc, }, { name: "exp", class: FUNC_CLASS_MATH, help: "Exponential function e**x.", - unaryFunc: types.BIF_exp, + unaryFunc: bifs.BIF_exp, }, { name: "expm1", class: FUNC_CLASS_MATH, help: "e**x - 1.", - unaryFunc: types.BIF_expm1, + unaryFunc: bifs.BIF_expm1, }, { name: "floor", class: FUNC_CLASS_MATH, help: "Floor: nearest integer at or below.", - unaryFunc: types.BIF_floor, + unaryFunc: bifs.BIF_floor, }, { @@ -665,112 +665,112 @@ Arrays are new in Miller 6; the substr function is older.`, class: FUNC_CLASS_MATH, help: `Inverse of normal cumulative distribution function. Note that invqorm(urand()) is normally distributed.`, - unaryFunc: types.BIF_invqnorm, + unaryFunc: bifs.BIF_invqnorm, }, { name: "log", class: FUNC_CLASS_MATH, help: "Natural (base-e) logarithm.", - unaryFunc: types.BIF_log, + unaryFunc: bifs.BIF_log, }, { name: "log10", class: FUNC_CLASS_MATH, help: "Base-10 logarithm.", - unaryFunc: types.BIF_log10, + unaryFunc: bifs.BIF_log10, }, { name: "log1p", class: FUNC_CLASS_MATH, help: "log(1-x).", - unaryFunc: types.BIF_log1p, + unaryFunc: bifs.BIF_log1p, }, { name: "logifit", class: FUNC_CLASS_MATH, help: `Given m and b from logistic regression, compute fit: $yhat=logifit($x,$m,$b).`, - ternaryFunc: types.BIF_logifit, + ternaryFunc: bifs.BIF_logifit, }, { name: "max", class: FUNC_CLASS_MATH, help: `Max of n numbers; null loses.`, - variadicFunc: types.BIF_max_variadic, + variadicFunc: bifs.BIF_max_variadic, }, { name: "min", class: FUNC_CLASS_MATH, help: `Min of n numbers; null loses.`, - variadicFunc: types.BIF_min_variadic, + variadicFunc: bifs.BIF_min_variadic, }, { name: "qnorm", class: FUNC_CLASS_MATH, help: `Normal cumulative distribution function.`, - unaryFunc: types.BIF_qnorm, + unaryFunc: bifs.BIF_qnorm, }, { name: "round", class: FUNC_CLASS_MATH, help: "Round to nearest integer.", - unaryFunc: types.BIF_round, + unaryFunc: bifs.BIF_round, }, { name: "sgn", class: FUNC_CLASS_MATH, help: `+1, 0, -1 for positive, zero, negative input respectively.`, - unaryFunc: types.BIF_sgn, + unaryFunc: bifs.BIF_sgn, }, { name: "sin", class: FUNC_CLASS_MATH, help: "Trigonometric sine.", - unaryFunc: types.BIF_sin, + unaryFunc: bifs.BIF_sin, }, { name: "sinh", class: FUNC_CLASS_MATH, help: "Hyperbolic sine.", - unaryFunc: types.BIF_sinh, + unaryFunc: bifs.BIF_sinh, }, { name: "sqrt", class: FUNC_CLASS_MATH, help: "Square root.", - unaryFunc: types.BIF_sqrt, + unaryFunc: bifs.BIF_sqrt, }, { name: "tan", class: FUNC_CLASS_MATH, help: "Trigonometric tangent.", - unaryFunc: types.BIF_tan, + unaryFunc: bifs.BIF_tan, }, { name: "tanh", class: FUNC_CLASS_MATH, help: "Hyperbolic tangent.", - unaryFunc: types.BIF_tanh, + unaryFunc: bifs.BIF_tanh, }, { name: "roundm", class: FUNC_CLASS_MATH, help: `Round to nearest multiple of m: roundm($x,$m) is the same as round($x/$m)*$m.`, - binaryFunc: types.BIF_roundm, + binaryFunc: bifs.BIF_roundm, }, { @@ -780,28 +780,28 @@ is normally distributed.`, examples: []string{ "Int-valued example: '$n=floor(20+urand()*11)'.", }, - zaryFunc: types.BIF_urand, + zaryFunc: bifs.BIF_urand, }, { name: "urandint", class: FUNC_CLASS_MATH, help: `Integer uniformly distributed between inclusive integer endpoints.`, - binaryFunc: types.BIF_urandint, + binaryFunc: bifs.BIF_urandint, }, { name: "urandrange", class: FUNC_CLASS_MATH, help: `Floating-point numbers uniformly distributed on the interval [a, b).`, - binaryFunc: types.BIF_urandrange, + binaryFunc: bifs.BIF_urandrange, }, { name: "urand32", class: FUNC_CLASS_MATH, help: `Integer uniformly distributed 0 and 2**32-1 inclusive.`, - zaryFunc: types.BIF_urand32, + zaryFunc: bifs.BIF_urand32, }, // ---------------------------------------------------------------- @@ -814,7 +814,7 @@ is normally distributed.`, examples: []string{ `gmt2sec("2001-02-03T04:05:06Z") = 981173106`, }, - unaryFunc: types.BIF_gmt2sec, + unaryFunc: bifs.BIF_gmt2sec, }, { @@ -827,8 +827,8 @@ unless second argument is supplied.`, `localtime2sec("2001-02-03 04:05:06", "Asia/Istanbul") = 981165906"`, }, // TODO: help-string - unaryFunc: types.BIF_localtime2sec_unary, - binaryFunc: types.BIF_localtime2sec_binary, + unaryFunc: bifs.BIF_localtime2sec_unary, + binaryFunc: bifs.BIF_localtime2sec_binary, hasMultipleArities: true, }, @@ -842,8 +842,8 @@ argument n, includes n decimal places for the seconds part.`, `sec2gmt(1234567890.123456) = "2009-02-13T23:31:30Z"`, `sec2gmt(1234567890.123456, 6) = "2009-02-13T23:31:30.123456Z"`, }, - unaryFunc: types.BIF_sec2gmt_unary, - binaryFunc: types.BIF_sec2gmt_binary, + unaryFunc: bifs.BIF_sec2gmt_unary, + binaryFunc: bifs.BIF_sec2gmt_binary, hasMultipleArities: true, }, @@ -859,9 +859,9 @@ includes n decimal places for the seconds part`, `sec2localtime(1234567890.123456, 6) = "2009-02-14 01:31:30.123456" with TZ="Asia/Istanbul"`, `sec2localtime(1234567890.123456, 6, "Asia/Istanbul") = "2009-02-14 01:31:30.123456"`, }, - unaryFunc: types.BIF_sec2localtime_unary, - binaryFunc: types.BIF_sec2localtime_binary, - ternaryFunc: types.BIF_sec2localtime_ternary, + unaryFunc: bifs.BIF_sec2localtime_unary, + binaryFunc: bifs.BIF_sec2localtime_binary, + ternaryFunc: bifs.BIF_sec2localtime_ternary, hasMultipleArities: true, }, @@ -873,7 +873,7 @@ Leaves non-numbers as-is.`, examples: []string{ `sec2gmtdate(1440768801.7) = "2015-08-28".`, }, - unaryFunc: types.BIF_sec2gmtdate, + unaryFunc: bifs.BIF_sec2gmtdate, }, { @@ -885,8 +885,8 @@ Leaves non-numbers as-is. Consults $TZ environment variable unless second argume `sec2localdate(1440768801.7) = "2015-08-28" with TZ="Asia/Istanbul"`, `sec2localdate(1440768801.7, "Asia/Istanbul") = "2015-08-28"`, }, - unaryFunc: types.BIF_sec2localdate_unary, - binaryFunc: types.BIF_sec2localdate_binary, + unaryFunc: bifs.BIF_sec2localdate_unary, + binaryFunc: bifs.BIF_sec2localdate_binary, hasMultipleArities: true, }, @@ -899,8 +899,8 @@ is supplied.`, `localtime2gmt("2000-01-01 00:00:00") = "1999-12-31T22:00:00Z" with TZ="Asia/Istanbul"`, `localtime2gmt("2000-01-01 00:00:00", "Asia/Istanbul") = "1999-12-31T22:00:00Z"`, }, - unaryFunc: types.BIF_localtime2gmt_unary, - binaryFunc: types.BIF_localtime2gmt_binary, + unaryFunc: bifs.BIF_localtime2gmt_unary, + binaryFunc: bifs.BIF_localtime2gmt_binary, hasMultipleArities: true, }, @@ -913,8 +913,8 @@ is supplied.`, `gmt2localtime("1999-12-31T22:00:00Z") = "2000-01-01 00:00:00" with TZ="Asia/Istanbul"`, `gmt2localtime("1999-12-31T22:00:00Z", "Asia/Istanbul") = "2000-01-01 00:00:00"`, }, - unaryFunc: types.BIF_gmt2localtime_unary, - binaryFunc: types.BIF_gmt2localtime_binary, + unaryFunc: bifs.BIF_gmt2localtime_unary, + binaryFunc: bifs.BIF_gmt2localtime_binary, hasMultipleArities: true, }, @@ -929,7 +929,7 @@ strftime_local.`, `strftime(1440768801.7,"%Y-%m-%dT%H:%M:%SZ") = "2015-08-28T13:33:21Z"`, `strftime(1440768801.7,"%Y-%m-%dT%H:%M:%3SZ") = "2015-08-28T13:33:21.700Z"`, }, - binaryFunc: types.BIF_strftime, + binaryFunc: bifs.BIF_strftime, }, { @@ -943,7 +943,7 @@ strftime_local.`, `strptime("1970-01-01 00:00:00 EET", "%Y-%m-%d %H:%M:%S %Z") = -7200`, }, - binaryFunc: types.BIF_strptime, + binaryFunc: bifs.BIF_strptime, }, { @@ -955,8 +955,8 @@ strftime_local.`, `strftime_local(1440768801.7, "%Y-%m-%d %H:%M:%3S %z") = "2015-08-28 16:33:21.700 +0300" with TZ="Asia/Istanbul"`, `strftime_local(1440768801.7, "%Y-%m-%d %H:%M:%3S %z", "Asia/Istanbul") = "2015-08-28 16:33:21.700 +0300"`, }, - binaryFunc: types.BIF_strftime_local_binary, - ternaryFunc: types.BIF_strftime_local_ternary, + binaryFunc: bifs.BIF_strftime_local_binary, + ternaryFunc: bifs.BIF_strftime_local_ternary, hasMultipleArities: true, }, @@ -972,8 +972,8 @@ strftime_local.`, // TODO: fix parse error on decimal part //`strptime_local("2015-08-28 13:33:21.345","%Y-%m-%d %H:%M:%S") = 1440758001.345`, }, - binaryFunc: types.BIF_strptime_local_binary, - ternaryFunc: types.BIF_strptime_local_ternary, + binaryFunc: bifs.BIF_strptime_local_binary, + ternaryFunc: bifs.BIF_strptime_local_ternary, hasMultipleArities: true, }, @@ -981,77 +981,77 @@ strftime_local.`, name: "dhms2fsec", class: FUNC_CLASS_TIME, help: `Recovers floating-point seconds as in dhms2fsec("5d18h53m20.250000s") = 500000.250000`, - unaryFunc: types.BIF_dhms2fsec, + unaryFunc: bifs.BIF_dhms2fsec, }, { name: "dhms2sec", class: FUNC_CLASS_TIME, help: `Recovers integer seconds as in dhms2sec("5d18h53m20s") = 500000`, - unaryFunc: types.BIF_dhms2sec, + unaryFunc: bifs.BIF_dhms2sec, }, { name: "fsec2dhms", class: FUNC_CLASS_TIME, help: `Formats floating-point seconds as in fsec2dhms(500000.25) = "5d18h53m20.250000s"`, - unaryFunc: types.BIF_fsec2dhms, + unaryFunc: bifs.BIF_fsec2dhms, }, { name: "fsec2hms", class: FUNC_CLASS_TIME, help: `Formats floating-point seconds as in fsec2hms(5000.25) = "01:23:20.250000"`, - unaryFunc: types.BIF_fsec2hms, + unaryFunc: bifs.BIF_fsec2hms, }, { name: "hms2fsec", class: FUNC_CLASS_TIME, help: `Recovers floating-point seconds as in hms2fsec("01:23:20.250000") = 5000.250000`, - unaryFunc: types.BIF_hms2fsec, + unaryFunc: bifs.BIF_hms2fsec, }, { name: "hms2sec", class: FUNC_CLASS_TIME, help: `Recovers integer seconds as in hms2sec("01:23:20") = 5000`, - unaryFunc: types.BIF_hms2sec, + unaryFunc: bifs.BIF_hms2sec, }, { name: "sec2dhms", class: FUNC_CLASS_TIME, help: `Formats integer seconds as in sec2dhms(500000) = "5d18h53m20s"`, - unaryFunc: types.BIF_sec2dhms, + unaryFunc: bifs.BIF_sec2dhms, }, { name: "sec2hms", class: FUNC_CLASS_TIME, help: `Formats integer seconds as in sec2hms(5000) = "01:23:20"`, - unaryFunc: types.BIF_sec2hms, + unaryFunc: bifs.BIF_sec2hms, }, { name: "systime", class: FUNC_CLASS_TIME, help: "help string will go here", - zaryFunc: types.BIF_systime, + zaryFunc: bifs.BIF_systime, }, { name: "systimeint", class: FUNC_CLASS_TIME, help: "help string will go here", - zaryFunc: types.BIF_systimeint, + zaryFunc: bifs.BIF_systimeint, }, { name: "uptime", class: FUNC_CLASS_TIME, help: "help string will go here", - zaryFunc: types.BIF_uptime, + zaryFunc: bifs.BIF_uptime, }, // ---------------------------------------------------------------- @@ -1061,273 +1061,273 @@ strftime_local.`, name: "is_absent", class: FUNC_CLASS_TYPING, help: "False if field is present in input, true otherwise", - unaryFunc: types.BIF_is_absent, + unaryFunc: bifs.BIF_is_absent, }, { name: "is_array", class: FUNC_CLASS_TYPING, help: "True if argument is an array.", - unaryFunc: types.BIF_is_array, + unaryFunc: bifs.BIF_is_array, }, { name: "is_bool", class: FUNC_CLASS_TYPING, help: "True if field is present with boolean value. Synonymous with is_boolean.", - unaryFunc: types.BIF_is_bool, + unaryFunc: bifs.BIF_is_bool, }, { name: "is_boolean", class: FUNC_CLASS_TYPING, help: "True if field is present with boolean value. Synonymous with is_bool.", - unaryFunc: types.BIF_is_boolean, + unaryFunc: bifs.BIF_is_boolean, }, { name: "is_empty", class: FUNC_CLASS_TYPING, help: "True if field is present in input with empty string value, false otherwise.", - unaryFunc: types.BIF_is_empty, + unaryFunc: bifs.BIF_is_empty, }, { name: "is_empty_map", class: FUNC_CLASS_TYPING, help: "True if argument is a map which is empty.", - unaryFunc: types.BIF_is_emptymap, + unaryFunc: bifs.BIF_is_emptymap, }, { name: "is_error", class: FUNC_CLASS_TYPING, help: "True if if argument is an error, such as taking string length of an integer.", - unaryFunc: types.BIF_is_error, + unaryFunc: bifs.BIF_is_error, }, { name: "is_float", class: FUNC_CLASS_TYPING, help: "True if field is present with value inferred to be float", - unaryFunc: types.BIF_is_float, + unaryFunc: bifs.BIF_is_float, }, { name: "is_int", class: FUNC_CLASS_TYPING, help: "True if field is present with value inferred to be int", - unaryFunc: types.BIF_is_int, + unaryFunc: bifs.BIF_is_int, }, { name: "is_map", class: FUNC_CLASS_TYPING, help: "True if argument is a map.", - unaryFunc: types.BIF_is_map, + unaryFunc: bifs.BIF_is_map, }, { name: "is_nonempty_map", class: FUNC_CLASS_TYPING, help: "True if argument is a map which is non-empty.", - unaryFunc: types.BIF_is_nonemptymap, + unaryFunc: bifs.BIF_is_nonemptymap, }, { name: "is_not_empty", class: FUNC_CLASS_TYPING, help: "False if field is present in input with empty value, true otherwise", - unaryFunc: types.BIF_is_notempty, + unaryFunc: bifs.BIF_is_notempty, }, { name: "is_not_map", class: FUNC_CLASS_TYPING, help: "True if argument is not a map.", - unaryFunc: types.BIF_is_notmap, + unaryFunc: bifs.BIF_is_notmap, }, { name: "is_not_array", class: FUNC_CLASS_TYPING, help: "True if argument is not an array.", - unaryFunc: types.BIF_is_notarray, + unaryFunc: bifs.BIF_is_notarray, }, { name: "is_not_null", class: FUNC_CLASS_TYPING, help: "False if argument is null (empty or absent), true otherwise.", - unaryFunc: types.BIF_is_notnull, + unaryFunc: bifs.BIF_is_notnull, }, { name: "is_null", class: FUNC_CLASS_TYPING, help: "True if argument is null (empty or absent), false otherwise.", - unaryFunc: types.BIF_is_null, + unaryFunc: bifs.BIF_is_null, }, { name: "is_numeric", class: FUNC_CLASS_TYPING, help: "True if field is present with value inferred to be int or float", - unaryFunc: types.BIF_is_numeric, + unaryFunc: bifs.BIF_is_numeric, }, { name: "is_present", class: FUNC_CLASS_TYPING, help: "True if field is present in input, false otherwise.", - unaryFunc: types.BIF_is_present, + unaryFunc: bifs.BIF_is_present, }, { name: "is_string", class: FUNC_CLASS_TYPING, help: "True if field is present with string (including empty-string) value", - unaryFunc: types.BIF_is_string, + unaryFunc: bifs.BIF_is_string, }, { name: "asserting_absent", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_absent on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_absent, + unaryFuncWithContext: bifs.BIF_asserting_absent, }, { name: "asserting_array", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_array on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_array, + unaryFuncWithContext: bifs.BIF_asserting_array, }, { name: "asserting_bool", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_bool on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_bool, + unaryFuncWithContext: bifs.BIF_asserting_bool, }, { name: "asserting_boolean", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_boolean on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_boolean, + unaryFuncWithContext: bifs.BIF_asserting_boolean, }, { name: "asserting_error", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_error on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_error, + unaryFuncWithContext: bifs.BIF_asserting_error, }, { name: "asserting_empty", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_empty on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_empty, + unaryFuncWithContext: bifs.BIF_asserting_empty, }, { name: "asserting_empty_map", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_empty_map on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_emptyMap, + unaryFuncWithContext: bifs.BIF_asserting_emptyMap, }, { name: "asserting_float", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_float on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_float, + unaryFuncWithContext: bifs.BIF_asserting_float, }, { name: "asserting_int", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_int on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_int, + unaryFuncWithContext: bifs.BIF_asserting_int, }, { name: "asserting_map", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_map on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_map, + unaryFuncWithContext: bifs.BIF_asserting_map, }, { name: "asserting_nonempty_map", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_nonempty_map on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_nonempty_map, + unaryFuncWithContext: bifs.BIF_asserting_nonempty_map, }, { name: "asserting_not_empty", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_not_empty on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_not_empty, + unaryFuncWithContext: bifs.BIF_asserting_not_empty, }, { name: "asserting_not_map", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_not_map on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_not_map, + unaryFuncWithContext: bifs.BIF_asserting_not_map, }, { name: "asserting_not_array", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_not_array on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_not_array, + unaryFuncWithContext: bifs.BIF_asserting_not_array, }, { name: "asserting_not_null", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_not_null on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_not_null, + unaryFuncWithContext: bifs.BIF_asserting_not_null, }, { name: "asserting_null", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_null on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_null, + unaryFuncWithContext: bifs.BIF_asserting_null, }, { name: "asserting_numeric", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_numeric on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_numeric, + unaryFuncWithContext: bifs.BIF_asserting_numeric, }, { name: "asserting_present", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_present on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_present, + unaryFuncWithContext: bifs.BIF_asserting_present, }, { name: "asserting_string", class: FUNC_CLASS_TYPING, help: `Aborts with an error if is_string on the argument returns false, else returns its argument.`, - unaryFuncWithContext: types.BIF_asserting_string, + unaryFuncWithContext: bifs.BIF_asserting_string, }, { name: "typeof", class: FUNC_CLASS_TYPING, help: "Convert argument to type of argument (e.g. \"str\"). For debug.", - unaryFunc: types.BIF_typeof, + unaryFunc: bifs.BIF_typeof, }, // ---------------------------------------------------------------- @@ -1337,14 +1337,14 @@ strftime_local.`, name: "boolean", class: FUNC_CLASS_CONVERSION, help: "Convert int/float/bool/string to boolean.", - unaryFunc: types.BIF_boolean, + unaryFunc: bifs.BIF_boolean, }, { name: "float", class: FUNC_CLASS_CONVERSION, help: "Convert int/float/bool/string to float.", - unaryFunc: types.BIF_float, + unaryFunc: bifs.BIF_float, }, { @@ -1352,21 +1352,21 @@ strftime_local.`, class: FUNC_CLASS_CONVERSION, help: `Convert int/float/bool to string using printf-style format string, e.g. '$s = fmtnum($n, "%08d")' or '$t = fmtnum($n, "%.6e")'.`, - binaryFunc: types.BIF_fmtnum, + binaryFunc: bifs.BIF_fmtnum, }, { name: "hexfmt", class: FUNC_CLASS_CONVERSION, help: `Convert int to hex string, e.g. 255 to "0xff".`, - unaryFunc: types.BIF_hexfmt, + unaryFunc: bifs.BIF_hexfmt, }, { name: "int", class: FUNC_CLASS_CONVERSION, help: "Convert int/float/bool/string to int.", - unaryFunc: types.BIF_int, + unaryFunc: bifs.BIF_int, }, { @@ -1377,7 +1377,7 @@ strftime_local.`, `joink({"a":3,"b":4,"c":5}, ",") = "a,b,c".`, `joink([1,2,3], ",") = "1,2,3".`, }, - binaryFunc: types.BIF_joink, + binaryFunc: bifs.BIF_joink, }, { @@ -1388,7 +1388,7 @@ strftime_local.`, `joinv([3,4,5], ",") = "3,4,5"`, `joinv({"a":3,"b":4,"c":5}, ",") = "3,4,5"`, }, - binaryFunc: types.BIF_joinv, + binaryFunc: bifs.BIF_joinv, }, { @@ -1399,7 +1399,7 @@ strftime_local.`, `joinkv([3,4,5], "=", ",") = "1=3,2=4,3=5"`, `joinkv({"a":3,"b":4,"c":5}, "=", ",") = "a=3,b=4,c=5"`, }, - ternaryFunc: types.BIF_joinkv, + ternaryFunc: bifs.BIF_joinkv, }, { @@ -1409,7 +1409,7 @@ strftime_local.`, examples: []string{ `splita("3,4,5", ",") = [3,4,5]`, }, - binaryFunc: types.BIF_splita, + binaryFunc: bifs.BIF_splita, }, { @@ -1419,7 +1419,7 @@ strftime_local.`, examples: []string{ `splita("3,4,5", ",") = ["3","4","5"]`, }, - binaryFunc: types.BIF_splitax, + binaryFunc: bifs.BIF_splitax, }, { @@ -1429,7 +1429,7 @@ strftime_local.`, examples: []string{ `splitkv("a=3,b=4,c=5", "=", ",") = {"a":3,"b":4,"c":5}`, }, - ternaryFunc: types.BIF_splitkv, + ternaryFunc: bifs.BIF_splitkv, }, { @@ -1439,7 +1439,7 @@ strftime_local.`, examples: []string{ `splitkvx("a=3,b=4,c=5", "=", ",") = {"a":"3","b":"4","c":"5"}`, }, - ternaryFunc: types.BIF_splitkvx, + ternaryFunc: bifs.BIF_splitkvx, }, { @@ -1449,7 +1449,7 @@ strftime_local.`, examples: []string{ `splitnv("a,b,c", ",") = {"1":"a","2":"b","3":"c"}`, }, - binaryFunc: types.BIF_splitnv, + binaryFunc: bifs.BIF_splitnv, }, { @@ -1459,14 +1459,14 @@ strftime_local.`, examples: []string{ `splitnvx("3,4,5", ",") = {"1":"3","2":"4","3":"5"}`, }, - binaryFunc: types.BIF_splitnvx, + binaryFunc: bifs.BIF_splitnvx, }, { name: "string", class: FUNC_CLASS_CONVERSION, help: "Convert int/float/bool/string/array/map to string.", - unaryFunc: types.BIF_string, + unaryFunc: bifs.BIF_string, }, // ---------------------------------------------------------------- @@ -1476,7 +1476,7 @@ strftime_local.`, name: "append", class: FUNC_CLASS_COLLECTIONS, help: "Appends second argument to end of first argument, which must be an array.", - binaryFunc: types.BIF_append, + binaryFunc: bifs.BIF_append, }, { @@ -1484,14 +1484,14 @@ strftime_local.`, class: FUNC_CLASS_COLLECTIONS, help: `Walks through a nested map/array, converting any map with consecutive keys "1", "2", ... into an array. Useful to wrap the output of unflatten.`, - unaryFunc: types.BIF_arrayify, + unaryFunc: bifs.BIF_arrayify, }, { name: "depth", class: FUNC_CLASS_COLLECTIONS, help: "Prints maximum depth of map/array. Scalars have depth 0.", - unaryFunc: types.BIF_depth, + unaryFunc: bifs.BIF_depth, }, { @@ -1504,8 +1504,8 @@ for non-JSON file formats like CSV.`, `flatten("", ".", {"a": { "b": 3 }}) is {"a.b" : 3}.`, `Two-argument version: flatten($*, ".") is the same as flatten("", ".", $*).`, }, - binaryFunc: types.BIF_flatten_binary, - ternaryFunc: types.BIF_flatten, + binaryFunc: bifs.BIF_flatten_binary, + ternaryFunc: bifs.BIF_flatten, hasMultipleArities: true, }, @@ -1513,14 +1513,14 @@ for non-JSON file formats like CSV.`, name: "get_keys", class: FUNC_CLASS_COLLECTIONS, help: "Returns array of keys of map or array", - unaryFunc: types.BIF_get_keys, + unaryFunc: bifs.BIF_get_keys, }, { name: "get_values", class: FUNC_CLASS_COLLECTIONS, help: "Returns array of keys of map or array -- in the latter case, returns a copy of the array", - unaryFunc: types.BIF_get_values, + unaryFunc: bifs.BIF_get_values, }, { @@ -1529,22 +1529,22 @@ for non-JSON file formats like CSV.`, help: `True/false if map has/hasn't key, e.g. 'haskey($*, "a")' or 'haskey(mymap, mykey)', or true/false if array index is in bounds / out of bounds. Error if 1st argument is not a map or array. Note -n..-1 alias to 1..n in Miller arrays.`, - binaryFunc: types.BIF_haskey, + binaryFunc: bifs.BIF_haskey, }, { name: "json_parse", class: FUNC_CLASS_COLLECTIONS, help: `Converts value from JSON-formatted string.`, - unaryFunc: types.BIF_json_parse, + unaryFunc: bifs.BIF_json_parse, }, { name: "json_stringify", class: FUNC_CLASS_COLLECTIONS, help: `Converts value to JSON-formatted string. Default output is single-line. With optional second boolean argument set to true, produces multiline output.`, - unaryFunc: types.BIF_json_stringify_unary, - binaryFunc: types.BIF_json_stringify_binary, + unaryFunc: bifs.BIF_json_stringify_unary, + binaryFunc: bifs.BIF_json_stringify_binary, hasMultipleArities: true, }, @@ -1552,14 +1552,14 @@ With optional second boolean argument set to true, produces multiline output.`, name: "leafcount", class: FUNC_CLASS_COLLECTIONS, help: `Counts total number of terminal values in map/array. For single-level map/array, same as length.`, - unaryFunc: types.BIF_leafcount, + unaryFunc: bifs.BIF_leafcount, }, { name: "length", class: FUNC_CLASS_COLLECTIONS, help: "Counts number of top-level entries in array/map. Scalars have length 1.", - unaryFunc: types.BIF_length, + unaryFunc: bifs.BIF_length, }, { @@ -1567,7 +1567,7 @@ With optional second boolean argument set to true, produces multiline output.`, class: FUNC_CLASS_COLLECTIONS, help: `With 0 args, returns empty map. With 1 arg, returns copy of arg. With 2 or more, returns copy of arg 1 with all keys from any of remaining argument maps removed.`, - variadicFunc: types.BIF_mapdiff, + variadicFunc: bifs.BIF_mapdiff, }, { @@ -1576,7 +1576,7 @@ returns copy of arg 1 with all keys from any of remaining argument maps removed. help: `Returns a map with keys from remaining arguments, if any, unset. Remaining arguments can be strings or arrays of string. E.g. 'mapexcept({1:2,3:4,5:6}, 1, 5, 7)' is '{3:4}' and 'mapexcept({1:2,3:4,5:6}, [1, 5, 7])' is '{3:4}'.`, - variadicFunc: types.BIF_mapexcept, + variadicFunc: bifs.BIF_mapexcept, minimumVariadicArity: 1, }, @@ -1586,7 +1586,7 @@ and 'mapexcept({1:2,3:4,5:6}, [1, 5, 7])' is '{3:4}'.`, help: `Returns a map with only keys from remaining arguments set. Remaining arguments can be strings or arrays of string. E.g. 'mapselect({1:2,3:4,5:6}, 1, 5, 7)' is '{1:2,5:6}' and 'mapselect({1:2,3:4,5:6}, [1, 5, 7])' is '{1:2,5:6}'.`, - variadicFunc: types.BIF_mapselect, + variadicFunc: bifs.BIF_mapselect, minimumVariadicArity: 1, }, @@ -1595,7 +1595,7 @@ Remaining arguments can be strings or arrays of string. E.g. 'mapselect({1:2,3: class: FUNC_CLASS_COLLECTIONS, help: `With 0 args, returns empty map. With >= 1 arg, returns a map with key-value pairs from all arguments. Rightmost collisions win, e.g. 'mapsum({1:2,3:4},{1:5})' is '{1:5,3:4}'.`, - variadicFunc: types.BIF_mapsum, + variadicFunc: bifs.BIF_mapsum, }, { @@ -1606,7 +1606,7 @@ See also arrayify.`, examples: []string{ `unflatten({"a.b.c" : 4}, ".") is {"a": "b": { "c": 4 }}.`, }, - binaryFunc: types.BIF_unflatten, + binaryFunc: bifs.BIF_unflatten, }, // ---------------------------------------------------------------- @@ -1727,28 +1727,28 @@ either case it should return a boolean.`, name: "hostname", class: FUNC_CLASS_SYSTEM, help: `Returns the hostname as a string.`, - zaryFunc: types.BIF_hostname, + zaryFunc: bifs.BIF_hostname, }, { name: "os", class: FUNC_CLASS_SYSTEM, help: `Returns the operating-system name as a string.`, - zaryFunc: types.BIF_os, + zaryFunc: bifs.BIF_os, }, { name: "system", class: FUNC_CLASS_SYSTEM, help: `Run command string, yielding its stdout minus final carriage return.`, - unaryFunc: types.BIF_system, + unaryFunc: bifs.BIF_system, }, { name: "version", class: FUNC_CLASS_SYSTEM, help: `Returns the Miller version as a string.`, - zaryFunc: types.BIF_version, + zaryFunc: bifs.BIF_version, }, } diff --git a/internal/pkg/dsl/cst/builtin_functions.go b/internal/pkg/dsl/cst/builtin_functions.go index 0a36f70ed..bc110b406 100644 --- a/internal/pkg/dsl/cst/builtin_functions.go +++ b/internal/pkg/dsl/cst/builtin_functions.go @@ -8,10 +8,11 @@ import ( "errors" "fmt" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -88,7 +89,7 @@ func (root *RootNode) BuildMultipleArityFunctionCallsiteNode( // ---------------------------------------------------------------- type ZaryFunctionCallsiteNode struct { - zaryFunc types.ZaryFunc + zaryFunc bifs.ZaryFunc } func (root *RootNode) BuildZaryFunctionCallsiteNode( @@ -116,13 +117,13 @@ func (root *RootNode) BuildZaryFunctionCallsiteNode( func (node *ZaryFunctionCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.zaryFunc() } // ---------------------------------------------------------------- type UnaryFunctionCallsiteNode struct { - unaryFunc types.UnaryFunc + unaryFunc bifs.UnaryFunc evaluable1 IEvaluable } @@ -157,13 +158,13 @@ func (root *RootNode) BuildUnaryFunctionCallsiteNode( func (node *UnaryFunctionCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.unaryFunc(node.evaluable1.Evaluate(state)) } // ---------------------------------------------------------------- type UnaryFunctionWithContextCallsiteNode struct { - unaryFuncWithContext types.UnaryFuncWithContext + unaryFuncWithContext bifs.UnaryFuncWithContext evaluable1 IEvaluable } @@ -198,13 +199,13 @@ func (root *RootNode) BuildUnaryFunctionWithContextCallsiteNode( func (node *UnaryFunctionWithContextCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.unaryFuncWithContext(node.evaluable1.Evaluate(state), state.Context) } // ---------------------------------------------------------------- type BinaryFunctionCallsiteNode struct { - binaryFunc types.BinaryFunc + binaryFunc bifs.BinaryFunc evaluable1 IEvaluable evaluable2 IEvaluable } @@ -271,7 +272,7 @@ func (root *RootNode) BuildBinaryFunctionCallsiteNode( func (node *BinaryFunctionCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.binaryFunc( node.evaluable1.Evaluate(state), node.evaluable2.Evaluate(state), @@ -321,7 +322,7 @@ func (root *RootNode) BuildBinaryFunctionWithStateCallsiteNode( func (node *BinaryFunctionWithStateCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.binaryFuncWithState( node.evaluable1.Evaluate(state), node.evaluable2.Evaluate(state), @@ -378,7 +379,7 @@ func (root *RootNode) BuildTernaryFunctionWithStateCallsiteNode( func (node *TernaryFunctionWithStateCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.ternaryFuncWithState( node.evaluable1.Evaluate(state), node.evaluable2.Evaluate(state), @@ -391,9 +392,9 @@ func (node *TernaryFunctionWithStateCallsiteNode) Evaluate( // RegexCaptureBinaryFunctionCallsiteNode special-cases the =~ and !=~ // operators which set the CST State object's captures array for "\1".."\9". // This is identical to BinaryFunctionCallsite except that -// BinaryFunctionCallsite's impl function takes two *types.Mlrval arguments and -// returns a *types.Mlrval, whereas RegexCaptureBinaryFunctionCallsiteNode's -// impl function takes two *types.Mlrval arguments but returns *types.Mlrval +// BinaryFunctionCallsite's impl function takes two *mlrval.Mlrval arguments and +// returns a *mlrval.Mlrval, whereas RegexCaptureBinaryFunctionCallsiteNode's +// impl function takes two *mlrval.Mlrval arguments but returns *mlrval.Mlrval // along with a []string captures array. The captures are stored in the State // object for use in subsequent statements. // @@ -420,7 +421,7 @@ func (node *TernaryFunctionWithStateCallsiteNode) Evaluate( // !=~ callsites only -- not sub/gsub, and not the capture-using replacement // statements like '$y = "\2:\1". type RegexCaptureBinaryFunctionCallsiteNode struct { - regexCaptureBinaryFunc types.RegexCaptureBinaryFunc + regexCaptureBinaryFunc bifs.RegexCaptureBinaryFunc evaluable1 IEvaluable evaluable2 IEvaluable } @@ -461,7 +462,7 @@ func (root *RootNode) BuildRegexCaptureBinaryFunctionCallsiteNode( func (node *RegexCaptureBinaryFunctionCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { output, captures := node.regexCaptureBinaryFunc( node.evaluable1.Evaluate(state), node.evaluable2.Evaluate(state), @@ -515,7 +516,7 @@ func (root *RootNode) BuildDotCallsiteNode( func (node *DotCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { value1 := node.evaluable1.Evaluate(state) mapvalue1 := value1.GetMap() @@ -524,23 +525,20 @@ func (node *DotCallsiteNode) Evaluate( // Case 1: map.attribute as shorthand for map["attribute"] value2 := mapvalue1.Get(node.string2) if value2 == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { return value2 } } else { // Case 2: string concatenation value2 := node.evaluable2.Evaluate(state) - return types.BIF_dot( - value1, - value2, - ) + return bifs.BIF_dot(value1, value2) } } // ---------------------------------------------------------------- type TernaryFunctionCallsiteNode struct { - ternaryFunc types.TernaryFunc + ternaryFunc bifs.TernaryFunc evaluable1 IEvaluable evaluable2 IEvaluable evaluable3 IEvaluable @@ -596,7 +594,7 @@ func (root *RootNode) BuildTernaryFunctionCallsiteNode( func (node *TernaryFunctionCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.ternaryFunc( node.evaluable1.Evaluate(state), node.evaluable2.Evaluate(state), @@ -606,7 +604,7 @@ func (node *TernaryFunctionCallsiteNode) Evaluate( // ---------------------------------------------------------------- type VariadicFunctionCallsiteNode struct { - variadicFunc types.VariadicFunc + variadicFunc bifs.VariadicFunc evaluables []IEvaluable } @@ -658,8 +656,8 @@ func (root *RootNode) BuildVariadicFunctionCallsiteNode( func (node *VariadicFunctionCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { - args := make([]*types.Mlrval, len(node.evaluables)) +) *mlrval.Mlrval { + args := make([]*mlrval.Mlrval, len(node.evaluables)) for i := range node.evaluables { args[i] = node.evaluables[i].Evaluate(state) } @@ -720,8 +718,8 @@ func (root *RootNode) BuildVariadicFunctionWithStateCallsiteNode( func (node *VariadicFunctionWithStateCallsiteNode) Evaluate( state *runtime.State, -) *types.Mlrval { - args := make([]*types.Mlrval, len(node.evaluables)) +) *mlrval.Mlrval { + args := make([]*mlrval.Mlrval, len(node.evaluables)) for i := range node.evaluables { args[i] = node.evaluables[i].Evaluate(state) } @@ -778,14 +776,14 @@ func (root *RootNode) BuildLogicalANDOperatorNode(a, b IEvaluable) *LogicalANDOp func (node *LogicalANDOperatorNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { aout := node.a.Evaluate(state) - atype := aout.GetType() - if !(atype == types.MT_ABSENT || atype == types.MT_BOOL) { - return types.MLRVAL_ERROR + atype := aout.Type() + if !(atype == mlrval.MT_ABSENT || atype == mlrval.MT_BOOL) { + return mlrval.ERROR } - if atype == types.MT_ABSENT { - return types.MLRVAL_ABSENT + if atype == mlrval.MT_ABSENT { + return mlrval.ABSENT } if aout.IsFalse() { // This means false && bogus type evaluates to true, which is sad but @@ -796,15 +794,15 @@ func (node *LogicalANDOperatorNode) Evaluate( } bout := node.b.Evaluate(state) - btype := bout.GetType() - if !(btype == types.MT_ABSENT || btype == types.MT_BOOL) { - return types.MLRVAL_ERROR + btype := bout.Type() + if !(btype == mlrval.MT_ABSENT || btype == mlrval.MT_BOOL) { + return mlrval.ERROR } - if btype == types.MT_ABSENT { - return types.MLRVAL_ABSENT + if btype == mlrval.MT_ABSENT { + return mlrval.ABSENT } - return types.MlrvalLogicalAND(aout, bout) + return bifs.BIF_logical_AND(aout, bout) } // ================================================================ @@ -826,14 +824,14 @@ func (root *RootNode) BuildLogicalOROperatorNode(a, b IEvaluable) *LogicalOROper // See the disposition-matrix discussion for LogicalANDOperator. func (node *LogicalOROperatorNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { aout := node.a.Evaluate(state) - atype := aout.GetType() - if !(atype == types.MT_ABSENT || atype == types.MT_BOOL) { - return types.MLRVAL_ERROR + atype := aout.Type() + if !(atype == mlrval.MT_ABSENT || atype == mlrval.MT_BOOL) { + return mlrval.ERROR } - if atype == types.MT_ABSENT { - return types.MLRVAL_ABSENT + if atype == mlrval.MT_ABSENT { + return mlrval.ABSENT } if aout.IsTrue() { // This means true || bogus type evaluates to true, which is sad but @@ -844,14 +842,14 @@ func (node *LogicalOROperatorNode) Evaluate( } bout := node.b.Evaluate(state) - btype := bout.GetType() - if !(btype == types.MT_ABSENT || btype == types.MT_BOOL) { - return types.MLRVAL_ERROR + btype := bout.Type() + if !(btype == mlrval.MT_ABSENT || btype == mlrval.MT_BOOL) { + return mlrval.ERROR } - if btype == types.MT_ABSENT { - return types.MLRVAL_ABSENT + if btype == mlrval.MT_ABSENT { + return mlrval.ABSENT } - return types.MlrvalLogicalOR(aout, bout) + return bifs.BIF_logical_OR(aout, bout) } // ================================================================ @@ -868,9 +866,9 @@ func (root *RootNode) BuildAbsentCoalesceOperatorNode(a, b IEvaluable) *AbsentCo // argument is not absent. func (node *AbsentCoalesceOperatorNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { aout := node.a.Evaluate(state) - if aout.GetType() != types.MT_ABSENT { + if aout.Type() != mlrval.MT_ABSENT { return aout } @@ -891,10 +889,10 @@ func (root *RootNode) BuildEmptyCoalesceOperatorNode(a, b IEvaluable) *EmptyCoal // argument is not absent. func (node *EmptyCoalesceOperatorNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { aout := node.a.Evaluate(state) - atype := aout.GetType() - if atype == types.MT_ABSENT || atype == types.MT_VOID || (atype == types.MT_STRING && aout.String() == "") { + atype := aout.Type() + if atype == mlrval.MT_ABSENT || atype == mlrval.MT_VOID || (atype == mlrval.MT_STRING && aout.String() == "") { return node.b.Evaluate(state) } else { return aout @@ -909,12 +907,12 @@ func (root *RootNode) BuildStandardTernaryOperatorNode(a, b, c IEvaluable) *Stan } func (node *StandardTernaryOperatorNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { aout := node.a.Evaluate(state) boolValue, isBool := aout.GetBoolValue() if !isBool { - return types.MLRVAL_ERROR + return mlrval.ERROR } // Short-circuit: defer evaluation unless needed @@ -936,12 +934,12 @@ func (node *StandardTernaryOperatorNode) Evaluate( // for the function-manager lookup table to indicate the arity of the function, // even though at runtime these functions should not get invoked. -func BinaryShortCircuitPlaceholder(input1, input2 *types.Mlrval) *types.Mlrval { +func BinaryShortCircuitPlaceholder(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval { lib.InternalCodingErrorPanic("Short-circuting was not correctly implemented") - return types.MLRVAL_ERROR // not reached + return mlrval.ERROR // not reached } -func TernaryShortCircuitPlaceholder(input1, input2, input3 *types.Mlrval) *types.Mlrval { +func TernaryShortCircuitPlaceholder(input1, input2, input3 *mlrval.Mlrval) *mlrval.Mlrval { lib.InternalCodingErrorPanic("Short-circuting was not correctly implemented") - return types.MLRVAL_ERROR // not reached + return mlrval.ERROR // not reached } diff --git a/internal/pkg/dsl/cst/collections.go b/internal/pkg/dsl/cst/collections.go index 6dd0c32a1..be2ca67e1 100644 --- a/internal/pkg/dsl/cst/collections.go +++ b/internal/pkg/dsl/cst/collections.go @@ -6,10 +6,11 @@ package cst import ( + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -42,12 +43,12 @@ func (node *RootNode) BuildArrayLiteralNode( func (node *ArrayLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { - mlrvals := make([]types.Mlrval, len(node.evaluables)) +) *mlrval.Mlrval { + mlrvals := make([]mlrval.Mlrval, len(node.evaluables)) for i := range node.evaluables { mlrvals[i] = *node.evaluables[i].Evaluate(state) } - return types.MlrvalFromArrayReference(mlrvals) + return mlrval.FromArray(mlrvals) } // ---------------------------------------------------------------- @@ -82,7 +83,7 @@ func (node *RootNode) BuildArrayOrMapIndexAccessNode( func (node *CollectionIndexAccessNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { baseMlrval := node.baseEvaluable.Evaluate(state) indexMlrval := node.indexEvaluable.Evaluate(state) @@ -96,21 +97,21 @@ func (node *CollectionIndexAccessNode) Evaluate( } else if baseMlrval.IsStringOrVoid() { mindex, isInt := indexMlrval.GetIntValue() if !isInt { - return types.MLRVAL_ERROR + return mlrval.ERROR } // Handle UTF-8 correctly: len(input1.printrep) will count bytes, not runes. runes := []rune(baseMlrval.String()) // Miller uses 1-up, and negatively aliased, indexing for strings and arrays. - zindex, inBounds := types.UnaliasArrayLengthIndex(len(runes), mindex) + zindex, inBounds := mlrval.UnaliasArrayLengthIndex(len(runes), mindex) if !inBounds { - return types.MLRVAL_ERROR + return mlrval.ERROR } - return types.MlrvalFromString(string(runes[zindex])) + return mlrval.FromString(string(runes[zindex])) } else if baseMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } @@ -155,55 +156,55 @@ func (node *RootNode) BuildArraySliceAccessNode( func (node *ArraySliceAccessNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { baseMlrval := node.baseEvaluable.Evaluate(state) lowerIndexMlrval := node.lowerIndexEvaluable.Evaluate(state) upperIndexMlrval := node.upperIndexEvaluable.Evaluate(state) if baseMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } if baseMlrval.IsString() { - return types.BIF_substr_1_up(baseMlrval, lowerIndexMlrval, upperIndexMlrval) + return bifs.BIF_substr_1_up(baseMlrval, lowerIndexMlrval, upperIndexMlrval) } array := baseMlrval.GetArray() if array == nil { - return types.MLRVAL_ERROR + return mlrval.ERROR } n := len(array) if lowerIndexMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } if upperIndexMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } lowerIndex, ok := lowerIndexMlrval.GetIntValue() if !ok { - if lowerIndexMlrval.IsEmpty() { + if lowerIndexMlrval.IsVoid() { lowerIndex = 1 } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } upperIndex, ok := upperIndexMlrval.GetIntValue() if !ok { - if upperIndexMlrval.IsEmpty() { + if upperIndexMlrval.IsVoid() { upperIndex = n } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } // UnaliasArrayIndex returns a boolean second return value to indicate // whether the index is in range. But here, for the slicing operation, we // inspect the in-range-ness ourselves so we discard that 2nd return value. - lowerZindex, _ := types.UnaliasArrayIndex(&array, lowerIndex) - upperZindex, _ := types.UnaliasArrayIndex(&array, upperIndex) + lowerZindex, _ := mlrval.UnaliasArrayIndex(&array, lowerIndex) + upperZindex, _ := mlrval.UnaliasArrayIndex(&array, upperIndex) if lowerZindex > upperZindex { - return types.MlrvalFromArrayReference(make([]types.Mlrval, 0)) + return mlrval.FromArray(make([]mlrval.Mlrval, 0)) } // Semantics: say x=[1,2,3,4,5]. Then x[3:10] is [3,4,5]. @@ -220,13 +221,13 @@ func (node *ArraySliceAccessNode) Evaluate( if lowerZindex < 0 { lowerZindex = 0 if lowerZindex > upperZindex { - return types.MlrvalFromArrayReference(make([]types.Mlrval, 0)) + return mlrval.FromArray(make([]mlrval.Mlrval, 0)) } } if upperZindex > n-1 { upperZindex = n - 1 if lowerZindex > upperZindex { - return types.MlrvalFromArrayReference(make([]types.Mlrval, 0)) + return mlrval.FromArray(make([]mlrval.Mlrval, 0)) } } @@ -234,7 +235,7 @@ func (node *ArraySliceAccessNode) Evaluate( // Miller slices have inclusive lower bound, inclusive upper bound. var m = upperZindex - lowerZindex + 1 - retval := make([]types.Mlrval, m) + retval := make([]mlrval.Mlrval, m) di := 0 for si := lowerZindex; si <= upperZindex; si++ { @@ -242,7 +243,7 @@ func (node *ArraySliceAccessNode) Evaluate( di++ } - return types.MlrvalFromArrayReference(retval) + return mlrval.FromArray(retval) } // ================================================================ @@ -273,23 +274,23 @@ func (node *RootNode) BuildPositionalFieldNameNode( // TODO: code-dedupe these next four Evaluate methods func (node *PositionalFieldNameNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { indexMlrval := node.indexEvaluable.Evaluate(state) if indexMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } index, ok := indexMlrval.GetIntValue() if !ok { - return types.MLRVAL_ERROR + return mlrval.ERROR } name, ok := state.Inrec.GetNameAtPositionalIndex(index) if !ok { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } - return types.MlrvalFromString(name) + return mlrval.FromString(name) } // ================================================================ @@ -319,20 +320,20 @@ func (node *RootNode) BuildPositionalFieldValueNode( func (node *PositionalFieldValueNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { indexMlrval := node.indexEvaluable.Evaluate(state) if indexMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } index, ok := indexMlrval.GetIntValue() if !ok { - return types.MLRVAL_ERROR + return mlrval.ERROR } retval := state.Inrec.GetWithPositionalIndex(index) if retval == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } return retval @@ -372,41 +373,41 @@ func (node *RootNode) BuildArrayOrMapPositionalNameAccessNode( func (node *ArrayOrMapPositionalNameAccessNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { baseMlrval := node.baseEvaluable.Evaluate(state) indexMlrval := node.indexEvaluable.Evaluate(state) if indexMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } index, ok := indexMlrval.GetIntValue() if !ok { - return types.MLRVAL_ERROR + return mlrval.ERROR } if baseMlrval.IsArray() { n, _ := baseMlrval.GetArrayLength() - zindex, ok := types.UnaliasArrayLengthIndex(int(n), index) + zindex, ok := mlrval.UnaliasArrayLengthIndex(int(n), index) if ok { - return types.MlrvalFromInt(zindex + 1) // Miller user-space indices are 1-up + return mlrval.FromInt(zindex + 1) // Miller user-space indices are 1-up } else { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } } else if baseMlrval.IsMap() { name, ok := baseMlrval.GetMap().GetNameAtPositionalIndex(index) if !ok { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { - return types.MlrvalFromString(name) + return mlrval.FromString(name) } } else if baseMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } @@ -444,17 +445,17 @@ func (node *RootNode) BuildArrayOrMapPositionalValueAccessNode( func (node *ArrayOrMapPositionalValueAccessNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { baseMlrval := node.baseEvaluable.Evaluate(state) indexMlrval := node.indexEvaluable.Evaluate(state) if indexMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } index, ok := indexMlrval.GetIntValue() if !ok { - return types.MLRVAL_ERROR + return mlrval.ERROR } if baseMlrval.IsArray() { @@ -465,16 +466,16 @@ func (node *ArrayOrMapPositionalValueAccessNode) Evaluate( } else if baseMlrval.IsMap() { value := baseMlrval.GetMap().GetWithPositionalIndex(index) if value == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } return value } else if baseMlrval.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } @@ -536,8 +537,8 @@ func (node *RootNode) BuildMapLiteralNode( func (node *MapLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { - output := types.MlrvalFromEmptyMap() +) *mlrval.Mlrval { + output := mlrval.FromEmptyMap() for i := range node.evaluablePairs { key := node.evaluablePairs[i].Key.Evaluate(state) diff --git a/internal/pkg/dsl/cst/cond.go b/internal/pkg/dsl/cst/cond.go index ba7f251f9..28e6cf336 100644 --- a/internal/pkg/dsl/cst/cond.go +++ b/internal/pkg/dsl/cst/cond.go @@ -10,8 +10,8 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) type CondBlockNode struct { @@ -46,7 +46,7 @@ func (root *RootNode) BuildCondBlockNode(astNode *dsl.ASTNode) (*CondBlockNode, func (node *CondBlockNode) Execute( state *runtime.State, ) (*BlockExitPayload, error) { - condition := types.MLRVAL_TRUE + condition := mlrval.TRUE if node.conditionNode != nil { condition = node.conditionNode.Evaluate(state) } diff --git a/internal/pkg/dsl/cst/emit_emitp.go b/internal/pkg/dsl/cst/emit_emitp.go index dd92ad2d5..9433ca51b 100644 --- a/internal/pkg/dsl/cst/emit_emitp.go +++ b/internal/pkg/dsl/cst/emit_emitp.go @@ -45,6 +45,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/output" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" @@ -54,13 +55,13 @@ import ( // Shared by emit and emitp type tEmitToRedirectFunc func( - newrec *types.Mlrmap, + newrec *mlrval.Mlrmap, state *runtime.State, ) error type tEmitExecutorFunc func( names []string, - values []*types.Mlrval, + values []*mlrval.Mlrval, state *runtime.State, ) error @@ -299,7 +300,7 @@ func (node *EmitXStatementNode) Execute(state *runtime.State) (*BlockExitPayload if node.topLevelEvaluableMap == nil { // 'emit @a', 'emit (@a, @b)', etc. names := node.topLevelNameList - values := make([]*types.Mlrval, len(names)) + values := make([]*mlrval.Mlrval, len(names)) for i, evaluable := range node.topLevelEvaluableList { values[i] = evaluable.Evaluate(state) } @@ -317,7 +318,7 @@ func (node *EmitXStatementNode) Execute(state *runtime.State) (*BlockExitPayload return nil, nil } names := make([]string, parentMapValue.FieldCount) - values := make([]*types.Mlrval, parentMapValue.FieldCount) + values := make([]*mlrval.Mlrval, parentMapValue.FieldCount) i := 0 for pe := parentMapValue.Head; pe != nil; pe = pe.Next { @@ -362,7 +363,7 @@ func (node *EmitXStatementNode) Execute(state *runtime.State) (*BlockExitPayload func (node *EmitXStatementNode) executeNonIndexedNonLashedEmit( names []string, - values []*types.Mlrval, + values []*mlrval.Mlrval, state *runtime.State, ) error { for i, value := range values { @@ -373,7 +374,7 @@ func (node *EmitXStatementNode) executeNonIndexedNonLashedEmit( valueAsMap := value.GetMap() // nil if not a map if valueAsMap == nil { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() newrec.PutCopy(names[i], value) err := node.emitToRedirectFunc(newrec, state) if err != nil { @@ -383,7 +384,7 @@ func (node *EmitXStatementNode) executeNonIndexedNonLashedEmit( } else { recurse := valueAsMap.IsNested() if !recurse { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for pe := valueAsMap.Head; pe != nil; pe = pe.Next { newrec.PutCopy(pe.Key, pe.Value) } @@ -394,7 +395,7 @@ func (node *EmitXStatementNode) executeNonIndexedNonLashedEmit( } else { // recurse nextLevelNames := make([]string, 0) - nextLevelValues := make([]*types.Mlrval, 0) + nextLevelValues := make([]*mlrval.Mlrval, 0) for pe := value.GetMap().Head; pe != nil; pe = pe.Next { nextLevelNames = append(nextLevelNames, pe.Key) nextLevelValues = append(nextLevelValues, pe.Value.Copy()) @@ -409,14 +410,14 @@ func (node *EmitXStatementNode) executeNonIndexedNonLashedEmit( func (node *EmitXStatementNode) executeNonIndexedNonLashedEmitP( names []string, - values []*types.Mlrval, + values []*mlrval.Mlrval, state *runtime.State, ) error { for i, value := range values { if value.IsAbsent() { continue } - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() newrec.PutCopy(names[i], value) err := node.emitToRedirectFunc(newrec, state) if err != nil { @@ -433,14 +434,14 @@ func (node *EmitXStatementNode) executeNonIndexedNonLashedEmitP( func (node *EmitXStatementNode) executeNonIndexedLashedEmit( names []string, - values []*types.Mlrval, + values []*mlrval.Mlrval, state *runtime.State, ) error { lib.InternalCodingErrorIf(len(values) < 1) leadingValueAsMap := values[0].GetMap() if leadingValueAsMap == nil { // Emit a record like a=1,b=2 - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for i, value := range values { if value.IsAbsent() { continue @@ -462,7 +463,7 @@ func (node *EmitXStatementNode) executeNonIndexedLashedEmit( valueAsMap := value.GetMap() // nil if not a map if valueAsMap == nil { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() newrec.PutCopy(names[i], value) err := node.emitToRedirectFunc(newrec, state) if err != nil { @@ -472,7 +473,7 @@ func (node *EmitXStatementNode) executeNonIndexedLashedEmit( } else { recurse := valueAsMap.IsNested() if !recurse { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for pe := valueAsMap.Head; pe != nil; pe = pe.Next { newrec.PutCopy(pe.Key, pe.Value) } @@ -483,7 +484,7 @@ func (node *EmitXStatementNode) executeNonIndexedLashedEmit( } else { // recurse nextLevelNames := make([]string, 0) - nextLevelValues := make([]*types.Mlrval, 0) + nextLevelValues := make([]*mlrval.Mlrval, 0) for pe := value.GetMap().Head; pe != nil; pe = pe.Next { nextLevelNames = append(nextLevelNames, pe.Key) nextLevelValues = append(nextLevelValues, pe.Value.Copy()) @@ -499,10 +500,10 @@ func (node *EmitXStatementNode) executeNonIndexedLashedEmit( func (node *EmitXStatementNode) executeNonIndexedLashedEmitP( names []string, - values []*types.Mlrval, + values []*mlrval.Mlrval, state *runtime.State, ) error { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for i, value := range values { if value.IsAbsent() { @@ -518,11 +519,11 @@ func (node *EmitXStatementNode) executeNonIndexedLashedEmitP( // ---------------------------------------------------------------- func (node *EmitXStatementNode) executeIndexed( names []string, - values []*types.Mlrval, + values []*mlrval.Mlrval, state *runtime.State, ) error { - emittableMaps := make([]*types.Mlrmap, len(values)) + emittableMaps := make([]*mlrval.Mlrmap, len(values)) for i, value := range values { if value.IsAbsent() { return nil @@ -535,7 +536,7 @@ func (node *EmitXStatementNode) executeIndexed( } // TODO: libify this - indices := make([]*types.Mlrval, len(node.indexEvaluables)) + indices := make([]*mlrval.Mlrval, len(node.indexEvaluables)) for i := range node.indexEvaluables { indices[i] = node.indexEvaluables[i].Evaluate(state) if indices[i].IsAbsent() { @@ -550,7 +551,7 @@ func (node *EmitXStatementNode) executeIndexed( if !node.isEmitP { if !node.isLashed { return node.executeIndexedNonLashedEmitAux( - types.NewMlrmapAsRecord(), + mlrval.NewMlrmapAsRecord(), names, emittableMaps, indices, @@ -558,7 +559,7 @@ func (node *EmitXStatementNode) executeIndexed( ) } else { return node.executeIndexedLashedEmitAux( - types.NewMlrmapAsRecord(), + mlrval.NewMlrmapAsRecord(), names, emittableMaps, indices, @@ -568,7 +569,7 @@ func (node *EmitXStatementNode) executeIndexed( } else { if !node.isLashed { return node.executeIndexedNonLashedEmitPAux( - types.NewMlrmapAsRecord(), + mlrval.NewMlrmapAsRecord(), names, emittableMaps, indices, @@ -576,7 +577,7 @@ func (node *EmitXStatementNode) executeIndexed( ) } else { return node.executeIndexedLashedEmitPAux( - types.NewMlrmapAsRecord(), + mlrval.NewMlrmapAsRecord(), names, emittableMaps, indices, @@ -650,10 +651,10 @@ func (node *EmitXStatementNode) executeIndexed( // * indices = ["b"] func (node *EmitXStatementNode) executeIndexedNonLashedEmitAux( - templateRecord *types.Mlrmap, + templateRecord *mlrval.Mlrmap, names []string, - emittableMaps []*types.Mlrmap, - indices []*types.Mlrval, + emittableMaps []*mlrval.Mlrmap, + indices []*mlrval.Mlrval, state *runtime.State, ) error { lib.InternalCodingErrorIf(len(indices) < 1) @@ -663,7 +664,7 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitAux( for i, emittableMap := range emittableMaps { for pe := emittableMap.Head; pe != nil; pe = pe.Next { newrec := templateRecord.Copy() - newrec.PutCopy(indexString, types.MlrvalFromString(pe.Key)) + newrec.PutCopy(indexString, mlrval.FromString(pe.Key)) if len(indices) == 1 { valueAsMap := pe.Value.GetMap() @@ -690,7 +691,7 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitAux( node.executeIndexedNonLashedEmitPAux( newrec, []string{names[i]}, - []*types.Mlrmap{valueAsMap}, + []*mlrval.Mlrmap{valueAsMap}, indices[1:], state, ) @@ -796,10 +797,10 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitAux( // o indices ["b"] func (node *EmitXStatementNode) executeIndexedLashedEmitAux( - templateRecord *types.Mlrmap, + templateRecord *mlrval.Mlrmap, names []string, - emittableMaps []*types.Mlrmap, - indices []*types.Mlrval, + emittableMaps []*mlrval.Mlrmap, + indices []*mlrval.Mlrval, state *runtime.State, ) error { lib.InternalCodingErrorIf(len(indices) < 1) @@ -810,11 +811,11 @@ func (node *EmitXStatementNode) executeIndexedLashedEmitAux( for pe := leadingMap.Head; pe != nil; pe = pe.Next { newrec := templateRecord.Copy() - indexValue := types.MlrvalFromString(pe.Key) + indexValue := mlrval.FromString(pe.Key) newrec.PutCopy(indexString, indexValue) - nextLevelValues := make([]*types.Mlrval, len(emittableMaps)) - nextLevelMaps := make([]*types.Mlrmap, len(emittableMaps)) + nextLevelValues := make([]*mlrval.Mlrval, len(emittableMaps)) + nextLevelMaps := make([]*mlrval.Mlrmap, len(emittableMaps)) for i, emittableMap := range emittableMaps { if emittableMap != nil { nextLevelValues[i] = emittableMaps[i].Get(pe.Key) @@ -865,10 +866,10 @@ func (node *EmitXStatementNode) executeIndexedLashedEmitAux( // Recurses over indices. func (node *EmitXStatementNode) executeIndexedNonLashedEmitPAux( - templateRecord *types.Mlrmap, + templateRecord *mlrval.Mlrmap, names []string, - emittableMaps []*types.Mlrmap, - indices []*types.Mlrval, + emittableMaps []*mlrval.Mlrmap, + indices []*mlrval.Mlrval, state *runtime.State, ) error { lib.InternalCodingErrorIf(len(indices) < 1) @@ -878,7 +879,7 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitPAux( for i, emittableMap := range emittableMaps { for pe := emittableMap.Head; pe != nil; pe = pe.Next { newrec := templateRecord.Copy() - newrec.PutCopy(indexString, types.MlrvalFromString(pe.Key)) + newrec.PutCopy(indexString, mlrval.FromString(pe.Key)) if len(indices) == 1 { newrec.PutCopy(names[i], pe.Value) @@ -898,7 +899,7 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitPAux( node.executeIndexedNonLashedEmitPAux( newrec, []string{names[i]}, - []*types.Mlrmap{valueAsMap}, + []*mlrval.Mlrmap{valueAsMap}, indices[1:], state, ) @@ -911,10 +912,10 @@ func (node *EmitXStatementNode) executeIndexedNonLashedEmitPAux( } func (node *EmitXStatementNode) executeIndexedLashedEmitPAux( - templateRecord *types.Mlrmap, + templateRecord *mlrval.Mlrmap, names []string, - emittableMaps []*types.Mlrmap, - indices []*types.Mlrval, + emittableMaps []*mlrval.Mlrmap, + indices []*mlrval.Mlrval, state *runtime.State, ) error { lib.InternalCodingErrorIf(len(indices) < 1) @@ -926,12 +927,12 @@ func (node *EmitXStatementNode) executeIndexedLashedEmitPAux( for pe := leadingMap.Head; pe != nil; pe = pe.Next { newrec := templateRecord.Copy() - indexValue := types.MlrvalFromString(pe.Key) + indexValue := mlrval.FromString(pe.Key) newrec.PutCopy(indexString, indexValue) indexValueString := indexValue.String() - nextLevels := make([]*types.Mlrval, len(emittableMaps)) - nextLevelMaps := make([]*types.Mlrmap, len(emittableMaps)) + nextLevels := make([]*mlrval.Mlrval, len(emittableMaps)) + nextLevelMaps := make([]*mlrval.Mlrmap, len(emittableMaps)) for i, emittableMap := range emittableMaps { if emittableMap != nil { nextLevel := emittableMap.Get(indexValueString) @@ -979,7 +980,7 @@ func (node *EmitXStatementNode) executeIndexedLashedEmitPAux( // ---------------------------------------------------------------- func (node *EmitXStatementNode) emitRecordToRecordStream( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, state *runtime.State, ) error { // The output channel is always non-nil, except for the Miller REPL. @@ -993,7 +994,7 @@ func (node *EmitXStatementNode) emitRecordToRecordStream( // ---------------------------------------------------------------- func (node *EmitXStatementNode) emitRecordToFileOrPipe( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, state *runtime.State, ) error { redirectorTarget := node.redirectorTargetEvaluable.Evaluate(state) diff --git a/internal/pkg/dsl/cst/emitf.go b/internal/pkg/dsl/cst/emitf.go index 05a91d7a0..33426f784 100644 --- a/internal/pkg/dsl/cst/emitf.go +++ b/internal/pkg/dsl/cst/emitf.go @@ -11,6 +11,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/output" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" @@ -27,7 +28,7 @@ import ( // in the CST logic, to keep this parser/AST logic simpler. type tEmitFToRedirectFunc func( - newrec *types.Mlrmap, + newrec *mlrval.Mlrmap, state *runtime.State, ) error @@ -139,7 +140,7 @@ func (root *RootNode) BuildEmitFStatementNode(astNode *dsl.ASTNode) (IExecutable } func (node *EmitFStatementNode) Execute(state *runtime.State) (*BlockExitPayload, error) { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for i, emitfEvaluable := range node.emitfEvaluables { emitfName := node.emitfNames[i] emitfValue := emitfEvaluable.Evaluate(state) @@ -178,7 +179,7 @@ func getNameFromNamedNode(astNode *dsl.ASTNode, description string) (string, err // ---------------------------------------------------------------- func (node *EmitFStatementNode) emitfToRecordStream( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, state *runtime.State, ) error { // The output channel is always non-nil, except for the Miller REPL. @@ -192,7 +193,7 @@ func (node *EmitFStatementNode) emitfToRecordStream( // ---------------------------------------------------------------- func (node *EmitFStatementNode) emitfToFileOrPipe( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, state *runtime.State, ) error { redirectorTarget := node.redirectorTargetEvaluable.Evaluate(state) diff --git a/internal/pkg/dsl/cst/env.go b/internal/pkg/dsl/cst/env.go index 371467235..d53fb80d9 100644 --- a/internal/pkg/dsl/cst/env.go +++ b/internal/pkg/dsl/cst/env.go @@ -12,8 +12,8 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) type EnvironmentVariableNode struct { @@ -35,14 +35,14 @@ func (root *RootNode) BuildEnvironmentVariableNode(astNode *dsl.ASTNode) (*Envir func (node *EnvironmentVariableNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { name := node.nameEvaluable.Evaluate(state) if name.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } if !name.IsString() { - return types.MLRVAL_ERROR + return mlrval.ERROR } - return types.MlrvalFromString(os.Getenv(name.String())) + return mlrval.FromString(os.Getenv(name.String())) } diff --git a/internal/pkg/dsl/cst/evaluable.go b/internal/pkg/dsl/cst/evaluable.go index 03722d4d2..bcca85559 100644 --- a/internal/pkg/dsl/cst/evaluable.go +++ b/internal/pkg/dsl/cst/evaluable.go @@ -13,8 +13,8 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -107,10 +107,10 @@ func (root *RootNode) BuildIndirectFieldValueNode( func (node *IndirectFieldValueNode) Evaluate( state *runtime.State, -) *types.Mlrval { // TODO: err +) *mlrval.Mlrval { // TODO: err fieldName := node.fieldNameEvaluable.Evaluate(state) if fieldName.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } // For normal DSL use the CST validator will prohibit this from being @@ -119,7 +119,7 @@ func (node *IndirectFieldValueNode) Evaluate( // print inrec attributes. Also, a UDF/UDS invoked from begin/end could try // to access the inrec, and that would get past the validator. if state.Inrec == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } value, err := state.Inrec.GetWithMlrvalIndex(fieldName) @@ -130,7 +130,7 @@ func (node *IndirectFieldValueNode) Evaluate( os.Exit(1) } if value == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } return value } @@ -157,15 +157,15 @@ func (root *RootNode) BuildIndirectOosvarValueNode( func (node *IndirectOosvarValueNode) Evaluate( state *runtime.State, -) *types.Mlrval { // TODO: err +) *mlrval.Mlrval { // TODO: err oosvarName := node.oosvarNameEvaluable.Evaluate(state) if oosvarName.IsAbsent() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } value := state.Oosvars.Get(oosvarName.String()) if value == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } return value diff --git a/internal/pkg/dsl/cst/for.go b/internal/pkg/dsl/cst/for.go index f312e092a..14008bf3f 100644 --- a/internal/pkg/dsl/cst/for.go +++ b/internal/pkg/dsl/cst/for.go @@ -9,8 +9,8 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -126,7 +126,7 @@ func (node *ForLoopOneVariableNode) Execute(state *runtime.State) (*BlockExitPay state.Stack.PushStackFrame() defer state.Stack.PopStackFrame() for pe := mapval.Head; pe != nil; pe = pe.Next { - mapkey := types.MlrvalFromString(pe.Key) + mapkey := mlrval.FromString(pe.Key) err := state.Stack.SetAtScope(node.indexVariable, mapkey) if err != nil { @@ -311,7 +311,7 @@ func (node *ForLoopTwoVariableNode) Execute(state *runtime.State) (*BlockExitPay state.Stack.PushStackFrame() defer state.Stack.PopStackFrame() for pe := mapval.Head; pe != nil; pe = pe.Next { - mapkey := types.MlrvalFromString(pe.Key) + mapkey := mlrval.FromString(pe.Key) err := state.Stack.SetAtScope(node.keyIndexVariable, mapkey) if err != nil { @@ -352,7 +352,7 @@ func (node *ForLoopTwoVariableNode) Execute(state *runtime.State) (*BlockExitPay state.Stack.PushStackFrame() defer state.Stack.PopStackFrame() for zindex, element := range arrayval { - mindex := types.MlrvalFromInt(int(zindex + 1)) + mindex := mlrval.FromInt(int(zindex + 1)) err := state.Stack.SetAtScope(node.keyIndexVariable, mindex) if err != nil { @@ -524,20 +524,20 @@ func (node *ForLoopMultivariableNode) Execute(state *runtime.State) (*BlockExitP // ---------------------------------------------------------------- func (node *ForLoopMultivariableNode) executeOuter( - mlrval *types.Mlrval, + mv *mlrval.Mlrval, keyIndexVariables []*runtime.StackVariable, state *runtime.State, ) (*BlockExitPayload, error) { if len(keyIndexVariables) == 1 { - return node.executeInner(mlrval, keyIndexVariables[0], state) + return node.executeInner(mv, keyIndexVariables[0], state) } // else, recurse - if mlrval.IsMap() { - mapval := mlrval.GetMap() + if mv.IsMap() { + mapval := mv.GetMap() for pe := mapval.Head; pe != nil; pe = pe.Next { - mapkey := types.MlrvalFromString(pe.Key) + mapkey := mlrval.FromString(pe.Key) err := state.Stack.SetAtScope(keyIndexVariables[0], mapkey) if err != nil { @@ -563,14 +563,14 @@ func (node *ForLoopMultivariableNode) executeOuter( } } - } else if mlrval.IsArray() { - arrayval := mlrval.GetArray() + } else if mv.IsArray() { + arrayval := mv.GetArray() // Note: Miller user-space array indices ("mindex") are 1-up. Internal // Go storage ("zindex") is 0-up. for zindex, element := range arrayval { - mindex := types.MlrvalFromInt(int(zindex + 1)) + mindex := mlrval.FromInt(int(zindex + 1)) err := state.Stack.SetAtScope(keyIndexVariables[0], mindex) if err != nil { @@ -597,7 +597,7 @@ func (node *ForLoopMultivariableNode) executeOuter( } } - } else if mlrval.IsAbsent() { + } else if mv.IsAbsent() { // Data-heterogeneity no-op } @@ -608,7 +608,7 @@ func (node *ForLoopMultivariableNode) executeOuter( // return nil, errors.New( // fmt.Sprintf( // "mlr: looped-over item is not a map or array; got %s", - // mlrval.GetTypeName(), + // mv.GetTypeName(), // ), // ) @@ -617,15 +617,15 @@ func (node *ForLoopMultivariableNode) executeOuter( // ---------------------------------------------------------------- func (node *ForLoopMultivariableNode) executeInner( - mlrval *types.Mlrval, + mv *mlrval.Mlrval, keyIndexVariable *runtime.StackVariable, state *runtime.State, ) (*BlockExitPayload, error) { - if mlrval.IsMap() { - mapval := mlrval.GetMap() + if mv.IsMap() { + mapval := mv.GetMap() for pe := mapval.Head; pe != nil; pe = pe.Next { - mapkey := types.MlrvalFromString(pe.Key) + mapkey := mlrval.FromString(pe.Key) err := state.Stack.SetAtScope(keyIndexVariable, mapkey) if err != nil { @@ -656,14 +656,14 @@ func (node *ForLoopMultivariableNode) executeInner( } } - } else if mlrval.IsArray() { - arrayval := mlrval.GetArray() + } else if mv.IsArray() { + arrayval := mv.GetArray() // Note: Miller user-space array indices ("mindex") are 1-up. Internal // Go storage ("zindex") is 0-up. for zindex, element := range arrayval { - mindex := types.MlrvalFromInt(int(zindex + 1)) + mindex := mlrval.FromInt(int(zindex + 1)) err := state.Stack.SetAtScope(keyIndexVariable, mindex) if err != nil { @@ -694,7 +694,7 @@ func (node *ForLoopMultivariableNode) executeInner( } } - } else if mlrval.IsAbsent() { + } else if mv.IsAbsent() { // Data-heterogeneity no-op } @@ -705,7 +705,7 @@ func (node *ForLoopMultivariableNode) executeInner( // return nil, errors.New( // fmt.Sprintf( // "mlr: looped-over item is not a map or array; got %s", - // mlrval.GetTypeName(), + // mv.GetTypeName(), // ), // ) diff --git a/internal/pkg/dsl/cst/hofs.go b/internal/pkg/dsl/cst/hofs.go index 4e067685a..0f4a60f11 100644 --- a/internal/pkg/dsl/cst/hofs.go +++ b/internal/pkg/dsl/cst/hofs.go @@ -13,8 +13,8 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // Most function types are in the github.com/johnkerl/miller/internal/pkg/types package. These types, though, @@ -24,29 +24,29 @@ import ( // BinaryFuncWithState is for select, apply, and reduce. type BinaryFuncWithState func( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval +) *mlrval.Mlrval // TernaryFuncWithState is for fold. type TernaryFuncWithState func( - input1 *types.Mlrval, - input2 *types.Mlrval, - input3 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, + input3 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval +) *mlrval.Mlrval // VariadicFuncWithState is for sort. type VariadicFuncWithState func( - inputs []*types.Mlrval, + inputs []*mlrval.Mlrval, state *runtime.State, -) *types.Mlrval +) *mlrval.Mlrval // tHOFSpace is the datatype for the getHOFSpace cache-manager. type tHOFSpace struct { udfCallsite *UDFCallsite - argsArray []*types.Mlrval + argsArray []*mlrval.Mlrval } // hofCache is persistent data for the getHOFSpace cache-manager. @@ -56,7 +56,7 @@ var hofCache map[string]*tHOFSpace = make(map[string]*tHOFSpace) // Those functions may be invoked on every record of a big data file, so we try // to cache data they need for UDF-callsite setup. func getHOFSpace( - funcVal *types.Mlrval, + funcVal *mlrval.Mlrval, arity int, hofName string, arrayOrMap string, @@ -120,7 +120,7 @@ func getHOFSpace( } udfCallsite := NewUDFCallsiteForHigherOrderFunction(udf, arity) - argsArray := make([]*types.Mlrval, arity) + argsArray := make([]*mlrval.Mlrval, arity) entry = &tHOFSpace{ udfCallsite: udfCallsite, argsArray: argsArray, @@ -131,7 +131,7 @@ func getHOFSpace( } // mustBeNonAbsent checks that a UDF for array reduce/fold/apply returned a value. -func isNonAbsentOrDie(mlrval *types.Mlrval, hofName string) *types.Mlrval { +func isNonAbsentOrDie(mlrval *mlrval.Mlrval, hofName string) *mlrval.Mlrval { if mlrval.IsAbsent() { hofCheckDie(mlrval, hofName, "second-argument function must return a value") } @@ -140,7 +140,7 @@ func isNonAbsentOrDie(mlrval *types.Mlrval, hofName string) *types.Mlrval { // getKVPairForAccumulatorOrDie checks that a user-supplied accumulator value // for a map fold is indeed a single-element map. -func getKVPairForAccumulatorOrDie(mlrval *types.Mlrval, hofName string) *types.Mlrmap { +func getKVPairForAccumulatorOrDie(mlrval *mlrval.Mlrval, hofName string) *mlrval.Mlrmap { kvPair := getKVPair(mlrval) if kvPair == nil { hofCheckDie(mlrval, hofName, "accumulator value must be a single-element map") @@ -150,7 +150,7 @@ func getKVPairForAccumulatorOrDie(mlrval *types.Mlrval, hofName string) *types.M // getKVPairForCallbackOrDie checks that a return value from a UDF for map // reduce/fold/apply is indeed a single-element map. -func getKVPairForCallbackOrDie(mlrval *types.Mlrval, hofName string) *types.Mlrmap { +func getKVPairForCallbackOrDie(mlrval *mlrval.Mlrval, hofName string) *mlrval.Mlrmap { kvPair := getKVPair(mlrval) if kvPair == nil { hofCheckDie(mlrval, hofName, "second-argument function must return single-element map") @@ -160,7 +160,7 @@ func getKVPairForCallbackOrDie(mlrval *types.Mlrval, hofName string) *types.Mlrm // hofCheckDie is a helper function for HOFs on maps, to check that the // user-supplied UDF returned a single-entry map. -func hofCheckDie(mlrval *types.Mlrval, hofName string, message string) { +func hofCheckDie(mlrval *mlrval.Mlrval, hofName string, message string) { fmt.Fprintf( os.Stderr, "mlr: %s: %s; got \"%s\".\n", @@ -172,7 +172,7 @@ func hofCheckDie(mlrval *types.Mlrval, hofName string, message string) { } // getKVPair is a helper function getKVPairOrDie. -func getKVPair(mlrval *types.Mlrval) *types.Mlrmap { +func getKVPair(mlrval *mlrval.Mlrval) *mlrval.Mlrmap { mapval := mlrval.GetMap() if mapval == nil { return nil @@ -183,7 +183,7 @@ func getKVPair(mlrval *types.Mlrval) *types.Mlrmap { return mapval } -func isFunctionOrDie(mlrval *types.Mlrval, hofName string) { +func isFunctionOrDie(mlrval *mlrval.Mlrval, hofName string) { if !mlrval.IsFunction() { fmt.Fprintf(os.Stderr, "mlr: %s: second argument must be a function; got %s.\n", hofName, mlrval.GetTypeName(), @@ -196,27 +196,27 @@ func isFunctionOrDie(mlrval *types.Mlrval, hofName string) { // SELECT HOF func SelectHOF( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { if input1.IsArray() { return selectArray(input1, input2, state) } else if input1.IsMap() { return selectMap(input1, input2, state) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } func selectArray( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "select") @@ -224,7 +224,7 @@ func selectArray( udfCallsite := hofSpace.udfCallsite argsArray := hofSpace.argsArray - outputArray := make([]types.Mlrval, 0, len(inputArray)) + outputArray := make([]mlrval.Mlrval, 0, len(inputArray)) for i := range inputArray { argsArray[0] = &inputArray[i] @@ -242,17 +242,17 @@ func selectArray( outputArray = append(outputArray, *inputArray[i].Copy()) } } - return types.MlrvalFromArrayReference(outputArray) + return mlrval.FromArray(outputArray) } func selectMap( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "select") @@ -260,10 +260,10 @@ func selectMap( udfCallsite := hofSpace.udfCallsite argsArray := hofSpace.argsArray - outputMap := types.NewMlrmap() + outputMap := mlrval.NewMlrmap() for pe := inputMap.Head; pe != nil; pe = pe.Next { - argsArray[0] = types.MlrvalFromString(pe.Key) + argsArray[0] = mlrval.FromString(pe.Key) argsArray[1] = pe.Value mret := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) bret, ok := mret.GetBoolValue() @@ -280,34 +280,34 @@ func selectMap( } } - return types.MlrvalFromMap(outputMap) + return mlrval.FromMap(outputMap) } // ================================================================ // APPLY HOF func ApplyHOF( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { if input1.IsArray() { return applyArray(input1, input2, state) } else if input1.IsMap() { return applyMap(input1, input2, state) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } func applyArray( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "apply") @@ -315,7 +315,7 @@ func applyArray( udfCallsite := hofSpace.udfCallsite argsArray := hofSpace.argsArray - outputArray := make([]types.Mlrval, len(inputArray)) + outputArray := make([]mlrval.Mlrval, len(inputArray)) for i := range inputArray { argsArray[0] = &inputArray[i] @@ -323,17 +323,17 @@ func applyArray( isNonAbsentOrDie(&retval, "apply") outputArray[i] = retval } - return types.MlrvalFromArrayReference(outputArray) + return mlrval.FromArray(outputArray) } func applyMap( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "apply") @@ -341,43 +341,43 @@ func applyMap( udfCallsite := hofSpace.udfCallsite argsArray := hofSpace.argsArray - outputMap := types.NewMlrmap() + outputMap := mlrval.NewMlrmap() for pe := inputMap.Head; pe != nil; pe = pe.Next { - argsArray[0] = types.MlrvalFromString(pe.Key) + argsArray[0] = mlrval.FromString(pe.Key) argsArray[1] = pe.Value retval := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) kvPair := getKVPairForCallbackOrDie(retval, "apply") outputMap.PutReference(kvPair.Head.Key, kvPair.Head.Value) } - return types.MlrvalFromMap(outputMap) + return mlrval.FromMap(outputMap) } // ================================================================ // REDUCE HOF func ReduceHOF( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { if input1.IsArray() { return reduceArray(input1, input2, state) } else if input1.IsMap() { return reduceMap(input1, input2, state) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } func reduceArray( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "reduce") @@ -401,13 +401,13 @@ func reduceArray( } func reduceMap( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "reduce") @@ -421,44 +421,44 @@ func reduceMap( } for pe := inputMap.Head.Next; pe != nil; pe = pe.Next { - argsArray[0] = types.MlrvalFromString(accumulator.Head.Key) + argsArray[0] = mlrval.FromString(accumulator.Head.Key) argsArray[1] = accumulator.Head.Value - argsArray[2] = types.MlrvalFromString(pe.Key) + argsArray[2] = mlrval.FromString(pe.Key) argsArray[3] = pe.Value.Copy() retval := (udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray)) kvPair := getKVPairForCallbackOrDie(retval, "reduce") accumulator = kvPair } - return types.MlrvalFromMap(accumulator) + return mlrval.FromMap(accumulator) } // ================================================================ // FOLD HOF func FoldHOF( - input1 *types.Mlrval, - input2 *types.Mlrval, - input3 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, + input3 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { if input1.IsArray() { return foldArray(input1, input2, input3, state) } else if input1.IsMap() { return foldMap(input1, input2, input3, state) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } func foldArray( - input1 *types.Mlrval, - input2 *types.Mlrval, - input3 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, + input3 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "fold") @@ -478,14 +478,14 @@ func foldArray( } func foldMap( - input1 *types.Mlrval, - input2 *types.Mlrval, - input3 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, + input3 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "fold") @@ -494,30 +494,30 @@ func foldMap( argsArray := hofSpace.argsArray if inputMap.IsEmpty() { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } accumulator := getKVPairForAccumulatorOrDie(input3, "reduce").Copy() for pe := inputMap.Head; pe != nil; pe = pe.Next { - argsArray[0] = types.MlrvalFromString(accumulator.Head.Key) + argsArray[0] = mlrval.FromString(accumulator.Head.Key) argsArray[1] = accumulator.Head.Value - argsArray[2] = types.MlrvalFromString(pe.Key) + argsArray[2] = mlrval.FromString(pe.Key) argsArray[3] = pe.Value.Copy() retval := (udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray)) kvPair := getKVPairForCallbackOrDie(retval, "reduce") accumulator = kvPair } - return types.MlrvalFromMap(accumulator) + return mlrval.FromMap(accumulator) } // ================================================================ // SORT HOF func SortHOF( - inputs []*types.Mlrval, + inputs []*mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { if len(inputs) == 1 { if inputs[0].IsArray() { @@ -525,7 +525,7 @@ func SortHOF( } else if inputs[0].IsMap() { return sortMK(inputs[0], "") } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } else if inputs[1].IsString() { @@ -534,7 +534,7 @@ func SortHOF( } else if inputs[0].IsMap() { return sortMK(inputs[0], inputs[1].String()) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } else if inputs[1].IsFunction() { @@ -543,7 +543,7 @@ func SortHOF( } else if inputs[0].IsMap() { return sortMF(inputs[0], inputs[1], state) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } else { @@ -552,7 +552,7 @@ func SortHOF( ) os.Exit(1) } - return types.MLRVAL_ERROR + return mlrval.ERROR } // ---------------------------------------------------------------- @@ -588,11 +588,11 @@ func decodeSortFlags(flags string) (tSortType, bool) { // sortA implements sort on array, with string flags rather than callback UDF. func sortA( - input1 *types.Mlrval, + input1 *mlrval.Mlrval, flags string, -) *types.Mlrval { +) *mlrval.Mlrval { if input1.GetArray() == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } output := input1.Copy() @@ -612,19 +612,19 @@ func sortA( return output } -func sortANumerical(array []types.Mlrval, reverse bool) { +func sortANumerical(array []mlrval.Mlrval, reverse bool) { if !reverse { sort.Slice(array, func(i, j int) bool { - return types.MlrvalLessThanAsBool(&array[i], &array[j]) + return mlrval.LessThan(&array[i], &array[j]) }) } else { sort.Slice(array, func(i, j int) bool { - return types.MlrvalGreaterThanAsBool(&array[i], &array[j]) + return mlrval.GreaterThan(&array[i], &array[j]) }) } } -func sortALexical(array []types.Mlrval, reverse bool) { +func sortALexical(array []mlrval.Mlrval, reverse bool) { if !reverse { sort.Slice(array, func(i, j int) bool { return array[i].String() < array[j].String() @@ -636,7 +636,7 @@ func sortALexical(array []types.Mlrval, reverse bool) { } } -func sortACaseFold(array []types.Mlrval, reverse bool) { +func sortACaseFold(array []mlrval.Mlrval, reverse bool) { if !reverse { sort.Slice(array, func(i, j int) bool { return strings.ToLower(array[i].String()) < strings.ToLower(array[j].String()) @@ -650,12 +650,12 @@ func sortACaseFold(array []types.Mlrval, reverse bool) { // sortA implements sort on map, with string flags rather than callback UDF. func sortMK( - input1 *types.Mlrval, + input1 *mlrval.Mlrval, flags string, -) *types.Mlrval { +) *mlrval.Mlrval { inmap := input1.GetMap() if inmap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } // Copy the keys to an array for sorting. @@ -682,13 +682,13 @@ func sortMK( } // Make a new map with keys in the new sort order. - outmap := types.NewMlrmap() + outmap := mlrval.NewMlrmap() for i := 0; i < n; i++ { key := keys[i] outmap.PutCopy(key, inmap.Get(key)) } - return types.MlrvalFromMapReferenced(outmap) + return mlrval.FromMap(outmap) } func sortMKNumerical(array []string, reverse bool) { @@ -742,30 +742,30 @@ func sortMKCaseFold(array []string, reverse bool) { // sortAF implements sort on arrays with callback UDF. func sortAF( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsFunction() { - return types.MLRVAL_ERROR + return mlrval.ERROR } hofSpace := getHOFSpace(input2, 2, "sort", "array") udfCallsite := hofSpace.udfCallsite argsArray := hofSpace.argsArray - outputArray := types.CopyMlrvalArray(inputArray) + outputArray := mlrval.CopyMlrvalArray(inputArray) sort.Slice(outputArray, func(i, j int) bool { argsArray[0] = &outputArray[i] argsArray[1] = &outputArray[j] // Call the user's comparator function. mret := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) - // Unpack the types.Mlrval return value into a number. + // Unpack the mlrval.Mlrval return value into a number. nret, ok := mret.GetNumericToFloatValue() if !ok { fmt.Fprintf( @@ -780,21 +780,21 @@ func sortAF( // Go sort-callback conventions: true if a < b, false otherwise. return nret < 0 }) - return types.MlrvalFromArrayReference(outputArray) + return mlrval.FromArray(outputArray) } // sortAF implements sort on arrays with callback UDF. func sortMF( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } if !input2.IsFunction() { - return types.MLRVAL_ERROR + return mlrval.ERROR } pairsArray := inputMap.ToPairsArray() @@ -804,14 +804,14 @@ func sortMF( argsArray := hofSpace.argsArray sort.Slice(pairsArray, func(i, j int) bool { - argsArray[0] = types.MlrvalFromString(pairsArray[i].Key) + argsArray[0] = mlrval.FromString(pairsArray[i].Key) argsArray[1] = pairsArray[i].Value - argsArray[2] = types.MlrvalFromString(pairsArray[j].Key) + argsArray[2] = mlrval.FromString(pairsArray[j].Key) argsArray[3] = pairsArray[j].Value // Call the user's comparator function. mret := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) - // Unpack the types.Mlrval return value into a number. + // Unpack the mlrval.Mlrval return value into a number. nret, ok := mret.GetNumericToFloatValue() if !ok { fmt.Fprintf( @@ -827,35 +827,35 @@ func sortMF( return nret < 0 }) - sortedMap := types.MlrmapFromPairsArray(pairsArray) - return types.MlrvalFromMapReferenced(sortedMap) + sortedMap := mlrval.MlrmapFromPairsArray(pairsArray) + return mlrval.FromMap(sortedMap) } // ================================================================ // ANY HOF func AnyHOF( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { if input1.IsArray() { return anyArray(input1, input2, state) } else if input1.IsMap() { return anyMap(input1, input2, state) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } func anyArray( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "any") @@ -881,17 +881,17 @@ func anyArray( break } } - return types.MlrvalFromBool(boolAny) + return mlrval.FromBool(boolAny) } func anyMap( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "any") @@ -902,7 +902,7 @@ func anyMap( boolAny := false for pe := inputMap.Head; pe != nil; pe = pe.Next { - argsArray[0] = types.MlrvalFromString(pe.Key) + argsArray[0] = mlrval.FromString(pe.Key) argsArray[1] = pe.Value mret := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) bret, ok := mret.GetBoolValue() @@ -920,34 +920,34 @@ func anyMap( } } - return types.MlrvalFromBool(boolAny) + return mlrval.FromBool(boolAny) } // ================================================================ // EVERY HOF func EveryHOF( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { if input1.IsArray() { return everyArray(input1, input2, state) } else if input1.IsMap() { return everyMap(input1, input2, state) } else { - return types.MLRVAL_ERROR + return mlrval.ERROR } } func everyArray( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputArray := input1.GetArray() if inputArray == nil { // not an array - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "every") @@ -973,17 +973,17 @@ func everyArray( break } } - return types.MlrvalFromBool(boolEvery) + return mlrval.FromBool(boolEvery) } func everyMap( - input1 *types.Mlrval, - input2 *types.Mlrval, + input1 *mlrval.Mlrval, + input2 *mlrval.Mlrval, state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { inputMap := input1.GetMap() if inputMap == nil { // not a map - return types.MLRVAL_ERROR + return mlrval.ERROR } isFunctionOrDie(input2, "every") @@ -994,7 +994,7 @@ func everyMap( boolEvery := true for pe := inputMap.Head; pe != nil; pe = pe.Next { - argsArray[0] = types.MlrvalFromString(pe.Key) + argsArray[0] = mlrval.FromString(pe.Key) argsArray[1] = pe.Value mret := udfCallsite.EvaluateWithArguments(state, udfCallsite.udf, argsArray) bret, ok := mret.GetBoolValue() @@ -1012,5 +1012,5 @@ func everyMap( } } - return types.MlrvalFromBool(boolEvery) + return mlrval.FromBool(boolEvery) } diff --git a/internal/pkg/dsl/cst/if.go b/internal/pkg/dsl/cst/if.go index 6b2fa223e..853cb8b95 100644 --- a/internal/pkg/dsl/cst/if.go +++ b/internal/pkg/dsl/cst/if.go @@ -9,8 +9,8 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -120,7 +120,7 @@ func (root *RootNode) BuildIfChainNode(astNode *dsl.ASTNode) (*IfChainNode, erro // ---------------------------------------------------------------- func (node *IfChainNode) Execute(state *runtime.State) (*BlockExitPayload, error) { for _, ifItem := range node.ifItems { - condition := types.MLRVAL_TRUE + condition := mlrval.TRUE if ifItem.conditionNode != nil { condition = ifItem.conditionNode.Evaluate(state) } diff --git a/internal/pkg/dsl/cst/leaves.go b/internal/pkg/dsl/cst/leaves.go index b39b0b16f..7726493f8 100644 --- a/internal/pkg/dsl/cst/leaves.go +++ b/internal/pkg/dsl/cst/leaves.go @@ -10,8 +10,8 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -114,18 +114,18 @@ func (root *RootNode) BuildDirectFieldRvalueNode(fieldName string) *DirectFieldR } func (node *DirectFieldRvalueNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { // For normal DSL use the CST validator will prohibit this from being // called in places the current record is undefined (begin and end blocks). // However in the REPL people can read past end of stream and still try to // print inrec attributes. Also, a UDF/UDS invoked from begin/end could try // to access the inrec, and that would get past the validator. if state.Inrec == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } value := state.Inrec.Get(node.fieldName) if value == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { return value } @@ -140,16 +140,16 @@ func (root *RootNode) BuildFullSrecRvalueNode() *FullSrecRvalueNode { } func (node *FullSrecRvalueNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { // For normal DSL use the CST validator will prohibit this from being // called in places the current record is undefined (begin and end blocks). // However in the REPL people can read past end of stream and still try to // print inrec attributes. Also, a UDF/UDS invoked from begin/end could try // to access the inrec, and that would get past the validator. if state.Inrec == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { - return types.MlrvalFromMap(state.Inrec) + return mlrval.FromMap(state.Inrec) } } @@ -165,10 +165,10 @@ func (root *RootNode) BuildDirectOosvarRvalueNode(variableName string) *DirectOo } func (node *DirectOosvarRvalueNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { value := state.Oosvars.Get(node.variableName) if value == nil { - return types.MLRVAL_ABSENT + return mlrval.ABSENT } else { return value } @@ -183,8 +183,8 @@ func (root *RootNode) BuildFullOosvarRvalueNode() *FullOosvarRvalueNode { } func (node *FullOosvarRvalueNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromMap(state.Oosvars) +) *mlrval.Mlrval { + return mlrval.FromMap(state.Oosvars) } // ---------------------------------------------------------------- @@ -208,7 +208,7 @@ func (root *RootNode) BuildLocalVariableNode(variableName string) *LocalVariable } func (node *LocalVariableNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { value := state.Stack.Get(node.stackVariable) if value != nil { return value @@ -218,14 +218,14 @@ func (node *LocalVariableNode) Evaluate( udf := node.udfManager.LookUpDisregardingArity(functionName) if udf != nil { - return types.MlrvalFromFunction(udf, functionName) + return mlrval.FromFunction(udf, functionName) } // TODO: allow built-in functions as well. Needs some API-merging as a // prerequisite since UDFs and BIFs are managed in quite different // structures. - return types.MLRVAL_ABSENT + return mlrval.ABSENT } // ---------------------------------------------------------------- @@ -237,18 +237,18 @@ func (node *LocalVariableNode) Evaluate( // regex literals differently from those for non-regex string literals. type RegexLiteralNode struct { - literal *types.Mlrval + literal *mlrval.Mlrval } func (root *RootNode) BuildRegexLiteralNode(literal string) IEvaluable { return &RegexLiteralNode{ - literal: types.MlrvalFromString(literal), + literal: mlrval.FromString(literal), } } func (node *RegexLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.literal } @@ -256,7 +256,7 @@ func (node *RegexLiteralNode) Evaluate( // StringLiteralNode is for any string literal that doesn't have any "\0" .. // "\9" in it. type StringLiteralNode struct { - literal *types.Mlrval + literal *mlrval.Mlrval } // RegexCaptureReplacementNode is for any string literal that has any "\0" .. @@ -288,7 +288,7 @@ func (root *RootNode) BuildStringLiteralNode(literal string) IEvaluable { hasCaptures, replacementCaptureMatrix := lib.RegexReplacementHasCaptures(literal) if !hasCaptures { return &StringLiteralNode{ - literal: types.MlrvalFromString(literal), + literal: mlrval.FromString(literal), } } else { return &RegexCaptureReplacementNode{ @@ -300,7 +300,7 @@ func (root *RootNode) BuildStringLiteralNode(literal string) IEvaluable { func (node *StringLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.literal } @@ -316,8 +316,8 @@ func (node *StringLiteralNode) Evaluate( // set on some previous invocation of =~ or !=~. func (node *RegexCaptureReplacementNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString( +) *mlrval.Mlrval { + return mlrval.FromString( lib.InterpolateCaptures( node.replacementString, node.replacementCaptureMatrix, @@ -328,65 +328,69 @@ func (node *RegexCaptureReplacementNode) Evaluate( // ---------------------------------------------------------------- type IntLiteralNode struct { - literal *types.Mlrval + literal *mlrval.Mlrval } func (root *RootNode) BuildIntLiteralNode(literal string) *IntLiteralNode { + ival, ok := lib.TryIntFromString(literal) + lib.InternalCodingErrorIf(!ok) return &IntLiteralNode{ - literal: types.MlrvalFromPrevalidatedIntString(literal), + literal: mlrval.FromPrevalidatedIntString(literal, ival), } } func (node *IntLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.literal } // ---------------------------------------------------------------- type FloatLiteralNode struct { - literal *types.Mlrval + literal *mlrval.Mlrval } func (root *RootNode) BuildFloatLiteralNode(literal string) *FloatLiteralNode { + fval, ok := lib.TryFloatFromString(literal) + lib.InternalCodingErrorIf(!ok) return &FloatLiteralNode{ - literal: types.MlrvalFromPrevalidatedFloat64String(literal), + literal: mlrval.FromPrevalidatedFloatString(literal, fval), } } func (node *FloatLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.literal } // ---------------------------------------------------------------- type BoolLiteralNode struct { - literal *types.Mlrval + literal *mlrval.Mlrval } func (root *RootNode) BuildBoolLiteralNode(literal string) *BoolLiteralNode { return &BoolLiteralNode{ - literal: types.MlrvalFromBoolString(literal), + literal: mlrval.FromBoolString(literal), } } func (node *BoolLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.literal } // ---------------------------------------------------------------- type NullLiteralNode struct { - literal *types.Mlrval + literal *mlrval.Mlrval } func (root *RootNode) BuildNullLiteralNode() *NullLiteralNode { return &NullLiteralNode{ - literal: types.MLRVAL_NULL, + literal: mlrval.NULL, } } func (node *NullLiteralNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { return node.literal } @@ -453,8 +457,8 @@ func (root *RootNode) BuildFILENAMENode() *FILENAMENode { } func (node *FILENAMENode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Context.FILENAME) +) *mlrval.Mlrval { + return mlrval.FromString(state.Context.FILENAME) } // ---------------------------------------------------------------- @@ -466,8 +470,8 @@ func (root *RootNode) BuildFILENUMNode() *FILENUMNode { } func (node *FILENUMNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromInt(state.Context.FILENUM) +) *mlrval.Mlrval { + return mlrval.FromInt(state.Context.FILENUM) } // ---------------------------------------------------------------- @@ -479,8 +483,8 @@ func (root *RootNode) BuildNFNode() *NFNode { } func (node *NFNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromInt(state.Inrec.FieldCount) +) *mlrval.Mlrval { + return mlrval.FromInt(state.Inrec.FieldCount) } // ---------------------------------------------------------------- @@ -492,8 +496,8 @@ func (root *RootNode) BuildNRNode() *NRNode { } func (node *NRNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromInt(state.Context.NR) +) *mlrval.Mlrval { + return mlrval.FromInt(state.Context.NR) } // ---------------------------------------------------------------- @@ -505,8 +509,8 @@ func (root *RootNode) BuildFNRNode() *FNRNode { } func (node *FNRNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromInt(state.Context.FNR) +) *mlrval.Mlrval { + return mlrval.FromInt(state.Context.FNR) } // ---------------------------------------------------------------- @@ -518,8 +522,8 @@ func (root *RootNode) BuildIRSNode() *IRSNode { } func (node *IRSNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Options.ReaderOptions.IRS) +) *mlrval.Mlrval { + return mlrval.FromString(state.Options.ReaderOptions.IRS) } // ---------------------------------------------------------------- @@ -531,8 +535,8 @@ func (root *RootNode) BuildIFSNode() *IFSNode { } func (node *IFSNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Options.ReaderOptions.IFS) +) *mlrval.Mlrval { + return mlrval.FromString(state.Options.ReaderOptions.IFS) } // ---------------------------------------------------------------- @@ -544,8 +548,8 @@ func (root *RootNode) BuildIPSNode() *IPSNode { } func (node *IPSNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Options.ReaderOptions.IPS) +) *mlrval.Mlrval { + return mlrval.FromString(state.Options.ReaderOptions.IPS) } // ---------------------------------------------------------------- @@ -557,8 +561,8 @@ func (root *RootNode) BuildORSNode() *ORSNode { } func (node *ORSNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Options.WriterOptions.ORS) +) *mlrval.Mlrval { + return mlrval.FromString(state.Options.WriterOptions.ORS) } // ---------------------------------------------------------------- @@ -570,8 +574,8 @@ func (root *RootNode) BuildOFSNode() *OFSNode { } func (node *OFSNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Options.WriterOptions.OFS) +) *mlrval.Mlrval { + return mlrval.FromString(state.Options.WriterOptions.OFS) } // ---------------------------------------------------------------- @@ -583,8 +587,8 @@ func (root *RootNode) BuildOPSNode() *OPSNode { } func (node *OPSNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Options.WriterOptions.OPS) +) *mlrval.Mlrval { + return mlrval.FromString(state.Options.WriterOptions.OPS) } // ---------------------------------------------------------------- @@ -596,8 +600,8 @@ func (root *RootNode) BuildFLATSEPNode() *FLATSEPNode { } func (node *FLATSEPNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString(state.Options.WriterOptions.FLATSEP) +) *mlrval.Mlrval { + return mlrval.FromString(state.Options.WriterOptions.FLATSEP) } // ================================================================ @@ -630,8 +634,8 @@ func (root *RootNode) BuildMathPINode() *MathPINode { } func (node *MathPINode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromFloat64(math.Pi) +) *mlrval.Mlrval { + return mlrval.FromFloat(math.Pi) } // ---------------------------------------------------------------- @@ -643,8 +647,8 @@ func (root *RootNode) BuildMathENode() *MathENode { } func (node *MathENode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromFloat64(math.E) +) *mlrval.Mlrval { + return mlrval.FromFloat(math.E) } // ================================================================ @@ -661,8 +665,8 @@ func (root *RootNode) BuildArraySliceEmptyLowerIndexNode( } func (node *LiteralOneNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromInt(1) +) *mlrval.Mlrval { + return mlrval.FromInt(1) } // ================================================================ @@ -681,8 +685,8 @@ func (root *RootNode) BuildArraySliceEmptyUpperIndexNode( } func (node *LiteralEmptyStringNode) Evaluate( state *runtime.State, -) *types.Mlrval { - return types.MlrvalFromString("") +) *mlrval.Mlrval { + return mlrval.FromString("") } // ---------------------------------------------------------------- @@ -698,7 +702,7 @@ func (root *RootNode) BuildPanicNode(astNode *dsl.ASTNode) (*PanicNode, error) { } func (node *PanicNode) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { lib.InternalCodingErrorPanic("Panic token was evaluated, not short-circuited.") return nil // not reached } diff --git a/internal/pkg/dsl/cst/lvalues.go b/internal/pkg/dsl/cst/lvalues.go index 0e3572c54..34dae1532 100644 --- a/internal/pkg/dsl/cst/lvalues.go +++ b/internal/pkg/dsl/cst/lvalues.go @@ -12,8 +12,8 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -84,7 +84,7 @@ func (root *RootNode) BuildAssignableNode( // ---------------------------------------------------------------- type DirectFieldValueLvalueNode struct { - lhsFieldName *types.Mlrval + lhsFieldName *mlrval.Mlrval } func (root *RootNode) BuildDirectFieldValueLvalueNode( @@ -92,26 +92,26 @@ func (root *RootNode) BuildDirectFieldValueLvalueNode( ) (IAssignable, error) { lib.InternalCodingErrorIf(astNode.Type != dsl.NodeTypeDirectFieldValue) - lhsFieldName := types.MlrvalFromString(string(astNode.Token.Lit)) + lhsFieldName := mlrval.FromString(string(astNode.Token.Lit)) return NewDirectFieldValueLvalueNode(lhsFieldName), nil } -func NewDirectFieldValueLvalueNode(lhsFieldName *types.Mlrval) *DirectFieldValueLvalueNode { +func NewDirectFieldValueLvalueNode(lhsFieldName *mlrval.Mlrval) *DirectFieldValueLvalueNode { return &DirectFieldValueLvalueNode{ lhsFieldName: lhsFieldName, } } func (node *DirectFieldValueLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *DirectFieldValueLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { @@ -134,7 +134,7 @@ func (node *DirectFieldValueLvalueNode) AssignIndexed( return nil } else { return state.Inrec.PutIndexed( - append([]*types.Mlrval{node.lhsFieldName}, indices...), + append([]*mlrval.Mlrval{node.lhsFieldName}, indices...), rvalue, ) } @@ -147,7 +147,7 @@ func (node *DirectFieldValueLvalueNode) Unassign( } func (node *DirectFieldValueLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { @@ -166,7 +166,7 @@ func (node *DirectFieldValueLvalueNode) UnassignIndexed( state.Inrec.Remove(name) } else { state.Inrec.RemoveIndexed( - append([]*types.Mlrval{node.lhsFieldName}, indices...), + append([]*mlrval.Mlrval{node.lhsFieldName}, indices...), ) } } @@ -199,15 +199,15 @@ func NewIndirectFieldValueLvalueNode( } func (node *IndirectFieldValueLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *IndirectFieldValueLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absentness of the rvalue, so we just assign @@ -233,7 +233,7 @@ func (node *IndirectFieldValueLvalueNode) AssignIndexed( return nil } else { return state.Inrec.PutIndexed( - append([]*types.Mlrval{lhsFieldName.Copy()}, indices...), + append([]*mlrval.Mlrval{lhsFieldName.Copy()}, indices...), rvalue, ) } @@ -246,7 +246,7 @@ func (node *IndirectFieldValueLvalueNode) Unassign( } func (node *IndirectFieldValueLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { // For normal DSL use the CST validator will prohibit this from being @@ -264,7 +264,7 @@ func (node *IndirectFieldValueLvalueNode) UnassignIndexed( state.Inrec.Remove(name) } else { state.Inrec.RemoveIndexed( - append([]*types.Mlrval{lhsFieldName.Copy()}, indices...), + append([]*mlrval.Mlrval{lhsFieldName.Copy()}, indices...), ) } } @@ -300,7 +300,7 @@ func NewPositionalFieldNameLvalueNode( } func (node *PositionalFieldNameLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absentness of the rvalue, so we just assign @@ -334,8 +334,8 @@ func (node *PositionalFieldNameLvalueNode) Assign( } func (node *PositionalFieldNameLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // TODO: reconsider this if /when we decide to allow string-slice @@ -352,7 +352,7 @@ func (node *PositionalFieldNameLvalueNode) Unassign( } func (node *PositionalFieldNameLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { lhsFieldIndex := node.lhsFieldIndexExpression.Evaluate(state) @@ -376,7 +376,7 @@ func (node *PositionalFieldNameLvalueNode) UnassignIndexed( } else { // xxx positional state.Inrec.RemoveIndexed( - append([]*types.Mlrval{lhsFieldIndex}, indices...), + append([]*mlrval.Mlrval{lhsFieldIndex}, indices...), ) } } @@ -412,15 +412,15 @@ func NewPositionalFieldValueLvalueNode( } func (node *PositionalFieldValueLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *PositionalFieldValueLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absentness of the rvalue, so we just assign @@ -460,7 +460,7 @@ func (node *PositionalFieldValueLvalueNode) AssignIndexed( } else { // xxx positional return state.Inrec.PutIndexed( - append([]*types.Mlrval{lhsFieldIndex}, indices...), + append([]*mlrval.Mlrval{lhsFieldIndex}, indices...), rvalue, ) } @@ -475,7 +475,7 @@ func (node *PositionalFieldValueLvalueNode) Unassign( } func (node *PositionalFieldValueLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { // For normal DSL use the CST validator will prohibit this from being @@ -498,7 +498,7 @@ func (node *PositionalFieldValueLvalueNode) UnassignIndexed( } else { // xxx positional state.Inrec.RemoveIndexed( - append([]*types.Mlrval{lhsFieldIndex}, indices...), + append([]*mlrval.Mlrval{lhsFieldIndex}, indices...), ) } } @@ -519,15 +519,15 @@ func NewFullSrecLvalueNode() *FullSrecLvalueNode { } func (node *FullSrecLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *FullSrecLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // For normal DSL use the CST validator will prohibit this from being @@ -558,7 +558,7 @@ func (node *FullSrecLvalueNode) Unassign( } func (node *FullSrecLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { // For normal DSL use the CST validator will prohibit this from being @@ -579,32 +579,32 @@ func (node *FullSrecLvalueNode) UnassignIndexed( // ---------------------------------------------------------------- type DirectOosvarValueLvalueNode struct { - lhsOosvarName *types.Mlrval + lhsOosvarName *mlrval.Mlrval } func (root *RootNode) BuildDirectOosvarValueLvalueNode(astNode *dsl.ASTNode) (IAssignable, error) { lib.InternalCodingErrorIf(astNode.Type != dsl.NodeTypeDirectOosvarValue) - lhsOosvarName := types.MlrvalFromString(string(astNode.Token.Lit)) + lhsOosvarName := mlrval.FromString(string(astNode.Token.Lit)) return NewDirectOosvarValueLvalueNode(lhsOosvarName), nil } -func NewDirectOosvarValueLvalueNode(lhsOosvarName *types.Mlrval) *DirectOosvarValueLvalueNode { +func NewDirectOosvarValueLvalueNode(lhsOosvarName *mlrval.Mlrval) *DirectOosvarValueLvalueNode { return &DirectOosvarValueLvalueNode{ lhsOosvarName: lhsOosvarName, } } func (node *DirectOosvarValueLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *DirectOosvarValueLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absent, so we just assign whatever we get @@ -617,7 +617,7 @@ func (node *DirectOosvarValueLvalueNode) AssignIndexed( return nil } else { return state.Oosvars.PutIndexed( - append([]*types.Mlrval{node.lhsOosvarName}, indices...), + append([]*mlrval.Mlrval{node.lhsOosvarName}, indices...), rvalue, ) } @@ -630,7 +630,7 @@ func (node *DirectOosvarValueLvalueNode) Unassign( } func (node *DirectOosvarValueLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { if indices == nil { @@ -638,7 +638,7 @@ func (node *DirectOosvarValueLvalueNode) UnassignIndexed( state.Oosvars.Remove(name) } else { state.Oosvars.RemoveIndexed( - append([]*types.Mlrval{node.lhsOosvarName}, indices...), + append([]*mlrval.Mlrval{node.lhsOosvarName}, indices...), ) } } @@ -672,15 +672,15 @@ func NewIndirectOosvarValueLvalueNode( } func (node *IndirectOosvarValueLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *IndirectOosvarValueLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absentness of the rvalue, so we just assign @@ -697,7 +697,7 @@ func (node *IndirectOosvarValueLvalueNode) AssignIndexed( return nil } else { return state.Oosvars.PutIndexed( - append([]*types.Mlrval{lhsOosvarName.Copy()}, indices...), + append([]*mlrval.Mlrval{lhsOosvarName.Copy()}, indices...), rvalue, ) } @@ -710,7 +710,7 @@ func (node *IndirectOosvarValueLvalueNode) Unassign( } func (node *IndirectOosvarValueLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { lhsOosvarName := node.lhsOosvarNameExpression.Evaluate(state) @@ -720,7 +720,7 @@ func (node *IndirectOosvarValueLvalueNode) UnassignIndexed( state.Oosvars.Remove(sname) } else { state.Oosvars.RemoveIndexed( - append([]*types.Mlrval{lhsOosvarName}, indices...), + append([]*mlrval.Mlrval{lhsOosvarName}, indices...), ) } } @@ -741,15 +741,15 @@ func NewFullOosvarLvalueNode() *FullOosvarLvalueNode { } func (node *FullOosvarLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *FullOosvarLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absentness of the rvalue, so we just assign @@ -771,7 +771,7 @@ func (node *FullOosvarLvalueNode) Unassign( } func (node *FullOosvarLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { if indices == nil { @@ -827,15 +827,15 @@ func NewLocalVariableLvalueNode( } func (node *LocalVariableLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { return node.AssignIndexed(rvalue, nil, state) } func (node *LocalVariableLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absent, so we just assign whatever we get @@ -863,7 +863,7 @@ func (node *LocalVariableLvalueNode) Unassign( } func (node *LocalVariableLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { if indices == nil { @@ -1006,10 +1006,10 @@ func NewIndexedLvalueNode( // '$x[1][2] = 3' or '@x[1][2] = 3', the indices are [1,2], and the baseLvalue // is '$x' or '@x' respectively. func (node *IndexedLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { - indices := make([]*types.Mlrval, len(node.indexEvaluables)) + indices := make([]*mlrval.Mlrval, len(node.indexEvaluables)) for i := range node.indexEvaluables { indices[i] = node.indexEvaluables[i].Evaluate(state) @@ -1021,15 +1021,15 @@ func (node *IndexedLvalueNode) Assign( // This lets the user do '$y[ ["a", "b", "c"] ] = $x' in lieu of // '$y["a"]["b"]["c"] = $x'. if len(indices) == 1 && indices[0].IsArray() { - indices = types.MakePointerArray(indices[0].GetArray()) + indices = mlrval.MakePointerArray(indices[0].GetArray()) } return node.baseLvalue.AssignIndexed(rvalue, indices, state) } func (node *IndexedLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { // We are the delegator, not the delegatee @@ -1040,7 +1040,7 @@ func (node *IndexedLvalueNode) AssignIndexed( func (node *IndexedLvalueNode) Unassign( state *runtime.State, ) { - indices := make([]*types.Mlrval, len(node.indexEvaluables)) + indices := make([]*mlrval.Mlrval, len(node.indexEvaluables)) for i := range node.indexEvaluables { indices[i] = node.indexEvaluables[i].Evaluate(state) } @@ -1049,7 +1049,7 @@ func (node *IndexedLvalueNode) Unassign( } func (node *IndexedLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { // We are the delegator, not the delegatee @@ -1082,7 +1082,7 @@ func NewEnvironmentVariableLvalueNode( } func (node *EnvironmentVariableLvalueNode) Assign( - rvalue *types.Mlrval, + rvalue *mlrval.Mlrval, state *runtime.State, ) error { // AssignmentNode checks for absentness of the rvalue, so we just assign @@ -1114,8 +1114,8 @@ func (node *EnvironmentVariableLvalueNode) Assign( } func (node *EnvironmentVariableLvalueNode) AssignIndexed( - rvalue *types.Mlrval, - indices []*types.Mlrval, + rvalue *mlrval.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) error { return errors.New("mlr: ENV[...] cannot be indexed.") @@ -1138,7 +1138,7 @@ func (node *EnvironmentVariableLvalueNode) Unassign( } func (node *EnvironmentVariableLvalueNode) UnassignIndexed( - indices []*types.Mlrval, + indices []*mlrval.Mlrval, state *runtime.State, ) { // TODO: needs error return diff --git a/internal/pkg/dsl/cst/root.go b/internal/pkg/dsl/cst/root.go index 350efd720..27636bc02 100644 --- a/internal/pkg/dsl/cst/root.go +++ b/internal/pkg/dsl/cst/root.go @@ -14,11 +14,11 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/dsl" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/output" "github.com/johnkerl/miller/internal/pkg/parsing/lexer" "github.com/johnkerl/miller/internal/pkg/parsing/parser" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // NewEmptyRoot sets up an empty CST, before ingesting any DSL strings. For @@ -464,7 +464,7 @@ func (root *RootNode) ExecuteBeginBlocks(state *runtime.State) error { return nil } -func (root *RootNode) ExecuteMainBlock(state *runtime.State) (outrec *types.Mlrmap, err error) { +func (root *RootNode) ExecuteMainBlock(state *runtime.State) (outrec *mlrval.Mlrmap, err error) { _, err = root.mainBlock.Execute(state) return state.Inrec, err } @@ -488,7 +488,7 @@ func (root *RootNode) ExecuteEndBlocks(state *runtime.State) error { // executed once, and then discarded. // This is the 'execute once' part of that. -func (root *RootNode) ExecuteREPLImmediate(state *runtime.State) (outrec *types.Mlrmap, err error) { +func (root *RootNode) ExecuteREPLImmediate(state *runtime.State) (outrec *mlrval.Mlrmap, err error) { _, err = root.replImmediateBlock.ExecuteFrameless(state) return state.Inrec, err } diff --git a/internal/pkg/dsl/cst/tee.go b/internal/pkg/dsl/cst/tee.go index 2f2fdfdbf..c433bce02 100644 --- a/internal/pkg/dsl/cst/tee.go +++ b/internal/pkg/dsl/cst/tee.go @@ -10,6 +10,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/output" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" @@ -58,7 +59,7 @@ import ( // ================================================================ type tTeeToRedirectFunc func( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, state *runtime.State, ) error @@ -156,7 +157,7 @@ func (node *TeeStatementNode) Execute(state *runtime.State) (*BlockExitPayload, // ---------------------------------------------------------------- func (node *TeeStatementNode) teeToFileOrPipe( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, state *runtime.State, ) error { redirectorTarget := node.redirectorTargetEvaluable.Evaluate(state) diff --git a/internal/pkg/dsl/cst/types.go b/internal/pkg/dsl/cst/types.go index f5ad98530..85dd7e52a 100644 --- a/internal/pkg/dsl/cst/types.go +++ b/internal/pkg/dsl/cst/types.go @@ -9,8 +9,8 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/dsl" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" - "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- @@ -67,16 +67,16 @@ type Executor func(state *runtime.State) (*BlockExitPayload, error) // ================================================================ // This is for any left-hand side (LHS or Lvalue) of an assignment statement. type IAssignable interface { - Assign(rvalue *types.Mlrval, state *runtime.State) error + Assign(rvalue *mlrval.Mlrval, state *runtime.State) error // 'foo = "bar"' or 'foo[3]["abc"] = "bar"' // For non-indexed assignment, which is the normal case, indices can be // zero-length or nil. - AssignIndexed(rvalue *types.Mlrval, indices []*types.Mlrval, state *runtime.State) error + AssignIndexed(rvalue *mlrval.Mlrval, indices []*mlrval.Mlrval, state *runtime.State) error Unassign(state *runtime.State) - UnassignIndexed(indices []*types.Mlrval, state *runtime.State) + UnassignIndexed(indices []*mlrval.Mlrval, state *runtime.State) } // ================================================================ @@ -84,7 +84,7 @@ type IAssignable interface { // Also, for computed field names on the left-hand side, like '$a . $b' in mlr // put '$[$a . $b]' = $x + $y'. type IEvaluable interface { - Evaluate(state *runtime.State) *types.Mlrval + Evaluate(state *runtime.State) *mlrval.Mlrval } // ================================================================ @@ -119,5 +119,5 @@ type BlockExitPayload struct { blockExitStatus BlockExitStatus // No multiple return yet in the Miller DSL -- if there were, this would be // an array. - blockReturnValue *types.Mlrval // TODO: TypeGatedMlrval + blockReturnValue *mlrval.Mlrval // TODO: TypeGatedMlrval } diff --git a/internal/pkg/dsl/cst/udf.go b/internal/pkg/dsl/cst/udf.go index b7f069665..734f0f72d 100644 --- a/internal/pkg/dsl/cst/udf.go +++ b/internal/pkg/dsl/cst/udf.go @@ -11,6 +11,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -83,7 +84,7 @@ func NewUDFCallsite( // NewUDFCallsiteForHigherOrderFunction is for UDF callsites such as // sortaf/sortmf. Here, the array/map to be sorted has already been evaluated -// and is an array of *types.Mlrval. The UDF needs to be invoked on pairs of +// and is an array of *mlrval.Mlrval. The UDF needs to be invoked on pairs of // array elements. func NewUDFCallsiteForHigherOrderFunction( udf *UDF, @@ -123,7 +124,7 @@ func (site *UDFCallsite) findUDF(state *runtime.State) *UDF { // See comments above NewUDFCallsite. func (site *UDFCallsite) Evaluate( state *runtime.State, -) *types.Mlrval { +) *mlrval.Mlrval { udf := site.findUDF(state) if udf == nil { @@ -188,7 +189,7 @@ func (site *UDFCallsite) Evaluate( os.Exit(1) } - arguments := make([]*types.Mlrval, numArguments) + arguments := make([]*mlrval.Mlrval, numArguments) for i := range udf.signature.typeGatedParameterNames { arguments[i] = site.argumentNodes[i].Evaluate(state) @@ -210,8 +211,8 @@ func (site *UDFCallsite) Evaluate( func (site *UDFCallsite) EvaluateWithArguments( state *runtime.State, udf *UDF, - arguments []*types.Mlrval, -) *types.Mlrval { + arguments []*mlrval.Mlrval, +) *mlrval.Mlrval { // Bind the arguments to the parameters. Function literals can access // locals in their enclosing scope; named functions cannot. Hence stack @@ -250,34 +251,34 @@ func (site *UDFCallsite) EvaluateWithArguments( // being MT_ERROR should be mapped to MT_ERROR here (nominally, // data-dependent). But error-return could be something not data-dependent. if err != nil { - err = udf.signature.typeGatedReturnValue.Check(types.MLRVAL_ERROR) + err = udf.signature.typeGatedReturnValue.Check(mlrval.ERROR) if err != nil { fmt.Fprint(os.Stderr, err) os.Exit(1) } - return types.MLRVAL_ERROR + return mlrval.ERROR } // Fell off end of function with no return if blockExitPayload == nil { - err = udf.signature.typeGatedReturnValue.Check(types.MLRVAL_ABSENT) + err = udf.signature.typeGatedReturnValue.Check(mlrval.ABSENT) if err != nil { fmt.Fprint(os.Stderr, err) os.Exit(1) } - return types.MLRVAL_ABSENT + return mlrval.ABSENT } // TODO: should be an internal coding error. This would be break or // continue not in a loop, or return-void, both of which should have been // reported as syntax errors during the parsing pass. if blockExitPayload.blockExitStatus != BLOCK_EXIT_RETURN_VALUE { - err = udf.signature.typeGatedReturnValue.Check(types.MLRVAL_ABSENT) + err = udf.signature.typeGatedReturnValue.Check(mlrval.ABSENT) if err != nil { fmt.Fprint(os.Stderr, err) os.Exit(1) } - return types.MLRVAL_ABSENT + return mlrval.ABSENT } // Definitely a Miller internal coding error if the user put 'return x' in @@ -434,7 +435,7 @@ func genFunctionLiteralName() string { // UnnamedUDFNode holds function literals like 'func (a, b) { return b - a }'. type UnnamedUDFNode struct { - udfAsMlrval *types.Mlrval + udfAsMlrval *mlrval.Mlrval } func (root *RootNode) BuildUnnamedUDFNode(astNode *dsl.ASTNode) (IEvaluable, error) { @@ -447,14 +448,14 @@ func (root *RootNode) BuildUnnamedUDFNode(astNode *dsl.ASTNode) (IEvaluable, err return nil, err } - udfAsMlrval := types.MlrvalFromFunction(udf, name) + udfAsMlrval := mlrval.FromFunction(udf, name) return &UnnamedUDFNode{ udfAsMlrval: udfAsMlrval, }, nil } -func (node *UnnamedUDFNode) Evaluate(state *runtime.State) *types.Mlrval { +func (node *UnnamedUDFNode) Evaluate(state *runtime.State) *mlrval.Mlrval { return node.udfAsMlrval } diff --git a/internal/pkg/dsl/cst/uds.go b/internal/pkg/dsl/cst/uds.go index 394c3fd88..7bc8ec91a 100644 --- a/internal/pkg/dsl/cst/uds.go +++ b/internal/pkg/dsl/cst/uds.go @@ -10,6 +10,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/runtime" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -106,7 +107,7 @@ func (site *UDSCallsite) Execute(state *runtime.State) (*BlockExitPayload, error // we push a new frameset and DefineTypedAtScope using the callee's frameset. // Evaluate the arguments - arguments := make([]*types.Mlrval, len(site.uds.signature.typeGatedParameterNames)) + arguments := make([]*mlrval.Mlrval, len(site.uds.signature.typeGatedParameterNames)) for i, typeGatedParameterName := range site.uds.signature.typeGatedParameterNames { arguments[i] = site.argumentNodes[i].Evaluate(state) diff --git a/internal/pkg/input/pseudo_reader_gen.go b/internal/pkg/input/pseudo_reader_gen.go index 143b2f5e9..21172583d 100644 --- a/internal/pkg/input/pseudo_reader_gen.go +++ b/internal/pkg/input/pseudo_reader_gen.go @@ -5,7 +5,9 @@ import ( "errors" "fmt" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -62,9 +64,9 @@ func (reader *PseudoReaderGen) process( return } - var doneComparator types.BinaryFunc = types.BIF_greater_than + var doneComparator mlrval.CmpFuncBool = mlrval.GreaterThan if step.GetNumericNegativeorDie() { - doneComparator = types.BIF_less_than + doneComparator = mlrval.LessThan } key := reader.readerOptions.GeneratorOptions.FieldName @@ -75,13 +77,11 @@ func (reader *PseudoReaderGen) process( eof := false for !eof { - mdone := doneComparator(value, stop) - done, _ := mdone.GetBoolValue() - if done { + if doneComparator(value, stop) { break } - record := types.NewMlrmap() + record := mlrval.NewMlrmap() record.PutCopy(key, value) context.UpdateForInputRecord() @@ -109,7 +109,7 @@ func (reader *PseudoReaderGen) process( } - value = types.BIF_plus_binary(value, step) + value = bifs.BIF_plus_binary(value, step) } if recordsAndContexts.Len() > 0 { @@ -121,8 +121,8 @@ func (reader *PseudoReaderGen) process( func (reader *PseudoReaderGen) tryParse( name string, svalue string, -) (*types.Mlrval, error) { - mvalue := types.MlrvalFromInferredType(svalue) +) (*mlrval.Mlrval, error) { + mvalue := mlrval.FromDeferredType(svalue) if mvalue == nil || !mvalue.IsNumeric() { return nil, errors.New( fmt.Sprintf("mlr: gen: %s \"%s\" is not parseable as number", name, svalue), diff --git a/internal/pkg/input/record_reader.go b/internal/pkg/input/record_reader.go index f8a140ae1..bf6542314 100644 --- a/internal/pkg/input/record_reader.go +++ b/internal/pkg/input/record_reader.go @@ -14,7 +14,7 @@ const CSV_BOM = "\xef\xbb\xbf" // FILENAME, NF, NF, FNR, etc.) needs to be duplicated and passed through the // channels along with each record. Hence the initial context, which readers // update on each new file/record, and the channel of types.RecordAndContext -// rather than channel of types.Mlrmap. +// rather than channel of mlrval.Mlrmap. type IRecordReader interface { Read( diff --git a/internal/pkg/input/record_reader_csv.go b/internal/pkg/input/record_reader_csv.go index a961e057c..05c945c11 100644 --- a/internal/pkg/input/record_reader_csv.go +++ b/internal/pkg/input/record_reader_csv.go @@ -12,6 +12,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -27,7 +28,6 @@ type RecordReaderCSV struct { header []string } -// ---------------------------------------------------------------- func NewRecordReaderCSV( readerOptions *cli.TReaderOptions, recordsPerBatch int, @@ -45,7 +45,6 @@ func NewRecordReaderCSV( }, nil } -// ---------------------------------------------------------------- func (reader *RecordReaderCSV) Read( filenames []string, context types.Context, @@ -218,7 +217,7 @@ func (reader *RecordReaderCSV) getRecordBatch( } } - record := types.NewMlrmapAsRecord() + record := mlrval.NewMlrmapAsRecord() nh := len(reader.header) nd := len(csvRecord) @@ -226,7 +225,7 @@ func (reader *RecordReaderCSV) getRecordBatch( if nh == nd { for i := 0; i < nh; i++ { key := reader.header[i] - value := types.MlrvalFromInferredTypeForDataFiles(csvRecord[i]) + value := mlrval.FromDeferredType(csvRecord[i]) record.PutReference(key, value) } @@ -246,19 +245,19 @@ func (reader *RecordReaderCSV) getRecordBatch( n := lib.IntMin2(nh, nd) for i = 0; i < n; i++ { key := reader.header[i] - value := types.MlrvalFromInferredTypeForDataFiles(csvRecord[i]) + value := mlrval.FromDeferredType(csvRecord[i]) record.PutReference(key, value) } if nh < nd { // if header shorter than data: use 1-up itoa keys key := strconv.Itoa(i + 1) - value := types.MlrvalFromInferredTypeForDataFiles(csvRecord[i]) + value := mlrval.FromDeferredType(csvRecord[i]) record.PutCopy(key, value) } if nh > nd { // if header longer than data: use "" values for i = nd; i < nh; i++ { - record.PutCopy(reader.header[i], types.MLRVAL_VOID) + record.PutCopy(reader.header[i], mlrval.VOID) } } } diff --git a/internal/pkg/input/record_reader_csvlite.go b/internal/pkg/input/record_reader_csvlite.go index b9f1c65d2..6e6939400 100644 --- a/internal/pkg/input/record_reader_csvlite.go +++ b/internal/pkg/input/record_reader_csvlite.go @@ -28,6 +28,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -242,10 +243,10 @@ func getRecordBatchExplicitCSVHeader( return } - record := types.NewMlrmapAsRecord() + record := mlrval.NewMlrmapAsRecord() if !reader.readerOptions.AllowRaggedCSVInput { for i, field := range fields { - value := types.MlrvalFromInferredTypeForDataFiles(field) + value := mlrval.FromDeferredType(field) record.PutCopy(reader.headerStrings[i], value) } } else { @@ -254,21 +255,21 @@ func getRecordBatchExplicitCSVHeader( n := lib.IntMin2(nh, nd) var i int for i = 0; i < n; i++ { - value := types.MlrvalFromInferredTypeForDataFiles(fields[i]) + value := mlrval.FromDeferredType(fields[i]) record.PutCopy(reader.headerStrings[i], value) } if nh < nd { // if header shorter than data: use 1-up itoa keys for i = nh; i < nd; i++ { key := strconv.Itoa(i + 1) - value := types.MlrvalFromInferredTypeForDataFiles(fields[i]) + value := mlrval.FromDeferredType(fields[i]) record.PutCopy(key, value) } } if nh > nd { // if header longer than data: use "" values for i = nd; i < nh; i++ { - record.PutCopy(reader.headerStrings[i], types.MLRVAL_VOID) + record.PutCopy(reader.headerStrings[i], mlrval.VOID) } } } @@ -359,10 +360,10 @@ func getRecordBatchImplicitCSVHeader( } } - record := types.NewMlrmapAsRecord() + record := mlrval.NewMlrmapAsRecord() if !reader.readerOptions.AllowRaggedCSVInput { for i, field := range fields { - value := types.MlrvalFromInferredTypeForDataFiles(field) + value := mlrval.FromDeferredType(field) record.PutCopy(reader.headerStrings[i], value) } } else { @@ -371,19 +372,19 @@ func getRecordBatchImplicitCSVHeader( n := lib.IntMin2(nh, nd) var i int for i = 0; i < n; i++ { - value := types.MlrvalFromInferredTypeForDataFiles(fields[i]) + value := mlrval.FromDeferredType(fields[i]) record.PutCopy(reader.headerStrings[i], value) } if nh < nd { // if header shorter than data: use 1-up itoa keys key := strconv.Itoa(i + 1) - value := types.MlrvalFromInferredTypeForDataFiles(fields[i]) + value := mlrval.FromDeferredType(fields[i]) record.PutCopy(key, value) } if nh > nd { // if header longer than data: use "" values for i = nd; i < nh; i++ { - record.PutCopy(reader.headerStrings[i], types.MLRVAL_VOID) + record.PutCopy(reader.headerStrings[i], mlrval.VOID) } } } diff --git a/internal/pkg/input/record_reader_dkvp_nidx.go b/internal/pkg/input/record_reader_dkvp_nidx.go index c2e3140ed..6c6039f9f 100644 --- a/internal/pkg/input/record_reader_dkvp_nidx.go +++ b/internal/pkg/input/record_reader_dkvp_nidx.go @@ -10,12 +10,13 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) // splitter_DKVP_NIDX is a function type for the one bit of code differing // between the DKVP reader and the NIDX reader, namely, how it splits lines. -type splitter_DKVP_NIDX func(reader *RecordReaderDKVPNIDX, line string) *types.Mlrmap +type splitter_DKVP_NIDX func(reader *RecordReaderDKVPNIDX, line string) *mlrval.Mlrmap type RecordReaderDKVPNIDX struct { readerOptions *cli.TReaderOptions @@ -150,8 +151,8 @@ func (reader *RecordReaderDKVPNIDX) getRecordBatch( return recordsAndContexts, false } -func recordFromDKVPLine(reader *RecordReaderDKVPNIDX, line string) *types.Mlrmap { - record := types.NewMlrmapAsRecord() +func recordFromDKVPLine(reader *RecordReaderDKVPNIDX, line string) *mlrval.Mlrmap { + record := mlrval.NewMlrmapAsRecord() var pairs []string // TODO: func-pointer this away @@ -179,19 +180,19 @@ func recordFromDKVPLine(reader *RecordReaderDKVPNIDX, line string) *types.Mlrmap // "a=". Here we use the positional index as the key. This way // DKVP is a generalization of NIDX. key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := types.MlrvalFromInferredTypeForDataFiles(kv[0]) + value := mlrval.FromDeferredType(kv[0]) record.PutReference(key, value) } else { key := kv[0] - value := types.MlrvalFromInferredTypeForDataFiles(kv[1]) + value := mlrval.FromDeferredType(kv[1]) record.PutReference(key, value) } } return record } -func recordFromNIDXLine(reader *RecordReaderDKVPNIDX, line string) *types.Mlrmap { - record := types.NewMlrmapAsRecord() +func recordFromNIDXLine(reader *RecordReaderDKVPNIDX, line string) *mlrval.Mlrmap { + record := mlrval.NewMlrmapAsRecord() var values []string // TODO: func-pointer this away @@ -208,7 +209,7 @@ func recordFromNIDXLine(reader *RecordReaderDKVPNIDX, line string) *types.Mlrmap for _, value := range values { i++ key := strconv.Itoa(i) - mval := types.MlrvalFromInferredTypeForDataFiles(value) + mval := mlrval.FromDeferredType(value) record.PutReference(key, mval) } return record diff --git a/internal/pkg/input/record_reader_dkvp_test.go b/internal/pkg/input/record_reader_dkvp_test.go new file mode 100644 index 000000000..33e9aa194 --- /dev/null +++ b/internal/pkg/input/record_reader_dkvp_test.go @@ -0,0 +1,59 @@ +package input + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/johnkerl/miller/internal/pkg/cli" +) + +func TestRecordFromDKVPLine(t *testing.T) { + readerOptions := cli.DefaultReaderOptions() + cli.FinalizeReaderOptions(&readerOptions) // compute IPS, IFS -> IPSRegex, IFSRegex + reader, err := NewRecordReaderDKVP(&readerOptions, 1) + assert.NotNil(t, reader) + assert.Nil(t, err) + + line := "" + record := recordFromDKVPLine(reader, line) + assert.NotNil(t, record) + assert.Equal(t, 0, record.FieldCount) + + line = "a=1,b=2,c=3" + record = recordFromDKVPLine(reader, line) + assert.NotNil(t, record) + assert.Equal(t, 3, record.FieldCount) + + assert.NotNil(t, record.Head) + assert.NotNil(t, record.Head.Next) + assert.NotNil(t, record.Head.Next.Next) + assert.Nil(t, record.Head.Next.Next.Next) + assert.Equal(t, record.Head.Key, "a") + assert.Equal(t, record.Head.Next.Key, "b") + assert.Equal(t, record.Head.Next.Next.Key, "c") + + line = "a=1,b=2,b=3" + record = recordFromDKVPLine(reader, line) + assert.NotNil(t, record) + assert.Equal(t, 2, record.FieldCount) + + assert.NotNil(t, record.Head) + assert.NotNil(t, record.Head.Next) + assert.Nil(t, record.Head.Next.Next) + assert.Equal(t, record.Head.Key, "a") + assert.Equal(t, record.Head.Next.Key, "b") + + line = "a,b,c" + record = recordFromDKVPLine(reader, line) + assert.NotNil(t, record) + assert.Equal(t, 3, record.FieldCount) + + assert.NotNil(t, record.Head) + assert.NotNil(t, record.Head.Next) + assert.NotNil(t, record.Head.Next.Next) + assert.Nil(t, record.Head.Next.Next.Next) + assert.Equal(t, record.Head.Key, "1") + assert.Equal(t, record.Head.Next.Key, "2") + assert.Equal(t, record.Head.Next.Next.Key, "3") +} diff --git a/internal/pkg/input/record_reader_json.go b/internal/pkg/input/record_reader_json.go index 13f76222e..7084e95c8 100644 --- a/internal/pkg/input/record_reader_json.go +++ b/internal/pkg/input/record_reader_json.go @@ -12,6 +12,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -107,7 +108,7 @@ func (reader *RecordReaderJSON) processHandle( } } - mlrval, eof, err := types.MlrvalDecodeFromJSON(decoder) + mlrval, eof, err := mlrval.MlrvalDecodeFromJSON(decoder) if eof { break } diff --git a/internal/pkg/input/record_reader_xtab.go b/internal/pkg/input/record_reader_xtab.go index 8213916ae..ec7530e58 100644 --- a/internal/pkg/input/record_reader_xtab.go +++ b/internal/pkg/input/record_reader_xtab.go @@ -9,6 +9,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -254,8 +255,8 @@ func (reader *RecordReaderXTAB) getRecordBatch( func (reader *RecordReaderXTAB) recordFromXTABLines( stanza *list.List, -) (*types.Mlrmap, error) { - record := types.NewMlrmapAsRecord() +) (*mlrval.Mlrmap, error) { + record := mlrval.NewMlrmapAsRecord() for e := stanza.Front(); e != nil; e = e.Next() { line := e.Value.(string) @@ -272,10 +273,10 @@ func (reader *RecordReaderXTAB) recordFromXTABLines( key := kv[0] if len(kv) == 1 { - value := types.MLRVAL_VOID + value := mlrval.VOID record.PutReference(key, value) } else { - value := types.MlrvalFromInferredTypeForDataFiles(kv[1]) + value := mlrval.FromDeferredType(kv[1]) record.PutReference(key, value) } } diff --git a/internal/pkg/lib/util.go b/internal/pkg/lib/util.go index c25b25597..232329180 100644 --- a/internal/pkg/lib/util.go +++ b/internal/pkg/lib/util.go @@ -114,7 +114,7 @@ func TryIntFromString(input string) (int, bool) { return 0, false } -func TryFloat64FromString(input string) (float64, bool) { +func TryFloatFromString(input string) (float64, bool) { fval, err := strconv.ParseFloat(input, 64) if err == nil { return fval, true diff --git a/internal/pkg/types/mlrmap.go b/internal/pkg/mlrval/mlrmap.go similarity index 99% rename from internal/pkg/types/mlrmap.go rename to internal/pkg/mlrval/mlrmap.go index be344050c..28e571df0 100644 --- a/internal/pkg/types/mlrmap.go +++ b/internal/pkg/mlrval/mlrmap.go @@ -51,7 +51,7 @@ // trivial. // ================================================================ -package types +package mlrval // For the C port having this off was a noticeable performance improvement (10-15%). // For the Go port having it off is a less-noticeable performance improvement (5%). diff --git a/internal/pkg/types/mlrmap_accessors.go b/internal/pkg/mlrval/mlrmap_accessors.go similarity index 92% rename from internal/pkg/types/mlrmap_accessors.go rename to internal/pkg/mlrval/mlrmap_accessors.go index 445d5865e..5fccacbae 100644 --- a/internal/pkg/types/mlrmap_accessors.go +++ b/internal/pkg/mlrval/mlrmap_accessors.go @@ -1,4 +1,4 @@ -package types +package mlrval import ( "bytes" @@ -12,12 +12,47 @@ func (mlrmap *Mlrmap) IsEmpty() bool { return mlrmap.Head == nil } -// ---------------------------------------------------------------- func (mlrmap *Mlrmap) Has(key string) bool { return mlrmap.findEntry(key) != nil } -// PutCpoy copies the key and value (deep-copying in case the value is array/map). +func (mlrmap *Mlrmap) Get(key string) *Mlrval { + pe := mlrmap.findEntry(key) + if pe == nil { + return nil + } else { + return pe.Value + } + return nil +} + +// PutReference copies the key but not the value. This is not safe for DSL use, +// where we could create undesired references between different objects. Only +// intended to be used at callsites which allocate a mlrval solely for the +// purpose of putting into a map, e.g. input-record readers. +func (mlrmap *Mlrmap) PutReference(key string, value *Mlrval) { + pe := mlrmap.findEntry(key) + if pe == nil { + pe = newMlrmapEntry(key, value) + if mlrmap.Head == nil { + mlrmap.Head = pe + mlrmap.Tail = pe + } else { + pe.Prev = mlrmap.Tail + pe.Next = nil + mlrmap.Tail.Next = pe + mlrmap.Tail = pe + } + if mlrmap.keysToEntries != nil { + mlrmap.keysToEntries[key] = pe + } + mlrmap.FieldCount++ + } else { + pe.Value = value + } +} + +// PutCopy copies the key and value (deep-copying in case the value is array/map). // This is safe for DSL use. See also PutReference. func (mlrmap *Mlrmap) PutCopy(key string, value *Mlrval) { pe := mlrmap.findEntry(key) @@ -41,22 +76,19 @@ func (mlrmap *Mlrmap) PutCopy(key string, value *Mlrval) { } } -// PutReference copies the key but not the value. This is not safe for DSL use, -// where we could create undesired references between different objects. Only -// intended to be used at callsites which allocate a mlrval solely for the -// purpose of putting into a map, e.g. input-record readers. -func (mlrmap *Mlrmap) PutReference(key string, value *Mlrval) { +// PrependReference is the same as PutReference, but puts a new entry first, not last. +func (mlrmap *Mlrmap) PrependReference(key string, value *Mlrval) { pe := mlrmap.findEntry(key) if pe == nil { pe = newMlrmapEntry(key, value) - if mlrmap.Head == nil { + if mlrmap.Tail == nil { mlrmap.Head = pe mlrmap.Tail = pe } else { - pe.Prev = mlrmap.Tail - pe.Next = nil - mlrmap.Tail.Next = pe - mlrmap.Tail = pe + pe.Prev = nil + pe.Next = mlrmap.Head + mlrmap.Head.Prev = pe + mlrmap.Head = pe } if mlrmap.keysToEntries != nil { mlrmap.keysToEntries[key] = pe @@ -123,11 +155,61 @@ func (mlrmap *Mlrmap) PutReferenceAfter( } } -func (mlrmap *Mlrmap) PutCopyWithMlrvalIndex(key *Mlrval, value *Mlrval) error { - if key.mvtype == MT_STRING { - mlrmap.PutCopy(key.printrep, value) +// findEntry is the basic hash-map accessor for Has, Put, Get, Remove, etc. +// Hashmaps in Go are a built-in type; Mlrmap is (a) +// insertion-order-preserving, and (b) supports either hashed or non-hashed +// access. +func (mlrmap *Mlrmap) findEntry(key string) *MlrmapEntry { + if mlrmap.keysToEntries != nil { + return mlrmap.keysToEntries[key] + } else { + for pe := mlrmap.Head; pe != nil; pe = pe.Next { + if pe.Key == key { + return pe + } + } return nil - } else if key.mvtype == MT_INT { + } +} + +// findEntryByPositionalIndex is for '$[1]' etc. in the DSL. +// +// Notes: +// * This is a linear search. +// * Indices are 1-up not 0-up +// * Indices -n..-1 are aliases for 1..n. In particular, it will be faster to +// get the -1st field than the nth. +// * Returns 0 on invalid index: 0, or < -n, or > n where n is the number of +// fields. +func (mlrmap *Mlrmap) findEntryByPositionalIndex(position int) *MlrmapEntry { + if position > mlrmap.FieldCount || position < -mlrmap.FieldCount || position == 0 { + return nil + } + if position > 0 { + var i int = 1 + for pe := mlrmap.Head; pe != nil; pe = pe.Next { + if i == position { + return pe + } + i++ + } + lib.InternalCodingErrorIf(true) + } else { + var i int = -1 + for pe := mlrmap.Tail; pe != nil; pe = pe.Prev { + if i == position { + return pe + } + i-- + } + lib.InternalCodingErrorIf(true) + } + lib.InternalCodingErrorIf(true) + return nil +} + +func (mlrmap *Mlrmap) PutCopyWithMlrvalIndex(key *Mlrval, value *Mlrval) error { + if key.IsStringOrInt() { mlrmap.PutCopy(key.String(), value) return nil } else { @@ -142,29 +224,6 @@ func (mlrmap *Mlrmap) PrependCopy(key string, value *Mlrval) { mlrmap.PrependReference(key, value.Copy()) } -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) PrependReference(key string, value *Mlrval) { - pe := mlrmap.findEntry(key) - if pe == nil { - pe = newMlrmapEntry(key, value) - if mlrmap.Tail == nil { - mlrmap.Head = pe - mlrmap.Tail = pe - } else { - pe.Prev = nil - pe.Next = mlrmap.Head - mlrmap.Head.Prev = pe - mlrmap.Head = pe - } - if mlrmap.keysToEntries != nil { - mlrmap.keysToEntries[key] = pe - } - mlrmap.FieldCount++ - } else { - pe.Value = value - } -} - // ---------------------------------------------------------------- // Merges that into mlrmap. func (mlrmap *Mlrmap) Merge(other *Mlrmap) { @@ -173,24 +232,12 @@ func (mlrmap *Mlrmap) Merge(other *Mlrmap) { } } -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) Get(key string) *Mlrval { - pe := mlrmap.findEntry(key) - if pe == nil { - return nil - } else { - return pe.Value - } - return nil -} - // ---------------------------------------------------------------- // Exposed for the 'nest' verb func (mlrmap *Mlrmap) GetEntry(key string) *MlrmapEntry { return mlrmap.findEntry(key) } -// ---------------------------------------------------------------- func (mlrmap *Mlrmap) GetKeys() []string { keys := make([]string, mlrmap.FieldCount) i := 0 @@ -202,15 +249,40 @@ func (mlrmap *Mlrmap) GetKeys() []string { } // ---------------------------------------------------------------- -// For '$[[[1]]]' etc. in the DSL. -// -// Notes: -// * This is a linear search. -// * Indices are 1-up not 0-up -// * Indices -n..-1 are aliases for 1..n. In particular, it will be faster to -// get the -1st field than the nth. -// * Returns 0 on invalid index: 0, or < -n, or > n where n is the number of -// fields. +// TODO: put error-return into this API +func (mlrmap *Mlrmap) PutNameWithPositionalIndex(position int, name *Mlrval) { + positionalEntry := mlrmap.findEntryByPositionalIndex(position) + + if positionalEntry == nil { + // TODO: handle out-of-bounds accesses + return + } + + s := "" + if name.IsString() || name.IsInt() { + s = name.String() + } else { + // TODO: return MlrvalFromError() + return + } + + // E.g. there are fields named 'a' and 'b', as positions 1 and 2, + // and the user does '$[[1]] = $[[2]]'. Then there would be two b's. + mapEntry := mlrmap.findEntry(s) + if mapEntry != nil && mapEntry != positionalEntry { + if mlrmap.keysToEntries != nil { + delete(mlrmap.keysToEntries, positionalEntry.Key) + } + mlrmap.Unlink(mapEntry) + } + + lib.InternalCodingErrorIf(s == "") + positionalEntry.Key = s + + if mlrmap.keysToEntries != nil { + mlrmap.keysToEntries[s] = positionalEntry + } +} func (mlrmap *Mlrmap) GetWithPositionalIndex(position int) *Mlrval { mapEntry := mlrmap.findEntryByPositionalIndex(position) @@ -221,7 +293,7 @@ func (mlrmap *Mlrmap) GetWithPositionalIndex(position int) *Mlrval { } func (mlrmap *Mlrmap) GetWithMlrvalIndex(index *Mlrval) (*Mlrval, error) { - if index.mvtype == MT_ARRAY { + if index.IsArray() { return mlrmap.getWithMlrvalArrayIndex(index) } else { return mlrmap.getWithMlrvalSingleIndex(index) @@ -233,7 +305,7 @@ func (mlrmap *Mlrmap) GetWithMlrvalIndex(index *Mlrval) (*Mlrval, error) { func (mlrmap *Mlrmap) getWithMlrvalArrayIndex(index *Mlrval) (*Mlrval, error) { current := mlrmap var retval *Mlrval = nil - lib.InternalCodingErrorIf(index.mvtype != MT_ARRAY) + lib.InternalCodingErrorIf(!index.IsArray()) array := index.arrayval n := len(array) for i, piece := range array { @@ -257,9 +329,9 @@ func (mlrmap *Mlrmap) getWithMlrvalArrayIndex(index *Mlrval) (*Mlrval, error) { } func (mlrmap *Mlrmap) getWithMlrvalSingleIndex(index *Mlrval) (*Mlrval, error) { - if index.mvtype == MT_STRING { + if index.IsString() { return mlrmap.Get(index.printrep), nil - } else if index.mvtype == MT_INT { + } else if index.IsInt() { return mlrmap.Get(index.String()), nil } else { return nil, errors.New( @@ -286,44 +358,6 @@ func (mlrmap *Mlrmap) GetNameAtPositionalIndex(position int) (string, bool) { return mapEntry.Key, true } -// ---------------------------------------------------------------- -// TODO: put error-return into this API -func (mlrmap *Mlrmap) PutNameWithPositionalIndex(position int, name *Mlrval) { - positionalEntry := mlrmap.findEntryByPositionalIndex(position) - - if positionalEntry == nil { - // TODO: handle out-of-bounds accesses - return - } - - s := "" - if name.mvtype == MT_STRING { - s = name.printrep - } else if name.mvtype == MT_INT { - s = name.String() - } else { - // TODO: return MlrvalFromError() - return - } - - // E.g. there are fields named 'a' and 'b', as positions 1 and 2, - // and the user does '$[[1]] = $[[2]]'. Then there would be two b's. - mapEntry := mlrmap.findEntry(s) - if mapEntry != nil && mapEntry != positionalEntry { - if mlrmap.keysToEntries != nil { - delete(mlrmap.keysToEntries, positionalEntry.Key) - } - mlrmap.Unlink(mapEntry) - } - - lib.InternalCodingErrorIf(s == "") - positionalEntry.Key = s - - if mlrmap.keysToEntries != nil { - mlrmap.keysToEntries[s] = positionalEntry - } -} - // ---------------------------------------------------------------- // Copies the key and value (deep-copying in case the value is array/map). // This is safe for DSL use. See also PutReference. @@ -371,17 +405,13 @@ func (mlrmap *Mlrmap) Contains(other *Mlrmap) bool { } thisval := mlrmap.Get(pe.Key) thatval := pe.Value - meq := BIF_equals(thisval, thatval) - eq, ok := meq.GetBoolValue() - lib.InternalCodingErrorIf(!ok) - if !eq { + if !Equals(thisval, thatval) { return false } } return true } -// ---------------------------------------------------------------- func (mlrmap *Mlrmap) Clear() { mlrmap.FieldCount = 0 // Assuming everything unreferenced is getting GC'ed by the Go runtime @@ -401,7 +431,6 @@ func (mlrmap *Mlrmap) Copy() *Mlrmap { return other } -// ---------------------------------------------------------------- // Returns true if it was found and removed func (mlrmap *Mlrmap) Remove(key string) bool { pe := mlrmap.findEntry(key) @@ -413,7 +442,6 @@ func (mlrmap *Mlrmap) Remove(key string) bool { } } -// ---------------------------------------------------------------- func (mlrmap *Mlrmap) MoveToHead(key string) { pe := mlrmap.findEntry(key) if pe != nil { @@ -422,7 +450,6 @@ func (mlrmap *Mlrmap) MoveToHead(key string) { } } -// ---------------------------------------------------------------- func (mlrmap *Mlrmap) MoveToTail(key string) { pe := mlrmap.findEntry(key) if pe != nil { @@ -439,6 +466,7 @@ func (mlrmap *Mlrmap) MoveToTail(key string) { // This is a Mlrmap (from string to Mlrval) so we handle the first level of // indexing here, then pass the remaining indices to the Mlrval at the desired // slot. + func (mlrmap *Mlrmap) PutIndexed(indices []*Mlrval, rvalue *Mlrval) error { return putIndexedOnMap(mlrmap, indices, rvalue) } @@ -485,6 +513,7 @@ func (mlrmap *Mlrmap) GetValuesJoined() string { // transformers which support a -g option but are invoked without it (e.g. 'mlr tail // -n 1' vs 'mlr tail -n 1 -g a,b,c'). In this case the return value is simply // the empty string. + func (mlrmap *Mlrmap) GetSelectedValuesJoined(selectedFieldNames []string) (string, bool) { if len(selectedFieldNames) == 0 { // The fall-through is functionally correct, but this is quicker with @@ -738,6 +767,7 @@ func (mlrmap *Mlrmap) IsNested() bool { if mlrmap.Head == nil { return false } else if mlrmap.Head.Value.GetMap() == nil { + //TODO: check IsArrayOrMap() return false } else { return true @@ -747,58 +777,6 @@ func (mlrmap *Mlrmap) IsNested() bool { // ================================================================ // PRIVATE METHODS -// ---------------------------------------------------------------- -func (mlrmap *Mlrmap) findEntry(key string) *MlrmapEntry { - if mlrmap.keysToEntries != nil { - return mlrmap.keysToEntries[key] - } else { - for pe := mlrmap.Head; pe != nil; pe = pe.Next { - if pe.Key == key { - return pe - } - } - return nil - } -} - -// ---------------------------------------------------------------- -// For '$[1]' etc. in the DSL. -// -// Notes: -// * This is a linear search. -// * Indices are 1-up not 0-up -// * Indices -n..-1 are aliases for 1..n. In particular, it will be faster to -// get the -1st field than the nth. -// * Returns 0 on invalid index: 0, or < -n, or > n where n is the number of -// fields. -func (mlrmap *Mlrmap) findEntryByPositionalIndex(position int) *MlrmapEntry { - if position > mlrmap.FieldCount || position < -mlrmap.FieldCount || position == 0 { - return nil - } - if position > 0 { - var i int = 1 - for pe := mlrmap.Head; pe != nil; pe = pe.Next { - if i == position { - return pe - } - i++ - } - lib.InternalCodingErrorIf(true) - } else { - var i int = -1 - for pe := mlrmap.Tail; pe != nil; pe = pe.Prev { - if i == position { - return pe - } - i-- - } - lib.InternalCodingErrorIf(true) - } - lib.InternalCodingErrorIf(true) - return nil -} - -// ---------------------------------------------------------------- func (mlrmap *Mlrmap) Unlink(pe *MlrmapEntry) { if pe == mlrmap.Head { if pe == mlrmap.Tail { @@ -822,7 +800,6 @@ func (mlrmap *Mlrmap) Unlink(pe *MlrmapEntry) { mlrmap.FieldCount-- } -// ---------------------------------------------------------------- // Does not check for duplicate keys func (mlrmap *Mlrmap) linkAtHead(pe *MlrmapEntry) { if mlrmap.Head == nil { diff --git a/internal/pkg/mlrval/mlrmap_accessors_test.go b/internal/pkg/mlrval/mlrmap_accessors_test.go new file mode 100644 index 000000000..5460c3feb --- /dev/null +++ b/internal/pkg/mlrval/mlrmap_accessors_test.go @@ -0,0 +1,49 @@ +package mlrval + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsEmpty(t *testing.T) { + mlrmap := NewMlrmap() + assert.Equal(t, true, mlrmap.IsEmpty()) +} + +func TestPutReference(t *testing.T) { + mlrmap := NewMlrmap() + key1 := "a" + val1 := FromInt(1) + mlrmap.PutReference(key1, val1) + + assert.False(t, mlrmap.IsEmpty()) + + assert.True(t, mlrmap.Has("a")) + assert.False(t, mlrmap.Has("b")) + assert.Equal(t, 1, mlrmap.FieldCount) + + read := mlrmap.Get("b") + assert.Nil(t, read) + + read = mlrmap.Get("a") + assert.NotNil(t, read) + intval, ok := read.GetIntValue() + assert.True(t, ok) + assert.Equal(t, 1, intval) + + key2 := "b" + val2 := FromBool(true) + mlrmap.PutReference(key2, val2) + + assert.True(t, mlrmap.Has("a")) + assert.True(t, mlrmap.Has("b")) + assert.Equal(t, 2, mlrmap.FieldCount) + + read = mlrmap.Get("a") + assert.NotNil(t, read) + read = mlrmap.Get("b") + assert.NotNil(t, read) +} + +// TODO: TestPrependReference diff --git a/internal/pkg/types/mlrmap_flatten_unflatten.go b/internal/pkg/mlrval/mlrmap_flatten_unflatten.go similarity index 92% rename from internal/pkg/types/mlrmap_flatten_unflatten.go rename to internal/pkg/mlrval/mlrmap_flatten_unflatten.go index bfea723a7..fbaeffc40 100644 --- a/internal/pkg/types/mlrmap_flatten_unflatten.go +++ b/internal/pkg/mlrval/mlrmap_flatten_unflatten.go @@ -20,7 +20,7 @@ // otherwise) when we convert to/from JSON. // ================================================================ -package types +package mlrval import ( "strings" @@ -136,7 +136,7 @@ func (mlrmap *Mlrmap) CopyUnflattened( for pe := mlrmap.Head; pe != nil; pe = pe.Next { // Is the field name something dot something? if strings.Contains(pe.Key, separator) { - arrayOfIndices := mlrvalSplitAXHelper(pe.Key, separator) + arrayOfIndices := SplitAXHelper(pe.Key, separator) lib.InternalCodingErrorIf(len(arrayOfIndices.arrayval) < 1) // If the input field name was "x.a" then remember the "x". baseIndex := arrayOfIndices.arrayval[0].String() @@ -157,7 +157,7 @@ func (mlrmap *Mlrmap) CopyUnflattened( for baseIndex := range affectedBaseIndices { oldValue := other.Get(baseIndex) lib.InternalCodingErrorIf(oldValue == nil) - newValue := BIF_arrayify(oldValue) + newValue := oldValue.Arrayify() other.PutReference(baseIndex, newValue) } @@ -186,7 +186,7 @@ func (mlrmap *Mlrmap) CopyUnflattenFields( for pe := mlrmap.Head; pe != nil; pe = pe.Next { // Is the field name something dot something? if strings.Contains(pe.Key, separator) { - arrayOfIndices := mlrvalSplitAXHelper(pe.Key, separator) + arrayOfIndices := SplitAXHelper(pe.Key, separator) lib.InternalCodingErrorIf(len(arrayOfIndices.arrayval) < 1) // If the input field name was "x.a" then remember the "x". baseIndex := arrayOfIndices.arrayval[0].String() @@ -211,7 +211,7 @@ func (mlrmap *Mlrmap) CopyUnflattenFields( for baseIndex := range affectedBaseIndices { oldValue := other.Get(baseIndex) lib.InternalCodingErrorIf(oldValue == nil) - newValue := BIF_arrayify(oldValue) + newValue := oldValue.Arrayify() other.PutReference(baseIndex, newValue) } @@ -231,10 +231,24 @@ func unflattenTerminal(input *Mlrval) *Mlrval { return input } if input.printrep == "{}" { - return MlrvalFromMapReferenced(NewMlrmap()) + return FromMap(NewMlrmap()) } if input.printrep == "[]" { - return MlrvalFromArrayReference(make([]Mlrval, 0)) + return FromArray(make([]Mlrval, 0)) } return input } + +// SplitAXHelper is split out for the benefit of BIF_splitax and +// BIF_unflatten. +func SplitAXHelper(input string, separator string) *Mlrval { + fields := lib.SplitString(input, separator) + + output := FromArray(make([]Mlrval, len(fields))) + + for i, field := range fields { + output.arrayval[i] = *FromString(field) + } + + return output +} diff --git a/internal/pkg/types/mlrmap_json.go b/internal/pkg/mlrval/mlrmap_json.go similarity index 95% rename from internal/pkg/types/mlrmap_json.go rename to internal/pkg/mlrval/mlrmap_json.go index 1a75edec7..7d8715302 100644 --- a/internal/pkg/types/mlrmap_json.go +++ b/internal/pkg/mlrval/mlrmap_json.go @@ -2,7 +2,7 @@ // See mlrval_json.go for details. This is the unmarshal/marshal solely for Mlrmap. // ================================================================ -package types +package mlrval import ( "bytes" @@ -76,7 +76,7 @@ func (mlrmap *Mlrmap) marshalJSONAuxMultiline( // Write the key which is necessarily string-valued in Miller, and in // JSON for that matter :) for i := 0; i < elementNestingDepth; i++ { - buffer.WriteString(MLRVAL_JSON_INDENT_STRING) + buffer.WriteString(JSON_INDENT_STRING) } encoded := string(millerJSONEncodeString(pe.Key)) colorized := colorizer.MaybeColorizeKey(encoded, outputIsStdout) @@ -102,7 +102,7 @@ func (mlrmap *Mlrmap) marshalJSONAuxMultiline( // Write empty map as '{}'. if mlrmap.Head != nil { for i := 0; i < elementNestingDepth-1; i++ { - buffer.WriteString(MLRVAL_JSON_INDENT_STRING) + buffer.WriteString(JSON_INDENT_STRING) } } buffer.WriteString("}") @@ -153,9 +153,9 @@ func (entry *MlrmapEntry) JSONStringifyInPlace( ) { outputBytes, err := entry.Value.MarshalJSON(jsonFormatting, false) if err != nil { - entry.Value = MLRVAL_ERROR + entry.Value = ERROR } else { - entry.Value = MlrvalFromString(string(outputBytes)) + entry.Value = FromString(string(outputBytes)) } } @@ -165,6 +165,6 @@ func (entry *MlrmapEntry) JSONParseInPlace() { input := entry.Value.String() err := entry.Value.UnmarshalJSON([]byte(input)) if err != nil { - entry.Value = MLRVAL_ERROR + entry.Value = ERROR } } diff --git a/internal/pkg/mlrval/mlrmap_new_test.go b/internal/pkg/mlrval/mlrmap_new_test.go new file mode 100644 index 000000000..2695daa58 --- /dev/null +++ b/internal/pkg/mlrval/mlrmap_new_test.go @@ -0,0 +1,17 @@ +package mlrval + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewMlrmapAsRecord(t *testing.T) { + mlrmap := newMlrmapUnhashed() + assert.Equal(t, false, mlrmap.isHashed()) +} + +func TestNewMlrmap(t *testing.T) { + mlrmap := NewMlrmap() + assert.Equal(t, true, mlrmap.isHashed()) +} diff --git a/internal/pkg/types/mlrmap_print.go b/internal/pkg/mlrval/mlrmap_print.go similarity index 99% rename from internal/pkg/types/mlrmap_print.go rename to internal/pkg/mlrval/mlrmap_print.go index ad9959a0a..9cfd8122a 100644 --- a/internal/pkg/types/mlrmap_print.go +++ b/internal/pkg/mlrval/mlrmap_print.go @@ -1,4 +1,4 @@ -package types +package mlrval import ( "bytes" diff --git a/internal/pkg/mlrval/mlrval_accessors.go b/internal/pkg/mlrval/mlrval_accessors.go new file mode 100644 index 000000000..90a6d73fc --- /dev/null +++ b/internal/pkg/mlrval/mlrval_accessors.go @@ -0,0 +1,103 @@ +package mlrval + +import ( + "strconv" + + "github.com/johnkerl/miller/internal/pkg/lib" +) + +func (mv *Mlrval) GetArrayLength() (int, bool) { + if mv.IsArray() { + return len(mv.arrayval), true + } else { + return -999, false + } +} + +func CopyMlrvalArray(input []Mlrval) []Mlrval { + output := make([]Mlrval, len(input)) + for i, element := range input { + output[i] = *element.Copy() + } + return output +} + +func CopyMlrvalPointerArray(input []*Mlrval) []*Mlrval { + output := make([]*Mlrval, len(input)) + for i, element := range input { + output[i] = element.Copy() + } + return output +} + +// --------------------------------------------------------------- +// For the flatten verb and DSL function. + +func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { + retval := NewMlrmap() + + if mv.IsMap() { + // Without this, the for-loop below is zero-pass and fields with "{}" + // values would disappear entirely in a JSON-to-CSV conversion. + if mv.mapval.IsEmpty() { + if prefix != "" { + retval.PutCopy(prefix, FromString("{}")) + } + } + + for pe := mv.mapval.Head; pe != nil; pe = pe.Next { + nextPrefix := pe.Key + if prefix != "" { + nextPrefix = prefix + delimiter + nextPrefix + } + if pe.Value.IsMap() || pe.Value.IsArray() { + nextResult := pe.Value.FlattenToMap(nextPrefix, delimiter) + lib.InternalCodingErrorIf(nextResult.mvtype != MT_MAP) + for pf := nextResult.mapval.Head; pf != nil; pf = pf.Next { + retval.PutCopy(pf.Key, pf.Value.Copy()) + } + } else { + retval.PutCopy(nextPrefix, pe.Value.Copy()) + } + } + + } else if mv.IsArray() { + // Without this, the for-loop below is zero-pass and fields with "[]" + // values would disappear entirely in a JSON-to-CSV conversion. + if len(mv.arrayval) == 0 { + if prefix != "" { + retval.PutCopy(prefix, FromString("[]")) + } + } + + for zindex, value := range mv.arrayval { + nextPrefix := strconv.Itoa(zindex + 1) // Miller user-space indices are 1-up + if prefix != "" { + nextPrefix = prefix + delimiter + nextPrefix + } + if value.IsMap() || value.IsArray() { + nextResult := value.FlattenToMap(nextPrefix, delimiter) + lib.InternalCodingErrorIf(nextResult.mvtype != MT_MAP) + for pf := nextResult.mapval.Head; pf != nil; pf = pf.Next { + retval.PutCopy(pf.Key, pf.Value.Copy()) + } + } else { + retval.PutCopy(nextPrefix, value.Copy()) + } + } + + } else { + retval.PutCopy(prefix, mv.Copy()) + } + + return *FromMap(retval) +} + +// Increment is used by stats1. +func (mv *Mlrval) Increment() { + if mv.mvtype == MT_INT { + mv.intval++ + } else if mv.mvtype == MT_FLOAT { + mv.floatval++ + } +} diff --git a/internal/pkg/mlrval/mlrval_cmp.go b/internal/pkg/mlrval/mlrval_cmp.go new file mode 100644 index 000000000..6b0907b83 --- /dev/null +++ b/internal/pkg/mlrval/mlrval_cmp.go @@ -0,0 +1,439 @@ +// ================================================================ +// Boolean expressions for ==, !=, >, >=, <, <=, <=> on Mlrvals +// ================================================================ + +// TODO: comment about mvtype; deferral; copying of deferrence. + +package mlrval + +import ( + "github.com/johnkerl/miller/internal/pkg/lib" +) + +type CmpFuncBool func(input1, input2 *Mlrval) bool +type CmpFuncInt func(input1, input2 *Mlrval) int // -1, 0, 1 for <=> + +// ---------------------------------------------------------------- +// Exported methods + +func Equals(input1, input2 *Mlrval) bool { + return eq_dispositions[input1.Type()][input2.Type()](input1, input2) +} +func NotEquals(input1, input2 *Mlrval) bool { + return ne_dispositions[input1.Type()][input2.Type()](input1, input2) +} +func GreaterThan(input1, input2 *Mlrval) bool { + return gt_dispositions[input1.Type()][input2.Type()](input1, input2) +} +func GreaterThanOrEquals(input1, input2 *Mlrval) bool { + return ge_dispositions[input1.Type()][input2.Type()](input1, input2) +} +func LessThan(input1, input2 *Mlrval) bool { + return lt_dispositions[input1.Type()][input2.Type()](input1, input2) +} +func LessThanOrEquals(input1, input2 *Mlrval) bool { + return le_dispositions[input1.Type()][input2.Type()](input1, input2) +} +func Cmp(input1, input2 *Mlrval) int { + return cmp_dispositions[input1.Type()][input2.Type()](input1, input2) +} + +// ---------------------------------------------------------------- +// Support routines for disposition-matrix entries + +// string_cmp implements the spaceship operator for strings. +func string_cmp(a, b string) int { + if a < b { + return -1 + } + if a > b { + return 1 + } + return 0 +} + +// int_cmp implements the spaceship operator for ints. +func int_cmp(a, b int) int { + if a < b { + return -1 + } + if a > b { + return 1 + } + return 0 +} + +// float_cmp implements the spaceship operator for floats. +func float_cmp(a, b float64) int { + if a < b { + return -1 + } + if a > b { + return 1 + } + return 0 +} + +// _true return boolean true as a binary-input function +func _true(input1, input2 *Mlrval) bool { + return true +} + +// _fals return boolean false as a binary-input function +func _fals(input1, input2 *Mlrval) bool { + return false +} + +// _i0__ return int 0 as a binary-input function +func _i0__(input1, input2 *Mlrval) int { + return 0 +} + +// _i1__ return int 1 as a binary-input function +func _i1__(input1, input2 *Mlrval) int { + return 1 +} + +// _n1__ return int -1 as a binary-input function +func _n1__(input1, input2 *Mlrval) int { + return -1 +} + +// ---------------------------------------------------------------- +// Disposition-matrix entries + +func eq_b_ss(input1, input2 *Mlrval) bool { + return input1.printrep == input2.printrep +} +func ne_b_ss(input1, input2 *Mlrval) bool { + return input1.printrep != input2.printrep +} +func gt_b_ss(input1, input2 *Mlrval) bool { + return input1.printrep > input2.printrep +} +func ge_b_ss(input1, input2 *Mlrval) bool { + return input1.printrep >= input2.printrep +} +func lt_b_ss(input1, input2 *Mlrval) bool { + return input1.printrep < input2.printrep +} +func le_b_ss(input1, input2 *Mlrval) bool { + return input1.printrep <= input2.printrep +} +func cmp_b_ss(input1, input2 *Mlrval) int { + return string_cmp(input1.printrep, input2.printrep) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_xs(input1, input2 *Mlrval) bool { + return input1.String() == input2.printrep +} +func ne_b_xs(input1, input2 *Mlrval) bool { + return input1.String() != input2.printrep +} +func gt_b_xs(input1, input2 *Mlrval) bool { + return input1.String() > input2.printrep +} +func ge_b_xs(input1, input2 *Mlrval) bool { + return input1.String() >= input2.printrep +} +func lt_b_xs(input1, input2 *Mlrval) bool { + return input1.String() < input2.printrep +} +func le_b_xs(input1, input2 *Mlrval) bool { + return input1.String() <= input2.printrep +} +func cmp_b_xs(input1, input2 *Mlrval) int { + return string_cmp(input1.String(), input2.printrep) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_sx(input1, input2 *Mlrval) bool { + return input1.printrep == input2.String() +} +func ne_b_sx(input1, input2 *Mlrval) bool { + return input1.printrep != input2.String() +} +func gt_b_sx(input1, input2 *Mlrval) bool { + return input1.printrep > input2.String() +} +func ge_b_sx(input1, input2 *Mlrval) bool { + return input1.printrep >= input2.String() +} +func lt_b_sx(input1, input2 *Mlrval) bool { + return input1.printrep < input2.String() +} +func le_b_sx(input1, input2 *Mlrval) bool { + return input1.printrep <= input2.String() +} +func cmp_b_sx(input1, input2 *Mlrval) int { + return string_cmp(input1.printrep, input2.String()) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_ii(input1, input2 *Mlrval) bool { + return input1.intval == input2.intval +} +func ne_b_ii(input1, input2 *Mlrval) bool { + return input1.intval != input2.intval +} +func gt_b_ii(input1, input2 *Mlrval) bool { + return input1.intval > input2.intval +} +func ge_b_ii(input1, input2 *Mlrval) bool { + return input1.intval >= input2.intval +} +func lt_b_ii(input1, input2 *Mlrval) bool { + return input1.intval < input2.intval +} +func le_b_ii(input1, input2 *Mlrval) bool { + return input1.intval <= input2.intval +} +func cmp_b_ii(input1, input2 *Mlrval) int { + return int_cmp(input1.intval, input2.intval) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_if(input1, input2 *Mlrval) bool { + return float64(input1.intval) == input2.floatval +} +func ne_b_if(input1, input2 *Mlrval) bool { + return float64(input1.intval) != input2.floatval +} +func gt_b_if(input1, input2 *Mlrval) bool { + return float64(input1.intval) > input2.floatval +} +func ge_b_if(input1, input2 *Mlrval) bool { + return float64(input1.intval) >= input2.floatval +} +func lt_b_if(input1, input2 *Mlrval) bool { + return float64(input1.intval) < input2.floatval +} +func le_b_if(input1, input2 *Mlrval) bool { + return float64(input1.intval) <= input2.floatval +} +func cmp_b_if(input1, input2 *Mlrval) int { + return float_cmp(float64(input1.intval), input2.floatval) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_fi(input1, input2 *Mlrval) bool { + return input1.floatval == float64(input2.intval) +} +func ne_b_fi(input1, input2 *Mlrval) bool { + return input1.floatval != float64(input2.intval) +} +func gt_b_fi(input1, input2 *Mlrval) bool { + return input1.floatval > float64(input2.intval) +} +func ge_b_fi(input1, input2 *Mlrval) bool { + return input1.floatval >= float64(input2.intval) +} +func lt_b_fi(input1, input2 *Mlrval) bool { + return input1.floatval < float64(input2.intval) +} +func le_b_fi(input1, input2 *Mlrval) bool { + return input1.floatval <= float64(input2.intval) +} +func cmp_b_fi(input1, input2 *Mlrval) int { + return float_cmp(input1.floatval, float64(input2.intval)) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_ff(input1, input2 *Mlrval) bool { + return input1.floatval == input2.floatval +} +func ne_b_ff(input1, input2 *Mlrval) bool { + return input1.floatval != input2.floatval +} +func gt_b_ff(input1, input2 *Mlrval) bool { + return input1.floatval > input2.floatval +} +func ge_b_ff(input1, input2 *Mlrval) bool { + return input1.floatval >= input2.floatval +} +func lt_b_ff(input1, input2 *Mlrval) bool { + return input1.floatval < input2.floatval +} +func le_b_ff(input1, input2 *Mlrval) bool { + return input1.floatval <= input2.floatval +} +func cmp_b_ff(input1, input2 *Mlrval) int { + return float_cmp(input1.floatval, input2.floatval) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_bb(input1, input2 *Mlrval) bool { + return input1.boolval == input2.boolval +} +func ne_b_bb(input1, input2 *Mlrval) bool { + return input1.boolval != input2.boolval +} + +func gt_b_bb(input1, input2 *Mlrval) bool { + return lib.BoolToInt(input1.boolval) > lib.BoolToInt(input2.boolval) +} +func ge_b_bb(input1, input2 *Mlrval) bool { + return lib.BoolToInt(input1.boolval) >= lib.BoolToInt(input2.boolval) +} +func lt_b_bb(input1, input2 *Mlrval) bool { + return lib.BoolToInt(input1.boolval) < lib.BoolToInt(input2.boolval) +} +func le_b_bb(input1, input2 *Mlrval) bool { + return lib.BoolToInt(input1.boolval) <= lib.BoolToInt(input2.boolval) +} +func cmp_b_bb(input1, input2 *Mlrval) int { + return int_cmp(lib.BoolToInt(input1.boolval), lib.BoolToInt(input2.boolval)) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_aa(input1, input2 *Mlrval) bool { + a := input1.arrayval + b := input2.arrayval + + // Different-length arrays are not equal + if len(a) != len(b) { + return false + } + + // Same-length arrays: return false if any slot is not equal, else true. + for i := range a { + if !Equals(&a[i], &b[i]) { + return false + } + } + + return true +} +func ne_b_aa(input1, input2 *Mlrval) bool { + return !eq_b_aa(input1, input2) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func eq_b_mm(input1, input2 *Mlrval) bool { + return input1.mapval.Equals(input2.mapval) +} +func ne_b_mm(input1, input2 *Mlrval) bool { + return !input1.mapval.Equals(input2.mapval) +} + +// We get a Golang "initialization loop" due to recursive depth computation +// if this is defined statically. So, we use a "package init" function. +var eq_dispositions = [MT_DIM][MT_DIM]CmpFuncBool{} + +func init() { + eq_dispositions = [MT_DIM][MT_DIM]CmpFuncBool{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*ABSENT */ {_fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*NULL */ {_fals, _fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*VOID */ {_fals, _fals, _fals, eq_b_ss, eq_b_ss, eq_b_sx, eq_b_sx, _fals, _fals, _fals, _fals}, + /*STRING */ {_fals, _fals, _fals, eq_b_ss, eq_b_ss, eq_b_sx, eq_b_sx, _fals, _fals, _fals, _fals}, + /*INT */ {_fals, _fals, _fals, eq_b_xs, eq_b_xs, eq_b_ii, eq_b_if, _fals, _fals, _fals, _fals}, + /*FLOAT */ {_fals, _fals, _fals, eq_b_xs, eq_b_xs, eq_b_fi, eq_b_ff, _fals, _fals, _fals, _fals}, + /*BOOL */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, eq_b_bb, _fals, _fals, _fals}, + /*ARRAY */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, eq_b_aa, _fals, _fals}, + /*MAP */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, eq_b_mm, _fals}, + /*FUNC */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _true}, + } +} + +var ne_dispositions = [MT_DIM][MT_DIM]CmpFuncBool{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*ABSENT */ {_fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*NULL */ {_fals, _fals, _fals, _true, _true, _true, _true, _true, _true, _true, _fals}, + /*VOID */ {_fals, _fals, _true, ne_b_ss, ne_b_ss, ne_b_sx, ne_b_sx, _true, _true, _true, _fals}, + /*STRING */ {_fals, _fals, _true, ne_b_ss, ne_b_ss, ne_b_sx, ne_b_sx, _true, _true, _true, _fals}, + /*INT */ {_fals, _fals, _true, ne_b_xs, ne_b_xs, ne_b_ii, ne_b_if, _true, _true, _true, _fals}, + /*FLOAT */ {_fals, _fals, _true, ne_b_xs, ne_b_xs, ne_b_fi, ne_b_ff, _true, _true, _true, _fals}, + /*BOOL */ {_fals, _fals, _true, _true, _true, _true, _true, ne_b_bb, _true, _true, _fals}, + /*ARRAY */ {_fals, _fals, _true, _true, _true, _true, _true, _true, ne_b_aa, _true, _fals}, + /*MAP */ {_fals, _fals, _true, _true, _true, _true, _true, _true, _true, ne_b_mm, _fals}, + /*FUNC */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _true}, +} + +var gt_dispositions = [MT_DIM][MT_DIM]CmpFuncBool{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*ABSENT */ {_fals, _true, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*NULL */ {_true, _fals, _fals, _true, _true, _true, _true, _true, _fals, _fals, _fals}, + /*VOID */ {_fals, _fals, _fals, gt_b_ss, gt_b_ss, gt_b_sx, gt_b_sx, _fals, _fals, _fals, _fals}, + /*STRING */ {_fals, _fals, _fals, gt_b_ss, gt_b_ss, gt_b_sx, gt_b_sx, _fals, _fals, _fals, _fals}, + /*INT */ {_fals, _fals, _fals, gt_b_xs, gt_b_xs, gt_b_ii, gt_b_if, _fals, _fals, _fals, _fals}, + /*FLOAT */ {_fals, _fals, _fals, gt_b_xs, gt_b_xs, gt_b_fi, gt_b_ff, _fals, _fals, _fals, _fals}, + /*BOOL */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, gt_b_bb, _fals, _fals, _fals}, + /*ARRAY */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*MAP */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*FUNC */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _true}, +} + +var ge_dispositions = [MT_DIM][MT_DIM]CmpFuncBool{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*ABSENT */ {_fals, _true, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*NULL */ {_true, _fals, _true, _true, _true, _true, _true, _true, _fals, _fals, _fals}, + /*VOID */ {_fals, _fals, _fals, ge_b_ss, ge_b_ss, ge_b_sx, ge_b_sx, _fals, _fals, _fals, _fals}, + /*STRING */ {_fals, _fals, _fals, ge_b_ss, ge_b_ss, ge_b_sx, ge_b_sx, _fals, _fals, _fals, _fals}, + /*INT */ {_fals, _fals, _fals, ge_b_xs, ge_b_xs, ge_b_ii, ge_b_if, _fals, _fals, _fals, _fals}, + /*FLOAT */ {_fals, _fals, _fals, ge_b_xs, ge_b_xs, ge_b_fi, ge_b_ff, _fals, _fals, _fals, _fals}, + /*BOOL */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, ge_b_bb, _fals, _fals, _fals}, + /*ARRAY */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*MAP */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*FUNC */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _true}, +} + +var lt_dispositions = [MT_DIM][MT_DIM]CmpFuncBool{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_true, _fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*ABSENT */ {_fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*NULL */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*VOID */ {_fals, _fals, _true, lt_b_ss, lt_b_ss, lt_b_sx, lt_b_sx, _fals, _fals, _fals, _fals}, + /*STRING */ {_fals, _fals, _true, lt_b_ss, lt_b_ss, lt_b_sx, lt_b_sx, _fals, _fals, _fals, _fals}, + /*INT */ {_fals, _fals, _true, lt_b_xs, lt_b_xs, lt_b_ii, lt_b_if, _fals, _fals, _fals, _fals}, + /*FLOAT */ {_fals, _fals, _true, lt_b_xs, lt_b_xs, lt_b_fi, lt_b_ff, _fals, _fals, _fals, _fals}, + /*BOOL */ {_fals, _fals, _true, _fals, _fals, _fals, _fals, lt_b_bb, _fals, _fals, _fals}, + /*ARRAY */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*MAP */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*FUNC */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _true}, +} + +var le_dispositions = [MT_DIM][MT_DIM]CmpFuncBool{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_true, _fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*ABSENT */ {_fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*NULL */ {_fals, _fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*VOID */ {_fals, _fals, _true, le_b_ss, le_b_ss, le_b_sx, le_b_sx, _fals, _fals, _fals, _fals}, + /*STRING */ {_fals, _fals, _true, le_b_ss, le_b_ss, le_b_sx, le_b_sx, _fals, _fals, _fals, _fals}, + /*INT */ {_fals, _fals, _true, le_b_xs, le_b_xs, le_b_ii, le_b_if, _fals, _fals, _fals, _fals}, + /*FLOAT */ {_fals, _fals, _true, le_b_xs, le_b_xs, le_b_fi, le_b_ff, _fals, _fals, _fals, _fals}, + /*BOOL */ {_fals, _fals, _true, _fals, _fals, _fals, _fals, le_b_bb, _fals, _fals, _fals}, + /*ARRAY */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*MAP */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals}, + /*FUNC */ {_fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _true}, +} + +// TODO: flesh these out for array and map +var cmp_dispositions = [MT_DIM][MT_DIM]CmpFuncInt{ + // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC + /*ERROR */ {_i0__, _i0__, _n1__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__}, + /*ABSENT */ {_i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__}, + /*NULL */ {_i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__}, + /*VOID */ {_i0__, _i0__, _n1__, cmp_b_ss, cmp_b_ss, cmp_b_sx, cmp_b_sx, _i0__, _i0__, _i0__, _i0__}, + /*STRING */ {_i0__, _i0__, _n1__, cmp_b_ss, cmp_b_ss, cmp_b_sx, cmp_b_sx, _i0__, _i0__, _i0__, _i0__}, + /*INT */ {_i0__, _i0__, _n1__, cmp_b_xs, cmp_b_xs, cmp_b_ii, cmp_b_if, _i0__, _i0__, _i0__, _i0__}, + /*FLOAT */ {_i0__, _i0__, _n1__, cmp_b_xs, cmp_b_xs, cmp_b_fi, cmp_b_ff, _i0__, _i0__, _i0__, _i0__}, + /*BOOL */ {_i0__, _i0__, _n1__, _i0__, _i0__, _i0__, _i0__, cmp_b_bb, _i0__, _i0__, _i0__}, + /*ARRAY */ {_i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__}, + /*MAP */ {_i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__}, + /*FUNC */ {_i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__, _i0__}, +} diff --git a/internal/pkg/types/mlrval_collections.go b/internal/pkg/mlrval/mlrval_collections.go similarity index 81% rename from internal/pkg/types/mlrval_collections.go rename to internal/pkg/mlrval/mlrval_collections.go index 75a36741e..fdf225d8b 100644 --- a/internal/pkg/types/mlrval_collections.go +++ b/internal/pkg/mlrval/mlrval_collections.go @@ -66,12 +66,13 @@ // // ================================================================ -package types +package mlrval import ( "errors" "fmt" "os" + "strconv" "github.com/johnkerl/miller/internal/pkg/lib" ) @@ -79,15 +80,15 @@ import ( // ================================================================ // TODO: copy-reduction refactor func (mv *Mlrval) ArrayGet(mindex *Mlrval) Mlrval { - if mv.mvtype != MT_ARRAY { - return *MLRVAL_ERROR + if !mv.IsArray() { + return *ERROR } - if mindex.mvtype != MT_INT { - return *MLRVAL_ERROR + if !mindex.IsInt() { + return *ERROR } value := arrayGetAliased(&mv.arrayval, mindex.intval) if value == nil { - return *MLRVAL_ABSENT + return *ABSENT } else { return *value } @@ -96,7 +97,7 @@ func (mv *Mlrval) ArrayGet(mindex *Mlrval) Mlrval { // ---------------------------------------------------------------- // TODO: make this return error so caller can do 'if err == nil { ... }' func (mv *Mlrval) ArrayPut(mindex *Mlrval, value *Mlrval) { - if mv.mvtype != MT_ARRAY { + if !mv.IsArray() { fmt.Fprintf( os.Stderr, "mlr: expected array as indexed item in ArrayPut; got %s\n", @@ -104,7 +105,7 @@ func (mv *Mlrval) ArrayPut(mindex *Mlrval, value *Mlrval) { ) os.Exit(1) } - if mindex.mvtype != MT_INT { + if !mindex.IsInt() { // TODO: need to be careful about semantics here. // Silent no-ops are not good UX ... fmt.Fprintf( @@ -208,7 +209,7 @@ func UnaliasArrayLengthIndex(n int, mindex int) (int, bool) { // ---------------------------------------------------------------- // TODO: thinking about capacity-resizing func (mv *Mlrval) ArrayAppend(value *Mlrval) { - if mv.mvtype != MT_ARRAY { + if !mv.IsArray() { // TODO: need to be careful about semantics here. // Silent no-ops are not good UX ... return @@ -218,16 +219,16 @@ func (mv *Mlrval) ArrayAppend(value *Mlrval) { // ================================================================ func (mv *Mlrval) MapGet(key *Mlrval) Mlrval { - if mv.mvtype != MT_MAP { - return *MLRVAL_ERROR + if !mv.IsMap() { + return *ERROR } mval, err := mv.mapval.GetWithMlrvalIndex(key) if err != nil { // xxx maybe error-return in the API - return *MLRVAL_ERROR + return *ERROR } if mval == nil { - return *MLRVAL_ABSENT + return *ABSENT } // This returns a reference, not a (deep) copy. In general in Miller, we // copy only on write/put. @@ -236,15 +237,15 @@ func (mv *Mlrval) MapGet(key *Mlrval) Mlrval { // ---------------------------------------------------------------- func (mv *Mlrval) MapPut(key *Mlrval, value *Mlrval) { - if mv.mvtype != MT_MAP { + if !mv.IsMap() { // TODO: need to be careful about semantics here. // Silent no-ops are not good UX ... return } - if key.mvtype == MT_STRING { + if key.IsString() { mv.mapval.PutCopy(key.printrep, value) - } else if key.mvtype == MT_INT { + } else if key.IsInt() { mv.mapval.PutCopy(key.String(), value) } // TODO: need to be careful about semantics here. @@ -290,19 +291,19 @@ func (mv *Mlrval) MapPut(key *Mlrval, value *Mlrval) { func (mv *Mlrval) PutIndexed(indices []*Mlrval, rvalue *Mlrval) error { lib.InternalCodingErrorIf(len(indices) < 1) - if mv.mvtype == MT_MAP { + if mv.IsMap() { return putIndexedOnMap(mv.mapval, indices, rvalue) - } else if mv.mvtype == MT_ARRAY { + } else if mv.IsArray() { return putIndexedOnArray(&mv.arrayval, indices, rvalue) } else { baseIndex := indices[0] - if baseIndex.mvtype == MT_STRING { - *mv = *MlrvalFromEmptyMap() + if baseIndex.IsString() { + *mv = *FromEmptyMap() return putIndexedOnMap(mv.mapval, indices, rvalue) - } else if baseIndex.mvtype == MT_INT { - *mv = MlrvalEmptyArray() + } else if baseIndex.IsInt() { + *mv = *FromEmptyArray() return putIndexedOnArray(&mv.arrayval, indices, rvalue) } else { return errors.New( @@ -340,7 +341,7 @@ func putIndexedOnMap(baseMap *Mlrmap, indices []*Mlrval, rvalue *Mlrval) error { } // If not last index, then recurse. - if baseIndex.mvtype != MT_STRING && baseIndex.mvtype != MT_INT { + if !baseIndex.IsString() && !baseIndex.IsInt() { // Base is map, index is invalid type return errors.New( "mlr: map indices must be string, int, or array thereof; got " + baseIndex.GetTypeName(), @@ -353,7 +354,7 @@ func putIndexedOnMap(baseMap *Mlrmap, indices []*Mlrval, rvalue *Mlrval) error { nextIndex := indices[1] var err error = nil - baseValue, err = NewMlrvalForAutoDeepen(nextIndex.mvtype) + baseValue, err = NewMlrvalForAutoDeepen(nextIndex.Type()) if err != nil { return err } @@ -374,7 +375,7 @@ func putIndexedOnArray( lib.InternalCodingErrorIf(numIndices < 1) mindex := indices[0] - if mindex.mvtype != MT_INT { + if !mindex.IsInt() { return errors.New( "Array index must be int, but was " + mindex.GetTypeName() + @@ -406,13 +407,13 @@ func putIndexedOnArray( nextIndex := indices[1] // Overwrite what's in this slot if it's the wrong type - if nextIndex.mvtype == MT_STRING { - if (*baseArray)[zindex].mvtype != MT_MAP { - (*baseArray)[zindex] = *MlrvalFromEmptyMap() + if nextIndex.IsString() { + if !(*baseArray)[zindex].IsMap() { + (*baseArray)[zindex] = *FromEmptyMap() } - } else if nextIndex.mvtype == MT_INT { - if (*baseArray)[zindex].mvtype != MT_ARRAY { - (*baseArray)[zindex] = MlrvalEmptyArray() + } else if nextIndex.IsInt() { + if !(*baseArray)[zindex].IsArray() { + (*baseArray)[zindex] = *FromEmptyArray() } } else { return errors.New( @@ -442,10 +443,10 @@ func putIndexedOnArray( func (mv *Mlrval) RemoveIndexed(indices []*Mlrval) error { lib.InternalCodingErrorIf(len(indices) < 1) - if mv.mvtype == MT_MAP { + if mv.IsMap() { return removeIndexedOnMap(mv.mapval, indices) - } else if mv.mvtype == MT_ARRAY { + } else if mv.IsArray() { return removeIndexedOnArray(&mv.arrayval, indices) } else { @@ -464,7 +465,7 @@ func removeIndexedOnMap(baseMap *Mlrmap, indices []*Mlrval) error { // If last index, then unset. if numIndices == 1 { - if baseIndex.mvtype == MT_STRING || baseIndex.mvtype == MT_INT { + if baseIndex.IsString() || baseIndex.IsInt() { baseMap.Remove(baseIndex.String()) return nil } else { @@ -476,7 +477,7 @@ func removeIndexedOnMap(baseMap *Mlrmap, indices []*Mlrval) error { } // If not last index, then recurse. - if baseIndex.mvtype == MT_STRING || baseIndex.mvtype == MT_INT { + if baseIndex.IsString() || baseIndex.IsInt() { // Base is map, index is string baseValue := baseMap.Get(baseIndex.String()) if baseValue != nil { @@ -503,7 +504,7 @@ func removeIndexedOnArray( lib.InternalCodingErrorIf(numIndices < 1) mindex := indices[0] - if mindex.mvtype != MT_INT { + if !mindex.IsInt() { return errors.New( "Array index must be int, but was " + mindex.GetTypeName() + @@ -575,18 +576,18 @@ func BsearchMlrvalArrayForDescendingInsert( return 0 } - if MlrvalGreaterThanAsBool(value, (*array)[0]) { + if GreaterThan(value, (*array)[0]) { return 0 } - if MlrvalLessThanAsBool(value, (*array)[hi]) { + if LessThan(value, (*array)[hi]) { return size } for lo < hi { middleElement := (*array)[mid] - if MlrvalEqualsAsBool(value, middleElement) { + if Equals(value, middleElement) { return mid - } else if MlrvalGreaterThanAsBool(value, middleElement) { + } else if GreaterThan(value, middleElement) { hi = mid newmid = (hi + lo) / 2 } else { @@ -594,9 +595,9 @@ func BsearchMlrvalArrayForDescendingInsert( newmid = (hi + lo) / 2 } if mid == newmid { - if MlrvalGreaterThanOrEqualsAsBool(value, (*array)[lo]) { + if GreaterThanOrEquals(value, (*array)[lo]) { return lo - } else if MlrvalGreaterThanOrEqualsAsBool(value, (*array)[hi]) { + } else if GreaterThanOrEquals(value, (*array)[hi]) { return hi } else { return hi + 1 @@ -622,18 +623,18 @@ func BsearchMlrvalArrayForAscendingInsert( return 0 } - if MlrvalLessThanAsBool(value, (*array)[0]) { + if LessThan(value, (*array)[0]) { return 0 } - if MlrvalGreaterThanAsBool(value, (*array)[hi]) { + if GreaterThan(value, (*array)[hi]) { return size } for lo < hi { middleElement := (*array)[mid] - if MlrvalEqualsAsBool(value, middleElement) { + if Equals(value, middleElement) { return mid - } else if MlrvalLessThanAsBool(value, middleElement) { + } else if LessThan(value, middleElement) { hi = mid newmid = (hi + lo) / 2 } else { @@ -641,9 +642,9 @@ func BsearchMlrvalArrayForAscendingInsert( newmid = (hi + lo) / 2 } if mid == newmid { - if MlrvalLessThanOrEqualsAsBool(value, (*array)[lo]) { + if LessThanOrEquals(value, (*array)[lo]) { return lo - } else if MlrvalLessThanOrEqualsAsBool(value, (*array)[hi]) { + } else if LessThanOrEquals(value, (*array)[hi]) { return hi } else { return hi + 1 @@ -654,3 +655,90 @@ func BsearchMlrvalArrayForAscendingInsert( return lo } + +// NewMlrvalForAutoDeepen is for auto-deepen of nested maps in things like +// +// $foo[1]["a"][2]["b"] = 3 +// +// Autocreated levels are maps. Array levels can be explicitly created e.g. +// +// $foo[1]["a"] ??= [] +// $foo[1]["a"][2]["b"] = 3 +func NewMlrvalForAutoDeepen(mvtype MVType) (*Mlrval, error) { + if mvtype == MT_STRING || mvtype == MT_INT { + empty := FromEmptyMap() + return empty, nil + } else { + return nil, errors.New( + "mlr: indices must be string, int, or array thereof; got " + GetTypeName(mvtype), + ) + } +} + +func (mv *Mlrval) Arrayify() *Mlrval { + if mv.IsMap() { + if mv.mapval.IsEmpty() { + return mv + } + + convertible := true + i := 0 + for pe := mv.mapval.Head; pe != nil; pe = pe.Next { + sval := strconv.Itoa(i + 1) // Miller user-space indices are 1-up + i++ + if pe.Key != sval { + convertible = false + } + pe.Value = pe.Value.Arrayify() + } + + if convertible { + arrayval := make([]Mlrval, mv.mapval.FieldCount) + i := 0 + for pe := mv.mapval.Head; pe != nil; pe = pe.Next { + arrayval[i] = *pe.Value.Copy() + i++ + } + return FromArray(arrayval) + + } else { + return mv + } + + } else if mv.IsArray() { + // TODO: comment (or rethink) that this modifies its inputs!! + output := mv.Copy() + for i := range mv.arrayval { + output.arrayval[i] = *output.arrayval[i].Arrayify() + } + return output + + } else { + return mv + } +} + +func LengthenMlrvalArray(array *[]Mlrval, newLength64 int) { + newLength := int(newLength64) + lib.InternalCodingErrorIf(newLength <= len(*array)) + + if newLength <= cap(*array) { + newArray := (*array)[:newLength] + for zindex := len(*array); zindex < newLength; zindex++ { + // TODO: comment why not MT_ABSENT or MT_VOID + newArray[zindex] = *NULL + } + *array = newArray + } else { + newArray := make([]Mlrval, newLength, 2*newLength) + zindex := 0 + for zindex = 0; zindex < len(*array); zindex++ { + newArray[zindex] = (*array)[zindex] + } + for zindex = len(*array); zindex < newLength; zindex++ { + // TODO: comment why not MT_ABSENT or MT_VOID + newArray[zindex] = *NULL + } + *array = newArray + } +} diff --git a/internal/pkg/mlrval/mlrval_constants.go b/internal/pkg/mlrval/mlrval_constants.go new file mode 100644 index 000000000..bfaacf77f --- /dev/null +++ b/internal/pkg/mlrval/mlrval_constants.go @@ -0,0 +1,59 @@ +// ================================================================ +// Constants/singletons of various types +// ================================================================ + +package mlrval + +// MlrvalFromPending is designed solely for the JSON API, for something +// intended to be mutated after construction once its type is (later) known. +// Whereas ERROR, ABSENT, etc are all singletons, this one +// must be mutable and therefor non-singleton. + +func MlrvalFromPending() Mlrval { + return Mlrval{ + mvtype: MT_PENDING, + printrep: "(bug-if-you-see-this:case-3)", // INVALID_PRINTREP, + } +} + +// These are made singletons as part of a copy-reduction effort. They're not +// marked const (I haven't figured out the right way to get that to compile; +// just using `const` isn't enough) but the gentelpersons' agreement is that +// the caller should never modify these. + +var ERROR = &Mlrval{ + mvtype: MT_ERROR, + printrep: ERROR_PRINTREP, + printrepValid: true, +} + +var ABSENT = &Mlrval{ + mvtype: MT_ABSENT, + printrep: ABSENT_PRINTREP, + printrepValid: true, +} + +var NULL = &Mlrval{ + mvtype: MT_NULL, + printrep: "null", + printrepValid: true, +} + +var VOID = &Mlrval{ + mvtype: MT_VOID, + printrep: "", + printrepValid: true, +} + +var TRUE = &Mlrval{ + mvtype: MT_BOOL, + printrep: "true", + printrepValid: true, + boolval: true, +} + +var FALSE = &Mlrval{ + mvtype: MT_BOOL, + printrep: "false", + printrepValid: true, +} diff --git a/internal/pkg/mlrval/mlrval_copy.go b/internal/pkg/mlrval/mlrval_copy.go new file mode 100644 index 000000000..115882c4e --- /dev/null +++ b/internal/pkg/mlrval/mlrval_copy.go @@ -0,0 +1,12 @@ +package mlrval + +// TODO: comment about mvtype; deferrence; copying of deferrence. +func (mv *Mlrval) Copy() *Mlrval { + other := *mv + if mv.mvtype == MT_MAP { + other.mapval = mv.mapval.Copy() + } else if mv.mvtype == MT_ARRAY { + other.arrayval = CopyMlrvalArray(mv.arrayval) + } + return &other +} diff --git a/internal/pkg/types/mlrval_format.go b/internal/pkg/mlrval/mlrval_format.go similarity index 61% rename from internal/pkg/types/mlrval_format.go rename to internal/pkg/mlrval/mlrval_format.go index 4c211441d..8ecc8df93 100644 --- a/internal/pkg/types/mlrval_format.go +++ b/internal/pkg/mlrval/mlrval_format.go @@ -1,4 +1,4 @@ -package types +package mlrval import ( "errors" @@ -38,40 +38,51 @@ import ( // - -i, -f, -s // ---------------------------------------------------------------- -// ---------------------------------------------------------------- -var mlrvalFormatterCache map[string]IMlrvalFormatter = make(map[string]IMlrvalFormatter) +// Nil means use default format. +// Set from the CLI parser using mlr --ofmt. +var floatOutputFormatter IFormatter = nil -func GetMlrvalFormatter( +func SetFloatOutputFormat(formatString string) error { + formatter, err := GetFormatter(formatString) + if err != nil { + return err + } + floatOutputFormatter = formatter + return nil +} + +var formatterCache map[string]IFormatter = make(map[string]IFormatter) + +type IFormatter interface { + Format(mlrval *Mlrval) *Mlrval + FormatFloat(floatValue float64) string // for --ofmt +} + +func GetFormatter( userLevelFormatString string, -) (IMlrvalFormatter, error) { +) (IFormatter, error) { // Cache hit - formatter, ok := mlrvalFormatterCache[userLevelFormatString] + formatter, ok := formatterCache[userLevelFormatString] if ok { return formatter, nil } // Cache miss - formatter, err := newMlrvalFormatter(userLevelFormatString) + formatter, err := newFormatter(userLevelFormatString) if err != nil { // TODO: temp exit fmt.Printf("mlr: %v\n", err) return nil, err } - mlrvalFormatterCache[userLevelFormatString] = formatter + formatterCache[userLevelFormatString] = formatter return formatter, nil } -// ---------------------------------------------------------------- -type IMlrvalFormatter interface { - Format(mlrval *Mlrval) *Mlrval - FormatFloat(floatValue float64) string // for --ofmt -} - // People can pass in things like "X%sX" unfortunately :( -func newMlrvalFormatter( +func newFormatter( userLevelFormatString string, -) (IMlrvalFormatter, error) { +) (IFormatter, error) { numPercents := strings.Count(userLevelFormatString, "%") if numPercents < 1 { return nil, errors.New( @@ -99,111 +110,112 @@ func newMlrvalFormatter( // and double-precision floats: e.g. "%08lld" and "%9.6lf". For Miller 6, // We must still accept these for backward compatibility. if strings.HasSuffix(goFormatString, "d") { - return newMlrvalFormatterToInt(goFormatString), nil + return newFormatterToInt(goFormatString), nil } if strings.HasSuffix(goFormatString, "x") { - return newMlrvalFormatterToInt(goFormatString), nil + return newFormatterToInt(goFormatString), nil } if strings.HasSuffix(goFormatString, "f") { - return newMlrvalFormatterToFloat(goFormatString), nil + return newFormatterToFloat(goFormatString), nil } if strings.HasSuffix(goFormatString, "e") { - return newMlrvalFormatterToFloat(goFormatString), nil + return newFormatterToFloat(goFormatString), nil } if strings.HasSuffix(goFormatString, "g") { - return newMlrvalFormatterToFloat(goFormatString), nil + return newFormatterToFloat(goFormatString), nil } if strings.HasSuffix(goFormatString, "s") { - return newMlrvalFormatterToString(goFormatString), nil + return newFormatterToString(goFormatString), nil } // TODO: // return nil, errors.New(fmt.Sprintf("unhandled format string \"%s\"", userLevelFormatString)) - return newMlrvalFormatterToString(goFormatString), nil + return newFormatterToString(goFormatString), nil } -//func regularizeFormat - // ---------------------------------------------------------------- -type mlrvalFormatterToFloat struct { + +type formatterToFloat struct { goFormatString string } -func newMlrvalFormatterToFloat(goFormatString string) IMlrvalFormatter { - return &mlrvalFormatterToFloat{ +func newFormatterToFloat(goFormatString string) IFormatter { + return &formatterToFloat{ goFormatString: goFormatString, } } -func (formatter *mlrvalFormatterToFloat) Format(mlrval *Mlrval) *Mlrval { - floatValue, isFloat := mlrval.GetFloatValue() +func (formatter *formatterToFloat) Format(mv *Mlrval) *Mlrval { + floatValue, isFloat := mv.GetFloatValue() if isFloat { formatted := fmt.Sprintf(formatter.goFormatString, floatValue) - return MlrvalTryPointerFromFloatString(formatted) + return TryFromFloatString(formatted) } - intValue, isInt := mlrval.GetIntValue() + intValue, isInt := mv.GetIntValue() if isInt { formatted := fmt.Sprintf(formatter.goFormatString, float64(intValue)) - return MlrvalTryPointerFromFloatString(formatted) + return TryFromFloatString(formatted) } - return mlrval + return mv } -func (formatter *mlrvalFormatterToFloat) FormatFloat(floatValue float64) string { +func (formatter *formatterToFloat) FormatFloat(floatValue float64) string { return fmt.Sprintf(formatter.goFormatString, floatValue) } // ---------------------------------------------------------------- -type mlrvalFormatterToInt struct { + +type formatterToInt struct { goFormatString string } -func newMlrvalFormatterToInt(goFormatString string) IMlrvalFormatter { - return &mlrvalFormatterToInt{ +func newFormatterToInt(goFormatString string) IFormatter { + return &formatterToInt{ goFormatString: goFormatString, } } -func (formatter *mlrvalFormatterToInt) Format(mlrval *Mlrval) *Mlrval { - intValue, isInt := mlrval.GetIntValue() +func (formatter *formatterToInt) Format(mv *Mlrval) *Mlrval { + intValue, isInt := mv.GetIntValue() if isInt { formatted := fmt.Sprintf(formatter.goFormatString, intValue) - return MlrvalTryPointerFromIntString(formatted) + return TryFromIntString(formatted) } - floatValue, isFloat := mlrval.GetFloatValue() + floatValue, isFloat := mv.GetFloatValue() if isFloat { formatted := fmt.Sprintf(formatter.goFormatString, int(floatValue)) - return MlrvalTryPointerFromIntString(formatted) + return TryFromIntString(formatted) } - return mlrval + return mv } -func (formatter *mlrvalFormatterToInt) FormatFloat(floatValue float64) string { +func (formatter *formatterToInt) FormatFloat(floatValue float64) string { return fmt.Sprintf(formatter.goFormatString, int(floatValue)) } // ---------------------------------------------------------------- -type mlrvalFormatterToString struct { + +type formatterToString struct { goFormatString string } -func newMlrvalFormatterToString(goFormatString string) IMlrvalFormatter { - return &mlrvalFormatterToString{ +func newFormatterToString(goFormatString string) IFormatter { + return &formatterToString{ goFormatString: goFormatString, } } -func (formatter *mlrvalFormatterToString) Format(mlrval *Mlrval) *Mlrval { - return MlrvalFromString( +func (formatter *formatterToString) Format(mv *Mlrval) *Mlrval { + return FromString( fmt.Sprintf( formatter.goFormatString, - mlrval.String(), + mv.String(), ), ) } -func (formatter *mlrvalFormatterToString) FormatFloat(floatValue float64) string { +func (formatter *formatterToString) FormatFloat(floatValue float64) string { return strconv.FormatFloat(floatValue, 'g', -1, 64) } diff --git a/internal/pkg/types/mlrval_format_test.go b/internal/pkg/mlrval/mlrval_format_test.go similarity index 59% rename from internal/pkg/types/mlrval_format_test.go rename to internal/pkg/mlrval/mlrval_format_test.go index 330732917..c26bf1f6d 100644 --- a/internal/pkg/types/mlrval_format_test.go +++ b/internal/pkg/mlrval/mlrval_format_test.go @@ -1,25 +1,18 @@ -// ================================================================ -// Most Miller tests (thousands of them) are command-line-driven via -// mlr regtest. Here are some cases needing special focus. -// ================================================================ - -// Invoke as: -// * cd internal/pkg/types -// * go test -// Or: -// * cd go -// * go test github.com/johnkerl/miller/internal/pkg/types/... - -package types +package mlrval import ( "testing" + + "github.com/stretchr/testify/assert" ) -func TestFormatterToString(t *testing.T) { +func TestFoo(t *testing.T) { + assert.Equal(t, true, true) +} - mv := MlrvalFromString("hello") - formatter := newMlrvalFormatterToString("%s") +func TestFormatterToString(t *testing.T) { + mv := FromString("hello") + formatter := newFormatterToString("%s") fmv := formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -28,8 +21,8 @@ func TestFormatterToString(t *testing.T) { t.Fatal() } - mv = MLRVAL_NULL - formatter = newMlrvalFormatterToString("%s") + mv = NULL + formatter = newFormatterToString("%s") fmv = formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -38,8 +31,8 @@ func TestFormatterToString(t *testing.T) { t.Fatal() } - mv = MLRVAL_TRUE - formatter = newMlrvalFormatterToString("%s") + mv = TRUE + formatter = newFormatterToString("%s") fmv = formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -48,8 +41,8 @@ func TestFormatterToString(t *testing.T) { t.Fatal() } - mv = MLRVAL_FALSE - formatter = newMlrvalFormatterToString("%s") + mv = FALSE + formatter = newFormatterToString("%s") fmv = formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -58,8 +51,8 @@ func TestFormatterToString(t *testing.T) { t.Fatal() } - mv = MlrvalFromInt(10) - formatter = newMlrvalFormatterToString("%s") + mv = FromInt(10) + formatter = newFormatterToString("%s") fmv = formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -68,8 +61,8 @@ func TestFormatterToString(t *testing.T) { t.Fatal() } - mv = MlrvalFromString("hello") - formatter = newMlrvalFormatterToString("[[%s]]") + mv = FromString("hello") + formatter = newFormatterToString("[[%s]]") fmv = formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -81,9 +74,8 @@ func TestFormatterToString(t *testing.T) { } func TestFormatterToInt(t *testing.T) { - - mv := MlrvalFromString("hello") - formatter := newMlrvalFormatterToInt("%d") + mv := FromString("hello") + formatter := newFormatterToInt("%d") fmv := formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -92,8 +84,8 @@ func TestFormatterToInt(t *testing.T) { t.Fatal() } - mv = MLRVAL_NULL - formatter = newMlrvalFormatterToInt("%d") + mv = NULL + formatter = newFormatterToInt("%d") fmv = formatter.Format(mv) if !fmv.IsNull() { t.Fatal() @@ -102,8 +94,8 @@ func TestFormatterToInt(t *testing.T) { t.Fatal() } - mv = MLRVAL_TRUE - formatter = newMlrvalFormatterToInt("%d") + mv = TRUE + formatter = newFormatterToInt("%d") fmv = formatter.Format(mv) if !fmv.IsBool() { t.Fatal() @@ -112,8 +104,8 @@ func TestFormatterToInt(t *testing.T) { t.Fatal() } - mv = MLRVAL_FALSE - formatter = newMlrvalFormatterToInt("%d") + mv = FALSE + formatter = newFormatterToInt("%d") fmv = formatter.Format(mv) if !fmv.IsBool() { t.Fatal() @@ -122,8 +114,8 @@ func TestFormatterToInt(t *testing.T) { t.Fatal() } - mv = MlrvalFromInt(10) - formatter = newMlrvalFormatterToInt("%d") + mv = FromInt(10) + formatter = newFormatterToInt("%d") fmv = formatter.Format(mv) if !fmv.IsInt() { t.Fatal() @@ -132,8 +124,8 @@ func TestFormatterToInt(t *testing.T) { t.Fatal() } - mv = MlrvalFromInt(10) - formatter = newMlrvalFormatterToInt("[[0x%x]]") + mv = FromInt(10) + formatter = newFormatterToInt("[[0x%x]]") fmv = formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -142,8 +134,8 @@ func TestFormatterToInt(t *testing.T) { t.Fatal() } - mv = MlrvalFromFloat64(10.1) - formatter = newMlrvalFormatterToInt("%d") + mv = FromFloat(10.1) + formatter = newFormatterToInt("%d") fmv = formatter.Format(mv) if !fmv.IsInt() { t.Fatal() @@ -156,8 +148,8 @@ func TestFormatterToInt(t *testing.T) { func TestFormatterToFloat(t *testing.T) { - mv := MlrvalFromString("hello") - formatter := newMlrvalFormatterToFloat("%.4f") + mv := FromString("hello") + formatter := newFormatterToFloat("%.4f") fmv := formatter.Format(mv) if !fmv.IsString() { t.Fatal() @@ -166,8 +158,8 @@ func TestFormatterToFloat(t *testing.T) { t.Fatal() } - mv = MLRVAL_NULL - formatter = newMlrvalFormatterToFloat("%.4f") + mv = NULL + formatter = newFormatterToFloat("%.4f") fmv = formatter.Format(mv) if !fmv.IsNull() { t.Fatal() @@ -176,8 +168,8 @@ func TestFormatterToFloat(t *testing.T) { t.Fatal() } - mv = MLRVAL_TRUE - formatter = newMlrvalFormatterToFloat("%.4f") + mv = TRUE + formatter = newFormatterToFloat("%.4f") fmv = formatter.Format(mv) if !fmv.IsBool() { t.Fatal() @@ -186,8 +178,8 @@ func TestFormatterToFloat(t *testing.T) { t.Fatal() } - mv = MLRVAL_FALSE - formatter = newMlrvalFormatterToFloat("%.4f") + mv = FALSE + formatter = newFormatterToFloat("%.4f") fmv = formatter.Format(mv) if !fmv.IsBool() { t.Fatal() @@ -196,8 +188,8 @@ func TestFormatterToFloat(t *testing.T) { t.Fatal() } - mv = MlrvalFromFloat64(10) - formatter = newMlrvalFormatterToFloat("%.4f") + mv = FromFloat(10) + formatter = newFormatterToFloat("%.4f") fmv = formatter.Format(mv) if !fmv.IsFloat() { t.Fatal() @@ -206,8 +198,8 @@ func TestFormatterToFloat(t *testing.T) { t.Fatal() } - mv = MlrvalFromFloat64(10.1) - formatter = newMlrvalFormatterToFloat("%.4f") + mv = FromFloat(10.1) + formatter = newFormatterToFloat("%.4f") fmv = formatter.Format(mv) if !fmv.IsFloat() { t.Fatal() @@ -219,8 +211,8 @@ func TestFormatterToFloat(t *testing.T) { func TestFormatter(t *testing.T) { - mv := MlrvalFromString("hello") - formatter, err := GetMlrvalFormatter("%d") + mv := FromString("hello") + formatter, err := GetFormatter("%d") if err != nil { t.Fatal() } @@ -232,8 +224,8 @@ func TestFormatter(t *testing.T) { t.Fatal() } - mv = MLRVAL_NULL - formatter, err = GetMlrvalFormatter("%d") + mv = NULL + formatter, err = GetFormatter("%d") if err != nil { t.Fatal() } @@ -245,8 +237,8 @@ func TestFormatter(t *testing.T) { t.Fatal() } - mv = MLRVAL_TRUE - formatter, err = GetMlrvalFormatter("%d") + mv = TRUE + formatter, err = GetFormatter("%d") if err != nil { t.Fatal() } @@ -258,8 +250,8 @@ func TestFormatter(t *testing.T) { t.Fatal() } - mv = MLRVAL_FALSE - formatter, err = GetMlrvalFormatter("%d") + mv = FALSE + formatter, err = GetFormatter("%d") if err != nil { t.Fatal() } @@ -271,8 +263,8 @@ func TestFormatter(t *testing.T) { t.Fatal() } - mv = MlrvalFromFloat64(10.123) - formatter, err = GetMlrvalFormatter("%d") + mv = FromFloat(10.123) + formatter, err = GetFormatter("%d") if err != nil { t.Fatal() } @@ -284,8 +276,8 @@ func TestFormatter(t *testing.T) { t.Fatal() } - mv = MlrvalFromFloat64(10.1) - formatter, err = GetMlrvalFormatter("%d") + mv = FromFloat(10.1) + formatter, err = GetFormatter("%d") if err != nil { t.Fatal() } diff --git a/internal/pkg/mlrval/mlrval_get.go b/internal/pkg/mlrval/mlrval_get.go new file mode 100644 index 000000000..072e75595 --- /dev/null +++ b/internal/pkg/mlrval/mlrval_get.go @@ -0,0 +1,148 @@ +package mlrval + +import ( + "fmt" + "os" + + "github.com/johnkerl/miller/internal/pkg/lib" +) + +// It's essential that we use mv.Type() not mv.mvtype, or use an Is...() +// predicate, or another Get...(), since types are JIT-computed on first access +// for most data-file values. See type.go for more information. + +func (mv *Mlrval) GetTypeBit() int { + return 1 << mv.Type() +} + +func (mv *Mlrval) GetStringValue() (stringValue string, isString bool) { + if mv.Type() == MT_STRING || mv.Type() == MT_VOID { + return mv.printrep, true + } else { + return "", false + } +} + +func (mv *Mlrval) GetIntValue() (intValue int, isInt bool) { + if mv.Type() == MT_INT { + return mv.intval, true + } else { + return -999, false + } +} + +func (mv *Mlrval) GetFloatValue() (floatValue float64, isFloat bool) { + if mv.Type() == MT_FLOAT { + return mv.floatval, true + } else { + return -777.0, false + } +} + +func (mv *Mlrval) GetNumericToFloatValue() (floatValue float64, isFloat bool) { + if mv.Type() == MT_FLOAT { + return mv.floatval, true + } else if mv.Type() == MT_INT { + return float64(mv.intval), true + } else { + return -888.0, false + } +} + +func (mv *Mlrval) GetNumericNegativeorDie() bool { + floatValue, ok := mv.GetNumericToFloatValue() + lib.InternalCodingErrorIf(!ok) + return floatValue < 0.0 +} + +func (mv *Mlrval) GetBoolValue() (boolValue bool, isBool bool) { + if mv.Type() == MT_BOOL { + return mv.boolval, true + } else { + return false, false + } +} + +func (mv *Mlrval) GetArray() []Mlrval { + if mv.IsArray() { + return mv.arrayval + } else { + return nil + } +} + +func (mv *Mlrval) GetMap() *Mlrmap { + if mv.IsMap() { + return mv.mapval + } else { + return nil + } +} + +func (mv *Mlrval) GetFunction() interface{} { + if mv.Type() == MT_FUNC { + return mv.funcval + } else { + return nil + } +} + +func (mv *Mlrval) GetTypeName() string { + return TYPE_NAMES[mv.Type()] +} + +func GetTypeName(mvtype MVType) string { + return TYPE_NAMES[mvtype] +} + +// These are for built-in functions operating within type-keyed +// disposition-vector/disposition-matrix context. They've already computed +// mv.Type() -- it's a fatal error if they haven't -- and they need the typed +// value. + +func (mv *Mlrval) AcquireStringValue() string { + lib.InternalCodingErrorIf(mv.mvtype != MT_STRING && mv.mvtype != MT_VOID) + return mv.printrep +} + +func (mv *Mlrval) AcquireIntValue() int { + lib.InternalCodingErrorIf(mv.mvtype != MT_INT) + return mv.intval +} + +func (mv *Mlrval) AcquireFloatValue() float64 { + lib.InternalCodingErrorIf(mv.mvtype != MT_FLOAT) + return mv.floatval +} + +func (mv *Mlrval) AcquireBoolValue() bool { + lib.InternalCodingErrorIf(mv.mvtype != MT_BOOL) + return mv.boolval +} + +func (mv *Mlrval) AcquireArrayValue() []Mlrval { + lib.InternalCodingErrorIf(mv.mvtype != MT_ARRAY) + return mv.arrayval +} + +func (mv *Mlrval) AcquireMapValue() *Mlrmap { + lib.InternalCodingErrorIf(mv.mvtype != MT_MAP) + return mv.mapval +} + +func (mv *Mlrval) GetNumericToFloatValueOrDie() (floatValue float64) { + floatValue, ok := mv.GetNumericToFloatValue() + if !ok { + fmt.Fprintf( + os.Stderr, + "%s: couldn't parse \"%s\" as number.", + "mlr", mv.String(), + ) + os.Exit(1) + } + return floatValue +} + +func (mv *Mlrval) AssertNumeric() { + _ = mv.GetNumericToFloatValueOrDie() +} diff --git a/internal/pkg/mlrval/mlrval_get_test.go b/internal/pkg/mlrval/mlrval_get_test.go new file mode 100644 index 000000000..33598c39a --- /dev/null +++ b/internal/pkg/mlrval/mlrval_get_test.go @@ -0,0 +1,188 @@ +// ================================================================ +// Tests mlrval typed-value extractors +// ================================================================ + +package mlrval + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetString(t *testing.T) { + mv := FromInferredType("234") + stringval, ok := mv.GetStringValue() + assert.False(t, ok) + + mv = FromDeferredType("234") + stringval, ok = mv.GetStringValue() + assert.False(t, ok) + + mv = FromInferredType("234.5") + stringval, ok = mv.GetStringValue() + assert.False(t, ok) + + mv = FromDeferredType("234.5") + stringval, ok = mv.GetStringValue() + assert.False(t, ok) + + mv = FromInferredType("abc") + stringval, ok = mv.GetStringValue() + assert.Equal(t, "abc", stringval) + assert.True(t, ok) + + mv = FromDeferredType("abc") + stringval, ok = mv.GetStringValue() + assert.Equal(t, "abc", stringval) + assert.True(t, ok) + + mv = FromInferredType("") + stringval, ok = mv.GetStringValue() + assert.Equal(t, "", stringval) + assert.True(t, ok) + + mv = FromDeferredType("") + stringval, ok = mv.GetStringValue() + assert.Equal(t, "", stringval) + assert.True(t, ok) +} + +func TestGetIntValue(t *testing.T) { + mv := FromInferredType("123") + intval, ok := mv.GetIntValue() + assert.Equal(t, 123, intval) + assert.True(t, ok) + + mv = FromDeferredType("123") + intval, ok = mv.GetIntValue() + assert.Equal(t, 123, intval) + assert.True(t, ok) + + mv = FromInferredType("123.4") + intval, ok = mv.GetIntValue() + assert.False(t, ok) + + mv = FromDeferredType("123.4") + intval, ok = mv.GetIntValue() + assert.False(t, ok) + + mv = FromInferredType("abc") + intval, ok = mv.GetIntValue() + assert.False(t, ok) + + mv = FromDeferredType("abc") + intval, ok = mv.GetIntValue() + assert.False(t, ok) +} + +func TestGetFloatValue(t *testing.T) { + mv := FromInferredType("234") + floatval, ok := mv.GetFloatValue() + assert.False(t, ok) + + mv = FromDeferredType("234") + floatval, ok = mv.GetFloatValue() + assert.False(t, ok) + + mv = FromInferredType("234.5") + floatval, ok = mv.GetFloatValue() + assert.Equal(t, 234.5, floatval) + assert.True(t, ok) + + mv = FromDeferredType("234.5") + floatval, ok = mv.GetFloatValue() + assert.Equal(t, 234.5, floatval) + assert.True(t, ok) + + mv = FromInferredType("abc") + floatval, ok = mv.GetFloatValue() + assert.False(t, ok) + + mv = FromDeferredType("abc") + floatval, ok = mv.GetFloatValue() + assert.False(t, ok) +} + +func TestGetNumericToFloatValue(t *testing.T) { + mv := FromInferredType("234") + floatval, ok := mv.GetNumericToFloatValue() + assert.Equal(t, 234.0, floatval) + assert.True(t, ok) + + mv = FromDeferredType("234") + floatval, ok = mv.GetNumericToFloatValue() + assert.Equal(t, 234.0, floatval) + assert.True(t, ok) + + mv = FromInferredType("234.5") + floatval, ok = mv.GetNumericToFloatValue() + assert.Equal(t, 234.5, floatval) + assert.True(t, ok) + + mv = FromDeferredType("234.5") + floatval, ok = mv.GetNumericToFloatValue() + assert.Equal(t, 234.5, floatval) + assert.True(t, ok) + + mv = FromInferredType("abc") + floatval, ok = mv.GetNumericToFloatValue() + assert.False(t, ok) + + mv = FromDeferredType("abc") + floatval, ok = mv.GetNumericToFloatValue() + assert.False(t, ok) +} + +func TestGetBoolValue(t *testing.T) { + mv := FromInferredType("234") + boolval, ok := mv.GetBoolValue() + assert.False(t, ok) + + mv = FromDeferredType("234") + boolval, ok = mv.GetBoolValue() + assert.False(t, ok) + + mv = FromInferredType("abc") + boolval, ok = mv.GetBoolValue() + assert.False(t, ok) + + mv = FromDeferredType("abc") + boolval, ok = mv.GetBoolValue() + assert.False(t, ok) + + mv = FromInferredType("true") + boolval, ok = mv.GetBoolValue() + assert.True(t, boolval) + assert.True(t, ok) + + mv = FromDeferredType("false") + boolval, ok = mv.GetBoolValue() + assert.False(t, ok, "from-data-file \"false\" should infer to string") +} + +func TestGetTypeName(t *testing.T) { + mv := FromInferredType("234") + assert.Equal(t, "int", mv.GetTypeName()) + + mv = FromDeferredType("234") + assert.Equal(t, "int", mv.GetTypeName()) + + mv = FromInferredType("234.5") + assert.Equal(t, "float", mv.GetTypeName()) + + mv = FromDeferredType("234.5") + assert.Equal(t, "float", mv.GetTypeName()) + + mv = FromInferredType("abc") + assert.Equal(t, "string", mv.GetTypeName()) + + mv = FromDeferredType("abc") + assert.Equal(t, "string", mv.GetTypeName()) + + mv = FromInferredType("") + assert.Equal(t, "empty", mv.GetTypeName()) + + mv = FromDeferredType("") + assert.Equal(t, "empty", mv.GetTypeName()) +} diff --git a/internal/pkg/mlrval/mlrval_infer.go b/internal/pkg/mlrval/mlrval_infer.go new file mode 100644 index 000000000..e445d04ad --- /dev/null +++ b/internal/pkg/mlrval/mlrval_infer.go @@ -0,0 +1,102 @@ +package mlrval + +import ( + "regexp" + "strings" + + "github.com/johnkerl/miller/internal/pkg/lib" +) + +// TODO: no infer-bool from data files. Always false in this path. + +// It's essential that we use mv.Type() not mv.mvtype since types are +// JIT-computed on first access for most data-file values. See type.go for more +// information. + +func (mv *Mlrval) Type() MVType { + if mv.mvtype == MT_PENDING { + packageLevelInferrer(mv, mv.printrep, false) + } + return mv.mvtype +} + +// Support for mlr -S, mlr -A, mlr -O. +type tInferrer func(mv *Mlrval, input string, inferBool bool) *Mlrval + +var packageLevelInferrer tInferrer = inferNormally + +func SetInferrerNoOctal() { + packageLevelInferrer = inferWithOctalSuppress +} +func SetInferrerIntAsFloat() { + packageLevelInferrer = inferWithIntAsFloat +} +func SetInferrerStringOnly() { + packageLevelInferrer = inferStringOnly +} + +// When loading data files, don't scan these words into floats -- even though +// the Go library is willing to do so. +var downcasedFloatNamesToNotInfer = map[string]bool{ + "inf": true, + "+inf": true, + "-inf": true, + "infinity": true, + "+infinity": true, + "-infinity": true, + "nan": true, +} + +func inferNormally(mv *Mlrval, input string, inferBool bool) *Mlrval { + if input == "" { + return mv.SetFromVoid() + } + + intval, iok := lib.TryIntFromString(input) + if iok { + return mv.SetFromPrevalidatedIntString(input, intval) + } + + if downcasedFloatNamesToNotInfer[strings.ToLower(input)] == false { + floatval, fok := lib.TryFloatFromString(input) + if fok { + return mv.SetFromPrevalidatedFloatString(input, floatval) + } + } + + if inferBool { + boolval, bok := lib.TryBoolFromBoolString(input) + if bok { + return mv.SetFromPrevalidatedBoolString(input, boolval) + } + } + return mv.SetFromString(input) +} + +var octalDetector = regexp.MustCompile("^-?0[0-9]+") + +func inferWithOctalSuppress(mv *Mlrval, input string, inferBool bool) *Mlrval { + inferNormally(mv, input, inferBool) + if mv.mvtype != MT_INT && mv.mvtype != MT_FLOAT { + return mv + } + + if octalDetector.MatchString(mv.printrep) { + return mv.SetFromString(input) + } else { + return mv + } +} + +func inferWithIntAsFloat(mv *Mlrval, input string, inferBool bool) *Mlrval { + inferNormally(mv, input, inferBool) + if mv.Type() == MT_INT { + mv.floatval = float64(mv.intval) + mv.mvtype = MT_FLOAT + } + return mv +} + +func inferStringOnly(mv *Mlrval, input string, inferBool bool) *Mlrval { + return mv.SetFromString(input) +} diff --git a/internal/pkg/mlrval/mlrval_is.go b/internal/pkg/mlrval/mlrval_is.go new file mode 100644 index 000000000..6fbb51f9b --- /dev/null +++ b/internal/pkg/mlrval/mlrval_is.go @@ -0,0 +1,135 @@ +package mlrval + +import ( + "github.com/johnkerl/miller/internal/pkg/lib" +) + +// It's essential that we use mv.Type() not mv.mvtype since types are +// JIT-computed on first access for most data-file values. See type.go for more +// information. + +func (mv *Mlrval) IsLegit() bool { + return mv.Type() >= MT_VOID +} + +// TODO: comment no JIT-infer here -- absent is non-inferrable and we needn't take the expense of JIT. +func (mv *Mlrval) IsErrorOrAbsent() bool { + t := mv.mvtype + return t == MT_ERROR || t == MT_ABSENT +} + +func (mv *Mlrval) IsError() bool { + return mv.Type() == MT_ERROR +} + +// TODO: comment no JIT-infer here -- absent is non-inferrable and we needn't take the expense of JIT. +func (mv *Mlrval) IsAbsent() bool { + return mv.mvtype == MT_ABSENT +} + +// TODO: comment no JIT-infer here -- NULL is non-inferrable and we needn't take the expense of JIT. +// This is a literal in JSON files, or else explicitly set to NULL. +func (mv *Mlrval) IsNull() bool { + return mv.mvtype == MT_NULL +} + +func (mv *Mlrval) IsVoid() bool { + if mv.mvtype == MT_VOID { + return true + } + if mv.mvtype == MT_PENDING && mv.printrep == "" { + lib.InternalCodingErrorIf(!mv.printrepValid) + return true + } + return false +} + +func (mv *Mlrval) IsErrorOrVoid() bool { + return mv.IsError() || mv.IsVoid() +} + +// * Error is non-empty +// * Absent is non-empty (shouldn't have been assigned in the first place; error should be surfaced) +// * Void is empty +// * Empty string is empty +// * Int/float/bool/array/map are all non-empty +func (mv *Mlrval) IsEmptyString() bool { + if mv.mvtype == MT_VOID { + return true + } + if mv.mvtype == MT_STRING && mv.printrep == "" { + return true + } + if mv.mvtype == MT_PENDING && mv.printrep == "" { + lib.InternalCodingErrorIf(!mv.printrepValid) + return true + } + return false +} + +func (mv *Mlrval) IsString() bool { + return mv.Type() == MT_STRING +} + +func (mv *Mlrval) IsStringOrVoid() bool { + t := mv.Type() + return t == MT_STRING || t == MT_VOID +} + +func (mv *Mlrval) IsStringOrInt() bool { + t := mv.Type() + return t == MT_STRING || t == MT_VOID || t == MT_INT +} + +func (mv *Mlrval) IsInt() bool { + return mv.Type() == MT_INT +} + +func (mv *Mlrval) IsFloat() bool { + return mv.Type() == MT_FLOAT +} + +func (mv *Mlrval) IsNumeric() bool { + t := mv.Type() + return t == MT_INT || t == MT_FLOAT +} + +func (mv *Mlrval) IsIntZero() bool { + return mv.Type() == MT_INT && mv.intval == 0 +} + +func (mv *Mlrval) IsBool() bool { + return mv.Type() == MT_BOOL +} + +func (mv *Mlrval) IsTrue() bool { + return mv.Type() == MT_BOOL && mv.boolval == true +} +func (mv *Mlrval) IsFalse() bool { + return mv.Type() == MT_BOOL && mv.boolval == false +} + +func (mv *Mlrval) IsArray() bool { + // TODO: comment non-deferrable type -- don't force a (potentially + // expensive in bulk) JIT-infer of other types + // return mv.Type() == MT_ARRAY + return mv.mvtype == MT_ARRAY +} +func (mv *Mlrval) IsMap() bool { + // TODO: comment non-deferrable type -- don't force a (potentially + // expensive in bulk) JIT-infer of other types + // return mv.Type() == MT_ARRAY + return mv.mvtype == MT_MAP +} +func (mv *Mlrval) IsArrayOrMap() bool { + // TODO: comment why not + // In flatten we don't want to type-infer things that don't need to be jitted. + // Arrays & maps are never from deferred type. + // t := mv.Type() + t := mv.mvtype + return t == MT_ARRAY || t == MT_MAP +} + +func (mv *Mlrval) IsFunction() bool { + return mv.mvtype == MT_FUNC +} diff --git a/internal/pkg/mlrval/mlrval_is_test.go b/internal/pkg/mlrval/mlrval_is_test.go new file mode 100644 index 000000000..5e89c7e3c --- /dev/null +++ b/internal/pkg/mlrval/mlrval_is_test.go @@ -0,0 +1,149 @@ +// ================================================================ +// Tests mlrval constructors. +// ================================================================ + +package mlrval + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsLegit(t *testing.T) { + assert.False(t, ERROR.IsLegit()) + assert.False(t, ABSENT.IsLegit()) + assert.False(t, NULL.IsLegit()) + assert.True(t, FromString("").IsLegit()) + assert.True(t, FromString("abc").IsLegit()) + assert.True(t, FromInt(123).IsLegit()) + assert.True(t, FromFloat(123.4).IsLegit()) + assert.True(t, FromBool(true).IsLegit()) + assert.True(t, FromArray([]Mlrval{*FromInt(10)}).IsLegit()) + assert.True(t, FromMap(NewMlrmap()).IsLegit()) +} + +func TestIsErrorOrAbsent(t *testing.T) { + assert.True(t, ERROR.IsErrorOrAbsent()) + assert.True(t, ABSENT.IsErrorOrAbsent()) + assert.False(t, NULL.IsErrorOrAbsent()) + assert.False(t, FromString("").IsErrorOrAbsent()) +} + +func TestIsError(t *testing.T) { + assert.True(t, ERROR.IsError()) + assert.False(t, ABSENT.IsError()) + assert.False(t, NULL.IsError()) + assert.False(t, FromString("").IsError()) +} + +func TestIsAbsent(t *testing.T) { + assert.False(t, ERROR.IsAbsent()) + assert.True(t, ABSENT.IsAbsent()) + assert.False(t, NULL.IsAbsent()) + assert.False(t, FromString("").IsAbsent()) +} + +func TestIsNull(t *testing.T) { + assert.False(t, ERROR.IsNull()) + assert.False(t, ABSENT.IsNull()) + assert.True(t, NULL.IsNull()) + assert.False(t, FromString("").IsNull()) +} + +func TestIsVoid(t *testing.T) { + assert.False(t, ERROR.IsVoid()) + assert.False(t, ABSENT.IsVoid()) + assert.False(t, NULL.IsVoid()) + assert.True(t, FromString("").IsVoid()) + assert.True(t, FromDeferredType("").IsVoid()) + assert.True(t, FromInferredType("").IsVoid()) + assert.False(t, FromDeferredType("abc").IsVoid()) + assert.False(t, FromInferredType("abc").IsVoid()) +} + +func TestIsEmptyString(t *testing.T) { + assert.False(t, ERROR.IsEmptyString()) + assert.False(t, ABSENT.IsEmptyString()) + assert.False(t, NULL.IsEmptyString()) + assert.True(t, FromString("").IsEmptyString()) + assert.True(t, FromDeferredType("").IsEmptyString()) + assert.True(t, FromInferredType("").IsEmptyString()) + assert.False(t, FromDeferredType("abc").IsEmptyString()) + assert.False(t, FromInferredType("abc").IsEmptyString()) +} + +func TestIsString(t *testing.T) { + assert.False(t, ERROR.IsString()) + assert.False(t, ABSENT.IsString()) + assert.False(t, NULL.IsString()) + assert.False(t, FromString("").IsString()) + assert.False(t, FromDeferredType("").IsString()) + assert.False(t, FromInferredType("").IsString()) + assert.True(t, FromDeferredType("abc").IsString()) + assert.True(t, FromInferredType("abc").IsString()) +} + +func TestIsStringOrVoid(t *testing.T) { + assert.False(t, ERROR.IsStringOrVoid()) + assert.False(t, ABSENT.IsStringOrVoid()) + assert.False(t, NULL.IsStringOrVoid()) + assert.True(t, FromString("").IsStringOrVoid()) + assert.True(t, FromDeferredType("").IsStringOrVoid()) + assert.True(t, FromInferredType("").IsStringOrVoid()) + assert.True(t, FromDeferredType("abc").IsStringOrVoid()) + assert.True(t, FromInferredType("abc").IsStringOrVoid()) +} + +func TestIsInt(t *testing.T) { + assert.True(t, FromDeferredType("123").IsInt()) + assert.True(t, FromInferredType("123").IsInt()) + assert.False(t, FromDeferredType("123.4").IsInt()) + assert.False(t, FromInferredType("123.4").IsInt()) + assert.False(t, FromDeferredType("abc").IsInt()) + assert.False(t, FromInferredType("abc").IsInt()) +} + +func TestIsIntZero(t *testing.T) { + assert.True(t, FromDeferredType("0").IsIntZero()) + assert.True(t, FromInferredType("0").IsIntZero()) + assert.True(t, FromDeferredType("-0").IsIntZero()) + assert.True(t, FromInferredType("-0").IsIntZero()) + assert.False(t, FromDeferredType("123").IsIntZero()) + assert.False(t, FromInferredType("123").IsIntZero()) + assert.False(t, FromDeferredType("abc").IsIntZero()) + assert.False(t, FromInferredType("abc").IsIntZero()) +} + +func TestIsFloat(t *testing.T) { + assert.False(t, FromDeferredType("123").IsFloat()) + assert.False(t, FromInferredType("123").IsFloat()) + assert.True(t, FromDeferredType("123.4").IsFloat()) + assert.True(t, FromInferredType("123.4").IsFloat()) + assert.False(t, FromDeferredType("abc").IsFloat()) + assert.False(t, FromInferredType("abc").IsFloat()) +} + +func TestIsNumeric(t *testing.T) { + assert.True(t, FromDeferredType("123").IsNumeric()) + assert.True(t, FromInferredType("123").IsNumeric()) + assert.True(t, FromDeferredType("123.4").IsNumeric()) + assert.True(t, FromInferredType("123.4").IsNumeric()) + assert.False(t, FromDeferredType("abc").IsNumeric()) + assert.False(t, FromInferredType("abc").IsNumeric()) +} + +func TestIsBool(t *testing.T) { + assert.False(t, FromDeferredType("123").IsBool()) + assert.False(t, FromInferredType("123").IsBool()) + assert.False(t, FromDeferredType("123.4").IsBool()) + assert.False(t, FromInferredType("123.4").IsBool()) + assert.False(t, FromDeferredType("abc").IsBool()) + assert.False(t, FromInferredType("abc").IsBool()) + assert.False(t, FromDeferredType("true").IsBool(), "from-data-file \"true\" should infer to string") + assert.True(t, FromDeferredType("true").IsString(), "from-data-file \"true\" should infer to string") + assert.True(t, FromInferredType("true").IsBool()) + assert.False(t, FromDeferredType("false").IsBool(), "from-data-file \"false\" should infer to string") + assert.True(t, FromDeferredType("false").IsString(), "from-data-file \"false\" should infer to string") + assert.True(t, FromInferredType("false").IsBool()) +} diff --git a/internal/pkg/types/mlrval_json.go b/internal/pkg/mlrval/mlrval_json.go similarity index 95% rename from internal/pkg/types/mlrval_json.go rename to internal/pkg/mlrval/mlrval_json.go index be2e95cbc..3da3368aa 100644 --- a/internal/pkg/types/mlrval_json.go +++ b/internal/pkg/mlrval/mlrval_json.go @@ -6,7 +6,7 @@ // Please see also https://golang.org/pkg/encoding/json/ // ================================================================ -package types +package mlrval import ( "bytes" @@ -18,7 +18,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/lib" ) -const MLRVAL_JSON_INDENT_STRING string = " " +const JSON_INDENT_STRING string = " " type TJSONFormatting int @@ -101,16 +101,16 @@ const ( // ---------------------------------------------------------------- func (mv *Mlrval) UnmarshalJSON(inputBytes []byte) error { - *mv = MlrvalFromPending() + *mv = *FromPending() decoder := json.NewDecoder(bytes.NewReader(inputBytes)) - mlrval, eof, err := MlrvalDecodeFromJSON(decoder) + pmv, eof, err := MlrvalDecodeFromJSON(decoder) if eof { return errors.New("Miller JSON parser: unexpected premature EOF.") } if err != nil { return err } - *mv = *mlrval + *mv = *pmv return nil } @@ -135,23 +135,23 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) ( delimiter, isDelim := startToken.(json.Delim) if !isDelim { if startToken == nil { - return MLRVAL_NULL, false, nil + return NULL, false, nil } sval, ok := startToken.(string) if ok { - mlrval := MlrvalFromString(sval) + mlrval := FromString(sval) return mlrval, false, nil } bval, ok := startToken.(bool) if ok { - return MlrvalFromBool(bval), false, nil + return FromBool(bval), false, nil } nval, ok := startToken.(json.Number) if ok { - mlrval := MlrvalFromInferredType(nval.String()) + mlrval := FromInferredType(nval.String()) return mlrval, false, nil } @@ -178,9 +178,9 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) ( ) } - mlrval := MlrvalFromPending() + mv := FromPending() if isArray { - mlrval = MlrvalEmptyArray() + mv = FromEmptyArray() for decoder.More() { element, eof, err := MlrvalDecodeFromJSON(decoder) @@ -191,11 +191,11 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) ( if err != nil { return nil, false, err } - mlrval.ArrayAppend(element) + mv.ArrayAppend(element) } } else { - mlrval = *MlrvalFromEmptyMap() + mv = FromEmptyMap() for decoder.More() { key, eof, err := MlrvalDecodeFromJSON(decoder) @@ -223,7 +223,7 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) ( } // xxx check here string-valued key - mlrval.MapPut(key, value) + mv.MapPut(key, value) } } @@ -252,7 +252,7 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) ( return nil, false, imbalanceError } - return &mlrval, false, nil + return mv, false, nil } return nil, false, errors.New("unimplemented") @@ -271,7 +271,7 @@ func (mv *Mlrval) marshalJSONAux( elementNestingDepth int, outputIsStdout bool, ) (string, error) { - switch mv.mvtype { + switch mv.Type() { case MT_PENDING: return mv.marshalJSONPending(outputIsStdout) break @@ -495,7 +495,7 @@ func (mv *Mlrval) marshalJSONArrayMultipleLines( return "", err } for i := 0; i < elementNestingDepth; i++ { - buffer.WriteString(MLRVAL_JSON_INDENT_STRING) + buffer.WriteString(JSON_INDENT_STRING) } buffer.WriteString(elementString) if i < n-1 { @@ -507,7 +507,7 @@ func (mv *Mlrval) marshalJSONArrayMultipleLines( // Write empty array as '[]' if n > 0 { for i := 0; i < elementNestingDepth-1; i++ { - buffer.WriteString(MLRVAL_JSON_INDENT_STRING) + buffer.WriteString(JSON_INDENT_STRING) } } diff --git a/internal/pkg/mlrval/mlrval_new.go b/internal/pkg/mlrval/mlrval_new.go new file mode 100644 index 000000000..507f5ad53 --- /dev/null +++ b/internal/pkg/mlrval/mlrval_new.go @@ -0,0 +1,234 @@ +// ================================================================ +// Constructors +// ================================================================ + +package mlrval + +import ( + //"errors" + + "github.com/johnkerl/miller/internal/pkg/lib" +) + +// TODO: comment for JSON-scanner context. +func FromPending() *Mlrval { + return &Mlrval{ + mvtype: MT_PENDING, + printrep: "(bug-if-you-see-this:case-1)", + printrepValid: false, + } +} + +// TODO: comment JIT context. Some things we already know are typed -- DSL +// things, or JSON contents. Others are deferred, e.g. items from any file +// format except JSON. +// TODO: comment re inferBool. +func FromDeferredType(input string) *Mlrval { + return &Mlrval{ + mvtype: MT_PENDING, + printrep: input, + printrepValid: true, + } +} + +// TODO: comment non-JIT context like mlr put -s. +// TODO: comment re inferBool. +func FromInferredType(input string) *Mlrval { + mv := &Mlrval{ + mvtype: MT_PENDING, + printrep: input, + printrepValid: true, + } + // TODO: comment re inferBool arg + packageLevelInferrer(mv, mv.printrep, true) + return mv +} + +func FromString(input string) *Mlrval { + if input == "" { + return VOID + } + return &Mlrval{ + mvtype: MT_STRING, + printrep: input, + printrepValid: true, + } +} + +func (mv *Mlrval) SetFromString(input string) *Mlrval { + mv.printrep = input + mv.printrepValid = true + if input == "" { + mv.mvtype = MT_VOID + } else { + mv.mvtype = MT_STRING + } + return mv +} + +func (mv *Mlrval) SetFromVoid() *Mlrval { + mv.printrep = "" + mv.printrepValid = true + mv.mvtype = MT_VOID + return mv +} + +func FromInt(input int) *Mlrval { + return &Mlrval{ + mvtype: MT_INT, + printrepValid: false, + intval: input, + } +} + +// TryFromIntString is used by the mlrval Formatter (fmtnum DSL function, +// format-values verb, etc). Each mlrval has printrep and a printrepValid for +// its original string, then a type-code like MT_INT or MT_FLOAT, and +// type-specific storage like intval or floatval. +// +// If the user has taken a mlrval with original string "314" and formatted it +// with "0x%04x" then its printrep will be "0x013a" but its type should still +// be MT_INT. +// +// If on the other hand the user has formatted the same mlrval with +// "[[%0x04x]]" then its printrep will be "[[0x013a]]" and it will be +// MT_STRING. This function supports that. +func TryFromIntString(input string) *Mlrval { + intval, ok := lib.TryIntFromString(input) + if ok { + return FromPrevalidatedIntString(input, intval) + } else { + return FromString(input) + } +} + +// TODO: comment +func (mv *Mlrval) SetFromPrevalidatedIntString(input string, intval int) *Mlrval { + mv.printrep = input + mv.printrepValid = true + mv.intval = intval + mv.mvtype = MT_INT + return mv +} + +// TODO: comment +func FromPrevalidatedIntString(input string, intval int) *Mlrval { + mv := &Mlrval{} + mv.SetFromPrevalidatedIntString(input, intval) + return mv +} + +func FromFloat(input float64) *Mlrval { + return &Mlrval{ + mvtype: MT_FLOAT, + printrepValid: false, + floatval: input, + } +} + +// TryFromFloatString is used by the mlrval Formatter (fmtnum DSL function, +// format-values verb, etc). Each mlrval has printrep and a printrepValid for +// its original string, then a type-code like MT_INT or MT_FLOAT, and +// type-specific storage like intval or floatval. +// +// If the user has taken a mlrval with original string "3.14" and formatted it +// with "%.4f" then its printrep will be "3.1400" but its type should still be +// MT_FLOAT. +// +// If on the other hand the user has formatted the same mlrval with "[[%.4f]]" +// then its printrep will be "[[3.1400]]" and it will be MT_STRING. This +// function supports that. +func TryFromFloatString(input string) *Mlrval { + floatval, ok := lib.TryFloatFromString(input) + if ok { + return FromPrevalidatedFloatString(input, floatval) + } else { + return FromString(input) + } +} + +// TODO: comment +func (mv *Mlrval) SetFromPrevalidatedFloatString(input string, floatval float64) *Mlrval { + mv.printrep = input + mv.printrepValid = true + mv.floatval = floatval + mv.mvtype = MT_FLOAT + return mv +} + +// TODO: comment +func FromPrevalidatedFloatString(input string, floatval float64) *Mlrval { + mv := &Mlrval{} + mv.SetFromPrevalidatedFloatString(input, floatval) + return mv +} + +func FromBool(input bool) *Mlrval { + if input == true { + return TRUE + } else { + return FALSE + } +} + +func FromBoolString(input string) *Mlrval { + if input == "true" { + return TRUE + } else if input == "false" { + return FALSE + } else { + lib.InternalCodingErrorIf(true) + return nil // not reached + } +} + +// TODO: comment +func (mv *Mlrval) SetFromPrevalidatedBoolString(input string, boolval bool) *Mlrval { + mv.printrep = input + mv.printrepValid = true + mv.boolval = boolval + mv.mvtype = MT_BOOL + return mv +} + +// The user-defined function is of type 'interface{}' here to avoid what would +// otherwise be a package-dependency cycle between this package and +// github.com/johnkerl/miller/internal/pkg/dsl/cst. +// +// Nominally the name argument is the user-specified name if `func f(a, b) { +// ... }`, or some autogenerated UUID like `fl0052` if `func (a, b) { ... }`. + +func FromFunction(funcval interface{}, name string) *Mlrval { + return &Mlrval{ + mvtype: MT_FUNC, + printrep: name, + printrepValid: true, + funcval: funcval, + } +} + +func FromArray(arrayval []Mlrval) *Mlrval { + return &Mlrval{ + mvtype: MT_ARRAY, + printrep: "(bug-if-you-see-this:case-4)", // INVALID_PRINTREP, + printrepValid: false, + arrayval: CopyMlrvalArray(arrayval), + } +} + +func FromEmptyArray() *Mlrval { + return FromArray(make([]Mlrval, 0)) +} + +func FromMap(mapval *Mlrmap) *Mlrval { + return &Mlrval{ + mvtype: MT_MAP, + printrep: "(bug-if-you-see-this:case-5)", // INVALID_PRINTREP, + printrepValid: false, + mapval: mapval.Copy(), + } +} + +func FromEmptyMap() *Mlrval { + return FromMap(NewMlrmap()) +} diff --git a/internal/pkg/mlrval/mlrval_new_test.go b/internal/pkg/mlrval/mlrval_new_test.go new file mode 100644 index 000000000..26dfeadc3 --- /dev/null +++ b/internal/pkg/mlrval/mlrval_new_test.go @@ -0,0 +1,142 @@ +// ================================================================ +// Tests mlrval constructors. +// ================================================================ + +package mlrval + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFromDeferredType(t *testing.T) { + mv := FromDeferredType("123") + assert.Equal(t, MVType(MT_PENDING), MVType(mv.mvtype)) + assert.Equal(t, "123", mv.printrep) + assert.True(t, mv.printrepValid) + + mv = FromDeferredType("true") + assert.Equal(t, MVType(MT_PENDING), MVType(mv.mvtype)) + assert.Equal(t, "true", mv.printrep) + assert.True(t, mv.printrepValid) + + mv = FromDeferredType("abc") + assert.Equal(t, MVType(MT_PENDING), MVType(mv.mvtype)) + assert.Equal(t, "abc", mv.printrep) + assert.True(t, mv.printrepValid) + + mv = FromDeferredType("") + assert.Equal(t, MVType(MT_PENDING), MVType(mv.mvtype)) + assert.Equal(t, "", mv.printrep) + assert.True(t, mv.printrepValid) +} + +func TestFromInferredType(t *testing.T) { + mv := FromInferredType("123") + assert.Equal(t, MVType(MT_INT), MVType(mv.mvtype)) + assert.Equal(t, "123", mv.printrep) + assert.True(t, mv.printrepValid) + assert.Equal(t, mv.intval, 123) + + mv = FromInferredType("true") + assert.Equal(t, MVType(MT_BOOL), MVType(mv.mvtype)) + assert.Equal(t, "true", mv.printrep) + assert.True(t, mv.printrepValid) + assert.Equal(t, mv.boolval, true) + + mv = FromInferredType("abc") + assert.Equal(t, MVType(MT_STRING), MVType(mv.mvtype)) + assert.Equal(t, "abc", mv.printrep) + assert.True(t, mv.printrepValid) + + mv = FromInferredType("") + assert.Equal(t, MVType(MT_VOID), MVType(mv.mvtype)) + assert.Equal(t, "", mv.printrep) + assert.True(t, mv.printrepValid) +} + +func TestFromString(t *testing.T) { + mv := FromString("123") + assert.Equal(t, MVType(MT_STRING), MVType(mv.mvtype)) + assert.Equal(t, "123", mv.printrep) + assert.True(t, mv.printrepValid) + + mv = FromString("") + assert.Equal(t, MVType(MT_VOID), MVType(mv.mvtype)) + assert.Equal(t, "", mv.printrep) + assert.True(t, mv.printrepValid) +} + +func TestFromInt(t *testing.T) { + mv := FromInt(123) + assert.Equal(t, MVType(MT_INT), MVType(mv.mvtype)) + assert.False(t, mv.printrepValid, "printrep should not be computed yet") +} + +func TestTryFromIntString(t *testing.T) { + mv := TryFromIntString("123") + assert.Equal(t, MVType(MT_INT), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid, "printrep should be computed") + + mv = TryFromIntString("[123]") + assert.Equal(t, MVType(MT_STRING), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid, "printrep should be computed") +} + +func TestFromFloat(t *testing.T) { + mv := FromFloat(123.4) + assert.Equal(t, MVType(MT_FLOAT), MVType(mv.mvtype)) + assert.False(t, mv.printrepValid, "printrep should not be computed yet") +} + +func TestTryFromFloatString(t *testing.T) { + mv := TryFromFloatString("123.4") + assert.Equal(t, MVType(MT_FLOAT), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid, "printrep should be computed") + + mv = TryFromIntString("[123.4]") + assert.Equal(t, MVType(MT_STRING), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid, "printrep should be computed") +} + +func TestFromBool(t *testing.T) { + mv := FromBool(true) + assert.Equal(t, MVType(MT_BOOL), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid) + + mv = FromBool(false) + assert.Equal(t, MVType(MT_BOOL), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid) +} + +func TestFromBoolString(t *testing.T) { + mv := FromBoolString("true") + assert.Equal(t, MVType(MT_BOOL), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid) + + mv = FromBoolString("false") + assert.Equal(t, MVType(MT_BOOL), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid) +} + +func TestFromFunction(t *testing.T) { + mv := FromFunction("test data", "f001") + assert.Equal(t, MVType(MT_FUNC), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid) + assert.Equal(t, "test data", mv.funcval.(string)) +} + +func TestFromArray(t *testing.T) { + mv := FromArray([]Mlrval{*FromInt(10)}) + assert.Equal(t, MVType(MT_ARRAY), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid) + assert.Equal(t, 1, len(mv.arrayval)) +} + +func TestFromMap(t *testing.T) { + mv := FromMap(NewMlrmap()) + assert.Equal(t, MVType(MT_MAP), MVType(mv.mvtype)) + assert.True(t, mv.printrepValid) + assert.True(t, mv.mapval.IsEmpty()) +} diff --git a/internal/pkg/types/mlrval_output.go b/internal/pkg/mlrval/mlrval_output.go similarity index 70% rename from internal/pkg/types/mlrval_output.go rename to internal/pkg/mlrval/mlrval_output.go index 022196758..b3cd4d10c 100644 --- a/internal/pkg/types/mlrval_output.go +++ b/internal/pkg/mlrval/mlrval_output.go @@ -1,4 +1,4 @@ -package types +package mlrval import ( "fmt" @@ -6,52 +6,60 @@ import ( "strconv" ) -// Empty string means use default format. -// Set from the CLI parser using mlr --ofmt. -var mlrvalFloatOutputFormatter IMlrvalFormatter = nil - -func SetMlrvalFloatOutputFormat(formatString string) error { - formatter, err := GetMlrvalFormatter(formatString) - if err != nil { - return err +// Must have non-pointer receiver in order to implement the fmt.Stringer +// interface to make this printable via fmt.Println et al. +func (mv Mlrval) String() string { + // TODO: comment re deferral -- important perf effect! + // if mv.IsFloat() && floatOutputFormatter != nil + // if mv.mvtype == MT_FLOAT && floatOutputFormatter != nil { + //if floatOutputFormatter != nil && (mv.mvtype == MT_FLOAT || mv.mvtype == MT_PENDING) { + if floatOutputFormatter != nil && mv.Type() == MT_FLOAT { + // Use the format string from global --ofmt, if supplied + return floatOutputFormatter.FormatFloat(mv.floatval) + } else { + mv.setPrintRep() + return mv.printrep } - mlrvalFloatOutputFormatter = formatter - return nil } // See mlrval.go for more about JIT-formatting of string backings func (mv *Mlrval) setPrintRep() { if !mv.printrepValid { - // xxx do it -- disposition vector - // xxx temp temp temp temp temp switch mv.mvtype { + case MT_PENDING: // Should not have gotten outside of the JSON decoder, so flag this // clearly visually if it should (buggily) slip through to // user-level visibility. - mv.printrep = "(bug-if-you-see-this)" // xxx constdef at top of file + mv.printrep = "(bug-if-you-see-this:case=3)" // xxx constdef at top of file break + case MT_ERROR: mv.printrep = "(error)" // xxx constdef at top of file break + case MT_ABSENT: // Callsites should be using absence to do non-assigns, so flag // this clearly visually if it should (buggily) slip through to // user-level visibility. - mv.printrep = "(bug-if-you-see-this)" // xxx constdef at top of file + mv.printrep = "(bug-if-you-see-this:case=4)" // xxx constdef at top of file break + case MT_VOID: mv.printrep = "" // xxx constdef at top of file break + case MT_STRING: - // panic i suppose break + case MT_INT: mv.printrep = strconv.Itoa(mv.intval) break + case MT_FLOAT: mv.printrep = strconv.FormatFloat(mv.floatval, 'f', -1, 64) break + case MT_BOOL: if mv.boolval == true { mv.printrep = "true" @@ -59,9 +67,8 @@ func (mv *Mlrval) setPrintRep() { mv.printrep = "false" } break - // TODO: handling indentation - case MT_ARRAY: + case MT_ARRAY: bytes, err := mv.MarshalJSON(JSON_MULTILINE, false) // maybe just InternalCodingErrorIf(err != nil) if err != nil { @@ -71,8 +78,8 @@ func (mv *Mlrval) setPrintRep() { mv.printrep = string(bytes) break - case MT_MAP: + case MT_MAP: bytes, err := mv.MarshalJSON(JSON_MULTILINE, false) // maybe just InternalCodingErrorIf(err != nil) if err != nil { @@ -86,15 +93,3 @@ func (mv *Mlrval) setPrintRep() { mv.printrepValid = true } } - -// Must have non-pointer receiver in order to implement the fmt.Stringer -// interface to make this printable via fmt.Println et al. -func (mv Mlrval) String() string { - if mv.mvtype == MT_FLOAT && mlrvalFloatOutputFormatter != nil { - // Use the format string from global --ofmt, if supplied - return mlrvalFloatOutputFormatter.FormatFloat(mv.floatval) - } else { - mv.setPrintRep() - return mv.printrep - } -} diff --git a/internal/pkg/mlrval/mlrval_output_test.go b/internal/pkg/mlrval/mlrval_output_test.go new file mode 100644 index 000000000..3bee43f52 --- /dev/null +++ b/internal/pkg/mlrval/mlrval_output_test.go @@ -0,0 +1,18 @@ +package mlrval + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestString(t *testing.T) { + assert.Equal(t, "234", FromInferredType("234").String()) + assert.Equal(t, "234", FromDeferredType("234").String()) + assert.Equal(t, "234.5", FromInferredType("234.5").String()) + assert.Equal(t, "234.5", FromDeferredType("234.5").String()) + assert.Equal(t, "abc", FromInferredType("abc").String()) + assert.Equal(t, "abc", FromDeferredType("abc").String()) + assert.Equal(t, "", FromInferredType("").String()) + assert.Equal(t, "", FromDeferredType("").String()) +} diff --git a/internal/pkg/types/mlrval.go b/internal/pkg/mlrval/mlrval_type.go similarity index 55% rename from internal/pkg/types/mlrval.go rename to internal/pkg/mlrval/mlrval_type.go index 9075637ab..4b5d9bef6 100644 --- a/internal/pkg/types/mlrval.go +++ b/internal/pkg/mlrval/mlrval_type.go @@ -1,5 +1,5 @@ // ================================================================ -// The `types.Mlrval` structure includes **string, int, float, boolean, void, +// The `Mlrval` structure includes **string, int, float, boolean, void, // absent, and error** types (not unlike PHP's `zval`) as well as // type-conversion logic for various operators. // @@ -8,7 +8,50 @@ // that is a bug. It would be great to be able to somehow lint for this. // ================================================================ -package types +// TODO: +// * why here carefully fenced +// * why interface for recursive/external items +// * why optimizing: +// o def string-to-type, e.g. 17-column-wide, million-row CSV file, +// '$z = $x + $y' kind of thing -- don't need to type-infer +// column $a "1234" as int only to write it back non-modified +// o defer type-to-string, e.g. '$z = a*b + c*d' w/ a,b,c,d ints, +// string-rep not needed -- defer string-formatting until write-out. +// * Careful fencing: want things to "just work" for as much of the +// code as possible. +// o Hide x.mvtype as x.Type() and that function can JIT-infer printrep +// "1234" and mvtype MT_PENDING to mvtype MT_INT and intval 1234. +// o Similarly MT_INT, intval 1235, printrepValid = false, only +// on to-string do printrep = "1235" and printrepValid=true. +// * For all but JSON: everything is string/int/float/bool. Any +// array-valued/map-valued are JSON-encoded _as strings_. +// * For JSON: no defer; everything is explicitly typed as part of +// the decode. + +// TODO: MERGE +// An int/float always starts from a string -- be it from record data from +// a file, or a literal within a DSL expression. The printrep is exactly +// that string, however the user formatted it, and the intval/floatval is +// computed from that -- and in sync with it -- at construction time. +// +// When a mlrval is computed from one or more others -- e.g. '$z = $x + 4' +// -- the printrep is not updated. That would be wasted CPU, since the +// string representation is not needed until when/if the value is printed +// as output. For computation methods the printrep is neglected and the +// printrepValid is set to false. +// +// In the String() method the printrep is computed from the intval/floatval +// and printrepValid is set back to true. +// +// Note that for MT_STRING, the printrep is always valid since it is the +// only payload for the mlrval. +// +// Thus we (a) keep user-specific input-formatting when possible, for the +// principle of least surprise; (b) avoid reformatting strings during +// intermediate arithmetic expressions; (c) resync arithmetic results to +// string formatting on a just-in-time basis when output is printed. + +package mlrval type Mlrval struct { @@ -16,46 +59,30 @@ type Mlrval struct { // I would call this "type" not "mvtype" but "type" is a keyword in Go. mvtype MVType - // An int/float always starts from a string -- be it from record data from - // a file, or a literal within a DSL expression. The printrep is exactly - // that string, however the user formatted it, and the intval/floatval is - // computed from that -- and in sync with it -- at construction time. - // - // When a mlrval is computed from one or more others -- e.g. '$z = $x + 4' - // -- the printrep is not updated. That would be wasted CPU, since the - // string representation is not needed until when/if the value is printed - // as output. For computation methods the printrep is neglected and the - // printrepValid is set to false. - // - // In the String() method the printrep is computed from the intval/floatval - // and printrepValid is set back to true. - // - // Note that for MT_STRING, the printrep is always valid since it is the - // only payload for the mlrval. - // - // Thus we (a) keep user-specific input-formatting when possible, for the - // principle of least surprise; (b) avoid reformatting strings during - // intermediate arithmetic expressions; (c) resync arithmetic results to - // string formatting on a just-in-time basis when output is printed. printrep string printrepValid bool intval int floatval float64 boolval bool - arrayval []Mlrval - mapval *Mlrmap - // These are first-class-function literals. Stored here as interface{} to - // avoid what would otherwise be a package-dependency cycle with the - // github.com/johnkerl/miller/internal/pkg/dsl/cst package. + + arrayval []Mlrval + mapval *Mlrmap + // First-class-function literals from internal/pkg/dsl/cst. + // Interfaced here to avoid package-dependency cycles. funcval interface{} } +const INVALID_PRINTREP = "(bug-if-you-see-this:case-2)" +const ERROR_PRINTREP = "(error)" +const ABSENT_PRINTREP = "(absent)" + // Enumeration for mlrval types // -// There are two kinds of null: ABSENT (key not present in a record) and VOID -// (key present with empty value). Note void is an acceptable string (empty -// string) but not an acceptable number. (In Javascript, similarly, there are -// undefined and null, respectively.) +// There are three kinds of null: ABSENT (key not present in a record) and VOID +// (key present with empty value); thirdly NULL for JSON null. Note void is an +// acceptable string (empty string) but not an acceptable number. (In +// JavaScript, similarly, there are undefined and null, respectively -- +// Miller's absent is more like JavaScript's undefined.) type MVType int @@ -63,7 +90,7 @@ type MVType int // matrices. If they are changed, it will break the disposition matrices, or // they will all need manual re-indexing. const ( - // Type not yet determined, during JSON decode. + // Type not yet determined: during JSON decode or TODO comment. MT_PENDING MVType = -1 // E.g. error encountered in one eval & it propagates up the AST at @@ -121,7 +148,6 @@ var TYPE_NAMES = [MT_DIM]string{ "funct", } -// ---------------------------------------------------------------- // For typed assignments in the DSL // TODO: comment more re typedecls @@ -166,7 +192,3 @@ func TypeNameToMask(typeName string) (mask int, present bool) { return 0, false } } - -func (mv *Mlrval) GetTypeBit() int { - return 1 << mv.mvtype -} diff --git a/internal/pkg/output/record_writer.go b/internal/pkg/output/record_writer.go index 667687132..30b17badb 100644 --- a/internal/pkg/output/record_writer.go +++ b/internal/pkg/output/record_writer.go @@ -3,7 +3,7 @@ package output import ( "bufio" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // IRecordWriter is the abstract interface for all record-writers. They are @@ -17,7 +17,7 @@ import ( // format. type IRecordWriter interface { Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) diff --git a/internal/pkg/output/record_writer_csv.go b/internal/pkg/output/record_writer_csv.go index 3f8196282..2ebdfd3e2 100644 --- a/internal/pkg/output/record_writer_csv.go +++ b/internal/pkg/output/record_writer_csv.go @@ -8,7 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/colorizer" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) type RecordWriterCSV struct { @@ -37,7 +37,7 @@ func NewRecordWriterCSV(writerOptions *cli.TWriterOptions) (*RecordWriterCSV, er } func (writer *RecordWriterCSV) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { diff --git a/internal/pkg/output/record_writer_csvlite.go b/internal/pkg/output/record_writer_csvlite.go index 40b6cd1f6..6acf0976c 100644 --- a/internal/pkg/output/record_writer_csvlite.go +++ b/internal/pkg/output/record_writer_csvlite.go @@ -6,7 +6,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/colorizer" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) type RecordWriterCSVLite struct { @@ -26,7 +26,7 @@ func NewRecordWriterCSVLite(writerOptions *cli.TWriterOptions) (*RecordWriterCSV } func (writer *RecordWriterCSVLite) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { diff --git a/internal/pkg/output/record_writer_dkvp.go b/internal/pkg/output/record_writer_dkvp.go index 5fd70362a..674388130 100644 --- a/internal/pkg/output/record_writer_dkvp.go +++ b/internal/pkg/output/record_writer_dkvp.go @@ -5,7 +5,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/colorizer" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) type RecordWriterDKVP struct { @@ -19,7 +19,7 @@ func NewRecordWriterDKVP(writerOptions *cli.TWriterOptions) (*RecordWriterDKVP, } func (writer *RecordWriterDKVP) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { diff --git a/internal/pkg/output/record_writer_json.go b/internal/pkg/output/record_writer_json.go index 3673a517e..b962f2dfa 100644 --- a/internal/pkg/output/record_writer_json.go +++ b/internal/pkg/output/record_writer_json.go @@ -6,14 +6,14 @@ import ( "os" "github.com/johnkerl/miller/internal/pkg/cli" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ---------------------------------------------------------------- type RecordWriterJSON struct { // Parameters: writerOptions *cli.TWriterOptions - jsonFormatting types.TJSONFormatting + jsonFormatting mlrval.TJSONFormatting // State: onFirst bool @@ -21,9 +21,9 @@ type RecordWriterJSON struct { // ---------------------------------------------------------------- func NewRecordWriterJSON(writerOptions *cli.TWriterOptions) (*RecordWriterJSON, error) { - var jsonFormatting types.TJSONFormatting = types.JSON_SINGLE_LINE + var jsonFormatting mlrval.TJSONFormatting = mlrval.JSON_SINGLE_LINE if writerOptions.JSONOutputMultiline { - jsonFormatting = types.JSON_MULTILINE + jsonFormatting = mlrval.JSON_MULTILINE } return &RecordWriterJSON{ writerOptions: writerOptions, @@ -34,7 +34,7 @@ func NewRecordWriterJSON(writerOptions *cli.TWriterOptions) (*RecordWriterJSON, // ---------------------------------------------------------------- func (writer *RecordWriterJSON) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { @@ -47,7 +47,7 @@ func (writer *RecordWriterJSON) Write( // ---------------------------------------------------------------- func (writer *RecordWriterJSON) writeWithListWrap( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { @@ -82,7 +82,7 @@ func (writer *RecordWriterJSON) writeWithListWrap( // ---------------------------------------------------------------- func (writer *RecordWriterJSON) writeWithoutListWrap( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { diff --git a/internal/pkg/output/record_writer_markdown.go b/internal/pkg/output/record_writer_markdown.go index 695cdd11c..c7bdf280d 100644 --- a/internal/pkg/output/record_writer_markdown.go +++ b/internal/pkg/output/record_writer_markdown.go @@ -6,7 +6,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/colorizer" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) type RecordWriterMarkdown struct { @@ -28,7 +28,7 @@ func NewRecordWriterMarkdown(writerOptions *cli.TWriterOptions) (*RecordWriterMa // ---------------------------------------------------------------- func (writer *RecordWriterMarkdown) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { diff --git a/internal/pkg/output/record_writer_nidx.go b/internal/pkg/output/record_writer_nidx.go index 1a224407b..d7625ca4b 100644 --- a/internal/pkg/output/record_writer_nidx.go +++ b/internal/pkg/output/record_writer_nidx.go @@ -4,7 +4,7 @@ import ( "bufio" "github.com/johnkerl/miller/internal/pkg/cli" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) type RecordWriterNIDX struct { @@ -20,7 +20,7 @@ func NewRecordWriterNIDX(writerOptions *cli.TWriterOptions) (*RecordWriterNIDX, } func (writer *RecordWriterNIDX) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { diff --git a/internal/pkg/output/record_writer_pprint.go b/internal/pkg/output/record_writer_pprint.go index c293aa97e..96be7f957 100644 --- a/internal/pkg/output/record_writer_pprint.go +++ b/internal/pkg/output/record_writer_pprint.go @@ -9,7 +9,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/colorizer" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) type RecordWriterPPRINT struct { @@ -34,7 +34,7 @@ func NewRecordWriterPPRINT(writerOptions *cli.TWriterOptions) (*RecordWriterPPRI // ---------------------------------------------------------------- func (writer *RecordWriterPPRINT) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { @@ -97,7 +97,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousList( var maxNR int = 0 for e := records.Front(); e != nil; e = e.Next() { - outrec := e.Value.(*types.Mlrmap) + outrec := e.Value.(*mlrval.Mlrmap) nr := outrec.FieldCount if maxNR < nr { maxNR = nr @@ -152,7 +152,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousListNonBarred( onFirst := true for e := records.Front(); e != nil; e = e.Next() { - outrec := e.Value.(*types.Mlrmap) + outrec := e.Value.(*mlrval.Mlrmap) // Print header line if onFirst && !writer.writerOptions.HeaderlessCSVOutput { @@ -254,7 +254,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousListBarred( onFirst := true for e := records.Front(); e != nil; e = e.Next() { - outrec := e.Value.(*types.Mlrmap) + outrec := e.Value.(*mlrval.Mlrmap) // Print header line if onFirst && !writer.writerOptions.HeaderlessCSVOutput { diff --git a/internal/pkg/output/record_writer_xtab.go b/internal/pkg/output/record_writer_xtab.go index 63a0f2fc2..a13536c1e 100644 --- a/internal/pkg/output/record_writer_xtab.go +++ b/internal/pkg/output/record_writer_xtab.go @@ -7,7 +7,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/colorizer" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ---------------------------------------------------------------- @@ -42,7 +42,7 @@ func NewRecordWriterXTAB(writerOptions *cli.TWriterOptions) (*RecordWriterXTAB, } func (writer *RecordWriterXTAB) Write( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { @@ -67,7 +67,7 @@ func (writer *RecordWriterXTAB) Write( } func (writer *RecordWriterXTAB) writeWithLeftAlignedValues( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, maxKeyLength int, @@ -100,7 +100,7 @@ func (writer *RecordWriterXTAB) writeWithLeftAlignedValues( } func (writer *RecordWriterXTAB) writeWithRightAlignedValues( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, bufferedOutputStream *bufio.Writer, outputIsStdout bool, maxKeyLength int, diff --git a/internal/pkg/runtime/stack.go b/internal/pkg/runtime/stack.go index 6ad9347be..d3db2b0a5 100644 --- a/internal/pkg/runtime/stack.go +++ b/internal/pkg/runtime/stack.go @@ -31,6 +31,7 @@ import ( "fmt" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -114,7 +115,7 @@ func (stack *Stack) PopStackFrame() { // Returns nil on no-such func (stack *Stack) Get( stackVariable *StackVariable, -) *types.Mlrval { +) *mlrval.Mlrval { return stack.head.get(stackVariable) } @@ -124,7 +125,7 @@ func (stack *Stack) Get( func (stack *Stack) DefineTypedAtScope( stackVariable *StackVariable, typeName string, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { return stack.head.defineTypedAtScope(stackVariable, typeName, mlrval) } @@ -136,7 +137,7 @@ func (stack *Stack) DefineTypedAtScope( // E.g. 'for (i = 0; i < 10; i += 1)' uses Set. func (stack *Stack) SetAtScope( stackVariable *StackVariable, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { return stack.head.setAtScope(stackVariable, mlrval) } @@ -150,7 +151,7 @@ func (stack *Stack) SetAtScope( // assignment is OK. func (stack *Stack) Set( stackVariable *StackVariable, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { return stack.head.set(stackVariable, mlrval) } @@ -158,8 +159,8 @@ func (stack *Stack) Set( // E.g. 'x[1] = 2' where the variable x may or may not have been already set. func (stack *Stack) SetIndexed( stackVariable *StackVariable, - indices []*types.Mlrval, - mlrval *types.Mlrval, + indices []*mlrval.Mlrval, + mlrval *mlrval.Mlrval, ) error { return stack.head.setIndexed(stackVariable, indices, mlrval) } @@ -174,7 +175,7 @@ func (stack *Stack) Unset( // E.g. 'unset x[1]' func (stack *Stack) UnsetIndexed( stackVariable *StackVariable, - indices []*types.Mlrval, + indices []*mlrval.Mlrval, ) { stack.head.unsetIndexed(stackVariable, indices) } @@ -225,7 +226,7 @@ func (frameset *StackFrameSet) dump() { // Returns nil on no-such func (frameset *StackFrameSet) get( stackVariable *StackVariable, -) *types.Mlrval { +) *mlrval.Mlrval { // Scope-walk numStackFrames := len(frameset.stackFrames) for offset := numStackFrames - 1; offset >= 0; offset-- { @@ -242,7 +243,7 @@ func (frameset *StackFrameSet) get( func (frameset *StackFrameSet) defineTypedAtScope( stackVariable *StackVariable, typeName string, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { offset := len(frameset.stackFrames) - 1 // TODO: comment @@ -254,7 +255,7 @@ func (frameset *StackFrameSet) defineTypedAtScope( // See Stack.SetAtScope comments above func (frameset *StackFrameSet) setAtScope( stackVariable *StackVariable, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { offset := len(frameset.stackFrames) - 1 return frameset.stackFrames[offset].set(stackVariable, mlrval) @@ -263,7 +264,7 @@ func (frameset *StackFrameSet) setAtScope( // See Stack.Set comments above func (frameset *StackFrameSet) set( stackVariable *StackVariable, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { // Scope-walk numStackFrames := len(frameset.stackFrames) @@ -279,8 +280,8 @@ func (frameset *StackFrameSet) set( // See Stack.SetIndexed comments above func (frameset *StackFrameSet) setIndexed( stackVariable *StackVariable, - indices []*types.Mlrval, - mlrval *types.Mlrval, + indices []*mlrval.Mlrval, + mlrval *mlrval.Mlrval, ) error { // Scope-walk numStackFrames := len(frameset.stackFrames) @@ -312,7 +313,7 @@ func (frameset *StackFrameSet) unset( // See Stack.UnsetIndexed comments above func (frameset *StackFrameSet) unsetIndexed( stackVariable *StackVariable, - indices []*types.Mlrval, + indices []*mlrval.Mlrval, ) { // Scope-walk numStackFrames := len(frameset.stackFrames) @@ -353,7 +354,7 @@ func newStackFrame() *StackFrame { // Returns nil on no such func (frame *StackFrame) get( stackVariable *StackVariable, -) *types.Mlrval { +) *mlrval.Mlrval { offset, ok := frame.namesToOffsets[stackVariable.name] if ok { return frame.vars[offset].GetValue() @@ -372,7 +373,7 @@ func (frame *StackFrame) has( // TODO: audit for honor of error-return at callsites func (frame *StackFrame) set( stackVariable *StackVariable, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { offset, ok := frame.namesToOffsets[stackVariable.name] if !ok { @@ -393,7 +394,7 @@ func (frame *StackFrame) set( func (frame *StackFrame) defineTyped( stackVariable *StackVariable, typeName string, - mlrval *types.Mlrval, + mlrval *mlrval.Mlrval, ) error { _, ok := frame.namesToOffsets[stackVariable.name] if !ok { @@ -418,16 +419,16 @@ func (frame *StackFrame) defineTyped( // TODO: audit for honor of error-return at callsites func (frame *StackFrame) setIndexed( stackVariable *StackVariable, - indices []*types.Mlrval, - mlrval *types.Mlrval, + indices []*mlrval.Mlrval, + mv *mlrval.Mlrval, ) error { value := frame.get(stackVariable) if value == nil { lib.InternalCodingErrorIf(len(indices) < 1) leadingIndex := indices[0] if leadingIndex.IsString() || leadingIndex.IsInt() { - newval := types.MlrvalFromEmptyMap() - newval.PutIndexed(indices, mlrval) + newval := mlrval.FromMap(mlrval.NewMlrmap()) + newval.PutIndexed(indices, mv) return frame.set(stackVariable, newval) } else { return errors.New( @@ -440,7 +441,7 @@ func (frame *StackFrame) setIndexed( } else { // For example maybe the variable exists and is an array but the // leading index is a string. - return value.PutIndexed(indices, mlrval) + return value.PutIndexed(indices, mv) } } @@ -455,7 +456,7 @@ func (frame *StackFrame) unset( func (frame *StackFrame) unsetIndexed( stackVariable *StackVariable, - indices []*types.Mlrval, + indices []*mlrval.Mlrval, ) { value := frame.get(stackVariable) if value == nil { diff --git a/internal/pkg/runtime/state.go b/internal/pkg/runtime/state.go index 19b11414c..7df865705 100644 --- a/internal/pkg/runtime/state.go +++ b/internal/pkg/runtime/state.go @@ -11,16 +11,18 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) type State struct { - Inrec *types.Mlrmap + Inrec *mlrval.Mlrmap Context *types.Context - Oosvars *types.Mlrmap - FilterExpression *types.Mlrval + Oosvars *mlrval.Mlrmap + FilterExpression *mlrval.Mlrval Stack *Stack OutputRecordsAndContexts *list.List // list of *types.RecordAndContext + // For holding "\0".."\9" between where they are set via things like // '$x =~ "(..)_(...)"', and interpolated via things like '$y = "\2:\1"'. RegexCaptures []string @@ -28,12 +30,12 @@ type State struct { } func NewEmptyState(options *cli.TOptions) *State { - oosvars := types.NewMlrmap() + oosvars := mlrval.NewMlrmap() return &State{ Inrec: nil, Context: nil, Oosvars: oosvars, - FilterExpression: types.MLRVAL_TRUE, + FilterExpression: mlrval.TRUE, Stack: NewStack(), // OutputRecordsAndContexts is assigned after construction @@ -45,7 +47,7 @@ func NewEmptyState(options *cli.TOptions) *State { } func (state *State) Update( - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, context *types.Context, ) { state.Inrec = inrec diff --git a/internal/pkg/transformers/altkv.go b/internal/pkg/transformers/altkv.go index 2fdde54ac..e7fddb3c9 100644 --- a/internal/pkg/transformers/altkv.go +++ b/internal/pkg/transformers/altkv.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -99,7 +100,7 @@ func (tr *TransformerAltkv) Transform( HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel) if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() outputFieldNumber := 1 for pe := inrec.Head; pe != nil; /* increment in loop body */ { diff --git a/internal/pkg/transformers/bar.go b/internal/pkg/transformers/bar.go index 993895b22..6172e6a73 100644 --- a/internal/pkg/transformers/bar.go +++ b/internal/pkg/transformers/bar.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -251,7 +252,7 @@ func (tr *TransformerBar) processNoAuto( if idx > tr.width { idx = tr.width } - inrec.PutReference(fieldName, types.MlrvalFromString(tr.bars[idx])) + inrec.PutReference(fieldName, mlrval.FromString(tr.bars[idx])) } outputRecordsAndContexts.PushBack(inrecAndContext) @@ -339,7 +340,7 @@ func (tr *TransformerBar) processAuto( buffer.WriteString("[") buffer.WriteString(shi) buffer.WriteString("]") - record.PutReference(fieldName, types.MlrvalFromString(buffer.String())) + record.PutReference(fieldName, mlrval.FromString(buffer.String())) } } diff --git a/internal/pkg/transformers/cat.go b/internal/pkg/transformers/cat.go index 8a0a5919a..c56b56d9c 100644 --- a/internal/pkg/transformers/cat.go +++ b/internal/pkg/transformers/cat.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -178,7 +179,7 @@ func (tr *TransformerCat) countersUngrouped( inrec := inrecAndContext.Record tr.counter++ key := tr.counterFieldName - inrec.PrependCopy(key, types.MlrvalFromInt(tr.counter)) + inrec.PrependCopy(key, mlrval.FromInt(tr.counter)) } outputRecordsAndContexts.PushBack(inrecAndContext) } @@ -210,7 +211,7 @@ func (tr *TransformerCat) countersGrouped( } key := tr.counterFieldName - inrec.PrependCopy(key, types.MlrvalFromInt(counter)) + inrec.PrependCopy(key, mlrval.FromInt(counter)) } outputRecordsAndContexts.PushBack(inrecAndContext) } diff --git a/internal/pkg/transformers/clean-whitespace.go b/internal/pkg/transformers/clean-whitespace.go index 85f349759..efb0a5db7 100644 --- a/internal/pkg/transformers/clean-whitespace.go +++ b/internal/pkg/transformers/clean-whitespace.go @@ -6,7 +6,9 @@ import ( "os" "strings" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -149,13 +151,13 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeysAndValues( outputDownstreamDoneChannel chan<- bool, ) { if !inrecAndContext.EndOfStream { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { - oldKey := types.MlrvalFromString(pe.Key) + oldKey := mlrval.FromString(pe.Key) // xxx temp - newKey := types.BIF_clean_whitespace(oldKey) - newValue := types.BIF_clean_whitespace(pe.Value) + newKey := bifs.BIF_clean_whitespace(oldKey) + newValue := bifs.BIF_clean_whitespace(pe.Value) // Transferring ownership from old record to new record; no copy needed newrec.PutReference(newKey.String(), newValue) } @@ -174,11 +176,11 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeys( outputDownstreamDoneChannel chan<- bool, ) { if !inrecAndContext.EndOfStream { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { - oldKey := types.MlrvalFromString(pe.Key) - newKey := types.BIF_clean_whitespace(oldKey) + oldKey := mlrval.FromString(pe.Key) + newKey := bifs.BIF_clean_whitespace(oldKey) // Transferring ownership from old record to new record; no copy needed newrec.PutReference(newKey.String(), pe.Value) } @@ -198,7 +200,7 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInValues( ) { if !inrecAndContext.EndOfStream { for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { - pe.Value = types.BIF_clean_whitespace(pe.Value) + pe.Value = bifs.BIF_clean_whitespace(pe.Value) } outputRecordsAndContexts.PushBack(inrecAndContext) } else { diff --git a/internal/pkg/transformers/count-similar.go b/internal/pkg/transformers/count-similar.go index 39d381c64..a3e45e16d 100644 --- a/internal/pkg/transformers/count-similar.go +++ b/internal/pkg/transformers/count-similar.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -154,7 +155,7 @@ func (tr *TransformerCountSimilar) Transform( recordListForGroup := outer.Value.(*list.List) // TODO: make 64-bit friendly groupSize := recordListForGroup.Len() - mgroupSize := types.MlrvalFromInt(int(groupSize)) + mgroupSize := mlrval.FromInt(int(groupSize)) for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() { recordAndContext := inner.Value.(*types.RecordAndContext) recordAndContext.Record.PutCopy(tr.counterFieldName, mgroupSize) diff --git a/internal/pkg/transformers/count.go b/internal/pkg/transformers/count.go index 02ddd80fe..88de43c59 100644 --- a/internal/pkg/transformers/count.go +++ b/internal/pkg/transformers/count.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -171,8 +172,8 @@ func (tr *TransformerCount) countUngrouped( if !inrecAndContext.EndOfStream { tr.ungroupedCount++ } else { - newrec := types.NewMlrmapAsRecord() - newrec.PutCopy(tr.outputFieldName, types.MlrvalFromInt(tr.ungroupedCount)) + newrec := mlrval.NewMlrmapAsRecord() + newrec.PutCopy(tr.outputFieldName, mlrval.FromInt(tr.ungroupedCount)) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker @@ -209,8 +210,8 @@ func (tr *TransformerCount) countGrouped( } else { if tr.showCountsOnly { - newrec := types.NewMlrmapAsRecord() - newrec.PutCopy(tr.outputFieldName, types.MlrvalFromInt(tr.groupedCounts.FieldCount)) + newrec := mlrval.NewMlrmapAsRecord() + newrec.PutCopy(tr.outputFieldName, mlrval.FromInt(tr.groupedCounts.FieldCount)) outrecAndContext := types.NewRecordAndContext(newrec, &inrecAndContext.Context) outputRecordsAndContexts.PushBack(outrecAndContext) @@ -218,7 +219,7 @@ func (tr *TransformerCount) countGrouped( } else { for outer := tr.groupedCounts.Head; outer != nil; outer = outer.Next { groupingKey := outer.Key - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() // Example: // * Suppose group-by fields are a,b. @@ -227,7 +228,7 @@ func (tr *TransformerCount) countGrouped( // * Grouping values for key is ["foo", "bar"] // Here we populate a record with "a=foo,b=bar". - groupingValuesForKey := tr.groupingValues.Get(groupingKey).([]*types.Mlrval) + groupingValuesForKey := tr.groupingValues.Get(groupingKey).([]*mlrval.Mlrval) i := 0 for _, groupingValueForKey := range groupingValuesForKey { newrec.PutCopy(tr.groupByFieldNames[i], groupingValueForKey) @@ -235,7 +236,7 @@ func (tr *TransformerCount) countGrouped( } countForGroup := outer.Value.(int) - newrec.PutCopy(tr.outputFieldName, types.MlrvalFromInt(countForGroup)) + newrec.PutCopy(tr.outputFieldName, mlrval.FromInt(countForGroup)) outrecAndContext := types.NewRecordAndContext(newrec, &inrecAndContext.Context) outputRecordsAndContexts.PushBack(outrecAndContext) diff --git a/internal/pkg/transformers/cut.go b/internal/pkg/transformers/cut.go index 8f90ee902..aff700163 100644 --- a/internal/pkg/transformers/cut.go +++ b/internal/pkg/transformers/cut.go @@ -9,6 +9,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -201,7 +202,7 @@ func (tr *TransformerCut) includeWithInputOrder( ) { if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - outrec := types.NewMlrmap() + outrec := mlrval.NewMlrmap() for pe := inrec.Head; pe != nil; pe = pe.Next { fieldName := pe.Key _, wanted := tr.fieldNameSet[fieldName] @@ -226,7 +227,7 @@ func (tr *TransformerCut) includeWithArgOrder( ) { if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - outrec := types.NewMlrmap() + outrec := mlrval.NewMlrmap() for _, fieldName := range tr.fieldNameList { value := inrec.Get(fieldName) if value != nil { @@ -268,7 +269,7 @@ func (tr *TransformerCut) processWithRegexes( ) { if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for pe := inrec.Head; pe != nil; pe = pe.Next { matchesAny := false for _, regex := range tr.regexes { diff --git a/internal/pkg/transformers/fill-down.go b/internal/pkg/transformers/fill-down.go index f96429a07..60dc26a5a 100644 --- a/internal/pkg/transformers/fill-down.go +++ b/internal/pkg/transformers/fill-down.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -122,7 +123,7 @@ type TransformerFillDown struct { onlyIfAbsent bool // state - lastNonNullValues map[string]*types.Mlrval + lastNonNullValues map[string]*mlrval.Mlrval recordTransformerFunc RecordTransformerFunc } @@ -135,7 +136,7 @@ func NewTransformerFillDown( tr := &TransformerFillDown{ fillDownFieldNames: fillDownFieldNames, onlyIfAbsent: onlyIfAbsent, - lastNonNullValues: make(map[string]*types.Mlrval), + lastNonNullValues: make(map[string]*mlrval.Mlrval), } if doAll { @@ -175,7 +176,7 @@ func (tr *TransformerFillDown) transformSpecified( if tr.onlyIfAbsent { present = value != nil } else { - present = value != nil && !value.IsEmpty() + present = value != nil && !value.IsVoid() } if present { @@ -214,7 +215,7 @@ func (tr *TransformerFillDown) transformAll( if tr.onlyIfAbsent { present = value != nil } else { - present = value != nil && !value.IsEmpty() + present = value != nil && !value.IsVoid() } if present { diff --git a/internal/pkg/transformers/fill-empty.go b/internal/pkg/transformers/fill-empty.go index 420656bd1..840b75ab1 100644 --- a/internal/pkg/transformers/fill-empty.go +++ b/internal/pkg/transformers/fill-empty.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -93,7 +94,7 @@ func transformerFillEmptyParseCLI( // ---------------------------------------------------------------- type TransformerFillEmpty struct { - fillValue *types.Mlrval + fillValue *mlrval.Mlrval } func NewTransformerFillEmpty( @@ -102,9 +103,9 @@ func NewTransformerFillEmpty( ) (*TransformerFillEmpty, error) { tr := &TransformerFillEmpty{} if inferType { - tr.fillValue = types.MlrvalFromInferredType(fillString) + tr.fillValue = mlrval.FromInferredType(fillString) } else { - tr.fillValue = types.MlrvalFromString(fillString) + tr.fillValue = mlrval.FromString(fillString) } return tr, nil } @@ -122,7 +123,7 @@ func (tr *TransformerFillEmpty) Transform( inrec := inrecAndContext.Record for pe := inrec.Head; pe != nil; pe = pe.Next { - if pe.Value.IsEmpty() { + if pe.Value.IsVoid() { pe.Value = tr.fillValue } } diff --git a/internal/pkg/transformers/format-values.go b/internal/pkg/transformers/format-values.go index 1a74ed14c..bcde9511f 100644 --- a/internal/pkg/transformers/format-values.go +++ b/internal/pkg/transformers/format-values.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -131,9 +132,9 @@ func transformerFormatValuesParseCLI( // ---------------------------------------------------------------- type TransformerFormatValues struct { - stringFormatter types.IMlrvalFormatter - intFormatter types.IMlrvalFormatter - floatFormatter types.IMlrvalFormatter + stringFormatter mlrval.IFormatter + intFormatter mlrval.IFormatter + floatFormatter mlrval.IFormatter coerceIntToFloat bool } @@ -143,17 +144,17 @@ func NewTransformerFormatValues( floatFormat string, coerceIntToFloat bool, ) (*TransformerFormatValues, error) { - stringFormatter, err := types.GetMlrvalFormatter(stringFormat) + stringFormatter, err := mlrval.GetFormatter(stringFormat) if err != nil { return nil, err } - intFormatter, err := types.GetMlrvalFormatter(intFormat) + intFormatter, err := mlrval.GetFormatter(intFormat) if err != nil { return nil, err } - floatFormatter, err := types.GetMlrvalFormatter(floatFormat) + floatFormatter, err := mlrval.GetFormatter(floatFormat) if err != nil { return nil, err } diff --git a/internal/pkg/transformers/fraction.go b/internal/pkg/transformers/fraction.go index 75a330ba4..462b139fb 100644 --- a/internal/pkg/transformers/fraction.go +++ b/internal/pkg/transformers/fraction.go @@ -6,8 +6,10 @@ import ( "os" "strings" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -135,12 +137,12 @@ type TransformerFraction struct { recordsAndContexts *list.List // Two-level map: Group-by field names are the first keyset; // fraction field names are keys into the second. - sums map[string]map[string]*types.Mlrval - cumus map[string]map[string]*types.Mlrval + sums map[string]map[string]*mlrval.Mlrval + cumus map[string]map[string]*mlrval.Mlrval - outputFieldNameSuffix string // "_fraction" or "_percent" - multiplier *types.Mlrval // 1.0 for fraction or 100.0 for percent - zero *types.Mlrval + outputFieldNameSuffix string // "_fraction" or "_percent" + multiplier *mlrval.Mlrval // 1.0 for fraction or 100.0 for percent + zero *mlrval.Mlrval } // ---------------------------------------------------------------- @@ -152,20 +154,20 @@ func NewTransformerFraction( ) (*TransformerFraction, error) { recordsAndContexts := list.New() - sums := make(map[string]map[string]*types.Mlrval) - cumus := make(map[string]map[string]*types.Mlrval) + sums := make(map[string]map[string]*mlrval.Mlrval) + cumus := make(map[string]map[string]*mlrval.Mlrval) - var multiplier *types.Mlrval + var multiplier *mlrval.Mlrval var outputFieldNameSuffix string if doPercents { - multiplier = types.MlrvalFromInt(100) + multiplier = mlrval.FromInt(100) if doCumu { outputFieldNameSuffix = "_cumulative_percent" } else { outputFieldNameSuffix = "_percent" } } else { - multiplier = types.MlrvalFromInt(1) + multiplier = mlrval.FromInt(1) if doCumu { outputFieldNameSuffix = "_cumulative_fraction" } else { @@ -173,7 +175,7 @@ func NewTransformerFraction( } } - zero := types.MlrvalFromInt(0) + zero := mlrval.FromInt(0) return &TransformerFraction{ fractionFieldNames: fractionFieldNames, @@ -208,11 +210,11 @@ func (tr *TransformerFraction) Transform( if hasAll { sumsForGroup := tr.sums[groupingKey] - var cumusForGroup map[string]*types.Mlrval = nil + var cumusForGroup map[string]*mlrval.Mlrval = nil if sumsForGroup == nil { - sumsForGroup = make(map[string]*types.Mlrval) + sumsForGroup = make(map[string]*mlrval.Mlrval) tr.sums[groupingKey] = sumsForGroup - cumusForGroup = make(map[string]*types.Mlrval) + cumusForGroup = make(map[string]*mlrval.Mlrval) tr.cumus[groupingKey] = cumusForGroup } for _, fractionFieldName := range tr.fractionFieldNames { @@ -224,7 +226,7 @@ func (tr *TransformerFraction) Transform( sumsForGroup[fractionFieldName] = value.Copy() cumusForGroup[fractionFieldName] = tr.zero } else { - sumsForGroup[fractionFieldName] = types.BIF_plus_binary(sum, value) + sumsForGroup[fractionFieldName] = bifs.BIF_plus_binary(sum, value) } } } @@ -254,23 +256,23 @@ func (tr *TransformerFraction) Transform( if value != nil { value.AssertNumeric() // may fatal the process - var numerator *types.Mlrval = nil - var cumu *types.Mlrval = nil - var outputValue *types.Mlrval = nil + var numerator *mlrval.Mlrval = nil + var cumu *mlrval.Mlrval = nil + var outputValue *mlrval.Mlrval = nil if tr.doCumu { cumu = cumusForGroup[fractionFieldName] - numerator = types.BIF_plus_binary(value, cumu) + numerator = bifs.BIF_plus_binary(value, cumu) } else { numerator = value } denominator := sumsForGroup[fractionFieldName] - if !types.MlrvalEqualsAsBool(value, tr.zero) { - outputValue = types.BIF_divide(numerator, denominator) - outputValue = types.BIF_times(outputValue, tr.multiplier) + if !mlrval.Equals(value, tr.zero) { + outputValue = bifs.BIF_divide(numerator, denominator) + outputValue = bifs.BIF_times(outputValue, tr.multiplier) } else { - outputValue = types.MLRVAL_ERROR + outputValue = mlrval.ERROR } outrec.PutCopy( @@ -279,7 +281,7 @@ func (tr *TransformerFraction) Transform( ) if tr.doCumu { - cumusForGroup[fractionFieldName] = types.BIF_plus_binary(cumu, value) + cumusForGroup[fractionFieldName] = bifs.BIF_plus_binary(cumu, value) } } } diff --git a/internal/pkg/transformers/gap.go b/internal/pkg/transformers/gap.go index 61e31b4d3..c0f6de7a0 100644 --- a/internal/pkg/transformers/gap.go +++ b/internal/pkg/transformers/gap.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -156,7 +157,7 @@ func (tr *TransformerGap) transformUnkeyed( ) { if !inrecAndContext.EndOfStream { if tr.recordCount > 0 && tr.recordCount%tr.gapCount == 0 { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } outputRecordsAndContexts.PushBack(inrecAndContext) @@ -183,7 +184,7 @@ func (tr *TransformerGap) transformKeyed( } if groupingKey != tr.previousGroupingKey && tr.recordCount > 0 { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } diff --git a/internal/pkg/transformers/histogram.go b/internal/pkg/transformers/histogram.go index 47f625b5f..57abf529f 100644 --- a/internal/pkg/transformers/histogram.go +++ b/internal/pkg/transformers/histogram.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -257,21 +258,21 @@ func (tr *TransformerHistogram) emitNonAuto( countFieldNames[valueFieldName] = tr.outputPrefix + valueFieldName + "_count" } for i := 0; i < tr.nbins; i++ { - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() outrec.PutReference( tr.outputPrefix+"bin_lo", - types.MlrvalFromFloat64((tr.lo+float64(i))/tr.mul), + mlrval.FromFloat((tr.lo+float64(i))/tr.mul), ) outrec.PutReference( tr.outputPrefix+"bin_hi", - types.MlrvalFromFloat64((tr.lo+float64(i+1))/tr.mul), + mlrval.FromFloat((tr.lo+float64(i+1))/tr.mul), ) for _, valueFieldName := range tr.valueFieldNames { outrec.PutReference( countFieldNames[valueFieldName], - types.MlrvalFromInt(tr.countsByField[valueFieldName][i]), + mlrval.FromInt(tr.countsByField[valueFieldName][i]), ) } @@ -363,21 +364,21 @@ func (tr *TransformerHistogram) emitAuto( } for i := 0; i < nbins; i++ { - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() outrec.PutReference( tr.outputPrefix+"bin_lo", - types.MlrvalFromFloat64((lo+float64(i))/mul), + mlrval.FromFloat((lo+float64(i))/mul), ) outrec.PutReference( tr.outputPrefix+"bin_hi", - types.MlrvalFromFloat64((lo+float64(i+1))/mul), + mlrval.FromFloat((lo+float64(i+1))/mul), ) for _, valueFieldName := range tr.valueFieldNames { outrec.PutReference( countFieldNames[valueFieldName], - types.MlrvalFromInt(tr.countsByField[valueFieldName][i]), + mlrval.FromInt(tr.countsByField[valueFieldName][i]), ) } diff --git a/internal/pkg/transformers/join.go b/internal/pkg/transformers/join.go index 7f9eca2e2..3b6147fa5 100644 --- a/internal/pkg/transformers/join.go +++ b/internal/pkg/transformers/join.go @@ -9,6 +9,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/input" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/transformers/utils" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -543,7 +544,7 @@ func (tr *TransformerJoin) formAndEmitPairs( rightrec := rightRecordAndContext.Record // Allocate a new output record which is the join of the left and right records. - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() // Add the joined-on fields to the new output record n := len(tr.opts.leftJoinFieldNames) diff --git a/internal/pkg/transformers/json-stringify.go b/internal/pkg/transformers/json-stringify.go index 68154e18f..045bc134a 100644 --- a/internal/pkg/transformers/json-stringify.go +++ b/internal/pkg/transformers/json-stringify.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -84,11 +85,11 @@ func transformerJSONStringifyParseCLI( } } - var jsonFormatting types.TJSONFormatting = types.JSON_SINGLE_LINE + var jsonFormatting mlrval.TJSONFormatting = mlrval.JSON_SINGLE_LINE if jvStack { - jsonFormatting = types.JSON_MULTILINE + jsonFormatting = mlrval.JSON_MULTILINE } else { - jsonFormatting = types.JSON_SINGLE_LINE + jsonFormatting = mlrval.JSON_SINGLE_LINE } *pargi = argi @@ -111,7 +112,7 @@ func transformerJSONStringifyParseCLI( // ---------------------------------------------------------------- type TransformerJSONStringify struct { // input - jsonFormatting types.TJSONFormatting + jsonFormatting mlrval.TJSONFormatting fieldNameSet map[string]bool // state @@ -119,7 +120,7 @@ type TransformerJSONStringify struct { } func NewTransformerJSONStringify( - jsonFormatting types.TJSONFormatting, + jsonFormatting mlrval.TJSONFormatting, fieldNames []string, ) (*TransformerJSONStringify, error) { var fieldNameSet map[string]bool = nil diff --git a/internal/pkg/transformers/merge-fields.go b/internal/pkg/transformers/merge-fields.go index bfdef6e9e..84f421b32 100644 --- a/internal/pkg/transformers/merge-fields.go +++ b/internal/pkg/transformers/merge-fields.go @@ -351,7 +351,7 @@ func (tr *TransformerMergeFields) transformByNameList( continue } - if mvalue.IsEmpty() { // key present with empty value + if mvalue.IsVoid() { // key present with empty value if !tr.keepInputFields { inrec.Remove(valueFieldName) } @@ -418,7 +418,7 @@ func (tr *TransformerMergeFields) transformByNameRegex( continue } - if mvalue.IsEmpty() { // Key present with empty value + if mvalue.IsVoid() { // Key present with empty value if !tr.keepInputFields { // We are modifying the record while iterating over it. next := pe.Next inrec.Unlink(pe) @@ -521,7 +521,7 @@ func (tr *TransformerMergeFields) transformByCollapsing( // The accumulator has been initialized with default values; continue // here. (If we were to continue before namedAccumulators.Put(...) we // would be failing to construct the accumulator.) - if mvalue.IsEmpty() { // key present with empty value + if mvalue.IsVoid() { // key present with empty value if !tr.keepInputFields { // We are modifying the record while iterating over it. next := pe.Next inrec.Unlink(pe) diff --git a/internal/pkg/transformers/most-or-least-frequent.go b/internal/pkg/transformers/most-or-least-frequent.go index fb7a6c7de..7ab896124 100644 --- a/internal/pkg/transformers/most-or-least-frequent.go +++ b/internal/pkg/transformers/most-or-least-frequent.go @@ -9,6 +9,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -178,7 +179,7 @@ type TransformerMostOrLeastFrequent struct { outputFieldName string descending bool countsByGroup *lib.OrderedMap // map[string]int - valuesForGroup map[string][]*types.Mlrval + valuesForGroup map[string][]*mlrval.Mlrval } type tMostOrLeastFrequentSortPair struct { @@ -201,7 +202,7 @@ func NewTransformerMostOrLeastFrequent( outputFieldName: outputFieldName, descending: descending, countsByGroup: lib.NewOrderedMap(), - valuesForGroup: make(map[string][]*types.Mlrval), + valuesForGroup: make(map[string][]*mlrval.Mlrval), } return tr, nil @@ -274,7 +275,7 @@ func (tr *TransformerMostOrLeastFrequent) Transform( outputLength = tr.maxOutputLength } for i := 0; i < outputLength; i++ { - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() groupByFieldValues := tr.valuesForGroup[sortPairs[i].groupingKey] for j := range tr.groupByFieldNames { outrec.PutCopy( @@ -284,7 +285,7 @@ func (tr *TransformerMostOrLeastFrequent) Transform( } if tr.showCounts { - outrec.PutReference(tr.outputFieldName, types.MlrvalFromInt(sortPairs[i].count)) + outrec.PutReference(tr.outputFieldName, mlrval.FromInt(sortPairs[i].count)) } outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } diff --git a/internal/pkg/transformers/nest.go b/internal/pkg/transformers/nest.go index 71e7e7fa6..e3cc86260 100644 --- a/internal/pkg/transformers/nest.go +++ b/internal/pkg/transformers/nest.go @@ -11,6 +11,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -335,7 +336,7 @@ func (tr *TransformerNest) explodeValuesAcrossFields( i := 1 for _, piece := range pieces { key := tr.fieldName + "_" + strconv.Itoa(i) - value := types.MlrvalFromString(piece) + value := mlrval.FromString(piece) recordEntry = inrec.PutReferenceAfter(recordEntry, key, value) i++ } @@ -368,7 +369,7 @@ func (tr *TransformerNest) explodeValuesAcrossRecords( pieces := strings.Split(svalue, tr.nestedFS) for _, piece := range pieces { outrec := inrec.Copy() - outrec.PutReference(tr.fieldName, types.MlrvalFromString(piece)) + outrec.PutReference(tr.fieldName, mlrval.FromString(piece)) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } @@ -404,13 +405,13 @@ func (tr *TransformerNest) explodePairsAcrossFields( recordEntry = inrec.PutReferenceAfter( recordEntry, pair[0], - types.MlrvalFromString(pair[1]), + mlrval.FromString(pair[1]), ) } else { // there is not a pair recordEntry = inrec.PutReferenceAfter( recordEntry, tr.fieldName, - types.MlrvalFromString(piece), + mlrval.FromString(piece), ) } } @@ -449,9 +450,9 @@ func (tr *TransformerNest) explodePairsAcrossRecords( // name, in which case replace its value. pair := strings.SplitN(piece, tr.nestedPS, 2) if len(pair) == 2 { // there is a pair - outrec.PutReferenceAfter(originalEntry, pair[0], types.MlrvalFromString(pair[1])) + outrec.PutReferenceAfter(originalEntry, pair[0], mlrval.FromString(pair[1])) } else { // there is not a pair - outrec.PutReferenceAfter(originalEntry, tr.fieldName, types.MlrvalFromString(piece)) + outrec.PutReferenceAfter(originalEntry, tr.fieldName, mlrval.FromString(piece)) } outrec.Unlink(originalEntry) @@ -473,7 +474,7 @@ func (tr *TransformerNest) implodeValuesAcrossFields( if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - var previousEntry *types.MlrmapEntry = nil + var previousEntry *mlrval.MlrmapEntry = nil fieldCount := 0 var buffer bytes.Buffer for pe := inrec.Head; pe != nil; /* increment in loop */ { @@ -497,7 +498,7 @@ func (tr *TransformerNest) implodeValuesAcrossFields( } if fieldCount > 0 { - newValue := types.MlrvalFromString(buffer.String()) + newValue := mlrval.FromString(buffer.String()) if previousEntry == nil { // No record before the unlinked one, i.e. list-head. inrec.PrependReference(tr.fieldName, newValue) } else { @@ -551,7 +552,7 @@ func (tr *TransformerNest) implodeValueAcrossRecords( bucket = iBucket.(*tNestBucket) } - pair := types.NewMlrmapAsRecord() + pair := mlrval.NewMlrmapAsRecord() pair.PutReference(tr.fieldName, fieldValueCopy) bucket.pairs.PushBack(pair) @@ -567,7 +568,7 @@ func (tr *TransformerNest) implodeValueAcrossRecords( i := 0 for pg := bucket.pairs.Front(); pg != nil; pg = pg.Next() { - pr := pg.Value.(*types.Mlrmap) + pr := pg.Value.(*mlrval.Mlrmap) if i > 0 { buffer.WriteString(tr.nestedFS) } @@ -576,7 +577,7 @@ func (tr *TransformerNest) implodeValueAcrossRecords( } // tr.fieldName was already present so we'll overwrite it in-place here. - outrec.PutReference(tr.fieldName, types.MlrvalFromString(buffer.String())) + outrec.PutReference(tr.fieldName, mlrval.FromString(buffer.String())) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } } @@ -586,11 +587,11 @@ func (tr *TransformerNest) implodeValueAcrossRecords( } type tNestBucket struct { - representative *types.Mlrmap + representative *mlrval.Mlrmap pairs *list.List } -func newNestBucket(representative *types.Mlrmap) *tNestBucket { +func newNestBucket(representative *mlrval.Mlrmap) *tNestBucket { return &tNestBucket{ representative: representative, pairs: list.New(), diff --git a/internal/pkg/transformers/put-or-filter.go b/internal/pkg/transformers/put-or-filter.go index 11d5755b4..c7a84901f 100644 --- a/internal/pkg/transformers/put-or-filter.go +++ b/internal/pkg/transformers/put-or-filter.go @@ -11,6 +11,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/dsl" "github.com/johnkerl/miller/internal/pkg/dsl/cst" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/parsing/lexer" "github.com/johnkerl/miller/internal/pkg/parsing/parser" "github.com/johnkerl/miller/internal/pkg/runtime" @@ -455,7 +456,7 @@ func NewTransformerPut( } key := pair[0] svalue := pair[1] - mvalue := types.MlrvalFromInferredType(svalue) + mvalue := mlrval.FromInferredType(svalue) runtimeState.Oosvars.PutCopy(key, mvalue) } } diff --git a/internal/pkg/transformers/regularize.go b/internal/pkg/transformers/regularize.go index 22ab2fbf8..f50c5c439 100644 --- a/internal/pkg/transformers/regularize.go +++ b/internal/pkg/transformers/regularize.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -112,7 +113,7 @@ func (tr *TransformerRegularize) Transform( tr.sortedToOriginal[currentSortedFieldNamesJoined] = currentFieldNames outputRecordsAndContexts.PushBack(inrecAndContext) } else { - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() for _, fieldName := range previousSortedFieldNames { outrec.PutReference(fieldName, inrec.Get(fieldName)) // inrec will be GC'ed } diff --git a/internal/pkg/transformers/remove-empty-columns.go b/internal/pkg/transformers/remove-empty-columns.go index d6b60f8bd..12fcd03e2 100644 --- a/internal/pkg/transformers/remove-empty-columns.go +++ b/internal/pkg/transformers/remove-empty-columns.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -107,7 +108,7 @@ func (tr *TransformerRemoveEmptyColumns) Transform( tr.recordsAndContexts.PushBack(inrecAndContext) for pe := inrec.Head; pe != nil; pe = pe.Next { - if !pe.Value.IsEmpty() { + if !pe.Value.IsVoid() { tr.namesWithNonEmptyValues[pe.Key] = true } } @@ -118,7 +119,7 @@ func (tr *TransformerRemoveEmptyColumns) Transform( outrecAndContext := e.Value.(*types.RecordAndContext) outrec := outrecAndContext.Record - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for pe := outrec.Head; pe != nil; pe = pe.Next { _, ok := tr.namesWithNonEmptyValues[pe.Key] diff --git a/internal/pkg/transformers/reorder.go b/internal/pkg/transformers/reorder.go index df1b419d7..39ac531be 100644 --- a/internal/pkg/transformers/reorder.go +++ b/internal/pkg/transformers/reorder.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -232,7 +233,7 @@ func (tr *TransformerReorder) reorderBefore( return } - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() pe := inrec.Head // * inrec will be GC'ed @@ -289,7 +290,7 @@ func (tr *TransformerReorder) reorderAfter( return } - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() pe := inrec.Head // * inrec will be GC'ed diff --git a/internal/pkg/transformers/reshape.go b/internal/pkg/transformers/reshape.go index 45de4430f..b2838874b 100644 --- a/internal/pkg/transformers/reshape.go +++ b/internal/pkg/transformers/reshape.go @@ -36,6 +36,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -301,7 +302,7 @@ func (tr *TransformerReshape) wideToLongNoRegex( ) { if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - pairs := types.NewMlrmap() + pairs := mlrval.NewMlrmap() for _, inputFieldName := range tr.inputFieldNames { value := inrec.Get(inputFieldName) if value != nil { @@ -320,7 +321,7 @@ func (tr *TransformerReshape) wideToLongNoRegex( } else { for pf := pairs.Head; pf != nil; pf = pf.Next { outrec := inrec.Copy() - outrec.PutReference(tr.outputKeyFieldName, types.MlrvalFromString(pf.Key)) + outrec.PutReference(tr.outputKeyFieldName, mlrval.FromString(pf.Key)) outrec.PutReference(tr.outputValueFieldName, pf.Value) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } @@ -340,7 +341,7 @@ func (tr *TransformerReshape) wideToLongRegex( ) { if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - pairs := types.NewMlrmap() + pairs := mlrval.NewMlrmap() for pd := inrec.Head; pd != nil; pd = pd.Next { for _, inputFieldRegex := range tr.inputFieldRegexes { @@ -362,7 +363,7 @@ func (tr *TransformerReshape) wideToLongRegex( } else { for pf := pairs.Head; pf != nil; pf = pf.Next { outrec := inrec.Copy() - outrec.PutReference(tr.outputKeyFieldName, types.MlrvalFromString(pf.Key)) + outrec.PutReference(tr.outputKeyFieldName, mlrval.FromString(pf.Key)) outrec.PutReference(tr.outputValueFieldName, pf.Value) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } @@ -439,13 +440,13 @@ func (tr *TransformerReshape) longToWide( } type tReshapeBucket struct { - representative *types.Mlrmap - pairs *types.Mlrmap + representative *mlrval.Mlrmap + pairs *mlrval.Mlrmap } -func newReshapeBucket(representative *types.Mlrmap) *tReshapeBucket { +func newReshapeBucket(representative *mlrval.Mlrmap) *tReshapeBucket { return &tReshapeBucket{ representative: representative, - pairs: types.NewMlrmap(), + pairs: mlrval.NewMlrmap(), } } diff --git a/internal/pkg/transformers/sec2gmt.go b/internal/pkg/transformers/sec2gmt.go index 9d99e9129..6973d2f02 100644 --- a/internal/pkg/transformers/sec2gmt.go +++ b/internal/pkg/transformers/sec2gmt.go @@ -7,6 +7,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -163,7 +164,7 @@ func (tr *TransformerSec2GMT) Transform( if value != nil { floatval, ok := value.GetNumericToFloatValue() if ok { - newValue := types.MlrvalFromString(lib.Sec2GMT( + newValue := mlrval.FromString(lib.Sec2GMT( floatval/tr.preDivide, tr.numDecimalPlaces, )) diff --git a/internal/pkg/transformers/sec2gmtdate.go b/internal/pkg/transformers/sec2gmtdate.go index d76ae16ef..1f886eba5 100644 --- a/internal/pkg/transformers/sec2gmtdate.go +++ b/internal/pkg/transformers/sec2gmtdate.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" "github.com/johnkerl/miller/internal/pkg/types" @@ -118,7 +119,7 @@ func (tr *TransformerSec2GMTDate) Transform( for _, fieldName := range tr.fieldNameList { value := inrec.Get(fieldName) if value != nil { - inrec.PutReference(fieldName, types.BIF_sec2gmtdate(value)) + inrec.PutReference(fieldName, bifs.BIF_sec2gmtdate(value)) } } outputRecordsAndContexts.PushBack(inrecAndContext) diff --git a/internal/pkg/transformers/seqgen.go b/internal/pkg/transformers/seqgen.go index 563501682..427fe1a36 100644 --- a/internal/pkg/transformers/seqgen.go +++ b/internal/pkg/transformers/seqgen.go @@ -7,7 +7,9 @@ import ( "os" "strings" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -117,11 +119,11 @@ func transformerSeqgenParseCLI( // ---------------------------------------------------------------- type TransformerSeqgen struct { fieldName string - start *types.Mlrval - stop *types.Mlrval - step *types.Mlrval - doneComparator types.BinaryFunc - mdone *types.Mlrval + start *mlrval.Mlrval + stop *mlrval.Mlrval + step *mlrval.Mlrval + doneComparator bifs.BinaryFunc + mdone *mlrval.Mlrval } // ---------------------------------------------------------------- @@ -131,10 +133,10 @@ func NewTransformerSeqgen( stopString string, stepString string, ) (*TransformerSeqgen, error) { - start := types.MlrvalFromInferredType(startString) - stop := types.MlrvalFromInferredType(stopString) - step := types.MlrvalFromInferredType(stepString) - var doneComparator types.BinaryFunc = nil + start := mlrval.FromInferredType(startString) + stop := mlrval.FromInferredType(stopString) + step := mlrval.FromInferredType(stepString) + var doneComparator bifs.BinaryFunc = nil fstart, startIsNumeric := start.GetNumericToFloatValue() if !startIsNumeric { @@ -167,12 +169,12 @@ func NewTransformerSeqgen( } if fstep > 0 { - doneComparator = types.BIF_greater_than + doneComparator = bifs.BIF_greater_than } else if fstep < 0 { - doneComparator = types.BIF_less_than + doneComparator = bifs.BIF_less_than } else { if fstart == fstop { - doneComparator = types.BIF_equals + doneComparator = bifs.BIF_equals } else { return nil, errors.New( "mlr seqgen: step must not be zero unless start == stop.", @@ -186,7 +188,7 @@ func NewTransformerSeqgen( stop: stop, step: step, doneComparator: doneComparator, - mdone: types.MLRVAL_FALSE, + mdone: mlrval.FALSE, }, nil } @@ -226,7 +228,7 @@ func (tr *TransformerSeqgen) Transform( break } - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() outrec.PutCopy(tr.fieldName, counter) context.UpdateForInputRecord() @@ -234,7 +236,7 @@ func (tr *TransformerSeqgen) Transform( outrecAndContext := types.NewRecordAndContext(outrec, context) outputRecordsAndContexts.PushBack(outrecAndContext) - counter = types.BIF_plus_binary(counter, tr.step) + counter = bifs.BIF_plus_binary(counter, tr.step) } outputRecordsAndContexts.PushBack(types.NewEndOfStreamMarker(context)) diff --git a/internal/pkg/transformers/sort.go b/internal/pkg/transformers/sort.go index a2b90223a..4365f9bf9 100644 --- a/internal/pkg/transformers/sort.go +++ b/internal/pkg/transformers/sort.go @@ -48,8 +48,10 @@ import ( "sort" "strings" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -109,7 +111,7 @@ func transformerSortParseCLI( argi++ groupByFieldNames := make([]string, 0) - comparatorFuncs := make([]types.ComparatorFunc, 0) + comparatorFuncs := make([]bifs.ComparatorFunc, 0) for argi < argc /* variable increment: 1 or 2 depending on flag */ { opt := args[argi] @@ -128,7 +130,7 @@ func transformerSortParseCLI( subList := cli.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.LexicalAscendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.LexicalAscendingComparator) } } else if opt == "-c" { @@ -139,14 +141,14 @@ func transformerSortParseCLI( subList := cli.VerbGetStringArrayArgOrDie(verb, "-nr", args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.CaseFoldDescendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.CaseFoldDescendingComparator) } } else { subList := cli.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.CaseFoldAscendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.CaseFoldAscendingComparator) } } @@ -154,7 +156,7 @@ func transformerSortParseCLI( subList := cli.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.LexicalDescendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.LexicalDescendingComparator) } } else if opt == "-n" { @@ -187,7 +189,7 @@ func transformerSortParseCLI( subList := cli.VerbGetStringArrayArgOrDie(verb, "-nf", args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.NumericAscendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.NumericAscendingComparator) } } else if args[argi] == "-r" { @@ -196,7 +198,7 @@ func transformerSortParseCLI( subList := cli.VerbGetStringArrayArgOrDie(verb, "-nr", args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.NumericDescendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.NumericDescendingComparator) } } else { @@ -204,7 +206,7 @@ func transformerSortParseCLI( subList := cli.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.NumericAscendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.NumericAscendingComparator) } } @@ -212,14 +214,14 @@ func transformerSortParseCLI( subList := cli.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.NumericAscendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.NumericAscendingComparator) } } else if opt == "-nr" { subList := cli.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) for _, item := range subList { groupByFieldNames = append(groupByFieldNames, item) - comparatorFuncs = append(comparatorFuncs, types.NumericDescendingComparator) + comparatorFuncs = append(comparatorFuncs, bifs.NumericDescendingComparator) } } else { @@ -268,7 +270,7 @@ func transformerSortParseCLI( type TransformerSort struct { // -- Input groupByFieldNames []string - comparatorFuncs []types.ComparatorFunc + comparatorFuncs []bifs.ComparatorFunc // -- State // Map from string to *list.List: @@ -280,7 +282,7 @@ type TransformerSort struct { func NewTransformerSort( groupByFieldNames []string, - comparatorFuncs []types.ComparatorFunc, + comparatorFuncs []bifs.ComparatorFunc, ) (*TransformerSort, error) { tr := &TransformerSort{ @@ -298,7 +300,7 @@ func NewTransformerSort( // ---------------------------------------------------------------- type GroupingKeysAndMlrvals struct { groupingKey string - mlrvals []*types.Mlrval + mlrvals []*mlrval.Mlrval } func (tr *TransformerSort) Transform( @@ -386,7 +388,7 @@ func groupHeadsToArray(groupHeads *lib.OrderedMap) []GroupingKeysAndMlrvals { for entry := groupHeads.Head; entry != nil; entry = entry.Next { retval[i] = GroupingKeysAndMlrvals{ groupingKey: entry.Key, - mlrvals: entry.Value.([]*types.Mlrval), + mlrvals: entry.Value.([]*mlrval.Mlrval), } i++ } diff --git a/internal/pkg/transformers/stats1.go b/internal/pkg/transformers/stats1.go index 602b2b357..d047c6bfb 100644 --- a/internal/pkg/transformers/stats1.go +++ b/internal/pkg/transformers/stats1.go @@ -11,6 +11,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/transformers/utils" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -253,7 +254,7 @@ type TransformerStats1 struct { // except we need maps that preserve insertion order. namedAccumulators *lib.OrderedMap - // map[string]OrderedMap[string]*types.Mlrval + // map[string]OrderedMap[string]*mlrval.Mlrval groupingKeysToGroupByFieldValues map[string]*lib.OrderedMap } @@ -382,7 +383,7 @@ func (tr *TransformerStats1) handleInputRecord( // E.g. if grouping by "a" and "b", and the current record has a=circle, b=blue, // then groupingKey is the string "circle,blue". var groupingKey string - var groupByFieldValues *lib.OrderedMap // OrderedMap[string]*types.Mlrval + var groupByFieldValues *lib.OrderedMap // OrderedMap[string]*mlrval.Mlrval var ok bool if tr.doRegexGroupByFieldNames { groupingKey, groupByFieldValues, ok = tr.getGroupByFieldNamesWithRegexes(inrec) @@ -425,13 +426,13 @@ func (tr *TransformerStats1) handleInputRecord( // regexed group-by field names, the group-by field names/values are the same // on every record. func (tr *TransformerStats1) getGroupByFieldNamesWithoutRegexes( - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, ) ( groupingKey string, - groupByFieldValues *lib.OrderedMap, // OrderedMap[string]*types.Mlrval, + groupByFieldValues *lib.OrderedMap, // OrderedMap[string]*mlrval.Mlrval, ok bool, ) { - var groupByFieldValuesArray []*types.Mlrval + var groupByFieldValuesArray []*mlrval.Mlrval groupingKey, groupByFieldValuesArray, ok = inrec.GetSelectedValuesAndJoined(tr.groupByFieldNameList) if !ok { return groupingKey, nil, false @@ -448,10 +449,10 @@ func (tr *TransformerStats1) getGroupByFieldNamesWithoutRegexes( // regexed group-by field names, the group-by field names/values may or may not // be the same on every record. func (tr *TransformerStats1) getGroupByFieldNamesWithRegexes( - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, ) ( groupingKey string, - groupByFieldValues *lib.OrderedMap, // OrderedMap[string]*types.Mlrval, + groupByFieldValues *lib.OrderedMap, // OrderedMap[string]*mlrval.Mlrval, ok bool, ) { @@ -480,7 +481,7 @@ func (tr *TransformerStats1) getGroupByFieldNamesWithRegexes( } func (tr *TransformerStats1) ingestWithoutValueFieldRegexes( - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, groupingKey string, level2 *lib.OrderedMap, ) { @@ -517,7 +518,7 @@ func (tr *TransformerStats1) ingestWithoutValueFieldRegexes( } func (tr *TransformerStats1) ingestWithValueFieldRegexes( - inrec *types.Mlrmap, + inrec *mlrval.Mlrmap, groupingKey string, level2 *lib.OrderedMap, ) { @@ -599,7 +600,7 @@ func (tr *TransformerStats1) handleEndOfRecordStream( level2 := pa.Value.(*lib.OrderedMap) groupByFieldValues := tr.groupingKeysToGroupByFieldValues[groupingKey] - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() tr.emitIntoOutputRecord( inrecAndContext.Record, @@ -615,17 +616,17 @@ func (tr *TransformerStats1) handleEndOfRecordStream( } func (tr *TransformerStats1) emitIntoOutputRecord( - inrec *types.Mlrmap, - groupByFieldValues *lib.OrderedMap, // OrderedMap[string]*types.Mlrval, + inrec *mlrval.Mlrmap, + groupByFieldValues *lib.OrderedMap, // OrderedMap[string]*mlrval.Mlrval, level2accumulators *lib.OrderedMap, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { for pa := tr.groupByFieldNamesForOutput.Head; pa != nil; pa = pa.Next { groupByFieldName := pa.Key iValue := groupByFieldValues.Get(groupByFieldName) if iValue != nil { - outrec.PutCopy(groupByFieldName, iValue.(*types.Mlrval)) + outrec.PutCopy(groupByFieldName, iValue.(*mlrval.Mlrval)) } } diff --git a/internal/pkg/transformers/stats2.go b/internal/pkg/transformers/stats2.go index 59d771abf..ca7b9be1f 100644 --- a/internal/pkg/transformers/stats2.go +++ b/internal/pkg/transformers/stats2.go @@ -9,6 +9,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/transformers/utils" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -393,11 +394,11 @@ func (tr *TransformerStats2) emit( context *types.Context, ) { for pa := tr.namedAccumulators.Head; pa != nil; pa = pa.Next { - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() // Add in a=s,b=t fields: groupingKey := pa.Key - groupByFieldValues := tr.groupingKeysToGroupByFieldValues.Get(groupingKey).([]*types.Mlrval) + groupByFieldValues := tr.groupingKeysToGroupByFieldValues.Get(groupingKey).([]*mlrval.Mlrval) for i, groupByFieldName := range tr.groupByFieldNameList { outrec.PutReference(groupByFieldName, groupByFieldValues[i].Copy()) } @@ -427,7 +428,7 @@ func (tr *TransformerStats2) emit( } func (tr *TransformerStats2) populateRecord( - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, valueFieldName1 string, valueFieldName2 string, valueFieldsToAccumulator *lib.OrderedMap, diff --git a/internal/pkg/transformers/step.go b/internal/pkg/transformers/step.go index b71b5bcf8..3f763eb45 100644 --- a/internal/pkg/transformers/step.go +++ b/internal/pkg/transformers/step.go @@ -7,8 +7,10 @@ import ( "os" "strings" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -163,7 +165,7 @@ type TransformerStep struct { // STATE // Scratch space used per-record - valueFieldValues []types.Mlrval + valueFieldValues []mlrval.Mlrval // Map from group-by field names to value-field names to array of // stepper objects. See the Transform method below for more details. groups map[string]map[string]map[string]tStepper @@ -316,7 +318,7 @@ type tStepperAllocator func( ) tStepper type tStepper interface { - process(valueFieldValue *types.Mlrval, inputRecord *types.Mlrmap) + process(valueFieldValue *mlrval.Mlrval, inputRecord *mlrval.Mlrmap) } type tStepperLookup struct { @@ -358,7 +360,7 @@ func allocateStepper( // ================================================================ type tStepperDelta struct { - previous *types.Mlrval + previous *mlrval.Mlrval outputFieldName string } @@ -374,17 +376,17 @@ func stepperDeltaAlloc( } func (stepper *tStepperDelta) process( - valueFieldValue *types.Mlrval, - inrec *types.Mlrmap, + valueFieldValue *mlrval.Mlrval, + inrec *mlrval.Mlrmap, ) { - if valueFieldValue.IsEmpty() { - inrec.PutCopy(stepper.outputFieldName, types.MLRVAL_VOID) + if valueFieldValue.IsVoid() { + inrec.PutCopy(stepper.outputFieldName, mlrval.VOID) return } - delta := types.MlrvalFromInt(0) + delta := mlrval.FromInt(0) if stepper.previous != nil { - delta = types.BIF_minus_binary(valueFieldValue, stepper.previous) + delta = bifs.BIF_minus_binary(valueFieldValue, stepper.previous) } inrec.PutCopy(stepper.outputFieldName, delta) @@ -393,7 +395,7 @@ func (stepper *tStepperDelta) process( // ================================================================ type tStepperShift struct { - previous *types.Mlrval + previous *mlrval.Mlrval outputFieldName string } @@ -409,11 +411,11 @@ func stepperShiftAlloc( } func (stepper *tStepperShift) process( - valueFieldValue *types.Mlrval, - inrec *types.Mlrmap, + valueFieldValue *mlrval.Mlrval, + inrec *mlrval.Mlrmap, ) { if stepper.previous == nil { - shift := types.MLRVAL_VOID + shift := mlrval.VOID inrec.PutCopy(stepper.outputFieldName, shift) } else { inrec.PutCopy(stepper.outputFieldName, stepper.previous) @@ -424,7 +426,7 @@ func (stepper *tStepperShift) process( // ================================================================ type tStepperFromFirst struct { - first *types.Mlrval + first *mlrval.Mlrval outputFieldName string } @@ -440,21 +442,21 @@ func stepperFromFirstAlloc( } func (stepper *tStepperFromFirst) process( - valueFieldValue *types.Mlrval, - inrec *types.Mlrmap, + valueFieldValue *mlrval.Mlrval, + inrec *mlrval.Mlrmap, ) { - fromFirst := types.MlrvalFromInt(0) + fromFirst := mlrval.FromInt(0) if stepper.first == nil { stepper.first = valueFieldValue.Copy() } else { - fromFirst = types.BIF_minus_binary(valueFieldValue, stepper.first) + fromFirst = bifs.BIF_minus_binary(valueFieldValue, stepper.first) } inrec.PutCopy(stepper.outputFieldName, fromFirst) } // ================================================================ type tStepperRatio struct { - previous *types.Mlrval + previous *mlrval.Mlrval outputFieldName string } @@ -470,17 +472,17 @@ func stepperRatioAlloc( } func (stepper *tStepperRatio) process( - valueFieldValue *types.Mlrval, - inrec *types.Mlrmap, + valueFieldValue *mlrval.Mlrval, + inrec *mlrval.Mlrmap, ) { - if valueFieldValue.IsEmpty() { - inrec.PutCopy(stepper.outputFieldName, types.MLRVAL_VOID) + if valueFieldValue.IsVoid() { + inrec.PutCopy(stepper.outputFieldName, mlrval.VOID) return } - ratio := types.MlrvalFromInt(1) + ratio := mlrval.FromInt(1) if stepper.previous != nil { - ratio = types.BIF_divide(valueFieldValue, stepper.previous) + ratio = bifs.BIF_divide(valueFieldValue, stepper.previous) } inrec.PutCopy(stepper.outputFieldName, ratio) @@ -489,7 +491,7 @@ func (stepper *tStepperRatio) process( // ================================================================ type tStepperRsum struct { - rsum *types.Mlrval + rsum *mlrval.Mlrval outputFieldName string } @@ -499,27 +501,27 @@ func stepperRsumAlloc( _unused2 []string, ) tStepper { return &tStepperRsum{ - rsum: types.MlrvalFromInt(0), + rsum: mlrval.FromInt(0), outputFieldName: inputFieldName + "_rsum", } } func (stepper *tStepperRsum) process( - valueFieldValue *types.Mlrval, - inrec *types.Mlrmap, + valueFieldValue *mlrval.Mlrval, + inrec *mlrval.Mlrmap, ) { - if valueFieldValue.IsEmpty() { - inrec.PutCopy(stepper.outputFieldName, types.MLRVAL_VOID) + if valueFieldValue.IsVoid() { + inrec.PutCopy(stepper.outputFieldName, mlrval.VOID) } else { - stepper.rsum = types.BIF_plus_binary(valueFieldValue, stepper.rsum) + stepper.rsum = bifs.BIF_plus_binary(valueFieldValue, stepper.rsum) inrec.PutCopy(stepper.outputFieldName, stepper.rsum) } } // ================================================================ type tStepperCounter struct { - counter *types.Mlrval - one *types.Mlrval + counter *mlrval.Mlrval + one *mlrval.Mlrval outputFieldName string } @@ -529,20 +531,20 @@ func stepperCounterAlloc( _unused2 []string, ) tStepper { return &tStepperCounter{ - counter: types.MlrvalFromInt(0), - one: types.MlrvalFromInt(1), + counter: mlrval.FromInt(0), + one: mlrval.FromInt(1), outputFieldName: inputFieldName + "_counter", } } func (stepper *tStepperCounter) process( - valueFieldValue *types.Mlrval, - inrec *types.Mlrmap, + valueFieldValue *mlrval.Mlrval, + inrec *mlrval.Mlrmap, ) { - if valueFieldValue.IsEmpty() { - inrec.PutCopy(stepper.outputFieldName, types.MLRVAL_VOID) + if valueFieldValue.IsVoid() { + inrec.PutCopy(stepper.outputFieldName, mlrval.VOID) } else { - stepper.counter = types.BIF_plus_binary(stepper.counter, stepper.one) + stepper.counter = bifs.BIF_plus_binary(stepper.counter, stepper.one) inrec.PutCopy(stepper.outputFieldName, stepper.counter) } } @@ -552,9 +554,9 @@ func (stepper *tStepperCounter) process( // ================================================================ type tStepperEWMA struct { - alphas []*types.Mlrval - oneMinusAlphas []*types.Mlrval - prevs []*types.Mlrval + alphas []*mlrval.Mlrval + oneMinusAlphas []*mlrval.Mlrval + prevs []*mlrval.Mlrval outputFieldNames []string havePrevs bool } @@ -569,9 +571,9 @@ func stepperEWMAAlloc( // len(ewmaSuffixes) in the CLI parser. n := len(stringAlphas) - alphas := make([]*types.Mlrval, n) - oneMinusAlphas := make([]*types.Mlrval, n) - prevs := make([]*types.Mlrval, n) + alphas := make([]*mlrval.Mlrval, n) + oneMinusAlphas := make([]*mlrval.Mlrval, n) + prevs := make([]*mlrval.Mlrval, n) outputFieldNames := make([]string, n) suffixes := stringAlphas @@ -582,7 +584,7 @@ func stepperEWMAAlloc( for i, stringAlpha := range stringAlphas { suffix := suffixes[i] - dalpha, ok := lib.TryFloat64FromString(stringAlpha) + dalpha, ok := lib.TryFloatFromString(stringAlpha) if !ok { fmt.Fprintf( os.Stderr, @@ -591,9 +593,9 @@ func stepperEWMAAlloc( ) os.Exit(1) } - alphas[i] = types.MlrvalFromFloat64(dalpha) - oneMinusAlphas[i] = types.MlrvalFromFloat64(1.0 - dalpha) - prevs[i] = types.MlrvalFromFloat64(0.0) + alphas[i] = mlrval.FromFloat(dalpha) + oneMinusAlphas[i] = mlrval.FromFloat(1.0 - dalpha) + prevs[i] = mlrval.FromFloat(0.0) outputFieldNames[i] = inputFieldName + "_ewma_" + suffix } @@ -607,8 +609,8 @@ func stepperEWMAAlloc( } func (stepper *tStepperEWMA) process( - valueFieldValue *types.Mlrval, - inrec *types.Mlrmap, + valueFieldValue *mlrval.Mlrval, + inrec *mlrval.Mlrmap, ) { if !stepper.havePrevs { for i := range stepper.alphas { @@ -620,9 +622,9 @@ func (stepper *tStepperEWMA) process( for i := range stepper.alphas { curr := valueFieldValue.Copy() // xxx pending pointer-output refactor - product1 := types.BIF_times(curr, stepper.alphas[i]) - product2 := types.BIF_times(stepper.prevs[i], stepper.oneMinusAlphas[i]) - next := types.BIF_plus_binary(product1, product2) + product1 := bifs.BIF_times(curr, stepper.alphas[i]) + product2 := bifs.BIF_times(stepper.prevs[i], stepper.oneMinusAlphas[i]) + next := bifs.BIF_plus_binary(product1, product2) inrec.PutCopy(stepper.outputFieldNames[i], next) stepper.prevs[i] = next } diff --git a/internal/pkg/transformers/template.go b/internal/pkg/transformers/template.go index 21a65c088..605d09cbb 100644 --- a/internal/pkg/transformers/template.go +++ b/internal/pkg/transformers/template.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -119,7 +120,7 @@ func transformerTemplateParseCLI( type TransformerTemplate struct { fieldNameList []string fieldNameSet map[string]bool - fillWith *types.Mlrval + fillWith *mlrval.Mlrval } func NewTransformerTemplate( @@ -129,7 +130,7 @@ func NewTransformerTemplate( return &TransformerTemplate{ fieldNameList: fieldNames, fieldNameSet: lib.StringListToSet(fieldNames), - fillWith: types.MlrvalFromString(fillWith), + fillWith: mlrval.FromString(fillWith), }, nil } @@ -144,7 +145,7 @@ func (tr *TransformerTemplate) Transform( HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel) if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - outrec := types.NewMlrmap() + outrec := mlrval.NewMlrmap() for _, fieldName := range tr.fieldNameList { value := inrec.Get(fieldName) if value != nil { diff --git a/internal/pkg/transformers/top.go b/internal/pkg/transformers/top.go index 647fab2c9..721768ee8 100644 --- a/internal/pkg/transformers/top.go +++ b/internal/pkg/transformers/top.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/transformers/utils" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -141,7 +142,7 @@ type TransformerTop struct { // Two-level map from grouping key (string of joined-together group-by field values), // to string value-field name, to *utils.TopKeeper groups *lib.OrderedMap - groupingKeysToGroupByFieldValues map[string][]*types.Mlrval + groupingKeysToGroupByFieldValues map[string][]*mlrval.Mlrval } // ---------------------------------------------------------------- @@ -163,7 +164,7 @@ func NewTransformerTop( outputFieldName: outputFieldName, groups: lib.NewOrderedMap(), - groupingKeysToGroupByFieldValues: make(map[string][]*types.Mlrval), + groupingKeysToGroupByFieldValues: make(map[string][]*mlrval.Mlrval), } return tr, nil @@ -259,7 +260,7 @@ func (tr *TransformerTop) emit( } else { for i := 0; i < tr.topCount; i++ { - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() // Add in a=s,b=t fields: for j := range tr.groupByFieldNames { @@ -273,11 +274,11 @@ func (tr *TransformerTop) emit( topKeeper := pb.Value.(*utils.TopKeeper) key := valueFieldName + "_top" if i < topKeeper.GetSize() { - newrec.PutReference(tr.outputFieldName, types.MlrvalFromInt(i+1)) + newrec.PutReference(tr.outputFieldName, mlrval.FromInt(i+1)) newrec.PutReference(key, topKeeper.TopValues[i].Copy()) } else { - newrec.PutReference(tr.outputFieldName, types.MlrvalFromInt(i+1)) - newrec.PutCopy(key, types.MLRVAL_VOID) + newrec.PutReference(tr.outputFieldName, mlrval.FromInt(i+1)) + newrec.PutCopy(key, mlrval.VOID) } } diff --git a/internal/pkg/transformers/uniq.go b/internal/pkg/transformers/uniq.go index d4f6ddd07..5cafd37f0 100644 --- a/internal/pkg/transformers/uniq.go +++ b/internal/pkg/transformers/uniq.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -372,7 +373,7 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowCounts( for pe := tr.uniqifiedRecords.Head; pe != nil; pe = pe.Next { outrecAndContext := pe.Value.(*types.RecordAndContext) icount := tr.uniqifiedRecordCounts.Get(pe.Key) - mcount := types.MlrvalFromInt(icount.(int)) + mcount := mlrval.FromInt(icount.(int)) outrecAndContext.Record.PrependReference(tr.outputFieldName, mcount) outputRecordsAndContexts.PushBack(outrecAndContext) } @@ -399,10 +400,10 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowNumDistinctOnly( } } else { // end of record stream - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() outrec.PutReference( tr.outputFieldName, - types.MlrvalFromInt(tr.uniqifiedRecordCounts.FieldCount), + mlrval.FromInt(tr.uniqifiedRecordCounts.FieldCount), ) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) @@ -473,13 +474,13 @@ func (tr *TransformerUniq) transformUnlashed( countsForFieldName := pe.Value.(*lib.OrderedMap) for pf := countsForFieldName.Head; pf != nil; pf = pf.Next { fieldValueString := pf.Key - outrec := types.NewMlrmapAsRecord() - outrec.PutReference("field", types.MlrvalFromString(fieldName)) + outrec := mlrval.NewMlrmapAsRecord() + outrec.PutReference("field", mlrval.FromString(fieldName)) outrec.PutCopy( "value", - tr.unlashedCountValues.Get(fieldName).(*lib.OrderedMap).Get(fieldValueString).(*types.Mlrval), + tr.unlashedCountValues.Get(fieldName).(*lib.OrderedMap).Get(fieldValueString).(*mlrval.Mlrval), ) - outrec.PutReference("count", types.MlrvalFromInt(pf.Value.(int))) + outrec.PutReference("count", mlrval.FromInt(pf.Value.(int))) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } } @@ -509,10 +510,10 @@ func (tr *TransformerUniq) transformNumDistinctOnly( } } else { - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() outrec.PutReference( "count", - types.MlrvalFromInt(tr.countsByGroup.FieldCount), + mlrval.FromInt(tr.countsByGroup.FieldCount), ) outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) @@ -544,8 +545,8 @@ func (tr *TransformerUniq) transformWithCounts( } else { // end of record stream for pa := tr.countsByGroup.Head; pa != nil; pa = pa.Next { - outrec := types.NewMlrmapAsRecord() - valuesForGroup := tr.valuesByGroup.Get(pa.Key).([]*types.Mlrval) + outrec := mlrval.NewMlrmapAsRecord() + valuesForGroup := tr.valuesByGroup.Get(pa.Key).([]*mlrval.Mlrval) for i, fieldName := range tr.fieldNames { outrec.PutCopy( fieldName, @@ -555,7 +556,7 @@ func (tr *TransformerUniq) transformWithCounts( if tr.showCounts { outrec.PutReference( tr.outputFieldName, - types.MlrvalFromInt(pa.Value.(int)), + mlrval.FromInt(pa.Value.(int)), ) } outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) @@ -584,7 +585,7 @@ func (tr *TransformerUniq) transformWithoutCounts( if !present { tr.countsByGroup.Put(groupingKey, 1) tr.valuesByGroup.Put(groupingKey, selectedValues) - outrec := types.NewMlrmapAsRecord() + outrec := mlrval.NewMlrmapAsRecord() for i, fieldName := range tr.fieldNames { outrec.PutCopy( diff --git a/internal/pkg/transformers/unsparsify.go b/internal/pkg/transformers/unsparsify.go index caaa3301a..365d86689 100644 --- a/internal/pkg/transformers/unsparsify.go +++ b/internal/pkg/transformers/unsparsify.go @@ -8,6 +8,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -110,7 +111,7 @@ func transformerUnsparsifyParseCLI( // ---------------------------------------------------------------- type TransformerUnsparsify struct { - fillerMlrval *types.Mlrval + fillerMlrval *mlrval.Mlrval recordsAndContexts *list.List fieldNamesSeen *lib.OrderedMap recordTransformerFunc RecordTransformerFunc @@ -127,7 +128,7 @@ func NewTransformerUnsparsify( } tr := &TransformerUnsparsify{ - fillerMlrval: types.MlrvalFromString(fillerString), + fillerMlrval: mlrval.FromString(fillerString), recordsAndContexts: list.New(), fieldNamesSeen: fieldNamesSeen, } @@ -174,7 +175,7 @@ func (tr *TransformerUnsparsify) transformNonStreaming( outrecAndContext := e.Value.(*types.RecordAndContext) outrec := outrecAndContext.Record - newrec := types.NewMlrmapAsRecord() + newrec := mlrval.NewMlrmapAsRecord() for pe := tr.fieldNamesSeen.Head; pe != nil; pe = pe.Next { fieldName := pe.Key if !outrec.Has(fieldName) { diff --git a/internal/pkg/transformers/utils/join-bucket-keeper.go b/internal/pkg/transformers/utils/join-bucket-keeper.go index c5be1253b..ea8b87e5c 100644 --- a/internal/pkg/transformers/utils/join-bucket-keeper.go +++ b/internal/pkg/transformers/utils/join-bucket-keeper.go @@ -116,6 +116,7 @@ import ( "github.com/johnkerl/miller/internal/pkg/cli" "github.com/johnkerl/miller/internal/pkg/input" "github.com/johnkerl/miller/internal/pkg/lib" + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) @@ -251,7 +252,7 @@ func (keeper *JoinBucketKeeper) computeState() tJoinBucketKeeperState { // will also be moved to keeper.leftUnpaired. func (keeper *JoinBucketKeeper) FindJoinBucket( - rightFieldValues []*types.Mlrval, // nil means right-file EOF + rightFieldValues []*mlrval.Mlrval, // nil means right-file EOF ) bool { // TODO: comment me isPaired := false @@ -367,7 +368,7 @@ func (keeper *JoinBucketKeeper) prepareForFirstJoinBucket() { // leftvals < rightvals && !eof. func (keeper *JoinBucketKeeper) prepareForNewJoinBucket( - rightFieldValues []*types.Mlrval, + rightFieldValues []*mlrval.Mlrval, ) { if !keeper.JoinBucket.WasPaired { moveRecordsAndContexts(keeper.leftUnpaireds, keeper.JoinBucket.RecordsAndContexts) @@ -465,7 +466,7 @@ func (keeper *JoinBucketKeeper) fillNextJoinBucket() { os.Exit(1) } - keeper.JoinBucket.leftFieldValues = types.CopyMlrvalPointerArray(peekFieldValues) + keeper.JoinBucket.leftFieldValues = mlrval.CopyMlrvalPointerArray(peekFieldValues) keeper.JoinBucket.RecordsAndContexts.PushBack(keeper.peekRecordAndContext) keeper.JoinBucket.WasPaired = false @@ -607,8 +608,8 @@ func moveRecordsAndContexts( // for numerical values). func compareLexically( - leftFieldValues []*types.Mlrval, - rightFieldValues []*types.Mlrval, + leftFieldValues []*mlrval.Mlrval, + rightFieldValues []*mlrval.Mlrval, ) int { lib.InternalCodingErrorIf(len(leftFieldValues) != len(rightFieldValues)) n := len(leftFieldValues) @@ -659,7 +660,7 @@ func (keeper *JoinBucketKeeper) dump(prefix string) { fmt.Printf("------------------------------------------------------\n") } -func dumpFieldValues(name string, values []*types.Mlrval) { +func dumpFieldValues(name string, values []*mlrval.Mlrval) { for i, value := range values { fmt.Printf("-- %s[%d] = %s\n", name, i, value.String()) } diff --git a/internal/pkg/transformers/utils/join-bucket.go b/internal/pkg/transformers/utils/join-bucket.go index ef9336cfb..92b169541 100644 --- a/internal/pkg/transformers/utils/join-bucket.go +++ b/internal/pkg/transformers/utils/join-bucket.go @@ -7,18 +7,18 @@ package utils import ( "container/list" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ---------------------------------------------------------------- type JoinBucket struct { - leftFieldValues []*types.Mlrval + leftFieldValues []*mlrval.Mlrval RecordsAndContexts *list.List WasPaired bool } func NewJoinBucket( - leftFieldValues []*types.Mlrval, + leftFieldValues []*mlrval.Mlrval, ) *JoinBucket { return &JoinBucket{ leftFieldValues: leftFieldValues, diff --git a/internal/pkg/transformers/utils/percentile-keeper.go b/internal/pkg/transformers/utils/percentile-keeper.go index 30e20cbd2..8200826c7 100644 --- a/internal/pkg/transformers/utils/percentile-keeper.go +++ b/internal/pkg/transformers/utils/percentile-keeper.go @@ -9,11 +9,12 @@ import ( "math" "sort" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/bifs" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) type PercentileKeeper struct { - data []*types.Mlrval + data []*mlrval.Mlrval sorted bool doInterpolatedPercentiles bool } @@ -23,21 +24,21 @@ func NewPercentileKeeper( doInterpolatedPercentiles bool, ) *PercentileKeeper { return &PercentileKeeper{ - data: make([]*types.Mlrval, 0, 1000), + data: make([]*mlrval.Mlrval, 0, 1000), sorted: false, doInterpolatedPercentiles: doInterpolatedPercentiles, } } func (keeper *PercentileKeeper) Reset() { - keeper.data = make([]*types.Mlrval, 0, 1000) + keeper.data = make([]*mlrval.Mlrval, 0, 1000) keeper.sorted = false } // ---------------------------------------------------------------- -func (keeper *PercentileKeeper) Ingest(value *types.Mlrval) { +func (keeper *PercentileKeeper) Ingest(value *mlrval.Mlrval) { if len(keeper.data) >= cap(keeper.data) { - newData := make([]*types.Mlrval, len(keeper.data), 2*cap(keeper.data)) + newData := make([]*mlrval.Mlrval, len(keeper.data), 2*cap(keeper.data)) copy(newData, keeper.data) keeper.data = newData } @@ -235,7 +236,7 @@ func computeIndexNoninterpolated(n int, p float64) int { } // xxx pending pointer-output refactor -func getPercentileLinearlyInterpolated(array []*types.Mlrval, n int, p float64) types.Mlrval { +func getPercentileLinearlyInterpolated(array []*mlrval.Mlrval, n int, p float64) mlrval.Mlrval { findex := (p / 100.0) * (float64(n) - 1) if findex < 0.0 { findex = 0.0 @@ -246,10 +247,10 @@ func getPercentileLinearlyInterpolated(array []*types.Mlrval, n int, p float64) } else { // array[iindex] + frac * (array[iindex+1] - array[iindex]) // TODO: just do this in float64. - frac := types.MlrvalFromFloat64(findex - float64(iindex)) - diff := types.BIF_minus_binary(array[iindex+1], array[iindex]) - prod := types.BIF_times(frac, diff) - return *types.BIF_plus_binary(array[iindex], prod) + frac := mlrval.FromFloat(findex - float64(iindex)) + diff := bifs.BIF_minus_binary(array[iindex+1], array[iindex]) + prod := bifs.BIF_times(frac, diff) + return *bifs.BIF_plus_binary(array[iindex], prod) } } @@ -257,14 +258,14 @@ func getPercentileLinearlyInterpolated(array []*types.Mlrval, n int, p float64) func (keeper *PercentileKeeper) sortIfNecessary() { if !keeper.sorted { sort.Slice(keeper.data, func(i, j int) bool { - return types.MlrvalLessThanAsBool(keeper.data[i], keeper.data[j]) + return mlrval.LessThan(keeper.data[i], keeper.data[j]) }) keeper.sorted = true } } // ---------------------------------------------------------------- -func (keeper *PercentileKeeper) Emit(percentile float64) *types.Mlrval { +func (keeper *PercentileKeeper) Emit(percentile float64) *mlrval.Mlrval { if keeper.doInterpolatedPercentiles { return keeper.EmitLinearlyInterpolated(percentile) } else { @@ -272,17 +273,17 @@ func (keeper *PercentileKeeper) Emit(percentile float64) *types.Mlrval { } } -func (keeper *PercentileKeeper) EmitNonInterpolated(percentile float64) *types.Mlrval { +func (keeper *PercentileKeeper) EmitNonInterpolated(percentile float64) *mlrval.Mlrval { if len(keeper.data) == 0 { - return types.MLRVAL_VOID + return mlrval.VOID } keeper.sortIfNecessary() return keeper.data[computeIndexNoninterpolated(int(len(keeper.data)), percentile)].Copy() } -func (keeper *PercentileKeeper) EmitLinearlyInterpolated(percentile float64) *types.Mlrval { +func (keeper *PercentileKeeper) EmitLinearlyInterpolated(percentile float64) *mlrval.Mlrval { if len(keeper.data) == 0 { - return types.MLRVAL_VOID + return mlrval.VOID } keeper.sortIfNecessary() output := getPercentileLinearlyInterpolated(keeper.data, int(len(keeper.data)), percentile) diff --git a/internal/pkg/transformers/utils/stats1-accumulators.go b/internal/pkg/transformers/utils/stats1-accumulators.go index 4bd3bb565..08f21a02e 100644 --- a/internal/pkg/transformers/utils/stats1-accumulators.go +++ b/internal/pkg/transformers/utils/stats1-accumulators.go @@ -9,14 +9,15 @@ import ( "os" "strings" + "github.com/johnkerl/miller/internal/pkg/bifs" "github.com/johnkerl/miller/internal/pkg/lib" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ---------------------------------------------------------------- type IStats1Accumulator interface { - Ingest(value *types.Mlrval) - Emit() *types.Mlrval + Ingest(value *mlrval.Mlrval) + Emit() *mlrval.Mlrval Reset() // for merge-fields where we reset after each record instead of replace/recreate } @@ -121,11 +122,11 @@ func NewStats1NamedAccumulator( } } -func (nacc *Stats1NamedAccumulator) Ingest(value *types.Mlrval) { +func (nacc *Stats1NamedAccumulator) Ingest(value *mlrval.Mlrval) { nacc.accumulator.Ingest(value) } -func (nacc *Stats1NamedAccumulator) Emit() (key string, value *types.Mlrval) { +func (nacc *Stats1NamedAccumulator) Emit() (key string, value *mlrval.Mlrval) { return nacc.outputFieldName, nacc.accumulator.Emit() } @@ -186,7 +187,7 @@ func tryPercentileFromName(accumulatorName string) (float64, bool) { return 50.0, true } if strings.HasPrefix(accumulatorName, "p") { - percentile, ok := lib.TryFloat64FromString(accumulatorName[1:]) + percentile, ok := lib.TryFloatFromString(accumulatorName[1:]) if !ok { return 0.0, false } @@ -288,11 +289,11 @@ func NewStats1CountAccumulator() IStats1Accumulator { count: 0, } } -func (acc *Stats1CountAccumulator) Ingest(value *types.Mlrval) { +func (acc *Stats1CountAccumulator) Ingest(value *mlrval.Mlrval) { acc.count++ } -func (acc *Stats1CountAccumulator) Emit() *types.Mlrval { - return types.MlrvalFromInt(acc.count) +func (acc *Stats1CountAccumulator) Emit() *mlrval.Mlrval { + return mlrval.FromInt(acc.count) } func (acc *Stats1CountAccumulator) Reset() { acc.count = 0 @@ -310,7 +311,7 @@ func NewStats1ModeAccumulator() IStats1Accumulator { countsByValue: lib.NewOrderedMap(), } } -func (acc *Stats1ModeAccumulator) Ingest(value *types.Mlrval) { +func (acc *Stats1ModeAccumulator) Ingest(value *mlrval.Mlrval) { key := value.String() // 1, 1.0, and 1.000 are distinct iPrevious, ok := acc.countsByValue.GetWithCheck(key) if !ok { @@ -319,9 +320,9 @@ func (acc *Stats1ModeAccumulator) Ingest(value *types.Mlrval) { acc.countsByValue.Put(key, iPrevious.(int)+1) } } -func (acc *Stats1ModeAccumulator) Emit() *types.Mlrval { +func (acc *Stats1ModeAccumulator) Emit() *mlrval.Mlrval { if acc.countsByValue.IsEmpty() { - return types.MLRVAL_VOID + return mlrval.VOID } maxValue := "" var maxCount = int(0) @@ -333,7 +334,7 @@ func (acc *Stats1ModeAccumulator) Emit() *types.Mlrval { maxCount = count } } - return types.MlrvalFromString(maxValue) + return mlrval.FromString(maxValue) } func (acc *Stats1ModeAccumulator) Reset() { acc.countsByValue = lib.NewOrderedMap() @@ -351,7 +352,7 @@ func NewStats1AntimodeAccumulator() IStats1Accumulator { countsByValue: lib.NewOrderedMap(), } } -func (acc *Stats1AntimodeAccumulator) Ingest(value *types.Mlrval) { +func (acc *Stats1AntimodeAccumulator) Ingest(value *mlrval.Mlrval) { key := value.String() // 1, 1.0, and 1.000 are distinct iPrevious, ok := acc.countsByValue.GetWithCheck(key) if !ok { @@ -360,9 +361,9 @@ func (acc *Stats1AntimodeAccumulator) Ingest(value *types.Mlrval) { acc.countsByValue.Put(key, iPrevious.(int)+1) } } -func (acc *Stats1AntimodeAccumulator) Emit() *types.Mlrval { +func (acc *Stats1AntimodeAccumulator) Emit() *mlrval.Mlrval { if acc.countsByValue.IsEmpty() { - return types.MLRVAL_VOID + return mlrval.VOID } minValue := "" var minCount = int(0) @@ -374,7 +375,7 @@ func (acc *Stats1AntimodeAccumulator) Emit() *types.Mlrval { minCount = count } } - return types.MlrvalFromString(minValue) + return mlrval.FromString(minValue) } func (acc *Stats1AntimodeAccumulator) Reset() { acc.countsByValue = lib.NewOrderedMap() @@ -382,261 +383,261 @@ func (acc *Stats1AntimodeAccumulator) Reset() { // ---------------------------------------------------------------- type Stats1SumAccumulator struct { - sum *types.Mlrval + sum *mlrval.Mlrval } func NewStats1SumAccumulator() IStats1Accumulator { return &Stats1SumAccumulator{ - sum: types.MlrvalFromInt(0), + sum: mlrval.FromInt(0), } } -func (acc *Stats1SumAccumulator) Ingest(value *types.Mlrval) { - acc.sum = types.BIF_plus_binary(acc.sum, value) +func (acc *Stats1SumAccumulator) Ingest(value *mlrval.Mlrval) { + acc.sum = bifs.BIF_plus_binary(acc.sum, value) } -func (acc *Stats1SumAccumulator) Emit() *types.Mlrval { +func (acc *Stats1SumAccumulator) Emit() *mlrval.Mlrval { return acc.sum.Copy() } func (acc *Stats1SumAccumulator) Reset() { - acc.sum = types.MlrvalFromInt(0) + acc.sum = mlrval.FromInt(0) } // ---------------------------------------------------------------- type Stats1MeanAccumulator struct { - sum *types.Mlrval + sum *mlrval.Mlrval count int } func NewStats1MeanAccumulator() IStats1Accumulator { return &Stats1MeanAccumulator{ - sum: types.MlrvalFromInt(0), + sum: mlrval.FromInt(0), count: 0, } } -func (acc *Stats1MeanAccumulator) Ingest(value *types.Mlrval) { - acc.sum = types.BIF_plus_binary(acc.sum, value) +func (acc *Stats1MeanAccumulator) Ingest(value *mlrval.Mlrval) { + acc.sum = bifs.BIF_plus_binary(acc.sum, value) acc.count++ } -func (acc *Stats1MeanAccumulator) Emit() *types.Mlrval { +func (acc *Stats1MeanAccumulator) Emit() *mlrval.Mlrval { if acc.count == 0 { - return types.MLRVAL_VOID + return mlrval.VOID } else { - return types.BIF_divide(acc.sum, types.MlrvalFromInt(acc.count)) + return bifs.BIF_divide(acc.sum, mlrval.FromInt(acc.count)) } } func (acc *Stats1MeanAccumulator) Reset() { - acc.sum = types.MlrvalFromInt(0) + acc.sum = mlrval.FromInt(0) acc.count = 0 } // ---------------------------------------------------------------- type Stats1MinAccumulator struct { - min *types.Mlrval + min *mlrval.Mlrval } func NewStats1MinAccumulator() IStats1Accumulator { return &Stats1MinAccumulator{ - min: types.MLRVAL_ABSENT, + min: mlrval.ABSENT, } } -func (acc *Stats1MinAccumulator) Ingest(value *types.Mlrval) { - acc.min = types.MlrvalBinaryMin(acc.min, value) +func (acc *Stats1MinAccumulator) Ingest(value *mlrval.Mlrval) { + acc.min = bifs.BIF_min_binary(acc.min, value) } -func (acc *Stats1MinAccumulator) Emit() *types.Mlrval { +func (acc *Stats1MinAccumulator) Emit() *mlrval.Mlrval { if acc.min.IsAbsent() { - return types.MLRVAL_VOID + return mlrval.VOID } else { return acc.min.Copy() } } func (acc *Stats1MinAccumulator) Reset() { - acc.min = types.MLRVAL_ABSENT + acc.min = mlrval.ABSENT } // ---------------------------------------------------------------- type Stats1MaxAccumulator struct { - max *types.Mlrval + max *mlrval.Mlrval } func NewStats1MaxAccumulator() IStats1Accumulator { return &Stats1MaxAccumulator{ - max: types.MLRVAL_ABSENT, + max: mlrval.ABSENT, } } -func (acc *Stats1MaxAccumulator) Ingest(value *types.Mlrval) { - acc.max = types.MlrvalBinaryMax(acc.max, value) +func (acc *Stats1MaxAccumulator) Ingest(value *mlrval.Mlrval) { + acc.max = bifs.BIF_max_binary(acc.max, value) } -func (acc *Stats1MaxAccumulator) Emit() *types.Mlrval { +func (acc *Stats1MaxAccumulator) Emit() *mlrval.Mlrval { if acc.max.IsAbsent() { - return types.MLRVAL_VOID + return mlrval.VOID } else { return acc.max.Copy() } } func (acc *Stats1MaxAccumulator) Reset() { - acc.max = types.MLRVAL_ABSENT + acc.max = mlrval.ABSENT } // ---------------------------------------------------------------- type Stats1VarAccumulator struct { count int - sum *types.Mlrval - sum2 *types.Mlrval + sum *mlrval.Mlrval + sum2 *mlrval.Mlrval } func NewStats1VarAccumulator() IStats1Accumulator { return &Stats1VarAccumulator{ count: 0, - sum: types.MlrvalFromInt(0), - sum2: types.MlrvalFromInt(0), + sum: mlrval.FromInt(0), + sum2: mlrval.FromInt(0), } } -func (acc *Stats1VarAccumulator) Ingest(value *types.Mlrval) { - value2 := types.BIF_times(value, value) +func (acc *Stats1VarAccumulator) Ingest(value *mlrval.Mlrval) { + value2 := bifs.BIF_times(value, value) acc.count++ - acc.sum = types.BIF_plus_binary(acc.sum, value) - acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) + acc.sum = bifs.BIF_plus_binary(acc.sum, value) + acc.sum2 = bifs.BIF_plus_binary(acc.sum2, value2) } -func (acc *Stats1VarAccumulator) Emit() *types.Mlrval { - return types.MlrvalGetVar(types.MlrvalFromInt(acc.count), acc.sum, acc.sum2) +func (acc *Stats1VarAccumulator) Emit() *mlrval.Mlrval { + return bifs.BIF_get_var(mlrval.FromInt(acc.count), acc.sum, acc.sum2) } func (acc *Stats1VarAccumulator) Reset() { acc.count = 0 - acc.sum = types.MlrvalFromInt(0) - acc.sum2 = types.MlrvalFromInt(0) + acc.sum = mlrval.FromInt(0) + acc.sum2 = mlrval.FromInt(0) } // ---------------------------------------------------------------- type Stats1StddevAccumulator struct { count int - sum *types.Mlrval - sum2 *types.Mlrval + sum *mlrval.Mlrval + sum2 *mlrval.Mlrval } func NewStats1StddevAccumulator() IStats1Accumulator { return &Stats1StddevAccumulator{ count: 0, - sum: types.MlrvalFromInt(0), - sum2: types.MlrvalFromInt(0), + sum: mlrval.FromInt(0), + sum2: mlrval.FromInt(0), } } -func (acc *Stats1StddevAccumulator) Ingest(value *types.Mlrval) { - value2 := types.BIF_times(value, value) +func (acc *Stats1StddevAccumulator) Ingest(value *mlrval.Mlrval) { + value2 := bifs.BIF_times(value, value) acc.count++ - acc.sum = types.BIF_plus_binary(acc.sum, value) - acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) + acc.sum = bifs.BIF_plus_binary(acc.sum, value) + acc.sum2 = bifs.BIF_plus_binary(acc.sum2, value2) } -func (acc *Stats1StddevAccumulator) Emit() *types.Mlrval { - return types.MlrvalGetStddev(types.MlrvalFromInt(acc.count), acc.sum, acc.sum2) +func (acc *Stats1StddevAccumulator) Emit() *mlrval.Mlrval { + return bifs.BIF_get_stddev(mlrval.FromInt(acc.count), acc.sum, acc.sum2) } func (acc *Stats1StddevAccumulator) Reset() { acc.count = 0 - acc.sum = types.MlrvalFromInt(0) - acc.sum2 = types.MlrvalFromInt(0) + acc.sum = mlrval.FromInt(0) + acc.sum2 = mlrval.FromInt(0) } // ---------------------------------------------------------------- type Stats1MeanEBAccumulator struct { count int - sum *types.Mlrval - sum2 *types.Mlrval + sum *mlrval.Mlrval + sum2 *mlrval.Mlrval } func NewStats1MeanEBAccumulator() IStats1Accumulator { return &Stats1MeanEBAccumulator{ count: 0, - sum: types.MlrvalFromInt(0), - sum2: types.MlrvalFromInt(0), + sum: mlrval.FromInt(0), + sum2: mlrval.FromInt(0), } } -func (acc *Stats1MeanEBAccumulator) Ingest(value *types.Mlrval) { - value2 := types.BIF_times(value, value) +func (acc *Stats1MeanEBAccumulator) Ingest(value *mlrval.Mlrval) { + value2 := bifs.BIF_times(value, value) acc.count++ - acc.sum = types.BIF_plus_binary(acc.sum, value) - acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) + acc.sum = bifs.BIF_plus_binary(acc.sum, value) + acc.sum2 = bifs.BIF_plus_binary(acc.sum2, value2) } -func (acc *Stats1MeanEBAccumulator) Emit() *types.Mlrval { - mcount := types.MlrvalFromInt(acc.count) - return types.MlrvalGetMeanEB(mcount, acc.sum, acc.sum2) +func (acc *Stats1MeanEBAccumulator) Emit() *mlrval.Mlrval { + mcount := mlrval.FromInt(acc.count) + return bifs.BIF_get_mean_EB(mcount, acc.sum, acc.sum2) } func (acc *Stats1MeanEBAccumulator) Reset() { acc.count = 0 - acc.sum = types.MlrvalFromInt(0) - acc.sum2 = types.MlrvalFromInt(0) + acc.sum = mlrval.FromInt(0) + acc.sum2 = mlrval.FromInt(0) } // ---------------------------------------------------------------- type Stats1SkewnessAccumulator struct { count int - sum *types.Mlrval - sum2 *types.Mlrval - sum3 *types.Mlrval + sum *mlrval.Mlrval + sum2 *mlrval.Mlrval + sum3 *mlrval.Mlrval } func NewStats1SkewnessAccumulator() IStats1Accumulator { return &Stats1SkewnessAccumulator{ count: 0, - sum: types.MlrvalFromInt(0), - sum2: types.MlrvalFromInt(0), - sum3: types.MlrvalFromInt(0), + sum: mlrval.FromInt(0), + sum2: mlrval.FromInt(0), + sum3: mlrval.FromInt(0), } } -func (acc *Stats1SkewnessAccumulator) Ingest(value *types.Mlrval) { - value2 := types.BIF_times(value, value) - value3 := types.BIF_times(value, value2) +func (acc *Stats1SkewnessAccumulator) Ingest(value *mlrval.Mlrval) { + value2 := bifs.BIF_times(value, value) + value3 := bifs.BIF_times(value, value2) acc.count++ - acc.sum = types.BIF_plus_binary(acc.sum, value) - acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) - acc.sum3 = types.BIF_plus_binary(acc.sum3, value3) + acc.sum = bifs.BIF_plus_binary(acc.sum, value) + acc.sum2 = bifs.BIF_plus_binary(acc.sum2, value2) + acc.sum3 = bifs.BIF_plus_binary(acc.sum3, value3) } -func (acc *Stats1SkewnessAccumulator) Emit() *types.Mlrval { - mcount := types.MlrvalFromInt(acc.count) - return types.MlrvalGetSkewness(mcount, acc.sum, acc.sum2, acc.sum3) +func (acc *Stats1SkewnessAccumulator) Emit() *mlrval.Mlrval { + mcount := mlrval.FromInt(acc.count) + return bifs.BIF_get_skewness(mcount, acc.sum, acc.sum2, acc.sum3) } func (acc *Stats1SkewnessAccumulator) Reset() { acc.count = 0 - acc.sum = types.MlrvalFromInt(0) - acc.sum2 = types.MlrvalFromInt(0) - acc.sum3 = types.MlrvalFromInt(0) + acc.sum = mlrval.FromInt(0) + acc.sum2 = mlrval.FromInt(0) + acc.sum3 = mlrval.FromInt(0) } // ---------------------------------------------------------------- type Stats1KurtosisAccumulator struct { count int - sum *types.Mlrval - sum2 *types.Mlrval - sum3 *types.Mlrval - sum4 *types.Mlrval + sum *mlrval.Mlrval + sum2 *mlrval.Mlrval + sum3 *mlrval.Mlrval + sum4 *mlrval.Mlrval } func NewStats1KurtosisAccumulator() IStats1Accumulator { return &Stats1KurtosisAccumulator{ count: 0, - sum: types.MlrvalFromInt(0), - sum2: types.MlrvalFromInt(0), - sum3: types.MlrvalFromInt(0), - sum4: types.MlrvalFromInt(0), + sum: mlrval.FromInt(0), + sum2: mlrval.FromInt(0), + sum3: mlrval.FromInt(0), + sum4: mlrval.FromInt(0), } } -func (acc *Stats1KurtosisAccumulator) Ingest(value *types.Mlrval) { - value2 := types.BIF_times(value, value) - value3 := types.BIF_times(value, value2) - value4 := types.BIF_times(value, value3) +func (acc *Stats1KurtosisAccumulator) Ingest(value *mlrval.Mlrval) { + value2 := bifs.BIF_times(value, value) + value3 := bifs.BIF_times(value, value2) + value4 := bifs.BIF_times(value, value3) acc.count++ - acc.sum = types.BIF_plus_binary(acc.sum, value) - acc.sum2 = types.BIF_plus_binary(acc.sum2, value2) - acc.sum3 = types.BIF_plus_binary(acc.sum3, value3) - acc.sum4 = types.BIF_plus_binary(acc.sum4, value4) + acc.sum = bifs.BIF_plus_binary(acc.sum, value) + acc.sum2 = bifs.BIF_plus_binary(acc.sum2, value2) + acc.sum3 = bifs.BIF_plus_binary(acc.sum3, value3) + acc.sum4 = bifs.BIF_plus_binary(acc.sum4, value4) } -func (acc *Stats1KurtosisAccumulator) Emit() *types.Mlrval { - mcount := types.MlrvalFromInt(acc.count) - return types.MlrvalGetKurtosis(mcount, acc.sum, acc.sum2, acc.sum3, acc.sum4) +func (acc *Stats1KurtosisAccumulator) Emit() *mlrval.Mlrval { + mcount := mlrval.FromInt(acc.count) + return bifs.BIF_get_kurtosis(mcount, acc.sum, acc.sum2, acc.sum3, acc.sum4) } func (acc *Stats1KurtosisAccumulator) Reset() { acc.count = 0 - acc.sum = types.MlrvalFromInt(0) - acc.sum2 = types.MlrvalFromInt(0) - acc.sum3 = types.MlrvalFromInt(0) - acc.sum4 = types.MlrvalFromInt(0) + acc.sum = mlrval.FromInt(0) + acc.sum2 = mlrval.FromInt(0) + acc.sum3 = mlrval.FromInt(0) + acc.sum4 = mlrval.FromInt(0) } // ---------------------------------------------------------------- @@ -663,13 +664,13 @@ func NewStats1PercentileAccumulator( } } -func (acc *Stats1PercentileAccumulator) Ingest(value *types.Mlrval) { +func (acc *Stats1PercentileAccumulator) Ingest(value *mlrval.Mlrval) { if acc.isPrimary { acc.percentileKeeper.Ingest(value) } } -func (acc *Stats1PercentileAccumulator) Emit() *types.Mlrval { +func (acc *Stats1PercentileAccumulator) Emit() *mlrval.Mlrval { return acc.percentileKeeper.Emit(acc.percentile) } diff --git a/internal/pkg/transformers/utils/stats2-accumulators.go b/internal/pkg/transformers/utils/stats2-accumulators.go index 13d86949a..9c360a861 100644 --- a/internal/pkg/transformers/utils/stats2-accumulators.go +++ b/internal/pkg/transformers/utils/stats2-accumulators.go @@ -10,7 +10,7 @@ import ( "os" "github.com/johnkerl/miller/internal/pkg/lib" - "github.com/johnkerl/miller/internal/pkg/types" + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ---------------------------------------------------------------- @@ -23,13 +23,13 @@ type IStats2Accumulator interface { Populate( valueFieldName1 string, valueFieldName2 string, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) Fit( x float64, y float64, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) } @@ -177,25 +177,25 @@ func (acc *Stats2LinRegOLSAccumulator) Ingest( func (acc *Stats2LinRegOLSAccumulator) Populate( valueFieldName1 string, valueFieldName2 string, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { if acc.count < 2 { - outrec.PutCopy(acc.mOutputFieldName, types.MLRVAL_VOID) - outrec.PutCopy(acc.bOutputFieldName, types.MLRVAL_VOID) + outrec.PutCopy(acc.mOutputFieldName, mlrval.VOID) + outrec.PutCopy(acc.bOutputFieldName, mlrval.VOID) } else { m, b := lib.GetLinearRegressionOLS(acc.count, acc.sumx, acc.sumx2, acc.sumxy, acc.sumy) - outrec.PutReference(acc.mOutputFieldName, types.MlrvalFromFloat64(m)) - outrec.PutReference(acc.bOutputFieldName, types.MlrvalFromFloat64(b)) + outrec.PutReference(acc.mOutputFieldName, mlrval.FromFloat(m)) + outrec.PutReference(acc.bOutputFieldName, mlrval.FromFloat(b)) } - outrec.PutReference(acc.nOutputFieldName, types.MlrvalFromInt(acc.count)) + outrec.PutReference(acc.nOutputFieldName, mlrval.FromInt(acc.count)) } func (acc *Stats2LinRegOLSAccumulator) Fit( x float64, y float64, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { if !acc.fitReady { @@ -209,10 +209,10 @@ func (acc *Stats2LinRegOLSAccumulator) Fit( } if acc.count < 2 { - outrec.PutCopy(acc.fitOutputFieldName, types.MLRVAL_VOID) + outrec.PutCopy(acc.fitOutputFieldName, mlrval.VOID) } else { yfit := acc.m*x + acc.b - outrec.PutReference(acc.fitOutputFieldName, types.MlrvalFromFloat64(yfit)) + outrec.PutReference(acc.fitOutputFieldName, mlrval.FromFloat(yfit)) } } @@ -262,24 +262,24 @@ func (acc *Stats2LogiRegAccumulator) Ingest( func (acc *Stats2LogiRegAccumulator) Populate( valueFieldName1 string, valueFieldName2 string, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { if len(acc.xs) < 2 { - outrec.PutCopy(acc.mOutputFieldName, types.MLRVAL_VOID) - outrec.PutCopy(acc.bOutputFieldName, types.MLRVAL_VOID) + outrec.PutCopy(acc.mOutputFieldName, mlrval.VOID) + outrec.PutCopy(acc.bOutputFieldName, mlrval.VOID) } else { m, b := lib.LogisticRegression(acc.xs, acc.ys) - outrec.PutCopy(acc.mOutputFieldName, types.MlrvalFromFloat64(m)) - outrec.PutCopy(acc.bOutputFieldName, types.MlrvalFromFloat64(b)) + outrec.PutCopy(acc.mOutputFieldName, mlrval.FromFloat(m)) + outrec.PutCopy(acc.bOutputFieldName, mlrval.FromFloat(b)) } - outrec.PutReference(acc.nOutputFieldName, types.MlrvalFromInt(len(acc.xs))) + outrec.PutReference(acc.nOutputFieldName, mlrval.FromInt(len(acc.xs))) } func (acc *Stats2LogiRegAccumulator) Fit( x float64, y float64, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { if !acc.fitReady { @@ -293,10 +293,10 @@ func (acc *Stats2LogiRegAccumulator) Fit( } if len(acc.xs) < 2 { - outrec.PutCopy(acc.fitOutputFieldName, types.MLRVAL_VOID) + outrec.PutCopy(acc.fitOutputFieldName, mlrval.VOID) } else { yfit := 1.0 / (1.0 + math.Exp(-acc.m*x-acc.b)) - outrec.PutReference(acc.fitOutputFieldName, types.MlrvalFromFloat64(yfit)) + outrec.PutReference(acc.fitOutputFieldName, mlrval.FromFloat(yfit)) } } @@ -347,11 +347,11 @@ func (acc *Stats2R2Accumulator) Ingest( func (acc *Stats2R2Accumulator) Populate( valueFieldName1 string, valueFieldName2 string, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { if acc.count < 2 { - outrec.PutCopy(acc.r2OutputFieldName, types.MLRVAL_VOID) + outrec.PutCopy(acc.r2OutputFieldName, mlrval.VOID) } else { n := float64(acc.count) sumx := acc.sumx @@ -363,7 +363,7 @@ func (acc *Stats2R2Accumulator) Populate( numerator = numerator * numerator denominator := (n*sumx2 - sumx*sumx) * (n*sumy2 - sumy*sumy) output := numerator / denominator - outrec.PutReference(acc.r2OutputFieldName, types.MlrvalFromFloat64(output)) + outrec.PutReference(acc.r2OutputFieldName, mlrval.FromFloat(output)) } } @@ -371,7 +371,7 @@ func (acc *Stats2R2Accumulator) Populate( func (acc *Stats2R2Accumulator) Fit( x float64, y float64, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { } @@ -488,7 +488,7 @@ func (acc *Stats2CorrCovAccumulator) Ingest( func (acc *Stats2CorrCovAccumulator) Populate( valueFieldName1 string, valueFieldName2 string, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { if acc.doWhich == DO_COVX { @@ -497,10 +497,10 @@ func (acc *Stats2CorrCovAccumulator) Populate( key10 := acc.covx10OutputFieldName key11 := acc.covx11OutputFieldName if acc.count < 2 { - outrec.PutCopy(key00, types.MLRVAL_VOID) - outrec.PutCopy(key01, types.MLRVAL_VOID) - outrec.PutCopy(key10, types.MLRVAL_VOID) - outrec.PutCopy(key11, types.MLRVAL_VOID) + outrec.PutCopy(key00, mlrval.VOID) + outrec.PutCopy(key01, mlrval.VOID) + outrec.PutCopy(key10, mlrval.VOID) + outrec.PutCopy(key11, mlrval.VOID) } else { Q := lib.GetCovMatrix( acc.count, @@ -510,10 +510,10 @@ func (acc *Stats2CorrCovAccumulator) Populate( acc.sumy2, acc.sumxy, ) - outrec.PutReference(key00, types.MlrvalFromFloat64(Q[0][0])) - outrec.PutReference(key01, types.MlrvalFromFloat64(Q[0][1])) - outrec.PutReference(key10, types.MlrvalFromFloat64(Q[1][0])) - outrec.PutReference(key11, types.MlrvalFromFloat64(Q[1][1])) + outrec.PutReference(key00, mlrval.FromFloat(Q[0][0])) + outrec.PutReference(key01, mlrval.FromFloat(Q[0][1])) + outrec.PutReference(key10, mlrval.FromFloat(Q[1][0])) + outrec.PutReference(key11, mlrval.FromFloat(Q[1][1])) } } else if acc.doWhich == DO_LINREG_PCA { @@ -530,19 +530,19 @@ func (acc *Stats2CorrCovAccumulator) Populate( keyv22 := acc.pca_v22OutputFieldName if acc.count < 2 { - outrec.PutCopy(keym, types.MLRVAL_VOID) - outrec.PutCopy(keyb, types.MLRVAL_VOID) - outrec.PutCopy(keyn, types.MLRVAL_VOID) - outrec.PutCopy(keyq, types.MLRVAL_VOID) + outrec.PutCopy(keym, mlrval.VOID) + outrec.PutCopy(keyb, mlrval.VOID) + outrec.PutCopy(keyn, mlrval.VOID) + outrec.PutCopy(keyq, mlrval.VOID) if acc.doVerbose { - outrec.PutCopy(keyl1, types.MLRVAL_VOID) - outrec.PutCopy(keyl2, types.MLRVAL_VOID) - outrec.PutCopy(keyv11, types.MLRVAL_VOID) - outrec.PutCopy(keyv12, types.MLRVAL_VOID) - outrec.PutCopy(keyv21, types.MLRVAL_VOID) - outrec.PutCopy(keyv22, types.MLRVAL_VOID) + outrec.PutCopy(keyl1, mlrval.VOID) + outrec.PutCopy(keyl2, mlrval.VOID) + outrec.PutCopy(keyv11, mlrval.VOID) + outrec.PutCopy(keyv12, mlrval.VOID) + outrec.PutCopy(keyv21, mlrval.VOID) + outrec.PutCopy(keyv22, mlrval.VOID) } } else { Q := lib.GetCovMatrix( @@ -560,18 +560,18 @@ func (acc *Stats2CorrCovAccumulator) Populate( y_mean := acc.sumy / float64(acc.count) m, b, q := lib.GetLinearRegressionPCA(l1, l2, v1, v2, x_mean, y_mean) - outrec.PutReference(keym, types.MlrvalFromFloat64(m)) - outrec.PutReference(keyb, types.MlrvalFromFloat64(b)) - outrec.PutReference(keyn, types.MlrvalFromInt(acc.count)) - outrec.PutReference(keyq, types.MlrvalFromFloat64(q)) + outrec.PutReference(keym, mlrval.FromFloat(m)) + outrec.PutReference(keyb, mlrval.FromFloat(b)) + outrec.PutReference(keyn, mlrval.FromInt(acc.count)) + outrec.PutReference(keyq, mlrval.FromFloat(q)) if acc.doVerbose { - outrec.PutReference(keyl1, types.MlrvalFromFloat64(l1)) - outrec.PutReference(keyl2, types.MlrvalFromFloat64(l2)) - outrec.PutReference(keyv11, types.MlrvalFromFloat64(v1[0])) - outrec.PutReference(keyv12, types.MlrvalFromFloat64(v1[1])) - outrec.PutReference(keyv21, types.MlrvalFromFloat64(v2[0])) - outrec.PutReference(keyv22, types.MlrvalFromFloat64(v2[1])) + outrec.PutReference(keyl1, mlrval.FromFloat(l1)) + outrec.PutReference(keyl2, mlrval.FromFloat(l2)) + outrec.PutReference(keyv11, mlrval.FromFloat(v1[0])) + outrec.PutReference(keyv12, mlrval.FromFloat(v1[1])) + outrec.PutReference(keyv21, mlrval.FromFloat(v2[0])) + outrec.PutReference(keyv22, mlrval.FromFloat(v2[1])) } } } else { @@ -580,7 +580,7 @@ func (acc *Stats2CorrCovAccumulator) Populate( key = acc.covOutputFieldName } if acc.count < 2 { - outrec.PutCopy(key, types.MLRVAL_VOID) + outrec.PutCopy(key, mlrval.VOID) } else { output := lib.GetCov(acc.count, acc.sumx, acc.sumy, acc.sumxy) if acc.doWhich == DO_CORR { @@ -588,7 +588,7 @@ func (acc *Stats2CorrCovAccumulator) Populate( sigmay := math.Sqrt(lib.GetVar(acc.count, acc.sumy, acc.sumy2)) output = output / sigmax / sigmay } - outrec.PutReference(key, types.MlrvalFromFloat64(output)) + outrec.PutReference(key, mlrval.FromFloat(output)) } } } @@ -596,7 +596,7 @@ func (acc *Stats2CorrCovAccumulator) Populate( func (acc *Stats2CorrCovAccumulator) Fit( x float64, y float64, - outrec *types.Mlrmap, + outrec *mlrval.Mlrmap, ) { if acc.doWhich != DO_LINREG_PCA { return @@ -619,10 +619,10 @@ func (acc *Stats2CorrCovAccumulator) Fit( acc.fitReady = true } if acc.count < 2 { - outrec.PutCopy(acc.pca_fitOutputFieldName, types.MLRVAL_VOID) + outrec.PutCopy(acc.pca_fitOutputFieldName, mlrval.VOID) } else { yfit := acc.m*x + acc.b - outrec.PutCopy(acc.pca_fitOutputFieldName, types.MlrvalFromFloat64(yfit)) + outrec.PutCopy(acc.pca_fitOutputFieldName, mlrval.FromFloat(yfit)) } } diff --git a/internal/pkg/transformers/utils/top-keeper.go b/internal/pkg/transformers/utils/top-keeper.go index d36700ad1..a8ecef371 100644 --- a/internal/pkg/transformers/utils/top-keeper.go +++ b/internal/pkg/transformers/utils/top-keeper.go @@ -5,30 +5,31 @@ package utils import ( + "github.com/johnkerl/miller/internal/pkg/mlrval" "github.com/johnkerl/miller/internal/pkg/types" ) // ---------------------------------------------------------------- type TopKeeper struct { - TopValues []*types.Mlrval + TopValues []*mlrval.Mlrval TopRecordsAndContexts []*types.RecordAndContext size int capacity int - bsearchFunc types.BsearchMlrvalArrayFunc + bsearchFunc mlrval.BsearchMlrvalArrayFunc } // ---------------------------------------------------------------- func NewTopKeeper(capacity int, doMax bool) *TopKeeper { keeper := &TopKeeper{ - TopValues: make([]*types.Mlrval, capacity), + TopValues: make([]*mlrval.Mlrval, capacity), TopRecordsAndContexts: make([]*types.RecordAndContext, capacity), size: 0, capacity: capacity, } if doMax { - keeper.bsearchFunc = types.BsearchMlrvalArrayForDescendingInsert + keeper.bsearchFunc = mlrval.BsearchMlrvalArrayForDescendingInsert } else { - keeper.bsearchFunc = types.BsearchMlrvalArrayForAscendingInsert + keeper.bsearchFunc = mlrval.BsearchMlrvalArrayForAscendingInsert } return keeper } @@ -69,7 +70,7 @@ func (keeper *TopKeeper) GetSize() int { // Our caller, the 'top' verb, feeds us records. We keep them or not; in the // latter case, the Go runtime GCs them. -func (keeper *TopKeeper) Add(value *types.Mlrval, recordAndContext *types.RecordAndContext) { +func (keeper *TopKeeper) Add(value *mlrval.Mlrval, recordAndContext *types.RecordAndContext) { destidx := keeper.bsearchFunc(&keeper.TopValues, keeper.size, value) if keeper.size < keeper.capacity { diff --git a/internal/pkg/types/context.go b/internal/pkg/types/context.go index 1aff93811..b64476116 100644 --- a/internal/pkg/types/context.go +++ b/internal/pkg/types/context.go @@ -4,6 +4,8 @@ import ( "bytes" "container/list" "strconv" + + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // Since Go is concurrent, the context struct (AWK-like variables such as @@ -17,14 +19,14 @@ import ( // output ordering. type RecordAndContext struct { - Record *Mlrmap + Record *mlrval.Mlrmap Context Context OutputString string EndOfStream bool } func NewRecordAndContext( - record *Mlrmap, + record *mlrval.Mlrmap, context *Context, ) *RecordAndContext { return &RecordAndContext{ diff --git a/internal/pkg/types/mlrval_accessors.go b/internal/pkg/types/mlrval_accessors.go deleted file mode 100644 index 09ec40a08..000000000 --- a/internal/pkg/types/mlrval_accessors.go +++ /dev/null @@ -1,315 +0,0 @@ -package types - -import ( - "fmt" - "os" - "strconv" - - "github.com/johnkerl/miller/internal/pkg/lib" -) - -// ---------------------------------------------------------------- -func (mv *Mlrval) GetType() MVType { - return mv.mvtype -} - -func (mv *Mlrval) GetTypeName() string { - return TYPE_NAMES[mv.mvtype] -} - -func GetTypeName(mvtype MVType) string { - return TYPE_NAMES[mvtype] -} - -// ---------------------------------------------------------------- -func (mv *Mlrval) IsLegit() bool { - return mv.mvtype >= MT_VOID -} - -func (mv *Mlrval) IsErrorOrAbsent() bool { - return mv.mvtype == MT_ERROR || mv.mvtype == MT_ABSENT -} - -func (mv *Mlrval) IsError() bool { - return mv.mvtype == MT_ERROR -} - -func (mv *Mlrval) IsAbsent() bool { - return mv.mvtype == MT_ABSENT -} - -func (mv *Mlrval) IsNull() bool { - return mv.mvtype == MT_NULL -} - -func (mv *Mlrval) IsVoid() bool { - return mv.mvtype == MT_VOID -} - -func (mv *Mlrval) IsErrorOrVoid() bool { - return mv.mvtype == MT_ERROR || mv.mvtype == MT_VOID -} - -// * Error is non-empty -// * Absent is non-empty (shouldn't have been assigned in the first place; error -// should be surfaced) -// * Void is empty -// * Empty string is empty -// * Int/float/bool/array/map are all non-empty -func (mv *Mlrval) IsEmpty() bool { - if mv.mvtype == MT_VOID { - return true - } else if mv.mvtype == MT_STRING { - return mv.printrep == "" - } else { - return false - } -} - -func (mv *Mlrval) IsString() bool { - return mv.mvtype == MT_STRING -} - -func (mv *Mlrval) IsStringOrVoid() bool { - return mv.mvtype == MT_STRING || mv.mvtype == MT_VOID -} - -func (mv *Mlrval) IsInt() bool { - return mv.mvtype == MT_INT -} - -func (mv *Mlrval) IsFloat() bool { - return mv.mvtype == MT_FLOAT -} - -func (mv *Mlrval) IsNumeric() bool { - return mv.mvtype == MT_INT || mv.mvtype == MT_FLOAT -} - -func (mv *Mlrval) IsIntZero() bool { - return mv.mvtype == MT_INT && mv.intval == 0 -} - -func (mv *Mlrval) IsBool() bool { - return mv.mvtype == MT_BOOL -} - -func (mv *Mlrval) IsTrue() bool { - return mv.mvtype == MT_BOOL && mv.boolval == true -} -func (mv *Mlrval) IsFalse() bool { - return mv.mvtype == MT_BOOL && mv.boolval == false -} - -func (mv *Mlrval) IsArray() bool { - return mv.mvtype == MT_ARRAY -} -func (mv *Mlrval) IsMap() bool { - return mv.mvtype == MT_MAP -} -func (mv *Mlrval) IsArrayOrMap() bool { - return mv.mvtype == MT_ARRAY || mv.mvtype == MT_MAP -} - -func (mv *Mlrval) IsFunction() bool { - return mv.mvtype == MT_FUNC -} - -// ---------------------------------------------------------------- -func (mv *Mlrval) GetIntValue() (intValue int, isInt bool) { - if mv.mvtype == MT_INT { - return mv.intval, true - } else { - return -999, false - } -} - -func (mv *Mlrval) GetFloatValue() (floatValue float64, isFloat bool) { - if mv.mvtype == MT_FLOAT { - return mv.floatval, true - } else { - return -777.0, false - } -} - -func (mv *Mlrval) GetNumericToFloatValue() (floatValue float64, isFloat bool) { - if mv.mvtype == MT_FLOAT { - return mv.floatval, true - } else if mv.mvtype == MT_INT { - return float64(mv.intval), true - } else { - return -888.0, false - } -} - -func (mv *Mlrval) GetNumericToFloatValueOrDie() (floatValue float64) { - floatValue, ok := mv.GetNumericToFloatValue() - if !ok { - fmt.Fprintf( - os.Stderr, - "%s: couldn't parse \"%s\" as number.", - "mlr", mv.String(), - ) - os.Exit(1) - } - return floatValue -} - -func (mv *Mlrval) GetNumericNegativeorDie() bool { - floatValue, ok := mv.GetNumericToFloatValue() - lib.InternalCodingErrorIf(!ok) - return floatValue < 0.0 -} - -func (mv *Mlrval) AssertNumeric() { - _ = mv.GetNumericToFloatValueOrDie() -} - -func (mv *Mlrval) GetString() (stringValue string, isString bool) { - if mv.mvtype == MT_STRING || mv.mvtype == MT_VOID { - return mv.printrep, true - } else { - return "", false - } -} - -func (mv *Mlrval) GetBoolValue() (boolValue bool, isBool bool) { - if mv.mvtype == MT_BOOL { - return mv.boolval, true - } else { - return false, false - } -} - -func (mv *Mlrval) GetArray() []Mlrval { - if mv.mvtype == MT_ARRAY { - return mv.arrayval - } else { - return nil - } -} - -func (mv *Mlrval) GetArrayLength() (int, bool) { - if mv.mvtype == MT_ARRAY { - return len(mv.arrayval), true - } else { - return -999, false - } -} - -func (mv *Mlrval) GetMap() *Mlrmap { - if mv.mvtype == MT_MAP { - return mv.mapval - } else { - return nil - } -} - -func (mv *Mlrval) GetFunction() interface{} { - if mv.mvtype == MT_FUNC { - return mv.funcval - } else { - return nil - } -} - -// ---------------------------------------------------------------- -func (mv *Mlrval) Copy() *Mlrval { - other := *mv - if mv.mvtype == MT_MAP { - other.mapval = mv.mapval.Copy() - } else if mv.mvtype == MT_ARRAY { - other.arrayval = CopyMlrvalArray(mv.arrayval) - } - return &other -} - -func CopyMlrvalArray(input []Mlrval) []Mlrval { - output := make([]Mlrval, len(input)) - for i, element := range input { - output[i] = *element.Copy() - } - return output -} - -func CopyMlrvalPointerArray(input []*Mlrval) []*Mlrval { - output := make([]*Mlrval, len(input)) - for i, element := range input { - output[i] = element.Copy() - } - return output -} - -// --------------------------------------------------------------- -// For the flatten verb and DSL function. - -func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval { - retval := NewMlrmap() - - if mv.mvtype == MT_MAP { - // Without this, the for-loop below is zero-pass and fields with "{}" - // values would disappear entirely in a JSON-to-CSV conversion. - if mv.mapval.IsEmpty() { - if prefix != "" { - retval.PutCopy(prefix, MlrvalFromString("{}")) - } - } - - for pe := mv.mapval.Head; pe != nil; pe = pe.Next { - nextPrefix := pe.Key - if prefix != "" { - nextPrefix = prefix + delimiter + nextPrefix - } - if pe.Value.mvtype == MT_MAP || pe.Value.mvtype == MT_ARRAY { - nextResult := pe.Value.FlattenToMap(nextPrefix, delimiter) - lib.InternalCodingErrorIf(nextResult.mvtype != MT_MAP) - for pf := nextResult.mapval.Head; pf != nil; pf = pf.Next { - retval.PutCopy(pf.Key, pf.Value.Copy()) - } - } else { - retval.PutCopy(nextPrefix, pe.Value.Copy()) - } - } - - } else if mv.mvtype == MT_ARRAY { - // Without this, the for-loop below is zero-pass and fields with "[]" - // values would disappear entirely in a JSON-to-CSV conversion. - if len(mv.arrayval) == 0 { - if prefix != "" { - retval.PutCopy(prefix, MlrvalFromString("[]")) - } - } - - for zindex, value := range mv.arrayval { - nextPrefix := strconv.Itoa(zindex + 1) // Miller user-space indices are 1-up - if prefix != "" { - nextPrefix = prefix + delimiter + nextPrefix - } - if value.mvtype == MT_MAP || value.mvtype == MT_ARRAY { - nextResult := value.FlattenToMap(nextPrefix, delimiter) - lib.InternalCodingErrorIf(nextResult.mvtype != MT_MAP) - for pf := nextResult.mapval.Head; pf != nil; pf = pf.Next { - retval.PutCopy(pf.Key, pf.Value.Copy()) - } - } else { - retval.PutCopy(nextPrefix, value.Copy()) - } - } - - } else { - retval.PutCopy(prefix, mv.Copy()) - } - - return *MlrvalFromMapReferenced(retval) -} - -// ---------------------------------------------------------------- -// Used by stats1. - -func (mv *Mlrval) Increment() { - if mv.mvtype == MT_INT { - mv.intval++ - } else if mv.mvtype == MT_FLOAT { - mv.floatval++ - } -} diff --git a/internal/pkg/types/mlrval_constants.go b/internal/pkg/types/mlrval_constants.go deleted file mode 100644 index ff903497a..000000000 --- a/internal/pkg/types/mlrval_constants.go +++ /dev/null @@ -1,93 +0,0 @@ -// ================================================================ -// Constants/singletons of various types -// ================================================================ - -package types - -// These are made singletons as part of a copy-reduction effort. They're not -// marked const (I haven't figured out the right way to get that to compile; -// just using `const` isn't enough) but the gentelpersons' agreement is that -// the caller should never modify these. - -// MlrvalFromPending is designed solely for the JSON API, for something -// intended to be mutated after construction once its type is (later) known. -// Whereas MLRVAL_ERROR, MLRVAL_ABSENT, etc are all singletons, this one -// must be mutable and therefor non-singleton. -func MlrvalFromPending() Mlrval { - return Mlrval{ - mvtype: MT_PENDING, - printrep: "(bug-if-you-see-this-pending-type)", - printrepValid: true, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: nil, - mapval: nil, - } -} - -var MLRVAL_ERROR = &Mlrval{ - mvtype: MT_ERROR, - printrep: "(error)", // xxx const somewhere - printrepValid: true, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: nil, - mapval: nil, -} - -var MLRVAL_ABSENT = &Mlrval{ - mvtype: MT_ABSENT, - printrep: "(absent)", - printrepValid: true, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: nil, - mapval: nil, -} - -var MLRVAL_NULL = &Mlrval{ - mvtype: MT_NULL, - printrep: "null", - printrepValid: true, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: nil, - mapval: nil, -} - -var MLRVAL_VOID = &Mlrval{ - mvtype: MT_VOID, - printrep: "", - printrepValid: true, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: nil, - mapval: nil, -} - -var MLRVAL_TRUE = &Mlrval{ - mvtype: MT_BOOL, - printrep: "true", - printrepValid: true, - intval: 0, - floatval: 0.0, - boolval: true, - arrayval: nil, - mapval: nil, -} - -var MLRVAL_FALSE = &Mlrval{ - mvtype: MT_BOOL, - printrep: "false", - printrepValid: true, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: nil, - mapval: nil, -} diff --git a/internal/pkg/types/mlrval_function_hashing.go b/internal/pkg/types/mlrval_function_hashing.go deleted file mode 100644 index 3a3adc7d0..000000000 --- a/internal/pkg/types/mlrval_function_hashing.go +++ /dev/null @@ -1,61 +0,0 @@ -package types - -import ( - "crypto/md5" - "crypto/sha1" - "crypto/sha256" - "crypto/sha512" - "fmt" -) - -func BIF_md5(input1 *Mlrval) *Mlrval { - if !input1.IsStringOrVoid() { - return MLRVAL_ERROR - } else { - return MlrvalFromString( - fmt.Sprintf( - "%x", - md5.Sum([]byte(input1.printrep)), - ), - ) - } -} - -func BIF_sha1(input1 *Mlrval) *Mlrval { - if !input1.IsStringOrVoid() { - return MLRVAL_ERROR - } else { - return MlrvalFromString( - fmt.Sprintf( - "%x", - sha1.Sum([]byte(input1.printrep)), - ), - ) - } -} - -func BIF_sha256(input1 *Mlrval) *Mlrval { - if !input1.IsStringOrVoid() { - return MLRVAL_ERROR - } else { - return MlrvalFromString( - fmt.Sprintf( - "%x", - sha256.Sum256([]byte(input1.printrep)), - ), - ) - } -} - -func BIF_sha512(input1 *Mlrval) *Mlrval { - if !input1.IsStringOrVoid() { - return MLRVAL_ERROR - } else { - return MlrvalFromString( - fmt.Sprintf( - "%x", - sha512.Sum512([]byte(input1.printrep)), - ), - ) - } -} diff --git a/internal/pkg/types/mlrval_functions_booleans.go b/internal/pkg/types/mlrval_functions_booleans.go deleted file mode 100644 index 3d00230ea..000000000 --- a/internal/pkg/types/mlrval_functions_booleans.go +++ /dev/null @@ -1,471 +0,0 @@ -// ================================================================ -// Boolean expressions for ==, !=, >, >=, <, <= -// ================================================================ - -package types - -import ( - "github.com/johnkerl/miller/internal/pkg/lib" -) - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -// string_cmp implements the spaceship operator for strings. -func string_cmp(a, b string) int { - if a < b { - return -1 - } - if a > b { - return 1 - } - return 0 -} - -// int_cmp implements the spaceship operator for ints. -func int_cmp(a, b int) int { - if a < b { - return -1 - } - if a > b { - return 1 - } - return 0 -} - -// float_cmp implements the spaceship operator for floats. -func float_cmp(a, b float64) int { - if a < b { - return -1 - } - if a > b { - return 1 - } - return 0 -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_ss(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep == input2.printrep) -} -func ne_b_ss(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep != input2.printrep) -} -func gt_b_ss(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep > input2.printrep) -} -func ge_b_ss(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep >= input2.printrep) -} -func lt_b_ss(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep < input2.printrep) -} -func le_b_ss(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep <= input2.printrep) -} -func cmp_b_ss(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(string_cmp(input1.printrep, input2.printrep)) -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_xs(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.String() == input2.printrep) -} -func ne_b_xs(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.String() != input2.printrep) -} -func gt_b_xs(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.String() > input2.printrep) -} -func ge_b_xs(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.String() >= input2.printrep) -} -func lt_b_xs(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.String() < input2.printrep) -} -func le_b_xs(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.String() <= input2.printrep) -} -func cmp_b_xs(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(string_cmp(input1.String(), input2.printrep)) -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_sx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep == input2.String()) -} -func ne_b_sx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep != input2.String()) -} -func gt_b_sx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep > input2.String()) -} -func ge_b_sx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep >= input2.String()) -} -func lt_b_sx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep < input2.String()) -} -func le_b_sx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.printrep <= input2.String()) -} -func cmp_b_sx(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(string_cmp(input1.printrep, input2.String())) -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.intval == input2.intval) -} -func ne_b_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.intval != input2.intval) -} -func gt_b_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.intval > input2.intval) -} -func ge_b_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.intval >= input2.intval) -} -func lt_b_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.intval < input2.intval) -} -func le_b_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.intval <= input2.intval) -} -func cmp_b_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(int_cmp(input1.intval, input2.intval)) -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(float64(input1.intval) == input2.floatval) -} -func ne_b_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(float64(input1.intval) != input2.floatval) -} -func gt_b_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(float64(input1.intval) > input2.floatval) -} -func ge_b_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(float64(input1.intval) >= input2.floatval) -} -func lt_b_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(float64(input1.intval) < input2.floatval) -} -func le_b_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(float64(input1.intval) <= input2.floatval) -} -func cmp_b_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(float_cmp(float64(input1.intval), input2.floatval)) -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval == float64(input2.intval)) -} -func ne_b_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval != float64(input2.intval)) -} -func gt_b_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval > float64(input2.intval)) -} -func ge_b_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval >= float64(input2.intval)) -} -func lt_b_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval < float64(input2.intval)) -} -func le_b_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval <= float64(input2.intval)) -} -func cmp_b_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(float_cmp(input1.floatval, float64(input2.intval))) -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval == input2.floatval) -} -func ne_b_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval != input2.floatval) -} -func gt_b_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval > input2.floatval) -} -func ge_b_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval >= input2.floatval) -} -func lt_b_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval < input2.floatval) -} -func le_b_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval <= input2.floatval) -} -func cmp_b_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(float_cmp(input1.floatval, input2.floatval)) -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_bb(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.boolval == input2.boolval) -} -func ne_b_bb(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.boolval != input2.boolval) -} - -// We could say ordering on bool is error, but, Miller allows -// sorting on bool so it should allow ordering on bool. - -func gt_b_bb(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(lib.BoolToInt(input1.boolval) > lib.BoolToInt(input2.boolval)) -} -func ge_b_bb(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(lib.BoolToInt(input1.boolval) >= lib.BoolToInt(input2.boolval)) -} -func lt_b_bb(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(lib.BoolToInt(input1.boolval) < lib.BoolToInt(input2.boolval)) -} -func le_b_bb(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(lib.BoolToInt(input1.boolval) <= lib.BoolToInt(input2.boolval)) -} -func cmp_b_bb(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(int_cmp(lib.BoolToInt(input1.boolval), lib.BoolToInt(input2.boolval))) -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_aa(input1, input2 *Mlrval) *Mlrval { - a := input1.arrayval - b := input2.arrayval - - // Different-length arrays are not equal - if len(a) != len(b) { - return MLRVAL_FALSE - } - - // Same-length arrays: return false if any slot is not equal, else true. - for i := range a { - eq := BIF_equals(&a[i], &b[i]) - lib.InternalCodingErrorIf(eq.mvtype != MT_BOOL) - if eq.boolval == false { - return MLRVAL_FALSE - } - } - - return MLRVAL_TRUE -} -func ne_b_aa(input1, input2 *Mlrval) *Mlrval { - output := eq_b_aa(input1, input2) - return MlrvalFromBool(!output.boolval) -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -func eq_b_mm(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mapval.Equals(input2.mapval)) -} -func ne_b_mm(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromBool(!input1.mapval.Equals(input2.mapval)) -} - -// We get a Golang "initialization loop" due to recursive depth computation -// if this is defined statically. So, we use a "package init" function. -var eq_dispositions = [MT_DIM][MT_DIM]BinaryFunc{} - -func init() { - eq_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ - // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC - /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, - /*ABSENT */ {_erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, - /*NULL */ {_erro, _absn, _true, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _erro}, - /*VOID */ {_erro, _absn, _fals, eq_b_ss, eq_b_ss, eq_b_sx, eq_b_sx, _fals, _fals, _fals, _erro}, - /*STRING */ {_erro, _absn, _fals, eq_b_ss, eq_b_ss, eq_b_sx, eq_b_sx, _fals, _fals, _fals, _erro}, - /*INT */ {_erro, _absn, _fals, eq_b_xs, eq_b_xs, eq_b_ii, eq_b_if, _fals, _fals, _fals, _erro}, - /*FLOAT */ {_erro, _absn, _fals, eq_b_xs, eq_b_xs, eq_b_fi, eq_b_ff, _fals, _fals, _fals, _erro}, - /*BOOL */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, eq_b_bb, _fals, _fals, _erro}, - /*ARRAY */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, _fals, eq_b_aa, _fals, _erro}, - /*MAP */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _fals, eq_b_mm, _erro}, - /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, - } -} - -var ne_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ - // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC - /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, - /*ABSENT */ {_erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, - /*NULL */ {_erro, _absn, _fals, _true, _true, _true, _true, _true, _true, _true, _erro}, - /*VOID */ {_erro, _absn, _true, ne_b_ss, ne_b_ss, ne_b_sx, ne_b_sx, _true, _true, _true, _erro}, - /*STRING */ {_erro, _absn, _true, ne_b_ss, ne_b_ss, ne_b_sx, ne_b_sx, _true, _true, _true, _erro}, - /*INT */ {_erro, _absn, _true, ne_b_xs, ne_b_xs, ne_b_ii, ne_b_if, _true, _true, _true, _erro}, - /*FLOAT */ {_erro, _absn, _true, ne_b_xs, ne_b_xs, ne_b_fi, ne_b_ff, _true, _true, _true, _erro}, - /*BOOL */ {_erro, _absn, _true, _true, _true, _true, _true, ne_b_bb, _true, _true, _erro}, - /*ARRAY */ {_erro, _absn, _true, _true, _true, _true, _true, _true, ne_b_aa, _true, _erro}, - /*MAP */ {_erro, _absn, _true, _true, _true, _true, _true, _true, _true, ne_b_mm, _erro}, - /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, -} - -var gt_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ - // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC - /*ERROR */ {_erro, _erro, _fals, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, - /*ABSENT */ {_erro, _absn, _true, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, - /*NULL */ {_true, _fals, _fals, _true, _true, _true, _true, _true, _absn, _absn, _erro}, - /*VOID */ {_erro, _absn, _fals, gt_b_ss, gt_b_ss, gt_b_sx, gt_b_sx, _fals, _fals, _fals, _erro}, - /*STRING */ {_erro, _absn, _fals, gt_b_ss, gt_b_ss, gt_b_sx, gt_b_sx, _fals, _fals, _fals, _erro}, - /*INT */ {_erro, _absn, _fals, gt_b_xs, gt_b_xs, gt_b_ii, gt_b_if, _fals, _fals, _fals, _erro}, - /*FLOAT */ {_erro, _absn, _fals, gt_b_xs, gt_b_xs, gt_b_fi, gt_b_ff, _fals, _fals, _fals, _erro}, - /*BOOL */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, gt_b_bb, _fals, _fals, _erro}, - /*ARRAY */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _erro, _fals, _erro}, - /*MAP */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _erro, _erro}, - /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, -} - -var ge_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ - // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC - /*ERROR */ {_erro, _erro, _fals, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, - /*ABSENT */ {_erro, _absn, _true, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, - /*NULL */ {_true, _fals, _true, _true, _true, _true, _true, _true, _absn, _absn, _erro}, - /*VOID */ {_erro, _absn, _fals, ge_b_ss, ge_b_ss, ge_b_sx, ge_b_sx, _fals, _fals, _fals, _erro}, - /*STRING */ {_erro, _absn, _fals, ge_b_ss, ge_b_ss, ge_b_sx, ge_b_sx, _fals, _fals, _fals, _erro}, - /*INT */ {_erro, _absn, _fals, ge_b_xs, ge_b_xs, ge_b_ii, ge_b_if, _fals, _fals, _fals, _erro}, - /*FLOAT */ {_erro, _absn, _fals, ge_b_xs, ge_b_xs, ge_b_fi, ge_b_ff, _fals, _fals, _fals, _erro}, - /*BOOL */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, ge_b_bb, _fals, _fals, _erro}, - /*ARRAY */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _erro, _fals, _erro}, - /*MAP */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _fals, _erro, _erro}, - /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, -} - -var lt_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ - // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC - /*ERROR */ {_erro, _erro, _true, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, - /*ABSENT */ {_erro, _absn, _fals, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, - /*NULL */ {_fals, _true, _fals, _fals, _fals, _fals, _fals, _fals, _absn, _absn, _erro}, - /*VOID */ {_erro, _absn, _true, lt_b_ss, lt_b_ss, lt_b_sx, lt_b_sx, _fals, _fals, _fals, _erro}, - /*STRING */ {_erro, _absn, _true, lt_b_ss, lt_b_ss, lt_b_sx, lt_b_sx, _fals, _fals, _fals, _erro}, - /*INT */ {_erro, _absn, _true, lt_b_xs, lt_b_xs, lt_b_ii, lt_b_if, _fals, _fals, _fals, _erro}, - /*FLOAT */ {_erro, _absn, _true, lt_b_xs, lt_b_xs, lt_b_fi, lt_b_ff, _fals, _fals, _fals, _erro}, - /*BOOL */ {_erro, _absn, _true, _fals, _fals, _fals, _fals, lt_b_bb, _fals, _fals, _erro}, - /*ARRAY */ {_erro, _absn, _absn, _fals, _fals, _fals, _fals, _fals, _erro, _fals, _erro}, - /*MAP */ {_erro, _absn, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _erro, _erro}, - /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, -} - -var le_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ - // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC - /*ERROR */ {_erro, _erro, _true, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, - /*ABSENT */ {_erro, _absn, _fals, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, - /*NULL */ {_fals, _true, _true, _fals, _fals, _fals, _fals, _fals, _absn, _absn, _erro}, - /*VOID */ {_erro, _absn, _true, le_b_ss, le_b_ss, le_b_sx, le_b_sx, _fals, _fals, _fals, _erro}, - /*STRING */ {_erro, _absn, _true, le_b_ss, le_b_ss, le_b_sx, le_b_sx, _fals, _fals, _fals, _erro}, - /*INT */ {_erro, _absn, _true, le_b_xs, le_b_xs, le_b_ii, le_b_if, _fals, _fals, _fals, _erro}, - /*FLOAT */ {_erro, _absn, _true, le_b_xs, le_b_xs, le_b_fi, le_b_ff, _fals, _fals, _fals, _erro}, - /*BOOL */ {_erro, _absn, _true, _fals, _fals, _fals, _fals, le_b_bb, _fals, _fals, _erro}, - /*ARRAY */ {_erro, _absn, _absn, _fals, _fals, _fals, _fals, _fals, _erro, _fals, _erro}, - /*MAP */ {_erro, _absn, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _erro, _erro}, - /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, -} - -var cmp_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ - // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC - /*ERROR */ {_erro, _erro, _true, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, - /*ABSENT */ {_erro, _absn, _fals, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, - /*NULL */ {_fals, _true, _true, _fals, _fals, _fals, _fals, _fals, _absn, _absn, _erro}, - /*VOID */ {_erro, _absn, _true, cmp_b_ss, cmp_b_ss, cmp_b_sx, cmp_b_sx, _fals, _fals, _fals, _erro}, - /*STRING */ {_erro, _absn, _true, cmp_b_ss, cmp_b_ss, cmp_b_sx, cmp_b_sx, _fals, _fals, _fals, _erro}, - /*INT */ {_erro, _absn, _true, cmp_b_xs, cmp_b_xs, cmp_b_ii, cmp_b_if, _fals, _fals, _fals, _erro}, - /*FLOAT */ {_erro, _absn, _true, cmp_b_xs, cmp_b_xs, cmp_b_fi, cmp_b_ff, _fals, _fals, _fals, _erro}, - /*BOOL */ {_erro, _absn, _true, _fals, _fals, _fals, _fals, cmp_b_bb, _fals, _fals, _erro}, - /*ARRAY */ {_erro, _absn, _absn, _fals, _fals, _fals, _fals, _fals, _erro, _fals, _erro}, - /*MAP */ {_erro, _absn, _absn, _fals, _fals, _fals, _fals, _fals, _fals, _erro, _erro}, - /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, -} - -func BIF_equals(input1, input2 *Mlrval) *Mlrval { - return eq_dispositions[input1.mvtype][input2.mvtype](input1, input2) -} -func BIF_not_equals(input1, input2 *Mlrval) *Mlrval { - return ne_dispositions[input1.mvtype][input2.mvtype](input1, input2) -} -func BIF_greater_than(input1, input2 *Mlrval) *Mlrval { - return gt_dispositions[input1.mvtype][input2.mvtype](input1, input2) -} -func BIF_greater_than_or_equals(input1, input2 *Mlrval) *Mlrval { - return ge_dispositions[input1.mvtype][input2.mvtype](input1, input2) -} -func BIF_less_than(input1, input2 *Mlrval) *Mlrval { - return lt_dispositions[input1.mvtype][input2.mvtype](input1, input2) -} -func BIF_less_than_or_equals(input1, input2 *Mlrval) *Mlrval { - return le_dispositions[input1.mvtype][input2.mvtype](input1, input2) -} -func BIF_cmp(input1, input2 *Mlrval) *Mlrval { - return cmp_dispositions[input1.mvtype][input2.mvtype](input1, input2) -} - -// For Go's sort.Slice. -func MlrvalLessThanAsBool(input1, input2 *Mlrval) bool { - // TODO refactor to avoid copy - // This is a hot path for sort GC and is worth significant hand-optimization - mretval := lt_dispositions[input1.mvtype][input2.mvtype](input1, input2) - retval, ok := mretval.GetBoolValue() - lib.InternalCodingErrorIf(!ok) - return retval -} - -// For Go's sort.Slice. -func MlrvalLessThanOrEqualsAsBool(input1, input2 *Mlrval) bool { - // TODO refactor to avoid copy - // This is a hot path for sort GC and is worth significant hand-optimization - mretval := le_dispositions[input1.mvtype][input2.mvtype](input1, input2) - retval, ok := mretval.GetBoolValue() - lib.InternalCodingErrorIf(!ok) - return retval -} - -// For top-keeper -func MlrvalGreaterThanAsBool(input1, input2 *Mlrval) bool { - // TODO refactor to avoid copy - // This is a hot path for sort GC and is worth significant hand-optimization - mretval := gt_dispositions[input1.mvtype][input2.mvtype](input1, input2) - retval, ok := mretval.GetBoolValue() - lib.InternalCodingErrorIf(!ok) - return retval -} - -// For top-keeper -func MlrvalGreaterThanOrEqualsAsBool(input1, input2 *Mlrval) bool { - // TODO refactor to avoid copy - // This is a hot path for sort GC and is worth significant hand-optimization - mretval := ge_dispositions[input1.mvtype][input2.mvtype](input1, input2) - retval, ok := mretval.GetBoolValue() - lib.InternalCodingErrorIf(!ok) - return retval -} - -// Convenience wrapper for non-DSL callsites that just want a bool -func MlrvalEqualsAsBool(input1, input2 *Mlrval) bool { - mretval := eq_dispositions[input1.mvtype][input2.mvtype](input1, input2) - retval, ok := mretval.GetBoolValue() - lib.InternalCodingErrorIf(!ok) - return retval -} - -// ---------------------------------------------------------------- -func MlrvalLogicalAND(input1, input2 *Mlrval) *Mlrval { - if input1.mvtype == MT_BOOL && input2.mvtype == MT_BOOL { - return MlrvalFromBool(input1.boolval && input2.boolval) - } else { - return MLRVAL_ERROR - } -} - -func MlrvalLogicalOR(input1, input2 *Mlrval) *Mlrval { - if input1.mvtype == MT_BOOL && input2.mvtype == MT_BOOL { - return MlrvalFromBool(input1.boolval || input2.boolval) - } else { - return MLRVAL_ERROR - } -} - -func BIF_logicalxor(input1, input2 *Mlrval) *Mlrval { - if input1.mvtype == MT_BOOL && input2.mvtype == MT_BOOL { - return MlrvalFromBool(input1.boolval != input2.boolval) - } else { - return MLRVAL_ERROR - } -} diff --git a/internal/pkg/types/mlrval_functions_collections.go b/internal/pkg/types/mlrval_functions_collections.go deleted file mode 100644 index 1fe9fd0fb..000000000 --- a/internal/pkg/types/mlrval_functions_collections.go +++ /dev/null @@ -1,782 +0,0 @@ -package types - -import ( - "bytes" - "strconv" - "strings" - - "github.com/johnkerl/miller/internal/pkg/lib" -) - -// ================================================================ -// Map/array count. Scalars (including strings) have length 1. -func BIF_length(input1 *Mlrval) *Mlrval { - switch input1.mvtype { - case MT_ERROR: - return MlrvalFromInt(0) - break - case MT_ABSENT: - return MlrvalFromInt(0) - break - case MT_ARRAY: - return MlrvalFromInt(int(len(input1.arrayval))) - break - case MT_MAP: - return MlrvalFromInt(int(input1.mapval.FieldCount)) - break - } - return MlrvalFromInt(1) -} - -// ================================================================ -func depth_from_array(input1 *Mlrval) *Mlrval { - maxChildDepth := 0 - for _, child := range input1.arrayval { - childDepth := BIF_depth(&child) - lib.InternalCodingErrorIf(childDepth.mvtype != MT_INT) - iChildDepth := int(childDepth.intval) - if iChildDepth > maxChildDepth { - maxChildDepth = iChildDepth - } - } - return MlrvalFromInt(int(1 + maxChildDepth)) -} - -func depth_from_map(input1 *Mlrval) *Mlrval { - maxChildDepth := 0 - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { - child := pe.Value - childDepth := BIF_depth(child) - lib.InternalCodingErrorIf(childDepth.mvtype != MT_INT) - iChildDepth := int(childDepth.intval) - if iChildDepth > maxChildDepth { - maxChildDepth = iChildDepth - } - } - return MlrvalFromInt(int(1 + maxChildDepth)) -} - -func depth_from_scalar(input1 *Mlrval) *Mlrval { - return MlrvalFromInt(0) -} - -// We get a Golang "initialization loop" due to recursive depth computation -// if this is defined statically. So, we use a "package init" function. -var depth_dispositions = [MT_DIM]UnaryFunc{} - -func init() { - depth_dispositions = [MT_DIM]UnaryFunc{ - /*ERROR */ _erro1, - /*ABSENT */ _absn1, - /*NULL */ _zero1, - /*VOID */ depth_from_scalar, - /*STRING */ depth_from_scalar, - /*INT */ depth_from_scalar, - /*FLOAT */ depth_from_scalar, - /*BOOL */ depth_from_scalar, - /*ARRAY */ depth_from_array, - /*MAP */ depth_from_map, - /*FUNC */ _erro1, - } -} - -func BIF_depth(input1 *Mlrval) *Mlrval { - return depth_dispositions[input1.mvtype](input1) -} - -// ================================================================ -func leafcount_from_array(input1 *Mlrval) *Mlrval { - sumChildLeafCount := 0 - for _, child := range input1.arrayval { - // Golang initialization loop if we do this :( - // childLeafCount := BIF_leafcount(&child) - - childLeafCount := MlrvalFromInt(1) - if child.mvtype == MT_ARRAY { - childLeafCount = leafcount_from_array(&child) - } else if child.mvtype == MT_MAP { - childLeafCount = leafcount_from_map(&child) - } - - lib.InternalCodingErrorIf(childLeafCount.mvtype != MT_INT) - iChildLeafCount := int(childLeafCount.intval) - sumChildLeafCount += iChildLeafCount - } - return MlrvalFromInt(int(sumChildLeafCount)) -} - -func leafcount_from_map(input1 *Mlrval) *Mlrval { - sumChildLeafCount := 0 - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { - child := pe.Value - - // Golang initialization loop if we do this :( - // childLeafCount := BIF_leafcount(child) - - childLeafCount := MlrvalFromInt(1) - if child.mvtype == MT_ARRAY { - childLeafCount = leafcount_from_array(child) - } else if child.mvtype == MT_MAP { - childLeafCount = leafcount_from_map(child) - } - - lib.InternalCodingErrorIf(childLeafCount.mvtype != MT_INT) - iChildLeafCount := int(childLeafCount.intval) - sumChildLeafCount += iChildLeafCount - } - return MlrvalFromInt(int(sumChildLeafCount)) -} - -func leafcount_from_scalar(input1 *Mlrval) *Mlrval { - return MlrvalFromInt(1) -} - -var leafcount_dispositions = [MT_DIM]UnaryFunc{ - /*ERROR */ _erro1, - /*ABSENT */ _absn1, - /*NULL */ _zero1, - /*VOID */ leafcount_from_scalar, - /*STRING */ leafcount_from_scalar, - /*INT */ leafcount_from_scalar, - /*FLOAT */ leafcount_from_scalar, - /*BOOL */ leafcount_from_scalar, - /*ARRAY */ leafcount_from_array, - /*MAP */ leafcount_from_map, - /*FUNC */ _erro1, -} - -func BIF_leafcount(input1 *Mlrval) *Mlrval { - return leafcount_dispositions[input1.mvtype](input1) -} - -// ---------------------------------------------------------------- -func has_key_in_array(input1, input2 *Mlrval) *Mlrval { - if input2.mvtype == MT_STRING { - return MLRVAL_FALSE - } - if input2.mvtype != MT_INT { - return MLRVAL_ERROR - } - _, ok := UnaliasArrayIndex(&input1.arrayval, input2.intval) - return MlrvalFromBool(ok) -} - -func has_key_in_map(input1, input2 *Mlrval) *Mlrval { - if input2.mvtype == MT_STRING || input2.mvtype == MT_INT { - return MlrvalFromBool(input1.mapval.Has(input2.String())) - } else { - return MLRVAL_ERROR - } -} - -func BIF_haskey(input1, input2 *Mlrval) *Mlrval { - if input1.mvtype == MT_ARRAY { - return has_key_in_array(input1, input2) - } else if input1.mvtype == MT_MAP { - return has_key_in_map(input1, input2) - } else { - return MLRVAL_ERROR - } -} - -// ================================================================ -func BIF_mapselect(mlrvals []*Mlrval) *Mlrval { - if len(mlrvals) < 1 { - return MLRVAL_ERROR - } - if mlrvals[0].mvtype != MT_MAP { - return MLRVAL_ERROR - } - oldmap := mlrvals[0].mapval - newMap := NewMlrmap() - - newKeys := make(map[string]bool) - for _, selectArg := range mlrvals[1:] { - if selectArg.mvtype == MT_STRING { - newKeys[selectArg.printrep] = true - } else if selectArg.mvtype == MT_INT { - newKeys[selectArg.String()] = true - } else if selectArg.mvtype == MT_ARRAY { - for _, element := range selectArg.arrayval { - if element.mvtype == MT_STRING { - newKeys[element.printrep] = true - } else { - return MLRVAL_ERROR - } - } - } else { - return MLRVAL_ERROR - } - } - - for pe := oldmap.Head; pe != nil; pe = pe.Next { - oldKey := pe.Key - _, present := newKeys[oldKey] - if present { - newMap.PutCopy(oldKey, oldmap.Get(oldKey)) - } - } - - return MlrvalFromMap(newMap) -} - -// ---------------------------------------------------------------- -func BIF_mapexcept(mlrvals []*Mlrval) *Mlrval { - if len(mlrvals) < 1 { - return MLRVAL_ERROR - } - if mlrvals[0].mvtype != MT_MAP { - return MLRVAL_ERROR - } - newMap := mlrvals[0].mapval.Copy() - - for _, exceptArg := range mlrvals[1:] { - if exceptArg.mvtype == MT_STRING { - newMap.Remove(exceptArg.printrep) - } else if exceptArg.mvtype == MT_INT { - newMap.Remove(exceptArg.String()) - } else if exceptArg.mvtype == MT_ARRAY { - for _, element := range exceptArg.arrayval { - if element.mvtype == MT_STRING { - newMap.Remove(element.printrep) - } else { - return MLRVAL_ERROR - } - } - } else { - return MLRVAL_ERROR - } - } - - return MlrvalFromMap(newMap) -} - -// ---------------------------------------------------------------- -func BIF_mapsum(mlrvals []*Mlrval) *Mlrval { - if len(mlrvals) == 0 { - return MlrvalFromEmptyMap() - } - if len(mlrvals) == 1 { - return mlrvals[0] - } - if mlrvals[0].mvtype != MT_MAP { - return MLRVAL_ERROR - } - newMap := mlrvals[0].mapval.Copy() - - for _, otherMapArg := range mlrvals[1:] { - if otherMapArg.mvtype != MT_MAP { - return MLRVAL_ERROR - } - - for pe := otherMapArg.mapval.Head; pe != nil; pe = pe.Next { - newMap.PutCopy(pe.Key, pe.Value) - } - } - - return MlrvalFromMap(newMap) -} - -// ---------------------------------------------------------------- -func BIF_mapdiff(mlrvals []*Mlrval) *Mlrval { - if len(mlrvals) == 0 { - return MlrvalFromEmptyMap() - } - if len(mlrvals) == 1 { - return mlrvals[0] - } - if mlrvals[0].mvtype != MT_MAP { - return MLRVAL_ERROR - } - newMap := mlrvals[0].mapval.Copy() - - for _, otherMapArg := range mlrvals[1:] { - if otherMapArg.mvtype != MT_MAP { - return MLRVAL_ERROR - } - - for pe := otherMapArg.mapval.Head; pe != nil; pe = pe.Next { - newMap.Remove(pe.Key) - } - } - - return MlrvalFromMap(newMap) -} - -// ================================================================ -// joink([1,2,3], ",") -> "1,2,3" -// joink({"a":3,"b":4,"c":5}, ",") -> "a,b,c" -func BIF_joink(input1, input2 *Mlrval) *Mlrval { - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR - } - fieldSeparator := input2.printrep - if input1.mvtype == MT_MAP { - var buffer bytes.Buffer - - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { - buffer.WriteString(pe.Key) - if pe.Next != nil { - buffer.WriteString(fieldSeparator) - } - } - - return MlrvalFromString(buffer.String()) - } else if input1.mvtype == MT_ARRAY { - var buffer bytes.Buffer - - for i := range input1.arrayval { - if i > 0 { - buffer.WriteString(fieldSeparator) - } - // Miller userspace array indices are 1-up - buffer.WriteString(strconv.Itoa(i + 1)) - } - - return MlrvalFromString(buffer.String()) - } else { - return MLRVAL_ERROR - } -} - -// ---------------------------------------------------------------- -// joinv([3,4,5], ",") -> "3,4,5" -// joinv({"a":3,"b":4,"c":5}, ",") -> "3,4,5" -func BIF_joinv(input1, input2 *Mlrval) *Mlrval { - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR - } - fieldSeparator := input2.printrep - - if input1.mvtype == MT_MAP { - var buffer bytes.Buffer - - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { - buffer.WriteString(pe.Value.String()) - if pe.Next != nil { - buffer.WriteString(fieldSeparator) - } - } - - return MlrvalFromString(buffer.String()) - } else if input1.mvtype == MT_ARRAY { - var buffer bytes.Buffer - - for i, element := range input1.arrayval { - if i > 0 { - buffer.WriteString(fieldSeparator) - } - buffer.WriteString(element.String()) - } - - return MlrvalFromString(buffer.String()) - } else { - return MLRVAL_ERROR - } -} - -// ---------------------------------------------------------------- -// joinkv([3,4,5], "=", ",") -> "1=3,2=4,3=5" -// joinkv({"a":3,"b":4,"c":5}, "=", ",") -> "a=3,b=4,c=5" -func BIF_joinkv(input1, input2, input3 *Mlrval) *Mlrval { - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR - } - pairSeparator := input2.printrep - if input3.mvtype != MT_STRING { - return MLRVAL_ERROR - } - fieldSeparator := input3.printrep - - if input1.mvtype == MT_MAP { - var buffer bytes.Buffer - - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { - buffer.WriteString(pe.Key) - buffer.WriteString(pairSeparator) - buffer.WriteString(pe.Value.String()) - if pe.Next != nil { - buffer.WriteString(fieldSeparator) - } - } - - return MlrvalFromString(buffer.String()) - } else if input1.mvtype == MT_ARRAY { - var buffer bytes.Buffer - - for i, element := range input1.arrayval { - if i > 0 { - buffer.WriteString(fieldSeparator) - } - // Miller userspace array indices are 1-up - buffer.WriteString(strconv.Itoa(i + 1)) - buffer.WriteString(pairSeparator) - buffer.WriteString(element.String()) - } - - return MlrvalFromString(buffer.String()) - } else { - return MLRVAL_ERROR - } -} - -// ================================================================ -// splitkv("a=3,b=4,c=5", "=", ",") -> {"a":3,"b":4,"c":5} -func BIF_splitkv(input1, input2, input3 *Mlrval) *Mlrval { - if !input1.IsStringOrVoid() { - return MLRVAL_ERROR - } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR - } - pairSeparator := input2.printrep - if input3.mvtype != MT_STRING { - return MLRVAL_ERROR - } - fieldSeparator := input3.printrep - - output := MlrvalFromEmptyMap() - - fields := lib.SplitString(input1.printrep, fieldSeparator) - for i, field := range fields { - pair := strings.SplitN(field, pairSeparator, 2) - if len(pair) == 1 { - key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up - value := MlrvalFromInferredType(pair[0]) - output.mapval.PutReference(key, value) - } else if len(pair) == 2 { - key := pair[0] - value := MlrvalFromInferredType(pair[1]) - output.mapval.PutReference(key, value) - } else { - lib.InternalCodingErrorIf(true) - } - } - return output -} - -// ---------------------------------------------------------------- -// splitkvx("a=3,b=4,c=5", "=", ",") -> {"a":"3","b":"4","c":"5"} -func BIF_splitkvx(input1, input2, input3 *Mlrval) *Mlrval { - if !input1.IsStringOrVoid() { - return MLRVAL_ERROR - } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR - } - pairSeparator := input2.printrep - if input3.mvtype != MT_STRING { - return MLRVAL_ERROR - } - fieldSeparator := input3.printrep - - output := MlrvalFromEmptyMap() - - fields := lib.SplitString(input1.printrep, fieldSeparator) - for i, field := range fields { - pair := strings.SplitN(field, pairSeparator, 2) - if len(pair) == 1 { - key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up - value := MlrvalFromString(pair[0]) - output.mapval.PutReference(key, value) - } else if len(pair) == 2 { - key := pair[0] - value := MlrvalFromString(pair[1]) - output.mapval.PutReference(key, value) - } else { - lib.InternalCodingErrorIf(true) - } - } - - return output -} - -// ---------------------------------------------------------------- -// splitnv("a,b,c", ",") -> {"1":"a","2":"b","3":"c"} -func BIF_splitnv(input1, input2 *Mlrval) *Mlrval { - if !input1.IsStringOrVoid() { - return MLRVAL_ERROR - } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR - } - - output := MlrvalFromEmptyMap() - - fields := lib.SplitString(input1.printrep, input2.printrep) - for i, field := range fields { - key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up - value := MlrvalFromInferredType(field) - output.mapval.PutReference(key, value) - } - - return output -} - -// ---------------------------------------------------------------- -// splitnvx("3,4,5", ",") -> {"1":"3","2":"4","3":"5"} -func BIF_splitnvx(input1, input2 *Mlrval) *Mlrval { - if !input1.IsStringOrVoid() { - return MLRVAL_ERROR - } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR - } - - output := MlrvalFromEmptyMap() - - fields := lib.SplitString(input1.printrep, input2.printrep) - for i, field := range fields { - key := strconv.Itoa(i + 1) // Miller user-space indices are 1-up - value := MlrvalFromString(field) - output.mapval.PutReference(key, value) - } - - return output -} - -// ---------------------------------------------------------------- -// splita("3,4,5", ",") -> [3,4,5] -func BIF_splita(input1, input2 *Mlrval) *Mlrval { - if !input1.IsStringOrVoid() { - return MLRVAL_ERROR - } - if !input2.IsString() { - return MLRVAL_ERROR - } - fieldSeparator := input2.printrep - - fields := lib.SplitString(input1.printrep, fieldSeparator) - - output := NewSizedMlrvalArray(int(len(fields))) - - for i, field := range fields { - value := MlrvalFromInferredType(field) - output.arrayval[i] = *value - } - - return output -} - -// ---------------------------------------------------------------- -// BIF_splitax splits a string to an array, without type-inference: -// e.g. splitax("3,4,5", ",") -> ["3","4","5"] -func BIF_splitax(input1, input2 *Mlrval) *Mlrval { - if !input1.IsStringOrVoid() { - return MLRVAL_ERROR - } - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR - } - input := input1.printrep - fieldSeparator := input2.printrep - - return mlrvalSplitAXHelper(input, fieldSeparator) -} - -// mlrvalSplitAXHelper is Split out for the benefit of BIF_splitax and -// BIF_unflatten. -func mlrvalSplitAXHelper(input string, separator string) *Mlrval { - fields := lib.SplitString(input, separator) - - output := NewSizedMlrvalArray(int(len(fields))) - - for i, field := range fields { - output.arrayval[i] = *MlrvalFromString(field) - } - - return output -} - -// ---------------------------------------------------------------- -func BIF_get_keys(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_MAP { - // TODO: make a ReferenceFrom with comments - output := NewSizedMlrvalArray(input1.mapval.FieldCount) - i := 0 - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { - output.arrayval[i] = *MlrvalFromString(pe.Key) - i++ - } - return output - - } else if input1.mvtype == MT_ARRAY { - output := NewSizedMlrvalArray(int(len(input1.arrayval))) - for i := range input1.arrayval { - output.arrayval[i] = *MlrvalFromInt(int(i + 1)) // Miller user-space indices are 1-up - } - return output - - } else { - return MLRVAL_ERROR - } -} - -// ---------------------------------------------------------------- -func BIF_get_values(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_MAP { - // TODO: make a ReferenceFrom with commenbs - output := NewSizedMlrvalArray(input1.mapval.FieldCount) - i := 0 - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { - output.arrayval[i] = *pe.Value.Copy() - i++ - } - return output - - } else if input1.mvtype == MT_ARRAY { - output := NewSizedMlrvalArray(int(len(input1.arrayval))) - for i, value := range input1.arrayval { - output.arrayval[i] = *value.Copy() - } - return output - - } else { - return MLRVAL_ERROR - } -} - -// ---------------------------------------------------------------- -func BIF_append(input1, input2 *Mlrval) *Mlrval { - if input1.mvtype != MT_ARRAY { - return MLRVAL_ERROR - } - - output := input1.Copy() - output.ArrayAppend(input2.Copy()) - return output -} - -// ---------------------------------------------------------------- -// First argumemnt is prefix. -// Second argument is delimiter. -// Third argument is map or array. -// flatten("a", ".", {"b": { "c": 4 }}) is {"a.b.c" : 4}. -// flatten("", ".", {"a": { "b": 3 }}) is {"a.b" : 3}. -func BIF_flatten(input1, input2, input3 *Mlrval) *Mlrval { - if input3.mvtype == MT_MAP || input3.mvtype == MT_ARRAY { - if input1.mvtype != MT_STRING && input1.mvtype != MT_VOID { - return MLRVAL_ERROR - } - prefix := input1.printrep - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR - } - delimiter := input2.printrep - - retval := input3.FlattenToMap(prefix, delimiter) - return &retval - } else { - return input3 - } -} - -// flatten($*, ".") is the same as flatten("", ".", $*) -func BIF_flatten_binary(input1, input2 *Mlrval) *Mlrval { - return BIF_flatten(MLRVAL_VOID, input2, input1) -} - -// ---------------------------------------------------------------- -// First argument is a map. -// Second argument is a delimiter string. -// unflatten({"a.b.c", ".") is {"a": { "b": { "c": 4}}}. -func BIF_unflatten(input1, input2 *Mlrval) *Mlrval { - if input2.mvtype != MT_STRING { - return MLRVAL_ERROR - } - if input1.mvtype != MT_MAP { - return input1 - } - oldmap := input1.mapval - separator := input2.printrep - newmap := oldmap.CopyUnflattened(separator) - return MlrvalFromMapReferenced(newmap) -} - -// ---------------------------------------------------------------- -// Converts maps with "1", "2", ... keys into arrays. Recurses nested data structures. -func BIF_arrayify(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_MAP { - if input1.mapval.IsEmpty() { - return input1 - } - - convertible := true - i := 0 - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { - sval := strconv.Itoa(i + 1) // Miller user-space indices are 1-up - i++ - if pe.Key != sval { - convertible = false - } - pe.Value = BIF_arrayify(pe.Value) - } - - if convertible { - arrayval := make([]Mlrval, input1.mapval.FieldCount) - i := 0 - for pe := input1.mapval.Head; pe != nil; pe = pe.Next { - arrayval[i] = *pe.Value.Copy() - i++ - } - return MlrvalFromArrayReference(arrayval) - - } else { - return input1 - } - - } else if input1.mvtype == MT_ARRAY { - // TODO: comment (or rethink) that this modifies its inputs!! - output := input1.Copy() - for i := range input1.arrayval { - output.arrayval[i] = *BIF_arrayify(&output.arrayval[i]) - } - return output - - } else { - return input1 - } -} - -// ---------------------------------------------------------------- -func BIF_json_parse(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_VOID { - return input1 - } else if input1.mvtype != MT_STRING { - return MLRVAL_ERROR - } else { - output := MlrvalFromPending() - err := output.UnmarshalJSON([]byte(input1.printrep)) - if err != nil { - return MLRVAL_ERROR - } - return &output - } -} - -func BIF_json_stringify_unary(input1 *Mlrval) *Mlrval { - outputBytes, err := input1.MarshalJSON(JSON_SINGLE_LINE, false) - if err != nil { - return MLRVAL_ERROR - } else { - return MlrvalFromString(string(outputBytes)) - } -} - -func BIF_json_stringify_binary(input1, input2 *Mlrval) *Mlrval { - var jsonFormatting TJSONFormatting = JSON_SINGLE_LINE - useMultiline, ok := input2.GetBoolValue() - if !ok { - return MLRVAL_ERROR - } - if useMultiline { - jsonFormatting = JSON_MULTILINE - } - - outputBytes, err := input1.MarshalJSON(jsonFormatting, false) - if err != nil { - return MLRVAL_ERROR - } else { - return MlrvalFromString(string(outputBytes)) - } -} diff --git a/internal/pkg/types/mlrval_functions_math.go b/internal/pkg/types/mlrval_functions_math.go deleted file mode 100644 index 2a291508e..000000000 --- a/internal/pkg/types/mlrval_functions_math.go +++ /dev/null @@ -1,21 +0,0 @@ -package types - -// Return error (unary math-library func) -func _math_unary_erro1(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval { - return MLRVAL_ERROR -} - -// Return absent (unary math-library func) -func _math_unary_absn1(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval { - return MLRVAL_ABSENT -} - -// Return null (unary math-library func) -func _math_unary_null1(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval { - return MLRVAL_NULL -} - -// Return void (unary math-library func) -func _math_unary_void1(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval { - return MLRVAL_VOID -} diff --git a/internal/pkg/types/mlrval_functions_mathlib.go b/internal/pkg/types/mlrval_functions_mathlib.go deleted file mode 100644 index 527528200..000000000 --- a/internal/pkg/types/mlrval_functions_mathlib.go +++ /dev/null @@ -1,225 +0,0 @@ -// ================================================================ -// Go math-library functions -// ================================================================ - -package types - -import ( - "math" - - "github.com/johnkerl/miller/internal/pkg/lib" -) - -// ---------------------------------------------------------------- -func math_unary_f_i(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval { - return MlrvalFromFloat64(f(float64(input1.intval))) -} -func math_unary_i_i(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval { - return MlrvalFromInt(int(f(float64(input1.intval)))) -} -func math_unary_f_f(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval { - return MlrvalFromFloat64(f(input1.floatval)) -} - -// Disposition vector for unary mathlib functions -var mudispo = [MT_DIM]mathLibUnaryFuncWrapper{ - /*ERROR */ _math_unary_erro1, - /*ABSENT */ _math_unary_absn1, - /*NULL */ _math_unary_null1, - /*VOID */ _math_unary_void1, - /*STRING */ _math_unary_erro1, - /*INT */ math_unary_f_i, - /*FLOAT */ math_unary_f_f, - /*BOOL */ _math_unary_erro1, - /*ARRAY */ _math_unary_absn1, - /*MAP */ _math_unary_absn1, - /*FUNC */ _math_unary_erro1, -} - -func BIF_acos(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Acos) } -func BIF_acosh(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Acosh) } -func BIF_asin(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Asin) } -func BIF_asinh(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Asinh) } -func BIF_atan(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Atan) } -func BIF_atanh(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Atanh) } -func BIF_cbrt(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Cbrt) } -func BIF_cos(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Cos) } -func BIF_cosh(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Cosh) } -func BIF_erf(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Erf) } -func BIF_erfc(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Erfc) } -func BIF_exp(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Exp) } -func BIF_expm1(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Expm1) } -func BIF_invqnorm(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, lib.Invqnorm) } -func BIF_log(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Log) } -func BIF_log10(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Log10) } -func BIF_log1p(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Log1p) } -func BIF_qnorm(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, lib.Qnorm) } -func BIF_sin(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Sin) } -func BIF_sinh(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Sinh) } -func BIF_sqrt(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Sqrt) } -func BIF_tan(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Tan) } -func BIF_tanh(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Tanh) } - -// Disposition vector for unary mathlib functions which are int-preserving -var imudispo = [MT_DIM]mathLibUnaryFuncWrapper{ - /*ERROR */ _math_unary_erro1, - /*ABSENT */ _math_unary_absn1, - /*NULL */ _math_unary_null1, - /*VOID */ _math_unary_void1, - /*STRING */ _math_unary_erro1, - /*INT */ math_unary_i_i, - /*FLOAT */ math_unary_f_f, - /*BOOL */ _math_unary_erro1, - /*ARRAY */ _math_unary_absn1, - /*MAP */ _math_unary_absn1, - /*FUNC */ _math_unary_erro1, -} - -// Int-preserving -func BIF_abs(input1 *Mlrval) *Mlrval { return imudispo[input1.mvtype](input1, math.Abs) } // xxx -func BIF_ceil(input1 *Mlrval) *Mlrval { return imudispo[input1.mvtype](input1, math.Ceil) } // xxx -func BIF_floor(input1 *Mlrval) *Mlrval { return imudispo[input1.mvtype](input1, math.Floor) } // xxx -func BIF_round(input1 *Mlrval) *Mlrval { return imudispo[input1.mvtype](input1, math.Round) } // xxx -func BIF_sgn(input1 *Mlrval) *Mlrval { return imudispo[input1.mvtype](input1, lib.Sgn) } // xxx - -// ================================================================ -// Exponentiation: DSL operator '**'. See also -// https://johnkerl.org/miller6/reference-main-arithmetic.html - -func pow_f_ii(input1, input2 *Mlrval) *Mlrval { - foutput := math.Pow(float64(input1.intval), float64(input2.intval)) - ioutput := int(foutput) - // Int raised to int power should be float if it can be (i.e. unless overflow) - if float64(ioutput) == foutput { - return MlrvalFromInt(ioutput) - } else { - return MlrvalFromFloat64(foutput) - } -} -func pow_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Pow(float64(input1.intval), input2.floatval)) -} -func pow_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Pow(input1.floatval, float64(input2.intval))) -} -func pow_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Pow(input1.floatval, input2.floatval)) -} - -var pow_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ - // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC - /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, - /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _i0__, _f0__, _erro, _absn, _absn, _erro}, - /*NULL */ {_erro, _absn, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, - /*VOID */ {_erro, _absn, _erro, _void, _erro, _void, _void, _erro, _absn, _absn, _erro}, - /*STRING */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, - /*INT */ {_erro, _1___, _erro, _void, _erro, pow_f_ii, pow_f_if, _erro, _absn, _absn, _erro}, - /*FLOAT */ {_erro, _1___, _erro, _void, _erro, pow_f_fi, pow_f_ff, _erro, _absn, _absn, _erro}, - /*BOOL */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, - /*ARRAY */ {_absn, _absn, _erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, - /*MAP */ {_absn, _absn, _erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, - /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, -} - -func BIF_pow(input1, input2 *Mlrval) *Mlrval { - return pow_dispositions[input1.mvtype][input2.mvtype](input1, input2) -} - -// ================================================================ -func atan2_f_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Atan2(float64(input1.intval), float64(input2.intval))) -} -func atan2_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Atan2(float64(input1.intval), input2.floatval)) -} -func atan2_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Atan2(input1.floatval, float64(input2.intval))) -} -func atan2_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(math.Atan2(input1.floatval, input2.floatval)) -} - -var atan2_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ - // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC - /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, - /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _i0__, _f0__, _erro, _absn, _absn, _erro}, - /*NULL */ {_erro, _absn, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, - /*VOID */ {_erro, _absn, _erro, _void, _erro, _void, _void, _erro, _absn, _absn, _erro}, - /*STRING */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, - /*INT */ {_erro, _1___, _erro, _void, _erro, atan2_f_ii, atan2_f_if, _erro, _absn, _absn, _erro}, - /*FLOAT */ {_erro, _1___, _erro, _void, _erro, atan2_f_fi, atan2_f_ff, _erro, _absn, _absn, _erro}, - /*BOOL */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, - /*ARRAY */ {_absn, _absn, _erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, - /*MAP */ {_absn, _absn, _erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, - /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, -} - -func BIF_atan2(input1, input2 *Mlrval) *Mlrval { - return atan2_dispositions[input1.mvtype][input2.mvtype](input1, input2) -} - -// ================================================================ -func mlr_roundm(x, m float64) float64 { - return math.Round(x/m) * m -} - -func roundm_f_ii(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromInt(int(mlr_roundm(float64(input1.intval), float64(input2.intval)))) -} -func roundm_f_if(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(mlr_roundm(float64(input1.intval), input2.floatval)) -} -func roundm_f_fi(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(mlr_roundm(input1.floatval, float64(input2.intval))) -} -func roundm_f_ff(input1, input2 *Mlrval) *Mlrval { - return MlrvalFromFloat64(mlr_roundm(input1.floatval, input2.floatval)) -} - -var roundm_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ - // . ERROR ABSENT NULL VOID STRING INT FLOAT BOOL ARRAY MAP FUNC - /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, - /*ABSENT */ {_erro, _absn, _absn, _absn, _erro, _i0__, _f0__, _erro, _absn, _absn, _erro}, - /*NULL */ {_erro, _absn, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, - /*VOID */ {_erro, _absn, _erro, _void, _erro, _void, _void, _erro, _absn, _absn, _erro}, - /*STRING */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, - /*INT */ {_erro, _1___, _erro, _void, _erro, roundm_f_ii, roundm_f_if, _erro, _absn, _absn, _erro}, - /*FLOAT */ {_erro, _1___, _erro, _void, _erro, roundm_f_fi, roundm_f_ff, _erro, _absn, _absn, _erro}, - /*BOOL */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _absn, _absn, _erro}, - /*ARRAY */ {_absn, _absn, _erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, - /*MAP */ {_absn, _absn, _erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _erro}, - /*FUNC */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, -} - -func BIF_roundm(input1, input2 *Mlrval) *Mlrval { - return roundm_dispositions[input1.mvtype][input2.mvtype](input1, input2) -} - -// ================================================================ -func BIF_logifit(input1, input2, input3 *Mlrval) *Mlrval { - if !input1.IsLegit() { - return input1 - } - if !input2.IsLegit() { - return input2 - } - if !input3.IsLegit() { - return input3 - } - - // int/float OK; rest not - x, xok := input1.GetNumericToFloatValue() - if !xok { - return MLRVAL_ERROR - } - m, mok := input2.GetNumericToFloatValue() - if !mok { - return MLRVAL_ERROR - } - b, bok := input3.GetNumericToFloatValue() - if !bok { - return MLRVAL_ERROR - } - - return MlrvalFromFloat64(1.0 / (1.0 + math.Exp(-m*x-b))) -} diff --git a/internal/pkg/types/mlrval_functions_relative_time.go b/internal/pkg/types/mlrval_functions_relative_time.go deleted file mode 100644 index fdf014940..000000000 --- a/internal/pkg/types/mlrval_functions_relative_time.go +++ /dev/null @@ -1,324 +0,0 @@ -package types - -import ( - "fmt" - "math" - "strings" -) - -func BIF_dhms2sec(input1 *Mlrval) *Mlrval { - if input1.mvtype != MT_STRING { - return MLRVAL_ERROR - } - var d, h, m, s int - - if strings.HasPrefix(input1.printrep, "-") { - - n, err := fmt.Sscanf(input1.printrep, "-%dd%dh%dm%ds", &d, &h, &m, &s) - if n == 4 && err == nil { - return MlrvalFromInt(-(s + m*60 + h*60*60 + d*60*60*24)) - } - n, err = fmt.Sscanf(input1.printrep, "-%dh%dm%ds", &h, &m, &s) - if n == 3 && err == nil { - return MlrvalFromInt(-(s + m*60 + h*60*60)) - } - n, err = fmt.Sscanf(input1.printrep, "-%dm%ds", &m, &s) - if n == 2 && err == nil { - return MlrvalFromInt(-(s + m*60)) - } - n, err = fmt.Sscanf(input1.printrep, "-%ds", &s) - if n == 1 && err == nil { - return MlrvalFromInt(-(s)) - } - - } else { - - n, err := fmt.Sscanf(input1.printrep, "%dd%dh%dm%ds", &d, &h, &m, &s) - if n == 4 && err == nil { - return MlrvalFromInt(s + m*60 + h*60*60 + d*60*60*24) - } - n, err = fmt.Sscanf(input1.printrep, "%dh%dm%ds", &h, &m, &s) - if n == 3 && err == nil { - return MlrvalFromInt(s + m*60 + h*60*60) - } - n, err = fmt.Sscanf(input1.printrep, "%dm%ds", &m, &s) - if n == 2 && err == nil { - return MlrvalFromInt(s + m*60) - } - n, err = fmt.Sscanf(input1.printrep, "%ds", &s) - if n == 1 && err == nil { - return MlrvalFromInt(s) - } - - } - return MLRVAL_ERROR -} - -func BIF_dhms2fsec(input1 *Mlrval) *Mlrval { - if input1.mvtype != MT_STRING { - return MLRVAL_ERROR - } - - var d, h, m int - var s float64 - - if strings.HasPrefix(input1.printrep, "-") { - - n, err := fmt.Sscanf(input1.printrep, "-%dd%dh%dm%fs", &d, &h, &m, &s) - if n == 4 && err == nil { - return MlrvalFromFloat64(-(s + float64(m*60+h*60*60+d*60*60*24))) - } - n, err = fmt.Sscanf(input1.printrep, "-%dh%dm%fs", &h, &m, &s) - if n == 3 && err == nil { - return MlrvalFromFloat64(-(s + float64(m*60+h*60*60))) - } - n, err = fmt.Sscanf(input1.printrep, "-%dm%fs", &m, &s) - if n == 2 && err == nil { - return MlrvalFromFloat64(-(s + float64(m*60))) - } - n, err = fmt.Sscanf(input1.printrep, "-%fs", &s) - if n == 1 && err == nil { - return MlrvalFromFloat64(-(s)) - } - - } else { - - n, err := fmt.Sscanf(input1.printrep, "%dd%dh%dm%fs", &d, &h, &m, &s) - if n == 4 && err == nil { - return MlrvalFromFloat64(s + float64(m*60+h*60*60+d*60*60*24)) - } - n, err = fmt.Sscanf(input1.printrep, "%dh%dm%fs", &h, &m, &s) - if n == 3 && err == nil { - return MlrvalFromFloat64(s + float64(m*60+h*60*60)) - } - n, err = fmt.Sscanf(input1.printrep, "%dm%fs", &m, &s) - if n == 2 && err == nil { - return MlrvalFromFloat64(s + float64(m*60)) - } - n, err = fmt.Sscanf(input1.printrep, "%fs", &s) - if n == 1 && err == nil { - return MlrvalFromFloat64(s) - } - - } - return MLRVAL_ERROR -} - -func BIF_hms2sec(input1 *Mlrval) *Mlrval { - if input1.mvtype != MT_STRING { - return MLRVAL_ERROR - } - if input1.printrep == "" { - return MLRVAL_ERROR - } - var h, m, s int - - if strings.HasPrefix(input1.printrep, "-") { - n, err := fmt.Sscanf(input1.printrep, "-%d:%d:%d", &h, &m, &s) - if n == 3 && err == nil { - return MlrvalFromInt(-(s + m*60 + h*60*60)) - } - } else { - n, err := fmt.Sscanf(input1.printrep, "%d:%d:%d", &h, &m, &s) - if n == 3 && err == nil { - return MlrvalFromInt(s + m*60 + h*60*60) - } - } - - return MLRVAL_ERROR -} - -func BIF_hms2fsec(input1 *Mlrval) *Mlrval { - if input1.mvtype != MT_STRING { - return MLRVAL_ERROR - } - - var h, m int - var s float64 - - if strings.HasPrefix(input1.printrep, "-") { - n, err := fmt.Sscanf(input1.printrep, "-%d:%d:%f", &h, &m, &s) - if n == 3 && err == nil { - return MlrvalFromFloat64(-(s + float64(m*60+h*60*60))) - } - } else { - n, err := fmt.Sscanf(input1.printrep, "%d:%d:%f", &h, &m, &s) - if n == 3 && err == nil { - return MlrvalFromFloat64(s + float64(m*60+h*60*60)) - } - } - - return MLRVAL_ERROR -} - -func BIF_sec2dhms(input1 *Mlrval) *Mlrval { - isec, ok := input1.GetIntValue() - if !ok { - return MLRVAL_ERROR - } - - var d, h, m, s int - - splitIntToDHMS(isec, &d, &h, &m, &s) - if d != 0 { - return MlrvalFromString( - fmt.Sprintf("%dd%02dh%02dm%02ds", d, h, m, s), - ) - } else if h != 0 { - return MlrvalFromString( - fmt.Sprintf("%dh%02dm%02ds", h, m, s), - ) - } else if m != 0 { - return MlrvalFromString( - fmt.Sprintf("%dm%02ds", m, s), - ) - } else { - return MlrvalFromString( - fmt.Sprintf("%ds", s), - ) - } - - return MLRVAL_ERROR -} - -func BIF_sec2hms(input1 *Mlrval) *Mlrval { - isec, ok := input1.GetIntValue() - if !ok { - return MLRVAL_ERROR - } - sign := "" - if isec < 0 { - sign = "-" - isec = -isec - } - - var d, h, m, s int - - splitIntToDHMS(isec, &d, &h, &m, &s) - h += d * 24 - - return MlrvalFromString( - fmt.Sprintf("%s%02d:%02d:%02d", sign, h, m, s), - ) - - return MLRVAL_ERROR -} - -func BIF_fsec2dhms(input1 *Mlrval) *Mlrval { - fsec, ok := input1.GetNumericToFloatValue() - if !ok { - return MLRVAL_ERROR - } - - sign := 1 - if fsec < 0 { - sign = -1 - fsec = -fsec - } - isec := int(math.Trunc(fsec)) - fractional := fsec - float64(isec) - - var d, h, m, s int - - splitIntToDHMS(isec, &d, &h, &m, &s) - - if d != 0 { - d = sign * d - return MlrvalFromString( - fmt.Sprintf( - "%dd%02dh%02dm%09.6fs", - d, h, m, float64(s)+fractional), - ) - } else if h != 0 { - h = sign * h - return MlrvalFromString( - fmt.Sprintf( - "%dh%02dm%09.6fs", - h, m, float64(s)+fractional), - ) - } else if m != 0 { - m = sign * m - return MlrvalFromString( - fmt.Sprintf( - "%dm%09.6fs", - m, float64(s)+fractional), - ) - } else { - s = sign * s - fractional = float64(sign) * fractional - return MlrvalFromString( - fmt.Sprintf( - "%.6fs", - float64(s)+fractional), - ) - } -} - -func BIF_fsec2hms(input1 *Mlrval) *Mlrval { - fsec, ok := input1.GetNumericToFloatValue() - if !ok { - return MLRVAL_ERROR - } - - sign := "" - if fsec < 0 { - sign = "-" - fsec = -fsec - } - isec := int(math.Trunc(fsec)) - fractional := fsec - float64(isec) - - var d, h, m, s int - - splitIntToDHMS(isec, &d, &h, &m, &s) - h += d * 24 - - // "%02.6f" does not exist so we have to do our own zero-pad - if s < 10 { - return MlrvalFromString( - fmt.Sprintf("%s%02d:%02d:0%.6f", sign, h, m, float64(s)+fractional), - ) - } else { - return MlrvalFromString( - fmt.Sprintf("%s%02d:%02d:%.6f", sign, h, m, float64(s)+fractional), - ) - } - - return MLRVAL_ERROR -} - -// Helper function -func splitIntToDHMS(u int, pd, ph, pm, ps *int) { - d := 0 - h := 0 - m := 0 - s := 0 - sign := 1 - if u < 0 { - u = -u - sign = -1 - } - s = u % 60 - u = u / 60 - if u == 0 { - s = s * sign - } else { - m = u % 60 - u = u / 60 - if u == 0 { - m = m * sign - } else { - h = u % 24 - u = u / 24 - if u == 0 { - h = h * sign - } else { - d = u * sign - } - } - } - *pd = d - *ph = h - *pm = m - *ps = s -} diff --git a/internal/pkg/types/mlrval_functions_system.go b/internal/pkg/types/mlrval_functions_system.go deleted file mode 100644 index 26a857ddb..000000000 --- a/internal/pkg/types/mlrval_functions_system.go +++ /dev/null @@ -1,45 +0,0 @@ -package types - -import ( - "os" - "os/exec" - "runtime" - "strings" - - "github.com/johnkerl/miller/internal/pkg/platform" - "github.com/johnkerl/miller/internal/pkg/version" -) - -func BIF_version() *Mlrval { - return MlrvalFromString(version.STRING) -} - -func BIF_os() *Mlrval { - return MlrvalFromString(runtime.GOOS) -} - -func BIF_hostname() *Mlrval { - hostname, err := os.Hostname() - if err != nil { - return MLRVAL_ERROR - } else { - return MlrvalFromString(hostname) - } -} - -func BIF_system(input1 *Mlrval) *Mlrval { - if !input1.IsStringOrVoid() { - return MLRVAL_ERROR - } - commandString := input1.printrep - - shellRunArray := platform.GetShellRunArray(commandString) - - outputBytes, err := exec.Command(shellRunArray[0], shellRunArray[1:]...).Output() - if err != nil { - return MLRVAL_ERROR - } - outputString := strings.TrimRight(string(outputBytes), "\n") - - return MlrvalFromString(outputString) -} diff --git a/internal/pkg/types/mlrval_functions_types.go b/internal/pkg/types/mlrval_functions_types.go deleted file mode 100644 index c06d06d44..000000000 --- a/internal/pkg/types/mlrval_functions_types.go +++ /dev/null @@ -1,286 +0,0 @@ -package types - -import ( - "fmt" - "os" - - "github.com/johnkerl/miller/internal/pkg/lib" -) - -// ================================================================ -func BIF_typeof(input1 *Mlrval) *Mlrval { - return MlrvalFromString(input1.GetTypeName()) -} - -// ---------------------------------------------------------------- -func string_to_int(input1 *Mlrval) *Mlrval { - i, ok := lib.TryIntFromString(input1.printrep) - if ok { - return MlrvalFromInt(i) - } else { - return MLRVAL_ERROR - } -} - -func float_to_int(input1 *Mlrval) *Mlrval { - return MlrvalFromInt(int(input1.floatval)) -} - -func bool_to_int(input1 *Mlrval) *Mlrval { - if input1.boolval == true { - return MlrvalFromInt(1) - } else { - return MlrvalFromInt(0) - } -} - -var to_int_dispositions = [MT_DIM]UnaryFunc{ - /*ERROR */ _erro1, - /*ABSENT */ _absn1, - /*NULL */ _null1, - /*VOID */ _void1, - /*STRING */ string_to_int, - /*INT */ _1u___, - /*FLOAT */ float_to_int, - /*BOOL */ bool_to_int, - /*ARRAY */ _erro1, - /*MAP */ _erro1, - /*FUNC */ _erro1, -} - -func BIF_int(input1 *Mlrval) *Mlrval { - return to_int_dispositions[input1.mvtype](input1) -} - -// ---------------------------------------------------------------- -func string_to_float(input1 *Mlrval) *Mlrval { - f, ok := lib.TryFloat64FromString(input1.printrep) - if ok { - return MlrvalFromFloat64(f) - } else { - return MLRVAL_ERROR - } -} - -func int_to_float(input1 *Mlrval) *Mlrval { - return MlrvalFromFloat64(float64(input1.intval)) -} - -func bool_to_float(input1 *Mlrval) *Mlrval { - if input1.boolval == true { - return MlrvalFromFloat64(1.0) - } else { - return MlrvalFromFloat64(0.0) - } -} - -var to_float_dispositions = [MT_DIM]UnaryFunc{ - /*ERROR */ _erro1, - /*ABSENT */ _absn1, - /*NULL */ _null1, - /*VOID */ _void1, - /*STRING */ string_to_float, - /*INT */ int_to_float, - /*FLOAT */ _1u___, - /*BOOL */ bool_to_float, - /*ARRAY */ _erro1, - /*MAP */ _erro1, - /*FUNC */ _erro1, -} - -func BIF_float(input1 *Mlrval) *Mlrval { - return to_float_dispositions[input1.mvtype](input1) -} - -// ---------------------------------------------------------------- -func string_to_boolean(input1 *Mlrval) *Mlrval { - b, ok := lib.TryBoolFromBoolString(input1.printrep) - if ok { - return MlrvalFromBool(b) - } else { - return MLRVAL_ERROR - } -} - -func int_to_bool(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.intval != 0) -} - -func float_to_bool(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.floatval != 0.0) -} - -var to_boolean_dispositions = [MT_DIM]UnaryFunc{ - /*ERROR */ _erro1, - /*ABSENT */ _absn1, - /*NULL */ _null1, - /*VOID */ _void1, - /*STRING */ string_to_boolean, - /*INT */ int_to_bool, - /*FLOAT */ float_to_bool, - /*BOOL */ _1u___, - /*ARRAY */ _erro1, - /*MAP */ _erro1, - /*FUNC */ _erro1, -} - -func BIF_boolean(input1 *Mlrval) *Mlrval { - return to_boolean_dispositions[input1.mvtype](input1) -} - -// ---------------------------------------------------------------- -func BIF_is_absent(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_ABSENT) -} -func BIF_is_error(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_ERROR) -} -func BIF_is_bool(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_BOOL) -} -func BIF_is_boolean(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_BOOL) -} -func BIF_is_empty(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_VOID { - return MLRVAL_TRUE - } else if input1.mvtype == MT_STRING { - if input1.printrep == "" { - return MLRVAL_TRUE - } else { - return MLRVAL_FALSE - } - } else { - return MLRVAL_FALSE - } -} -func BIF_is_emptymap(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_MAP && input1.mapval.IsEmpty()) -} -func BIF_is_float(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_FLOAT) -} -func BIF_is_int(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_INT) -} -func BIF_is_map(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_MAP) -} -func BIF_is_array(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_ARRAY) -} -func BIF_is_nonemptymap(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_MAP && input1.mapval.FieldCount != 0) -} -func BIF_is_notempty(input1 *Mlrval) *Mlrval { - if input1.mvtype == MT_VOID { - return MLRVAL_FALSE - } else if input1.mvtype == MT_STRING { - if input1.printrep == "" { - return MLRVAL_FALSE - } else { - return MLRVAL_TRUE - } - } else { - return MLRVAL_TRUE - } -} -func BIF_is_notmap(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype != MT_MAP) -} -func BIF_is_notarray(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype != MT_ARRAY) -} -func BIF_is_notnull(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype != MT_ABSENT && input1.mvtype != MT_VOID) -} -func BIF_is_null(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_ABSENT || input1.mvtype == MT_VOID) -} -func BIF_is_numeric(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_INT || input1.mvtype == MT_FLOAT) -} -func BIF_is_present(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype != MT_ABSENT) -} -func BIF_is_string(input1 *Mlrval) *Mlrval { - return MlrvalFromBool(input1.mvtype == MT_STRING || input1.mvtype == MT_VOID) -} - -// ---------------------------------------------------------------- -func assertingCommon(input1, check *Mlrval, description string, context *Context) *Mlrval { - if check.IsFalse() { - // TODO: get context as in the C impl - //fprintf(stderr, "%s: %s type-assertion failed at NR=%lld FNR=%lld FILENAME=%s\n", - //MLR_GLOBALS.bargv0, pstate->desc, pvars->pctx->nr, pvars->pctx->fnr, pvars->pctx->filename); - //exit(1); - fmt.Fprintf( - os.Stderr, - "mlr: %s type-assertion failed at NR=%d FNR=%d FILENAME=%s\n", - description, - context.NR, - context.FNR, - context.FILENAME, - ) - os.Exit(1) - } - return input1 -} - -func BIF_asserting_absent(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_absent(input1), "is_absent", context) -} -func BIF_asserting_error(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_error(input1), "is_error", context) -} -func BIF_asserting_bool(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_bool(input1), "is_bool", context) -} -func BIF_asserting_boolean(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_boolean(input1), "is_boolean", context) -} -func BIF_asserting_empty(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_empty(input1), "is_empty", context) -} -func BIF_asserting_emptyMap(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_emptymap(input1), "is_empty_map", context) -} -func BIF_asserting_float(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_float(input1), "is_float", context) -} -func BIF_asserting_int(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_int(input1), "is_int", context) -} -func BIF_asserting_map(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_map(input1), "is_map", context) -} -func BIF_asserting_array(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_array(input1), "is_array", context) -} -func BIF_asserting_nonempty_map(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_nonemptymap(input1), "is_non_empty_map", context) -} -func BIF_asserting_not_empty(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_notempty(input1), "is_not_empty", context) -} -func BIF_asserting_not_map(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_notmap(input1), "is_not_map", context) -} -func BIF_asserting_not_array(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_notarray(input1), "is_not_array", context) -} -func BIF_asserting_not_null(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_notnull(input1), "is_not_null", context) -} -func BIF_asserting_null(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_null(input1), "is_null", context) -} -func BIF_asserting_numeric(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_numeric(input1), "is_numeric", context) -} -func BIF_asserting_present(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_present(input1), "is_present", context) -} -func BIF_asserting_string(input1 *Mlrval, context *Context) *Mlrval { - return assertingCommon(input1, BIF_is_string(input1), "is_string", context) -} diff --git a/internal/pkg/types/mlrval_infer.go b/internal/pkg/types/mlrval_infer.go deleted file mode 100644 index 171a6bf06..000000000 --- a/internal/pkg/types/mlrval_infer.go +++ /dev/null @@ -1,198 +0,0 @@ -// ================================================================ -// Constructors -// ================================================================ - -package types - -import ( - "regexp" - "strings" - - "github.com/johnkerl/miller/internal/pkg/lib" -) - -// ---------------------------------------------------------------- - -// MlrvalFromPrevalidatedIntString is for situations where the string has -// already been determined to be parseable as int. For example, int literals in -// the Miller DSL fall into this category, as the parser has already matched -// them. -func MlrvalFromPrevalidatedIntString(input string) *Mlrval { - ival, ok := lib.TryIntFromString(input) - lib.InternalCodingErrorIf(!ok) - return &Mlrval{ - mvtype: MT_INT, - printrep: input, - printrepValid: true, - intval: ival, - } -} - -// MlrvalFromPrevalidatedFloat64String is for situations where the string has -// already been determined to be parseable as int. For example, int literals in -// the Miller DSL fall into this category, as the parser has already matched -// them. -func MlrvalFromPrevalidatedFloat64String(input string) *Mlrval { - fval, ok := lib.TryFloat64FromString(input) - lib.InternalCodingErrorIf(!ok) - return &Mlrval{ - mvtype: MT_FLOAT, - printrep: input, - printrepValid: true, - floatval: fval, - } -} - -// ---------------------------------------------------------------- - -// MlrvalTryPointerFromFloatString is used by MlrvalFormatter (fmtnum DSL -// function, format-values verb, etc). Each mlrval has printrep and a -// printrepValid for its original string, then a type-code like MT_INT or -// MT_FLOAT, and type-specific storage like intval or floatval. -// -// If the user has taken a mlrval with original string "3.14" and formatted it -// with "%.4f" then its printrep will be "3.1400" but its type should still be -// MT_FLOAT. -// -// If on the other hand the user has formatted the same mlrval with "[[%.4f]]" -// then its printrep will be "[[3.1400]]" and it will be MT_STRING. -// This function supports that. -func MlrvalTryPointerFromFloatString(input string) *Mlrval { - _, fok := lib.TryFloat64FromString(input) - if fok { - return MlrvalFromPrevalidatedFloat64String(input) - } else { - return MlrvalFromString(input) - } -} - -// MlrvalTryPointerFromIntString is used by MlrvalFormatter (fmtnum DSL -// function, format-values verb, etc). Each mlrval has printrep and a -// printrepValid for its original string, then a type-code like MT_INT or -// MT_FLOAT, and type-specific storage like intval or floatval. -// -// If the user has taken a mlrval with original string "314" and formatted it -// with "0x%04x" then its printrep will be "0x013a" but its type should still be -// MT_INT. -// -// If on the other hand the user has formatted the same mlrval with -// "[[%0x04x]]" then its printrep will be "[[0x013a]]" and it will be -// MT_STRING. This function supports that. -func MlrvalTryPointerFromIntString(input string) *Mlrval { - _, iok := lib.TryIntFromString(input) - if iok { - return MlrvalFromPrevalidatedIntString(input) - } else { - return MlrvalFromString(input) - } -} - -// ---------------------------------------------------------------- - -// When loading data files, don't scan these words into floats -- even though -// the Go library is willing to do so. -var downcasedFloatNamesToNotInfer = map[string]bool{ - "inf": true, - "+inf": true, - "-inf": true, - "infinity": true, - "+infinity": true, - "-infinity": true, - "nan": true, -} - -// ---------------------------------------------------------------- -type tInferrer func(input string, inferBool bool) *Mlrval - -var inferrer tInferrer = inferNormally - -func SetInferrerNoOctal() { - inferrer = inferWithOctalSuppress -} -func SetInferrerIntAsFloat() { - inferrer = inferWithIntAsFloat -} -func SetInferrerStringOnly() { - inferrer = inferStringOnly -} - -// MlrvalFromInferredTypeForDataFiles is for parsing field values directly from -// data files (except JSON, which is typed -- "true" and true are distinct). -// Mostly the same as MlrvalFromInferredType, except it doesn't auto-infer -// true/false to bool; don't auto-infer NaN/Inf to float; etc. -func MlrvalFromInferredTypeForDataFiles(input string) *Mlrval { - return inferrer(input, false) -} - -// MlrvalFromInferredType is for parsing field values not directly from data -// files. Mostly the same as MlrvalFromInferredTypeForDataFiles, except it -// auto-infers true/false to bool; don't auto-infer NaN/Inf to float; etc. -func MlrvalFromInferredType(input string) *Mlrval { - return inferrer(input, true) -} - -func inferNormally(input string, inferBool bool) *Mlrval { - if input == "" { - return MLRVAL_VOID - } - - _, iok := lib.TryIntFromString(input) - if iok { - return MlrvalFromPrevalidatedIntString(input) - } - - if downcasedFloatNamesToNotInfer[strings.ToLower(input)] == false { - _, fok := lib.TryFloat64FromString(input) - if fok { - return MlrvalFromPrevalidatedFloat64String(input) - } - } - - if inferBool { - _, bok := lib.TryBoolFromBoolString(input) - if bok { - return MlrvalFromBoolString(input) - } - } - - return MlrvalFromString(input) -} - -var octalDetector = regexp.MustCompile("^-?0[0-9]+") - -func inferWithOctalSuppress(input string, inferBool bool) *Mlrval { - output := inferNormally(input, inferBool) - if output.mvtype != MT_INT && output.mvtype != MT_FLOAT { - return output - } - - if octalDetector.MatchString(input) { - return MlrvalFromString(input) - } else { - return output - } - - return output -} - -func inferWithIntAsFloat(input string, inferBool bool) *Mlrval { - output := inferNormally(input, inferBool) - if output.mvtype == MT_INT { - return &Mlrval{ - mvtype: MT_FLOAT, - printrepValid: true, - printrep: input, - floatval: float64(output.intval), - } - } else { - return output - } -} - -func inferStringOnly(input string, inferBool bool) *Mlrval { - if input == "" { - return MLRVAL_VOID - } - - return MlrvalFromString(input) -} diff --git a/internal/pkg/types/mlrval_new.go b/internal/pkg/types/mlrval_new.go deleted file mode 100644 index 0ae2dc7ae..000000000 --- a/internal/pkg/types/mlrval_new.go +++ /dev/null @@ -1,199 +0,0 @@ -// ================================================================ -// Constructors -// ================================================================ - -package types - -import ( - "errors" - - "github.com/johnkerl/miller/internal/pkg/lib" -) - -func MlrvalFromString(input string) *Mlrval { - if input == "" { - return MLRVAL_VOID - } - return &Mlrval{ - mvtype: MT_STRING, - printrep: input, - printrepValid: true, - } -} - -func MlrvalFromInt(input int) *Mlrval { - return &Mlrval{ - mvtype: MT_INT, - printrepValid: false, - intval: input, - } -} - -func MlrvalFromFloat64(input float64) *Mlrval { - return &Mlrval{ - mvtype: MT_FLOAT, - printrepValid: false, - floatval: input, - } -} - -func MlrvalFromBool(input bool) *Mlrval { - if input == true { - return MLRVAL_TRUE - } else { - return MLRVAL_FALSE - } -} - -func MlrvalFromBoolString(input string) *Mlrval { - if input == "true" { - return MLRVAL_TRUE - } else if input == "false" { - return MLRVAL_FALSE - } else { - lib.InternalCodingErrorIf(true) - return MLRVAL_ERROR // not reached - } -} - -// The user-defined function is of type 'interface{}' here to avoid what would -// otherwise be a package-dependency cycle between this package and -// github.com/johnkerl/miller/internal/pkg/dsl/cst. -// -// Nominally the name argument is the user-specified name if `func f(a, b) { -// ... }`, or some autogenerated UUID like `fl0052` if `func (a, b) { ... }`. -func MlrvalFromFunction(funcval interface{}, name string) *Mlrval { - return &Mlrval{ - mvtype: MT_FUNC, - printrep: name, - printrepValid: true, - funcval: funcval, - } -} - -func MlrvalFromEmptyMap() *Mlrval { - return &Mlrval{ - mvtype: MT_MAP, - printrep: "(bug-if-you-see-this-map-type)", - printrepValid: false, - mapval: NewMlrmap(), - } -} - -func MlrvalFromMap(mlrmap *Mlrmap) *Mlrval { - mv := MlrvalFromEmptyMap() - if mlrmap == nil { - // TODO maybe return 2nd-arg error in the API - return MLRVAL_ERROR - } - - for pe := mlrmap.Head; pe != nil; pe = pe.Next { - mv.mapval.PutCopy(pe.Key, pe.Value) - } - return mv -} - -// Like previous but doesn't copy. Only safe when the argument's sole purpose -// is to be passed into here. -func MlrvalFromMapReferenced(mlrmap *Mlrmap) *Mlrval { - mv := MlrvalFromEmptyMap() - if mlrmap == nil { - // xxx maybe return 2nd-arg error in the API - return MLRVAL_ERROR - } - - for pe := mlrmap.Head; pe != nil; pe = pe.Next { - mv.mapval.PutReference(pe.Key, pe.Value) - } - return mv -} - -// Does not copy the data. We can make a MlrvalFromArrayLiteralCopy if needed, -// using values.CopyMlrvalArray(). -func MlrvalEmptyArray() Mlrval { - return Mlrval{ - mvtype: MT_ARRAY, - printrep: "(bug-if-you-see-this-array-type)", - printrepValid: false, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: make([]Mlrval, 0, 10), - mapval: nil, - } -} - -// Users can do things like '$new[1][2][3] = 4' even if '$new' isn't already -// allocated. This function supports that. -func NewSizedMlrvalArray(length int) *Mlrval { - arrayval := make([]Mlrval, length, 2*length) - - for i := 0; i < int(length); i++ { - arrayval[i] = *MLRVAL_VOID - } - - return &Mlrval{ - mvtype: MT_ARRAY, - printrep: "(bug-if-you-see-this-array-type)", - printrepValid: false, - intval: 0, - floatval: 0.0, - boolval: false, - arrayval: arrayval, - mapval: nil, - } -} - -// Does not copy the data. We can make a SetFromArrayLiteralCopy if needed -// using values.CopyMlrvalArray(). -func MlrvalFromArrayReference(input []Mlrval) *Mlrval { - return &Mlrval{ - mvtype: MT_ARRAY, - printrepValid: false, - arrayval: input, - } -} - -func LengthenMlrvalArray(array *[]Mlrval, newLength64 int) { - newLength := int(newLength64) - lib.InternalCodingErrorIf(newLength <= len(*array)) - - if newLength <= cap(*array) { - newArray := (*array)[:newLength] - for zindex := len(*array); zindex < newLength; zindex++ { - // TODO: comment why not MT_ABSENT or MT_VOID - newArray[zindex] = *MLRVAL_NULL - } - *array = newArray - } else { - newArray := make([]Mlrval, newLength, 2*newLength) - zindex := 0 - for zindex = 0; zindex < len(*array); zindex++ { - newArray[zindex] = (*array)[zindex] - } - for zindex = len(*array); zindex < newLength; zindex++ { - // TODO: comment why not MT_ABSENT or MT_VOID - newArray[zindex] = *MLRVAL_NULL - } - *array = newArray - } -} - -// NewMlrvalForAutoDeepen is for auto-deepen of nested maps in things like -// -// $foo[1]["a"][2]["b"] = 3 -// -// Autocreated levels are maps. Array levels can be explicitly created e.g. -// -// $foo[1]["a"] ??= [] -// $foo[1]["a"][2]["b"] = 3 -func NewMlrvalForAutoDeepen(mvtype MVType) (*Mlrval, error) { - if mvtype == MT_STRING || mvtype == MT_INT { - empty := MlrvalFromEmptyMap() - return empty, nil - } else { - return nil, errors.New( - "mlr: indices must be string, int, or array thereof; got " + GetTypeName(mvtype), - ) - } -} diff --git a/internal/pkg/types/mlrval_typing.go b/internal/pkg/types/mlrval_typing.go index a9ba8550e..334241c3c 100644 --- a/internal/pkg/types/mlrval_typing.go +++ b/internal/pkg/types/mlrval_typing.go @@ -8,6 +8,8 @@ package types import ( "errors" "fmt" + + "github.com/johnkerl/miller/internal/pkg/mlrval" ) // ---------------------------------------------------------------- @@ -21,7 +23,7 @@ func NewTypeGatedMlrvalName( name string, // e.g. "x" typeName string, // e.g. "num" ) (*TypeGatedMlrvalName, error) { - typeMask, ok := TypeNameToMask(typeName) + typeMask, ok := mlrval.TypeNameToMask(typeName) if !ok { return nil, errors.New( fmt.Sprintf( @@ -36,7 +38,7 @@ func NewTypeGatedMlrvalName( }, nil } -func (tname *TypeGatedMlrvalName) Check(value *Mlrval) error { +func (tname *TypeGatedMlrvalName) Check(value *mlrval.Mlrval) error { bit := value.GetTypeBit() if bit&tname.TypeMask != 0 { return nil @@ -53,13 +55,13 @@ func (tname *TypeGatedMlrvalName) Check(value *Mlrval) error { // ---------------------------------------------------------------- type TypeGatedMlrvalVariable struct { typeGatedMlrvalName *TypeGatedMlrvalName - value *Mlrval + value *mlrval.Mlrval } func NewTypeGatedMlrvalVariable( name string, // e.g. "x" typeName string, // e.g. "num" - value *Mlrval, + value *mlrval.Mlrval, ) (*TypeGatedMlrvalVariable, error) { typeGatedMlrvalName, err := NewTypeGatedMlrvalName(name, typeName) if err != nil { @@ -81,7 +83,7 @@ func (tvar *TypeGatedMlrvalVariable) GetName() string { return tvar.typeGatedMlrvalName.Name } -func (tvar *TypeGatedMlrvalVariable) GetValue() *Mlrval { +func (tvar *TypeGatedMlrvalVariable) GetValue() *mlrval.Mlrval { return tvar.value } @@ -89,7 +91,7 @@ func (tvar *TypeGatedMlrvalVariable) ValueString() string { return tvar.value.String() } -func (tvar *TypeGatedMlrvalVariable) Assign(value *Mlrval) error { +func (tvar *TypeGatedMlrvalVariable) Assign(value *mlrval.Mlrval) error { err := tvar.typeGatedMlrvalName.Check(value) if err != nil { return err @@ -101,5 +103,5 @@ func (tvar *TypeGatedMlrvalVariable) Assign(value *Mlrval) error { } func (tvar *TypeGatedMlrvalVariable) Unassign() { - tvar.value = MLRVAL_ABSENT + tvar.value = mlrval.ABSENT } diff --git a/scripts/chain-cmps.sh b/scripts/chain-cmps.sh index d818c8897..5b54e79fa 100755 --- a/scripts/chain-cmps.sh +++ b/scripts/chain-cmps.sh @@ -1,40 +1,40 @@ -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "./mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ check \ | md5sum done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "./mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ cat \ | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "./mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ head \ | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "./mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ tail \ | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "./mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ tac \ | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "./mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ sort -f shape \ | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "./mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ sort -n quantity \ | md5sum; diff --git a/scripts/chain-lengths.sh b/scripts/chain-lengths.sh index b6d2b6122..f059bec06 100755 --- a/scripts/chain-lengths.sh +++ b/scripts/chain-lengths.sh @@ -1,17 +1,17 @@ -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ then put -f scripts/chain-1.mlr \ | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ then put -f scripts/chain-1.mlr \ then put -f scripts/chain-1.mlr \ | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ then put -f scripts/chain-1.mlr \ then put -f scripts/chain-1.mlr \ @@ -19,7 +19,7 @@ echo; for m in mlr5 "./mlr -S"; do | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ then put -f scripts/chain-1.mlr \ then put -f scripts/chain-1.mlr \ @@ -28,7 +28,7 @@ echo; for m in mlr5 "./mlr -S"; do | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ then put -f scripts/chain-1.mlr \ then put -f scripts/chain-1.mlr \ @@ -38,7 +38,7 @@ echo; for m in mlr5 "./mlr -S"; do | md5sum; done -echo; for m in mlr5 "./mlr -S"; do +echo; for m in mlr5 ~/tmp/miller/mlr "mlr -S" mlr; do justtime $m --csv --from ~/tmp/big.csv \ then put -f scripts/chain-1.mlr \ then put -f scripts/chain-1.mlr \ diff --git a/scripts/time-big-files b/scripts/time-big-files index 3de09bf1c..e74f06922 100755 --- a/scripts/time-big-files +++ b/scripts/time-big-files @@ -3,24 +3,36 @@ ourdir=$(dirname $0) $ourdir/time-big-file dkvp mlr5 +$ourdir/time-big-file dkvp ~/tmp/miller/mlr $ourdir/time-big-file dkvp 'mlr -S' +$ourdir/time-big-file dkvp mlr echo $ourdir/time-big-file nidx mlr5 +$ourdir/time-big-file nidx ~/tmp/miller/mlr $ourdir/time-big-file nidx 'mlr -S' +$ourdir/time-big-file nidx mlr echo $ourdir/time-big-file xtab mlr5 +$ourdir/time-big-file xtab ~/tmp/miller/mlr $ourdir/time-big-file xtab 'mlr -S' +$ourdir/time-big-file xtab mlr echo $ourdir/time-big-file csv mlr5 +$ourdir/time-big-file csv ~/tmp/miller/mlr $ourdir/time-big-file csv 'mlr -S' +$ourdir/time-big-file csv mlr echo $ourdir/time-big-file csvlite mlr5 +$ourdir/time-big-file csvlite ~/tmp/miller/mlr $ourdir/time-big-file csvlite 'mlr -S' +$ourdir/time-big-file csvlite mlr echo $ourdir/time-big-file json mlr5 +$ourdir/time-big-file json ~/tmp/miller/mlr $ourdir/time-big-file json 'mlr -S' +$ourdir/time-big-file json mlr diff --git a/test/cases/verb-fill-empty/x b/test/cases/verb-fill-empty/x deleted file mode 100644 index f4bdb8b24..000000000 --- a/test/cases/verb-fill-empty/x +++ /dev/null @@ -1 +0,0 @@ -$tb = typeof($b) diff --git a/todo.txt b/todo.txt index 7f4449726..10b6dc53d 100644 --- a/todo.txt +++ b/todo.txt @@ -2,8 +2,31 @@ PUNCHDOWN LIST * --ifs-regex & --ips-regex -- guessing is not safe as evidence by '.' and '|' +* JIT next: + o JSON I/O UT + o krepl miller6 new doclink-comments + o IsArrayOrMap infers all -- it should not & it should say why not + o fix: + mlr --from test/input/abixy --opprint put -v -f test/cases/dsl-for-srec-loops/0005/mlr + -> for(k,v in $*) {...} while modifying $* in-loop: $* isn't being copied + o fix: + mlr --from data/small put 'dump $*' + ! fix + echo x=0xff | mlr --oxtab --ofmt '%f' cat + o look at pre/post FromMap; also check PutReference/PutCopy callsites + o UTs for gd-2 + - mlrval_cmp.go + - JSON I/O + - mv from-array/from-map w/ copy & mutate orig & check new -- & vice versa + - dash-O and octal infer + o grep tmiller map put ref vs map put copy -* big-picture item @ Rmd; also webdoc intro page +* perf next: + o batchify source-quench + ! hash-records back off + ? further channelize (CSV-first focus) mlrval infer vs record-put ? + +* big-picture item @ Rmd (csv memes; and beyond); also webdoc intro page * mr -S test/cases/dsl-array-map-indexing @@ -34,6 +57,11 @@ PUNCHDOWN LIST - make a 2nd/3rd cmd main w/ simple model & tweak that o dkvp-reader factor-out ... o mods: + ! need cmps for array/map -- no absent/error anymore here + ! moreover, proofread all mlrval_cmp.go dispo mxes + ! array/map fields: marshal as JSON_SINGLE_LINE + - replace rac-list with batch-array -- ? profile to see what value. + - unaliasArrayIndex in multiple packages for cycdep break -- ? - randsel for arrays - downstreamDone batchify > head @@ -58,6 +86,9 @@ PUNCHDOWN LIST - fix record/line sequencing regressions - maybe increase records-per-batch in join-bucket-keeper; and/or refactor - maybe increase records-per-batch in repl; and/or refactor + - lots of reading/neatening on types, bifs, mlrval split-up x all source files + - follow-ons for lots of UT per se on types, bifs, mlrval + ? []Mlrval -> []*Mlrval ? o goals: - keep goroutines -- including per-transformer -- for parallelism - look for flex ideas on how to structure that parallelism @@ -95,6 +126,68 @@ PUNCHDOWN LIST > try unconditional batching and hiding *first* to see how much best-case perf to be had ? lazy type-infer?? needs careful use of accessor-mutators in place of mv.type etc +---------------------------------------------------------------- +* JIT refactor + o x 'tens/dozens of UT cases' + o populate each bifs/X_test.go for each bifs/X.go from existing regression-test data + m internal/pkg/mlrval + k UT-driven mlrval defer/infer + - pkg/mlrval doc.go & README.md + o internal/pkg/types + k UT-driven mlrmap constructor refactor + - UT-driven JSON-parser refactor + o internal/pkg/input + k basic + mlrval.FromDeferredType + mlrmap.PutReference(key, value) + + k UT-driven DKVP-from-line + ~ UT-driven X-from-line: nidx, xtab, csvlite, csv + - UT-driven JSON-from-string + o internal/pkg/output + - basic + mlrval.String() + - UT-driven to-line + o internal/pkg/bifs + - more UT ... final authority is RT though + + ? stick w/ regtest ? + - internal/pkg/colorizer + - internal/pkg/platform + - internal/pkg/runtime + + - internal/pkg/stream + - internal/pkg/cli + - internal/pkg/climain + + - internal/pkg/transformers/utils + - internal/pkg/transformers + + - internal/pkg/parsing + - internal/pkg/dsl + - internal/pkg/dsl/cst + + - internal/pkg/auxents/repl + - internal/pkg/auxents/regtest + - internal/pkg/auxents + - internal/pkg/auxents/help +---------------------------------------------------------------- + +* perf more + o nim6 note on perf: baseline/cat; processor/pipelining; + o update https://miller.readthedocs.io/en/latest/cpu/, and/or + https://miller.readthedocs.io/en/latest/streaming-and-memory/, and/or + make a new page w/ info on per + o JIT: maasure + k pipelined reader + k batched reader separately + k buffered output + o re-check hashed/unhashed and do so periodically + - CLI switch + - check with more than just abixy + +---------------------------------------------------------------- + * blockers: - keep checking issues - verslink old relnotes @@ -195,6 +288,8 @@ PUNCHDOWN LIST ================================================================ NON-BLOCKERS +* handling for --cpuprofile not at args[1] slot + * try simpler-than-regex-split-string for repeated-single -- especially for XTAB reader * UT-per-se of XTAB channelizedStanzaScanner