diff --git a/go/reg-test/cases/case-positional-indexing.sh b/go/reg-test/cases/case-positional-indexing.sh new file mode 100644 index 000000000..01e5e0539 --- /dev/null +++ b/go/reg-test/cases/case-positional-indexing.sh @@ -0,0 +1,43 @@ + +run_mlr --oxtab --from reg-test/input/abixy head -n 1 then put ' + $_1 = ""; # just for visual output-spacing + $srec_keyed_by_2 = $[2]; + $srec_name_2 = $[[2]]; + $srec_value_2 = $[[[2]]]; + $_2 = ""; + $dollar_star_keyed_by_2 = $*[2]; + $dollar_star_name_2 = $*[[2]]; + $dollar_star_value_2 = $*[[[2]]]; + $_3 = ""; + mymap = {"a":7, "b":8, "c":9}; + $mymap_keyed_by_2 = mymap[2]; + $mymap_name_2 = mymap[[2]]; + $mymap_value_2 = mymap[[[2]]]; + $_4 = ""; + myarray = [7, 8, 9]; + $myarray_keyed_by_2 = myarray[2]; + $myarray_name_2 = myarray[[2]]; + $myarray_value_2 = myarray[[[2]]]; +' + +run_mlr --oxtab --from reg-test/input/abixy head -n 1 then put ' + $_1 = ""; # just for visual output-spacing + $srec_keyed_by_2 = $[900]; + $srec_name_2 = $[[900]]; + $srec_value_2 = $[[[900]]]; + $_2 = ""; + $dollar_star_keyed_by_2 = $*[900]; + $dollar_star_name_2 = $*[[900]]; + $dollar_star_value_2 = $*[[[900]]]; + $_3 = ""; + mymap = {"a":7, "b":8, "c":9}; + $mymap_keyed_by_2 = mymap[900]; + $mymap_name_2 = mymap[[900]]; + $mymap_value_2 = mymap[[[900]]]; + $_4 = ""; + myarray = [7, 8, 9]; + $myarray_keyed_by_2 = myarray[900]; + $myarray_name_2 = myarray[[900]]; + $myarray_value_2 = myarray[[[900]]]; +' + diff --git a/go/reg-test/expected/case-go-dsl-begin-end.sh.out b/go/reg-test/expected/case-go-dsl-begin-end.sh.out index 20786131e..84a6baf17 100644 --- a/go/reg-test/expected/case-go-dsl-begin-end.sh.out +++ b/go/reg-test/expected/case-go-dsl-begin-end.sh.out @@ -66,7 +66,7 @@ mlr --from ./reg-test/input/s.dkvp put -q begin{@sum=[0,0]} @sum[1+NR%2] += $x; mlr --from ./reg-test/input/s.dkvp put -q begin{@sum=[]} @sum[1+NR%2] += $x; end{dump} { - "sum": ["", (error)] + "sum": ["", 0.5513934501043855] } mlr --from ./reg-test/input/s.dkvp put -q begin{} @sum[1+(NR%2)] += $x; end{dump} diff --git a/go/reg-test/expected/case-go-dsl-type-predicates.sh.out b/go/reg-test/expected/case-go-dsl-type-predicates.sh.out index 16ac60b13..6a60053a2 100644 --- a/go/reg-test/expected/case-go-dsl-type-predicates.sh.out +++ b/go/reg-test/expected/case-go-dsl-type-predicates.sh.out @@ -444,12 +444,12 @@ mlr --from ./reg-test/input/s.dkvp head -n 1 then put -q dump; { - "is_absent_array_out_of_bounds": false, - "is_present_array_out_of_bounds": true, + "is_absent_array_out_of_bounds": true, + "is_present_array_out_of_bounds": false, "is_empty_array_out_of_bounds": false, "is_not_empty_array_out_of_bounds": true, - "is_null_array_out_of_bounds": false, - "is_not_null_array_out_of_bounds": true, + "is_null_array_out_of_bounds": true, + "is_not_null_array_out_of_bounds": false, "is_bool_array_out_of_bounds": false, "is_boolean_array_out_of_bounds": false, "is_float_array_out_of_bounds": false, @@ -487,12 +487,12 @@ mlr --from ./reg-test/input/s.dkvp head -n 1 then put -q dump; { - "is_absent_array_out_of_bounds": false, - "is_present_array_out_of_bounds": true, + "is_absent_array_out_of_bounds": true, + "is_present_array_out_of_bounds": false, "is_empty_array_out_of_bounds": false, "is_not_empty_array_out_of_bounds": true, - "is_null_array_out_of_bounds": false, - "is_not_null_array_out_of_bounds": true, + "is_null_array_out_of_bounds": true, + "is_not_null_array_out_of_bounds": false, "is_bool_array_out_of_bounds": false, "is_boolean_array_out_of_bounds": false, "is_float_array_out_of_bounds": false, diff --git a/go/reg-test/expected/case-positional-indexing.sh.out b/go/reg-test/expected/case-positional-indexing.sh.out new file mode 100644 index 000000000..478c62ee5 --- /dev/null +++ b/go/reg-test/expected/case-positional-indexing.sh.out @@ -0,0 +1,70 @@ + +mlr --oxtab --from reg-test/input/abixy head -n 1 then put + $_1 = ""; # just for visual output-spacing + $srec_keyed_by_2 = $[2]; + $srec_name_2 = $[[2]]; + $srec_value_2 = $[[[2]]]; + $_2 = ""; + $dollar_star_keyed_by_2 = $*[2]; + $dollar_star_name_2 = $*[[2]]; + $dollar_star_value_2 = $*[[[2]]]; + $_3 = ""; + mymap = {"a":7, "b":8, "c":9}; + $mymap_keyed_by_2 = mymap[2]; + $mymap_name_2 = mymap[[2]]; + $mymap_value_2 = mymap[[[2]]]; + $_4 = ""; + myarray = [7, 8, 9]; + $myarray_keyed_by_2 = myarray[2]; + $myarray_name_2 = myarray[[2]]; + $myarray_value_2 = myarray[[[2]]]; + +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +_1 +srec_name_2 b +srec_value_2 pan +_2 +dollar_star_name_2 b +dollar_star_value_2 pan +_3 +mymap_name_2 b +mymap_value_2 8 +_4 +myarray_keyed_by_2 8 +myarray_name_2 2 +myarray_value_2 8 + +mlr --oxtab --from reg-test/input/abixy head -n 1 then put + $_1 = ""; # just for visual output-spacing + $srec_keyed_by_2 = $[900]; + $srec_name_2 = $[[900]]; + $srec_value_2 = $[[[900]]]; + $_2 = ""; + $dollar_star_keyed_by_2 = $*[900]; + $dollar_star_name_2 = $*[[900]]; + $dollar_star_value_2 = $*[[[900]]]; + $_3 = ""; + mymap = {"a":7, "b":8, "c":9}; + $mymap_keyed_by_2 = mymap[900]; + $mymap_name_2 = mymap[[900]]; + $mymap_value_2 = mymap[[[900]]]; + $_4 = ""; + myarray = [7, 8, 9]; + $myarray_keyed_by_2 = myarray[900]; + $myarray_name_2 = myarray[[900]]; + $myarray_value_2 = myarray[[[900]]]; + +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +_1 +_2 +_3 +_4 + diff --git a/go/src/miller/dsl/ast_test.go b/go/src/miller/dsl/ast_test.go index b615329bf..a2eb63dc9 100644 --- a/go/src/miller/dsl/ast_test.go +++ b/go/src/miller/dsl/ast_test.go @@ -1,3 +1,5 @@ +// Most tests are in reg-test/run + package dsl import ( diff --git a/go/src/miller/dsl/cst/collections.go b/go/src/miller/dsl/cst/collections.go index 2fa69480d..124221395 100644 --- a/go/src/miller/dsl/cst/collections.go +++ b/go/src/miller/dsl/cst/collections.go @@ -102,17 +102,238 @@ func (this *RootNode) BuildArraySliceAccessNode( return this.BuildPanicNode(astNode) } -// if astNode.Type == dsl.NodeTypeArraySliceEmptyLowerIndex { -// return BuildPanicNode(astNode) // xxx temp -// } -// if astNode.Type == dsl.NodeTypeArraySliceEmptyUpperIndex { -// return BuildPanicNode(astNode) // xxx temp -// } +// ================================================================ +// For input record 'a=7,b=8,c=9', $[[2]] = "b" + +type PositionalFieldNameNode struct { + indexEvaluable IEvaluable +} + +func (this *RootNode) BuildPositionalFieldNameNode( + astNode *dsl.ASTNode, +) (IEvaluable, error) { + lib.InternalCodingErrorIf(astNode.Type != dsl.NodeTypePositionalFieldName) + lib.InternalCodingErrorIf(len(astNode.Children) != 1) + + indexASTNode := astNode.Children[0] + + indexEvaluable, err := this.BuildEvaluableNode(indexASTNode) + if err != nil { + return nil, err + } + + return &PositionalFieldNameNode{ + indexEvaluable: indexEvaluable, + }, nil +} + +// TODO: code-dedupe these next four Evaluate methods +func (this *PositionalFieldNameNode) Evaluate(state *State) types.Mlrval { + indexMlrval := this.indexEvaluable.Evaluate(state) + if indexMlrval.IsAbsent() { + return types.MlrvalFromAbsent() + } + + index, ok := indexMlrval.GetIntValue() + if !ok { + return types.MlrvalFromError() + } + + name := state.Inrec.GetNameAtPositionalIndex(index) + if name == nil { + return types.MlrvalFromAbsent() + } + + return types.MlrvalFromString(*name) +} + +// ================================================================ +// For input record 'a=7,b=8,c=9', $[[2]] = 8 + +type PositionalFieldValueNode struct { + indexEvaluable IEvaluable +} + +func (this *RootNode) BuildPositionalFieldValueNode( + astNode *dsl.ASTNode, +) (IEvaluable, error) { + lib.InternalCodingErrorIf(astNode.Type != dsl.NodeTypePositionalFieldValue) + lib.InternalCodingErrorIf(len(astNode.Children) != 1) + + indexASTNode := astNode.Children[0] + + indexEvaluable, err := this.BuildEvaluableNode(indexASTNode) + if err != nil { + return nil, err + } + + return &PositionalFieldValueNode{ + indexEvaluable: indexEvaluable, + }, nil +} + +func (this *PositionalFieldValueNode) Evaluate(state *State) types.Mlrval { + indexMlrval := this.indexEvaluable.Evaluate(state) + if indexMlrval.IsAbsent() { + return types.MlrvalFromAbsent() + } + + index, ok := indexMlrval.GetIntValue() + if !ok { + return types.MlrvalFromError() + } + + retval := state.Inrec.GetWithPositionalIndex(index) + if retval == nil { + return types.MlrvalFromAbsent() + } + + return *retval +} + +// ================================================================ +// For x = [7,8,9], x[[2]] = 2 +// For y = {"a":7,"b":8,"c":9}, y[[2]] = "b" +type ArrayOrMapPositionalNameAccessNode struct { + baseEvaluable IEvaluable + indexEvaluable IEvaluable +} + +func (this *RootNode) BuildArrayOrMapPositionalNameAccessNode( + astNode *dsl.ASTNode, +) (IEvaluable, error) { + lib.InternalCodingErrorIf(astNode.Type != dsl.NodeTypeArrayOrMapPositionalNameAccess) + lib.InternalCodingErrorIf(len(astNode.Children) != 2) + + baseASTNode := astNode.Children[0] + indexASTNode := astNode.Children[1] + + baseEvaluable, err := this.BuildEvaluableNode(baseASTNode) + if err != nil { + return nil, err + } + indexEvaluable, err := this.BuildEvaluableNode(indexASTNode) + if err != nil { + return nil, err + } + + return &ArrayOrMapPositionalNameAccessNode{ + baseEvaluable: baseEvaluable, + indexEvaluable: indexEvaluable, + }, nil +} + +func (this *ArrayOrMapPositionalNameAccessNode) Evaluate(state *State) types.Mlrval { + baseMlrval := this.baseEvaluable.Evaluate(state) + indexMlrval := this.indexEvaluable.Evaluate(state) + + if indexMlrval.IsAbsent() { + return types.MlrvalFromAbsent() + } + + index, ok := indexMlrval.GetIntValue() + if !ok { + return types.MlrvalFromError() + } + + if baseMlrval.IsArray() { + n, _ := baseMlrval.GetArrayLength() + zindex, ok := types.UnaliasArrayLengthIndex(int64(n), index) + if ok { + return types.MlrvalFromInt64(zindex + 1) // Miller user-space indices are 1-up + } else { + return types.MlrvalFromAbsent() + } + + } else if baseMlrval.IsMap() { + name := baseMlrval.GetMap().GetNameAtPositionalIndex(index) + if name == nil { + return types.MlrvalFromAbsent() + } else { + return types.MlrvalFromString(*name) + } + + } else if baseMlrval.IsAbsent() { + return baseMlrval + + } else { + return types.MlrvalFromError() + } +} + +// ================================================================ +// For x = [7,8,9], x[[2]] = 8 +// For y = {"a":7,"b":8,"c":9}, y[[2]] = 8 +type ArrayOrMapPositionalValueAccessNode struct { + baseEvaluable IEvaluable + indexEvaluable IEvaluable +} + +func (this *RootNode) BuildArrayOrMapPositionalValueAccessNode( + astNode *dsl.ASTNode, +) (IEvaluable, error) { + lib.InternalCodingErrorIf(astNode.Type != dsl.NodeTypeArrayOrMapPositionalValueAccess) + lib.InternalCodingErrorIf(len(astNode.Children) != 2) + + baseASTNode := astNode.Children[0] + indexASTNode := astNode.Children[1] + + baseEvaluable, err := this.BuildEvaluableNode(baseASTNode) + if err != nil { + return nil, err + } + indexEvaluable, err := this.BuildEvaluableNode(indexASTNode) + if err != nil { + return nil, err + } + + return &ArrayOrMapPositionalValueAccessNode{ + baseEvaluable: baseEvaluable, + indexEvaluable: indexEvaluable, + }, nil +} + +func (this *ArrayOrMapPositionalValueAccessNode) Evaluate(state *State) types.Mlrval { + baseMlrval := this.baseEvaluable.Evaluate(state) + indexMlrval := this.indexEvaluable.Evaluate(state) + + if indexMlrval.IsAbsent() { + return types.MlrvalFromAbsent() + } + + index, ok := indexMlrval.GetIntValue() + if !ok { + return types.MlrvalFromError() + } + + if baseMlrval.IsArray() { + return baseMlrval.ArrayGet(&indexMlrval) + + } else if baseMlrval.IsMap() { + value := baseMlrval.GetMap().GetWithPositionalIndex(index) + if value == nil { + return types.MlrvalFromAbsent() + } + + retval := value.Copy() + return *retval + + } else if baseMlrval.IsAbsent() { + return baseMlrval + + } else { + return types.MlrvalFromError() + } +} + +// ================================================================ +// This is for computing map entries at runtime. For example, in +// +// mlr put 'mymap = {"sum": $x + $y, "diff": $x - $y}; ...' +// +// the first pair would have key being string-literal "sum" and value being the +// evaluable expression '$x + $y'. -// ---------------------------------------------------------------- -// This is for computing map entries at runtime. For example, in mlr put 'mymap -// = {"sum": $x + $y, "diff": $x - $y}; ...', the first pair would have key -// being string-literal "sum" and value being the evaluable expression '$x + $y'. type EvaluablePair struct { Key IEvaluable Value IEvaluable diff --git a/go/src/miller/dsl/cst/evaluable.go b/go/src/miller/dsl/cst/evaluable.go index 05ffd3310..0f406db99 100644 --- a/go/src/miller/dsl/cst/evaluable.go +++ b/go/src/miller/dsl/cst/evaluable.go @@ -25,22 +25,34 @@ func (this *RootNode) BuildEvaluableNode(astNode *dsl.ASTNode) (IEvaluable, erro switch astNode.Type { - case dsl.NodeTypeArrayLiteral: + case dsl.NodeTypeArrayLiteral: // [...] return this.BuildArrayLiteralNode(astNode) - case dsl.NodeTypeMapLiteral: + case dsl.NodeTypeMapLiteral: // {...} return this.BuildMapLiteralNode(astNode) - case dsl.NodeTypeArrayOrMapIndexAccess: + case dsl.NodeTypeArrayOrMapIndexAccess: // x[...] return this.BuildArrayOrMapIndexAccessNode(astNode) - case dsl.NodeTypeArraySliceAccess: + case dsl.NodeTypeArraySliceAccess: // myarray[lo:hi] return this.BuildArraySliceAccessNode(astNode) - case dsl.NodeTypeIndirectFieldValue: + case dsl.NodeTypePositionalFieldName: // $[[...]] + return this.BuildPositionalFieldNameNode(astNode) + + case dsl.NodeTypePositionalFieldValue: // $[[[...]]] + return this.BuildPositionalFieldValueNode(astNode) + + case dsl.NodeTypeArrayOrMapPositionalNameAccess: // mymap[[...]]] + return this.BuildArrayOrMapPositionalNameAccessNode(astNode) + + case dsl.NodeTypeArrayOrMapPositionalValueAccess: // mymap[[[...]]] + return this.BuildArrayOrMapPositionalValueAccessNode(astNode) + + case dsl.NodeTypeIndirectFieldValue: // $[...] return this.BuildIndirectFieldValueNode(astNode) - case dsl.NodeTypeEnvironmentVariable: + case dsl.NodeTypeEnvironmentVariable: // ENV["NAME"] return this.BuildEnvironmentVariableNode(astNode) // Operators are just functions with infix syntax so we treat them like diff --git a/go/src/miller/types/mlrmap_accessors.go b/go/src/miller/types/mlrmap_accessors.go index d7b15100a..cf119b1cd 100644 --- a/go/src/miller/types/mlrmap_accessors.go +++ b/go/src/miller/types/mlrmap_accessors.go @@ -137,6 +137,8 @@ func (this *Mlrmap) GetKeys() []string { } // ---------------------------------------------------------------- +// TODO: update this comment regarding syntax +// // For '$[1]' etc. in the DSL. // // Notes: @@ -146,6 +148,7 @@ func (this *Mlrmap) GetKeys() []string { // get the -1st field than the nth. // * Returns 0 on invalid index: 0, or < -n, or > n where n is the number of // fields. + func (this *Mlrmap) GetWithPositionalIndex(position int64) *Mlrval { mapEntry := this.findEntryByPositionalIndex(position) if mapEntry == nil { @@ -154,6 +157,7 @@ func (this *Mlrmap) GetWithPositionalIndex(position int64) *Mlrval { return mapEntry.Value } +// TODO: update for array/map project func (this *Mlrmap) GetWithMlrvalIndex(index *Mlrval) (*Mlrval, error) { if index.mvtype == MT_STRING { return this.Get(&index.printrep), nil @@ -166,6 +170,14 @@ func (this *Mlrmap) GetWithMlrvalIndex(index *Mlrval) (*Mlrval, error) { } } +func (this *Mlrmap) GetNameAtPositionalIndex(position int64) *string { + mapEntry := this.findEntryByPositionalIndex(position) + if mapEntry == nil { + return nil + } + return mapEntry.Key +} + func (this *Mlrmap) RemoveWithPositionalIndex(position int64) { mapEntry := this.findEntryByPositionalIndex(position) if mapEntry != nil { diff --git a/go/src/miller/types/mlrval_accessors.go b/go/src/miller/types/mlrval_accessors.go index 018331fbb..cc5ca4a8f 100644 --- a/go/src/miller/types/mlrval_accessors.go +++ b/go/src/miller/types/mlrval_accessors.go @@ -130,6 +130,14 @@ func (this *Mlrval) GetArray() []Mlrval { } } +func (this *Mlrval) GetArrayLength() (int, bool) { + if this.mvtype == MT_ARRAY { + return len(this.arrayval), true + } else { + return -999, false + } +} + func (this *Mlrval) GetMap() *Mlrmap { if this.mvtype == MT_MAP { return this.mapval diff --git a/go/src/miller/types/mlrval_collections.go b/go/src/miller/types/mlrval_collections.go index fd6fdf3ea..53a233573 100644 --- a/go/src/miller/types/mlrval_collections.go +++ b/go/src/miller/types/mlrval_collections.go @@ -85,7 +85,7 @@ func (this *Mlrval) ArrayGet(mindex *Mlrval) Mlrval { } value := arrayGetAliased(&this.arrayval, mindex.intval) if value == nil { - return MlrvalFromError() + return MlrvalFromAbsent() } else { return *value } @@ -135,10 +135,10 @@ func arrayPutAliased(array *[]Mlrval, mindex int64, value *Mlrval) bool { func unaliasArrayIndex(array *[]Mlrval, mindex int64) (int64, bool) { n := int64(len(*array)) - return unaliasArrayLengthIndex(n, mindex) + return UnaliasArrayLengthIndex(n, mindex) } -func unaliasArrayLengthIndex(n int64, mindex int64) (int64, bool) { +func UnaliasArrayLengthIndex(n int64, mindex int64) (int64, bool) { if 1 <= mindex && mindex <= n { zindex := mindex - 1 return zindex, true diff --git a/go/src/miller/types/mlrval_functions_strings.go b/go/src/miller/types/mlrval_functions_strings.go index 09eed6948..ae783a754 100644 --- a/go/src/miller/types/mlrval_functions_strings.go +++ b/go/src/miller/types/mlrval_functions_strings.go @@ -81,8 +81,8 @@ func MlrvalSubstr(ma, mb, mc *Mlrval) Mlrval { strlen := int64(len(ma.printrep)) // Convert from negative-aliased 1-up to positive-only 0-up - m, mok := unaliasArrayLengthIndex(strlen, mb.intval) - n, nok := unaliasArrayLengthIndex(strlen, mc.intval) + m, mok := UnaliasArrayLengthIndex(strlen, mb.intval) + n, nok := UnaliasArrayLengthIndex(strlen, mc.intval) if !mok || !nok { return MlrvalFromString("") diff --git a/go/tester b/go/tester index a9eed04c3..ed6a698f5 100755 --- a/go/tester +++ b/go/tester @@ -10,3 +10,4 @@ export GOPATH=$(pwd) go test miller/types +go test src/miller/dsl/ast_test.go diff --git a/go/u/test-dsl b/go/u/test-dsl deleted file mode 100755 index 5479a769b..000000000 --- a/go/u/test-dsl +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -export GOPATH=$(pwd) - -# Build the bin/gocc executable: -go get github.com/goccmack/gocc - -# Read the Miller DSL grammar file and generate code: -bin/gocc -o src/miller/parsing src/miller/parsing/mlr.bnf -# Code-gen directories: -# src/miller/parsing/errors/ -# src/miller/parsing/lexer/ -# src/miller/parsing/parser/ -# src/miller/parsing/token/ -# src/miller/parsing/util/ - -# Run tests: -go test -v src/miller/dsl/ast_test.go