Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-08-02 12:32:21 +00:00
Code Issues Projects Releases Wiki Activity

iterate on [[...]] / [[[...]]]

Browse source
This commit is contained in:
John Kerl 2020-11-25 12:37:31 -05:00
parent fd222d69ee
commit ed2a4dc6ae
13 changed files with 399 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

43
go/reg-test/cases/case-positional-indexing.sh Normal file
View file

@ -0,0 +1,43 @@
run_mlr --oxtab --from reg-test/input/abixy head -n 1 then put '
$_1 = ""; # just for visual output-spacing
$srec_keyed_by_2 = $[2];
$srec_name_2 = $[[2]];
$srec_value_2 = $[[[2]]];
$_2 = "";
$dollar_star_keyed_by_2 = $*[2];
$dollar_star_name_2 = $*[[2]];
$dollar_star_value_2 = $*[[[2]]];
$_3 = "";
mymap = {"a":7, "b":8, "c":9};
$mymap_keyed_by_2 = mymap[2];
$mymap_name_2 = mymap[[2]];
$mymap_value_2 = mymap[[[2]]];
$_4 = "";
myarray = [7, 8, 9];
$myarray_keyed_by_2 = myarray[2];
$myarray_name_2 = myarray[[2]];
$myarray_value_2 = myarray[[[2]]];
'
run_mlr --oxtab --from reg-test/input/abixy head -n 1 then put '
$_1 = ""; # just for visual output-spacing
$srec_keyed_by_2 = $[900];
$srec_name_2 = $[[900]];
$srec_value_2 = $[[[900]]];
$_2 = "";
$dollar_star_keyed_by_2 = $*[900];
$dollar_star_name_2 = $*[[900]];
$dollar_star_value_2 = $*[[[900]]];
$_3 = "";
mymap = {"a":7, "b":8, "c":9};
$mymap_keyed_by_2 = mymap[900];
$mymap_name_2 = mymap[[900]];
$mymap_value_2 = mymap[[[900]]];
$_4 = "";
myarray = [7, 8, 9];
$myarray_keyed_by_2 = myarray[900];
$myarray_name_2 = myarray[[900]];
$myarray_value_2 = myarray[[[900]]];
'

2
go/reg-test/expected/case-go-dsl-begin-end.sh.out
View file

@ -66,7 +66,7 @@ mlr --from ./reg-test/input/s.dkvp put -q begin{@sum=[0,0]} @sum[1+NR%2] += $x;
mlr --from ./reg-test/input/s.dkvp put -q begin{@sum=[]} @sum[1+NR%2] += $x; end{dump}
{
"sum": ["", (error)]
"sum": ["", 0.5513934501043855]
}
mlr --from ./reg-test/input/s.dkvp put -q begin{} @sum[1+(NR%2)] += $x; end{dump}

16
go/reg-test/expected/case-go-dsl-type-predicates.sh.out
View file

@ -444,12 +444,12 @@ mlr --from ./reg-test/input/s.dkvp head -n 1 then put -q
dump;
{
"is_absent_array_out_of_bounds": false,
"is_present_array_out_of_bounds": true,
"is_absent_array_out_of_bounds": true,
"is_present_array_out_of_bounds": false,
"is_empty_array_out_of_bounds": false,
"is_not_empty_array_out_of_bounds": true,
"is_null_array_out_of_bounds": false,
"is_not_null_array_out_of_bounds": true,
"is_null_array_out_of_bounds": true,
"is_not_null_array_out_of_bounds": false,
"is_bool_array_out_of_bounds": false,
"is_boolean_array_out_of_bounds": false,
"is_float_array_out_of_bounds": false,
@ -487,12 +487,12 @@ mlr --from ./reg-test/input/s.dkvp head -n 1 then put -q
dump;
{
"is_absent_array_out_of_bounds": false,
"is_present_array_out_of_bounds": true,
"is_absent_array_out_of_bounds": true,
"is_present_array_out_of_bounds": false,
"is_empty_array_out_of_bounds": false,
"is_not_empty_array_out_of_bounds": true,
"is_null_array_out_of_bounds": false,
"is_not_null_array_out_of_bounds": true,
"is_null_array_out_of_bounds": true,
"is_not_null_array_out_of_bounds": false,
"is_bool_array_out_of_bounds": false,
"is_boolean_array_out_of_bounds": false,
"is_float_array_out_of_bounds": false,

70
go/reg-test/expected/case-positional-indexing.sh.out Normal file
View file

@ -0,0 +1,70 @@
mlr --oxtab --from reg-test/input/abixy head -n 1 then put
$_1 = ""; # just for visual output-spacing
$srec_keyed_by_2 = $[2];
$srec_name_2 = $[[2]];
$srec_value_2 = $[[[2]]];
$_2 = "";
$dollar_star_keyed_by_2 = $*[2];
$dollar_star_name_2 = $*[[2]];
$dollar_star_value_2 = $*[[[2]]];
$_3 = "";
mymap = {"a":7, "b":8, "c":9};
$mymap_keyed_by_2 = mymap[2];
$mymap_name_2 = mymap[[2]];
$mymap_value_2 = mymap[[[2]]];
$_4 = "";
myarray = [7, 8, 9];
$myarray_keyed_by_2 = myarray[2];
$myarray_name_2 = myarray[[2]];
$myarray_value_2 = myarray[[[2]]];
a pan
b pan
i 1
x 0.3467901443380824
y 0.7268028627434533
_1
srec_name_2 b
srec_value_2 pan
_2
dollar_star_name_2 b
dollar_star_value_2 pan
_3
mymap_name_2 b
mymap_value_2 8
_4
myarray_keyed_by_2 8
myarray_name_2 2
myarray_value_2 8
mlr --oxtab --from reg-test/input/abixy head -n 1 then put
$_1 = ""; # just for visual output-spacing
$srec_keyed_by_2 = $[900];
$srec_name_2 = $[[900]];
$srec_value_2 = $[[[900]]];
$_2 = "";
$dollar_star_keyed_by_2 = $*[900];
$dollar_star_name_2 = $*[[900]];
$dollar_star_value_2 = $*[[[900]]];
$_3 = "";
mymap = {"a":7, "b":8, "c":9};
$mymap_keyed_by_2 = mymap[900];
$mymap_name_2 = mymap[[900]];
$mymap_value_2 = mymap[[[900]]];
$_4 = "";
myarray = [7, 8, 9];
$myarray_keyed_by_2 = myarray[900];
$myarray_name_2 = myarray[[900]];
$myarray_value_2 = myarray[[[900]]];
a pan
b pan
i 1
x 0.3467901443380824
y 0.7268028627434533
_1
_2
_3
_4

2
go/src/miller/dsl/ast_test.go
View file

@ -1,3 +1,5 @@
// Most tests are in reg-test/run
package dsl
import (

241
go/src/miller/dsl/cst/collections.go
View file

@ -102,17 +102,238 @@ func (this *RootNode) BuildArraySliceAccessNode(
return this.BuildPanicNode(astNode)
}
// if astNode.Type == dsl.NodeTypeArraySliceEmptyLowerIndex {
// return BuildPanicNode(astNode) // xxx temp
// }
// if astNode.Type == dsl.NodeTypeArraySliceEmptyUpperIndex {
// return BuildPanicNode(astNode) // xxx temp
// }
// ================================================================
// For input record 'a=7,b=8,c=9', $[[2]] = "b"
type PositionalFieldNameNode struct {
indexEvaluable IEvaluable
}
func (this *RootNode) BuildPositionalFieldNameNode(
astNode *dsl.ASTNode,
) (IEvaluable, error) {
lib.InternalCodingErrorIf(astNode.Type != dsl.NodeTypePositionalFieldName)
lib.InternalCodingErrorIf(len(astNode.Children) != 1)
indexASTNode := astNode.Children[0]
indexEvaluable, err := this.BuildEvaluableNode(indexASTNode)
if err != nil {
return nil, err
}
return &PositionalFieldNameNode{
indexEvaluable: indexEvaluable,
}, nil
}
// TODO: code-dedupe these next four Evaluate methods
func (this *PositionalFieldNameNode) Evaluate(state *State) types.Mlrval {
indexMlrval := this.indexEvaluable.Evaluate(state)
if indexMlrval.IsAbsent() {
return types.MlrvalFromAbsent()
}
index, ok := indexMlrval.GetIntValue()
if !ok {
return types.MlrvalFromError()
}
name := state.Inrec.GetNameAtPositionalIndex(index)
if name == nil {
return types.MlrvalFromAbsent()
}
return types.MlrvalFromString(*name)
}
// ================================================================
// For input record 'a=7,b=8,c=9', $[[2]] = 8
type PositionalFieldValueNode struct {
indexEvaluable IEvaluable
}
func (this *RootNode) BuildPositionalFieldValueNode(
astNode *dsl.ASTNode,
) (IEvaluable, error) {
lib.InternalCodingErrorIf(astNode.Type != dsl.NodeTypePositionalFieldValue)
lib.InternalCodingErrorIf(len(astNode.Children) != 1)
indexASTNode := astNode.Children[0]
indexEvaluable, err := this.BuildEvaluableNode(indexASTNode)
if err != nil {
return nil, err
}
return &PositionalFieldValueNode{
indexEvaluable: indexEvaluable,
}, nil
}
func (this *PositionalFieldValueNode) Evaluate(state *State) types.Mlrval {
indexMlrval := this.indexEvaluable.Evaluate(state)
if indexMlrval.IsAbsent() {
return types.MlrvalFromAbsent()
}
index, ok := indexMlrval.GetIntValue()
if !ok {
return types.MlrvalFromError()
}
retval := state.Inrec.GetWithPositionalIndex(index)
if retval == nil {
return types.MlrvalFromAbsent()
}
return *retval
}
// ================================================================
// For x = [7,8,9], x[[2]] = 2
// For y = {"a":7,"b":8,"c":9}, y[[2]] = "b"
type ArrayOrMapPositionalNameAccessNode struct {
baseEvaluable IEvaluable
indexEvaluable IEvaluable
}
func (this *RootNode) BuildArrayOrMapPositionalNameAccessNode(
astNode *dsl.ASTNode,
) (IEvaluable, error) {
lib.InternalCodingErrorIf(astNode.Type != dsl.NodeTypeArrayOrMapPositionalNameAccess)
lib.InternalCodingErrorIf(len(astNode.Children) != 2)
baseASTNode := astNode.Children[0]
indexASTNode := astNode.Children[1]
baseEvaluable, err := this.BuildEvaluableNode(baseASTNode)
if err != nil {
return nil, err
}
indexEvaluable, err := this.BuildEvaluableNode(indexASTNode)
if err != nil {
return nil, err
}
return &ArrayOrMapPositionalNameAccessNode{
baseEvaluable: baseEvaluable,
indexEvaluable: indexEvaluable,
}, nil
}
func (this *ArrayOrMapPositionalNameAccessNode) Evaluate(state *State) types.Mlrval {
baseMlrval := this.baseEvaluable.Evaluate(state)
indexMlrval := this.indexEvaluable.Evaluate(state)
if indexMlrval.IsAbsent() {
return types.MlrvalFromAbsent()
}
index, ok := indexMlrval.GetIntValue()
if !ok {
return types.MlrvalFromError()
}
if baseMlrval.IsArray() {
n, _ := baseMlrval.GetArrayLength()
zindex, ok := types.UnaliasArrayLengthIndex(int64(n), index)
if ok {
return types.MlrvalFromInt64(zindex + 1) // Miller user-space indices are 1-up
} else {
return types.MlrvalFromAbsent()
}
} else if baseMlrval.IsMap() {
name := baseMlrval.GetMap().GetNameAtPositionalIndex(index)
if name == nil {
return types.MlrvalFromAbsent()
} else {
return types.MlrvalFromString(*name)
}
} else if baseMlrval.IsAbsent() {
return baseMlrval
} else {
return types.MlrvalFromError()
}
}
// ================================================================
// For x = [7,8,9], x[[2]] = 8
// For y = {"a":7,"b":8,"c":9}, y[[2]] = 8
type ArrayOrMapPositionalValueAccessNode struct {
baseEvaluable IEvaluable
indexEvaluable IEvaluable
}
func (this *RootNode) BuildArrayOrMapPositionalValueAccessNode(
astNode *dsl.ASTNode,
) (IEvaluable, error) {
lib.InternalCodingErrorIf(astNode.Type != dsl.NodeTypeArrayOrMapPositionalValueAccess)
lib.InternalCodingErrorIf(len(astNode.Children) != 2)
baseASTNode := astNode.Children[0]
indexASTNode := astNode.Children[1]
baseEvaluable, err := this.BuildEvaluableNode(baseASTNode)
if err != nil {
return nil, err
}
indexEvaluable, err := this.BuildEvaluableNode(indexASTNode)
if err != nil {
return nil, err
}
return &ArrayOrMapPositionalValueAccessNode{
baseEvaluable: baseEvaluable,
indexEvaluable: indexEvaluable,
}, nil
}
func (this *ArrayOrMapPositionalValueAccessNode) Evaluate(state *State) types.Mlrval {
baseMlrval := this.baseEvaluable.Evaluate(state)
indexMlrval := this.indexEvaluable.Evaluate(state)
if indexMlrval.IsAbsent() {
return types.MlrvalFromAbsent()
}
index, ok := indexMlrval.GetIntValue()
if !ok {
return types.MlrvalFromError()
}
if baseMlrval.IsArray() {
return baseMlrval.ArrayGet(&indexMlrval)
} else if baseMlrval.IsMap() {
value := baseMlrval.GetMap().GetWithPositionalIndex(index)
if value == nil {
return types.MlrvalFromAbsent()
}
retval := value.Copy()
return *retval
} else if baseMlrval.IsAbsent() {
return baseMlrval
} else {
return types.MlrvalFromError()
}
}
// ================================================================
// This is for computing map entries at runtime. For example, in
//
// mlr put 'mymap = {"sum": $x + $y, "diff": $x - $y}; ...'
//
// the first pair would have key being string-literal "sum" and value being the
// evaluable expression '$x + $y'.
// ----------------------------------------------------------------
// This is for computing map entries at runtime. For example, in mlr put 'mymap
// = {"sum": $x + $y, "diff": $x - $y}; ...', the first pair would have key
// being string-literal "sum" and value being the evaluable expression '$x + $y'.
type EvaluablePair struct {
Key IEvaluable
Value IEvaluable

24
go/src/miller/dsl/cst/evaluable.go
View file

@ -25,22 +25,34 @@ func (this *RootNode) BuildEvaluableNode(astNode *dsl.ASTNode) (IEvaluable, erro
switch astNode.Type {
case dsl.NodeTypeArrayLiteral:
case dsl.NodeTypeArrayLiteral: // [...]
return this.BuildArrayLiteralNode(astNode)
case dsl.NodeTypeMapLiteral:
case dsl.NodeTypeMapLiteral: // {...}
return this.BuildMapLiteralNode(astNode)
case dsl.NodeTypeArrayOrMapIndexAccess:
case dsl.NodeTypeArrayOrMapIndexAccess: // x[...]
return this.BuildArrayOrMapIndexAccessNode(astNode)
case dsl.NodeTypeArraySliceAccess:
case dsl.NodeTypeArraySliceAccess: // myarray[lo:hi]
return this.BuildArraySliceAccessNode(astNode)
case dsl.NodeTypeIndirectFieldValue:
case dsl.NodeTypePositionalFieldName: // $[[...]]
return this.BuildPositionalFieldNameNode(astNode)
case dsl.NodeTypePositionalFieldValue: // $[[[...]]]
return this.BuildPositionalFieldValueNode(astNode)
case dsl.NodeTypeArrayOrMapPositionalNameAccess: // mymap[[...]]]
return this.BuildArrayOrMapPositionalNameAccessNode(astNode)
case dsl.NodeTypeArrayOrMapPositionalValueAccess: // mymap[[[...]]]
return this.BuildArrayOrMapPositionalValueAccessNode(astNode)
case dsl.NodeTypeIndirectFieldValue: // $[...]
return this.BuildIndirectFieldValueNode(astNode)
case dsl.NodeTypeEnvironmentVariable:
case dsl.NodeTypeEnvironmentVariable: // ENV["NAME"]
return this.BuildEnvironmentVariableNode(astNode)
// Operators are just functions with infix syntax so we treat them like

12
go/src/miller/types/mlrmap_accessors.go
View file

@ -137,6 +137,8 @@ func (this *Mlrmap) GetKeys() []string {
}
// ----------------------------------------------------------------
// TODO: update this comment regarding syntax
//
// For '$[1]' etc. in the DSL.
//
// Notes:
@ -146,6 +148,7 @@ func (this *Mlrmap) GetKeys() []string {
// get the -1st field than the nth.
// * Returns 0 on invalid index: 0, or < -n, or > n where n is the number of
// fields.
func (this *Mlrmap) GetWithPositionalIndex(position int64) *Mlrval {
mapEntry := this.findEntryByPositionalIndex(position)
if mapEntry == nil {
@ -154,6 +157,7 @@ func (this *Mlrmap) GetWithPositionalIndex(position int64) *Mlrval {
return mapEntry.Value
}
// TODO: update for array/map project
func (this *Mlrmap) GetWithMlrvalIndex(index *Mlrval) (*Mlrval, error) {
if index.mvtype == MT_STRING {
return this.Get(&index.printrep), nil
@ -166,6 +170,14 @@ func (this *Mlrmap) GetWithMlrvalIndex(index *Mlrval) (*Mlrval, error) {
}
}
func (this *Mlrmap) GetNameAtPositionalIndex(position int64) *string {
mapEntry := this.findEntryByPositionalIndex(position)
if mapEntry == nil {
return nil
}
return mapEntry.Key
}
func (this *Mlrmap) RemoveWithPositionalIndex(position int64) {
mapEntry := this.findEntryByPositionalIndex(position)
if mapEntry != nil {

8
go/src/miller/types/mlrval_accessors.go
View file

@ -130,6 +130,14 @@ func (this *Mlrval) GetArray() []Mlrval {
}
}
func (this *Mlrval) GetArrayLength() (int, bool) {
if this.mvtype == MT_ARRAY {
return len(this.arrayval), true
} else {
return -999, false
}
}
func (this *Mlrval) GetMap() *Mlrmap {
if this.mvtype == MT_MAP {
return this.mapval

6
go/src/miller/types/mlrval_collections.go
View file

@ -85,7 +85,7 @@ func (this *Mlrval) ArrayGet(mindex *Mlrval) Mlrval {
}
value := arrayGetAliased(&this.arrayval, mindex.intval)
if value == nil {
return MlrvalFromError()
return MlrvalFromAbsent()
} else {
return *value
}
@ -135,10 +135,10 @@ func arrayPutAliased(array *[]Mlrval, mindex int64, value *Mlrval) bool {
func unaliasArrayIndex(array *[]Mlrval, mindex int64) (int64, bool) {
n := int64(len(*array))
return unaliasArrayLengthIndex(n, mindex)
return UnaliasArrayLengthIndex(n, mindex)
}
func unaliasArrayLengthIndex(n int64, mindex int64) (int64, bool) {
func UnaliasArrayLengthIndex(n int64, mindex int64) (int64, bool) {
if 1 <= mindex && mindex <= n {
zindex := mindex - 1
return zindex, true

4
go/src/miller/types/mlrval_functions_strings.go
View file

@ -81,8 +81,8 @@ func MlrvalSubstr(ma, mb, mc *Mlrval) Mlrval {
strlen := int64(len(ma.printrep))
// Convert from negative-aliased 1-up to positive-only 0-up
m, mok := unaliasArrayLengthIndex(strlen, mb.intval)
n, nok := unaliasArrayLengthIndex(strlen, mc.intval)
m, mok := UnaliasArrayLengthIndex(strlen, mb.intval)
n, nok := UnaliasArrayLengthIndex(strlen, mc.intval)
if !mok || !nok {
return MlrvalFromString("")

1
go/tester
View file

@ -10,3 +10,4 @@
export GOPATH=$(pwd)
go test miller/types
go test src/miller/dsl/ast_test.go

20
go/u/test-dsl
View file

@ -1,20 +0,0 @@
#!/bin/bash
set -euo pipefail
export GOPATH=$(pwd)
# Build the bin/gocc executable:
go get github.com/goccmack/gocc
# Read the Miller DSL grammar file and generate code:
bin/gocc -o src/miller/parsing src/miller/parsing/mlr.bnf
# Code-gen directories:
# src/miller/parsing/errors/
# src/miller/parsing/lexer/
# src/miller/parsing/parser/
# src/miller/parsing/token/
# src/miller/parsing/util/
# Run tests:
go test -v src/miller/dsl/ast_test.go