Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-01-23 02:14:13 +00:00
Code Issues Projects Releases Wiki Activity

More mlrval size-reduction (#1132)

Browse source
This commit is contained in:
John Kerl 2022-11-26 11:13:53 -05:00 • committed by GitHub
parent 6f22401d53
commit 1e6ae3fd1e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 96 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

3
cmd/sizes/main.go
View file

@ -7,6 +7,8 @@
package main
import (
"fmt"
"github.com/johnkerl/miller/internal/pkg/mlrval"
)
@ -15,5 +17,6 @@ func main() {
mvs[0] = *mlrval.FromString("hello")
mvs[1] = *mlrval.FromString("world")
mvs[0].ShowSizes()
fmt.Println()
mvs[1].ShowSizes()
}

6
internal/pkg/mlrval/mlrmap_accessors.go
View file

@ -339,7 +339,7 @@ func (mlrmap *Mlrmap) getWithMlrvalArrayIndex(index *Mlrval) (*Mlrval, error) {
current := mlrmap
var retval *Mlrval = nil
lib.InternalCodingErrorIf(!index.IsArray())
array := index.arrayval
array := index.x.arrayval
n := len(array)
for i, piece := range array {
next, err := current.GetWithMlrvalIndex(piece)
@ -350,7 +350,7 @@ func (mlrmap *Mlrmap) getWithMlrvalArrayIndex(index *Mlrval) (*Mlrval, error) {
if !next.IsMap() {
return nil, fmt.Errorf("mlr: cannot multi-index non-map.")
}
current = next.mapval
current = next.x.mapval
} else {
retval = next.Copy()
}
@ -784,7 +784,7 @@ func (mlrmap *Mlrmap) SortByKeyRecursively() {
// Old record will be GC'ed: just move pointers
value := mlrmap.Get(key)
if value.IsMap() {
value.mapval.SortByKeyRecursively()
value.x.mapval.SortByKeyRecursively()
}
other.PutReference(key, value)
}

14
internal/pkg/mlrval/mlrmap_flatten_unflatten.go
View file

@ -137,13 +137,13 @@ func (mlrmap *Mlrmap) CopyUnflattened(
// Is the field name something dot something?
if strings.Contains(pe.Key, separator) {
arrayOfIndices := SplitAXHelper(pe.Key, separator)
lib.InternalCodingErrorIf(len(arrayOfIndices.arrayval) < 1)
lib.InternalCodingErrorIf(len(arrayOfIndices.x.arrayval) < 1)
// If the input field name was "x.a" then remember the "x".
baseIndex := arrayOfIndices.arrayval[0].String()
baseIndex := arrayOfIndices.x.arrayval[0].String()
affectedBaseIndices[baseIndex] = true
// Use PutIndexed to assign $x["a"] = 7, or $x["b"] = 8, etc.
other.PutIndexed(
CopyMlrvalArray(arrayOfIndices.arrayval),
CopyMlrvalArray(arrayOfIndices.x.arrayval),
unflattenTerminal(pe.Value).Copy(),
)
} else {
@ -187,13 +187,13 @@ func (mlrmap *Mlrmap) CopyUnflattenFields(
// Is the field name something dot something?
if strings.Contains(pe.Key, separator) {
arrayOfIndices := SplitAXHelper(pe.Key, separator)
lib.InternalCodingErrorIf(len(arrayOfIndices.arrayval) < 1)
lib.InternalCodingErrorIf(len(arrayOfIndices.x.arrayval) < 1)
// If the input field name was "x.a" then remember the "x".
baseIndex := arrayOfIndices.arrayval[0].String()
baseIndex := arrayOfIndices.x.arrayval[0].String()
if fieldNameSet[baseIndex] {
// Use PutIndexed to assign $x["a"] = 7, or $x["b"] = 8, etc.
other.PutIndexed(
CopyMlrvalArray(arrayOfIndices.arrayval),
CopyMlrvalArray(arrayOfIndices.x.arrayval),
unflattenTerminal(pe.Value).Copy(),
)
affectedBaseIndices[baseIndex] = true
@ -247,7 +247,7 @@ func SplitAXHelper(input string, separator string) *Mlrval {
output := FromArray(make([]*Mlrval, len(fields)))
for i, field := range fields {
output.arrayval[i] = FromString(field)
output.x.arrayval[i] = FromString(field)
}
return output

14
internal/pkg/mlrval/mlrval_accessors.go
View file

@ -8,7 +8,7 @@ import (
func (mv *Mlrval) GetArrayLength() (int, bool) {
if mv.IsArray() {
return len(mv.arrayval), true
return len(mv.x.arrayval), true
} else {
return -999, false
}
@ -35,13 +35,13 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval {
if mv.IsMap() {
// Without this, the for-loop below is zero-pass and fields with "{}"
// values would disappear entirely in a JSON-to-CSV conversion.
if mv.mapval.IsEmpty() {
if mv.x.mapval.IsEmpty() {
if prefix != "" {
retval.PutCopy(prefix, FromString("{}"))
}
}
for pe := mv.mapval.Head; pe != nil; pe = pe.Next {
for pe := mv.x.mapval.Head; pe != nil; pe = pe.Next {
nextPrefix := pe.Key
if prefix != "" {
nextPrefix = prefix + delimiter + nextPrefix
@ -49,7 +49,7 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval {
if pe.Value.IsMap() || pe.Value.IsArray() {
nextResult := pe.Value.FlattenToMap(nextPrefix, delimiter)
lib.InternalCodingErrorIf(nextResult.mvtype != MT_MAP)
for pf := nextResult.mapval.Head; pf != nil; pf = pf.Next {
for pf := nextResult.x.mapval.Head; pf != nil; pf = pf.Next {
retval.PutCopy(pf.Key, pf.Value.Copy())
}
} else {
@ -60,13 +60,13 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval {
} else if mv.IsArray() {
// Without this, the for-loop below is zero-pass and fields with "[]"
// values would disappear entirely in a JSON-to-CSV conversion.
if len(mv.arrayval) == 0 {
if len(mv.x.arrayval) == 0 {
if prefix != "" {
retval.PutCopy(prefix, FromString("[]"))
}
}
for zindex, value := range mv.arrayval {
for zindex, value := range mv.x.arrayval {
nextPrefix := strconv.Itoa(zindex + 1) // Miller user-space indices are 1-up
if prefix != "" {
nextPrefix = prefix + delimiter + nextPrefix
@ -74,7 +74,7 @@ func (mv *Mlrval) FlattenToMap(prefix string, delimiter string) Mlrval {
if value.IsMap() || value.IsArray() {
nextResult := value.FlattenToMap(nextPrefix, delimiter)
lib.InternalCodingErrorIf(nextResult.mvtype != MT_MAP)
for pf := nextResult.mapval.Head; pf != nil; pf = pf.Next {
for pf := nextResult.x.mapval.Head; pf != nil; pf = pf.Next {
retval.PutCopy(pf.Key, pf.Value.Copy())
}
} else {

40
internal/pkg/mlrval/mlrval_collections.go
View file

@ -86,7 +86,7 @@ func (mv *Mlrval) ArrayGet(mindex *Mlrval) Mlrval {
if !mindex.IsInt() {
return *ERROR
}
value := arrayGetAliased(&mv.arrayval, int(mindex.intval))
value := arrayGetAliased(&mv.x.arrayval, int(mindex.intval))
if value == nil {
return *ABSENT
} else {
@ -116,12 +116,12 @@ func (mv *Mlrval) ArrayPut(mindex *Mlrval, value *Mlrval) {
os.Exit(1)
}
ok := arrayPutAliased(&mv.arrayval, int(mindex.intval), value)
ok := arrayPutAliased(&mv.x.arrayval, int(mindex.intval), value)
if !ok {
fmt.Fprintf(
os.Stderr,
"mlr: array index %d out of bounds %d..%d\n",
mindex.intval, 1, len(mv.arrayval),
mindex.intval, 1, len(mv.x.arrayval),
)
os.Exit(1)
}
@ -213,7 +213,7 @@ func (mv *Mlrval) ArrayAppend(value *Mlrval) {
// Silent no-ops are not good UX ...
return
}
mv.arrayval = append(mv.arrayval, value)
mv.x.arrayval = append(mv.x.arrayval, value)
}
// ================================================================
@ -222,7 +222,7 @@ func (mv *Mlrval) MapGet(key *Mlrval) Mlrval {
return *ERROR
}
mval, err := mv.mapval.GetWithMlrvalIndex(key)
mval, err := mv.x.mapval.GetWithMlrvalIndex(key)
if err != nil { // xxx maybe error-return in the API
return *ERROR
}
@ -243,9 +243,9 @@ func (mv *Mlrval) MapPut(key *Mlrval, value *Mlrval) {
}
if key.IsString() {
mv.mapval.PutCopy(key.printrep, value)
mv.x.mapval.PutCopy(key.printrep, value)
} else if key.IsInt() {
mv.mapval.PutCopy(key.String(), value)
mv.x.mapval.PutCopy(key.String(), value)
}
// TODO: need to be careful about semantics here.
// Silent no-ops are not good UX ...
@ -291,19 +291,19 @@ func (mv *Mlrval) PutIndexed(indices []*Mlrval, rvalue *Mlrval) error {
lib.InternalCodingErrorIf(len(indices) < 1)
if mv.IsMap() {
return putIndexedOnMap(mv.mapval, indices, rvalue)
return putIndexedOnMap(mv.x.mapval, indices, rvalue)
} else if mv.IsArray() {
return putIndexedOnArray(&mv.arrayval, indices, rvalue)
return putIndexedOnArray(&mv.x.arrayval, indices, rvalue)
} else {
baseIndex := indices[0]
if baseIndex.IsString() {
*mv = *FromEmptyMap()
return putIndexedOnMap(mv.mapval, indices, rvalue)
return putIndexedOnMap(mv.x.mapval, indices, rvalue)
} else if baseIndex.IsInt() {
*mv = *FromEmptyArray()
return putIndexedOnArray(&mv.arrayval, indices, rvalue)
return putIndexedOnArray(&mv.x.arrayval, indices, rvalue)
} else {
return errors.New(
"mlr: only maps and arrays are indexable; got " + mv.GetTypeName(),
@ -326,7 +326,7 @@ func putIndexedOnMap(baseMap *Mlrmap, indices []*Mlrval, rvalue *Mlrval) error {
".",
)
}
*baseMap = *rvalue.mapval.Copy()
*baseMap = *rvalue.x.mapval.Copy()
return nil
}
@ -438,10 +438,10 @@ func (mv *Mlrval) RemoveIndexed(indices []*Mlrval) error {
lib.InternalCodingErrorIf(len(indices) < 1)
if mv.IsMap() {
return removeIndexedOnMap(mv.mapval, indices)
return removeIndexedOnMap(mv.x.mapval, indices)
} else if mv.IsArray() {
return removeIndexedOnArray(&mv.arrayval, indices)
return removeIndexedOnArray(&mv.x.arrayval, indices)
} else {
return errors.New(
@ -659,13 +659,13 @@ func NewMlrvalForAutoDeepen(mvtype MVType) (*Mlrval, error) {
func (mv *Mlrval) Arrayify() *Mlrval {
if mv.IsMap() {
if mv.mapval.IsEmpty() {
if mv.x.mapval.IsEmpty() {
return mv
}
convertible := true
i := 0
for pe := mv.mapval.Head; pe != nil; pe = pe.Next {
for pe := mv.x.mapval.Head; pe != nil; pe = pe.Next {
sval := strconv.Itoa(i + 1) // Miller user-space indices are 1-up
i++
if pe.Key != sval {
@ -675,9 +675,9 @@ func (mv *Mlrval) Arrayify() *Mlrval {
}
if convertible {
arrayval := make([]*Mlrval, mv.mapval.FieldCount)
arrayval := make([]*Mlrval, mv.x.mapval.FieldCount)
i := 0
for pe := mv.mapval.Head; pe != nil; pe = pe.Next {
for pe := mv.x.mapval.Head; pe != nil; pe = pe.Next {
arrayval[i] = pe.Value.Copy()
i++
}
@ -690,8 +690,8 @@ func (mv *Mlrval) Arrayify() *Mlrval {
} else if mv.IsArray() {
// TODO: comment (or rethink) that this modifies its inputs!!
output := mv.Copy()
for i := range mv.arrayval {
output.arrayval[i] = output.arrayval[i].Arrayify()
for i := range mv.x.arrayval {
output.x.arrayval[i] = output.x.arrayval[i].Arrayify()
}
return output

12
internal/pkg/mlrval/mlrval_copy.go
View file

@ -4,9 +4,17 @@ package mlrval
func (mv *Mlrval) Copy() *Mlrval {
other := *mv
if mv.mvtype == MT_MAP {
other.mapval = mv.mapval.Copy()
other.x = &mlrvalExtended{
mapval: mv.x.mapval.Copy(),
}
} else if mv.mvtype == MT_ARRAY {
other.arrayval = CopyMlrvalArray(mv.arrayval)
other.x = &mlrvalExtended{
arrayval: CopyMlrvalArray(mv.x.arrayval),
}
} else if mv.mvtype == MT_FUNC {
other.x = &mlrvalExtended{
funcval: mv.x.funcval,
}
}
return &other
}

10
internal/pkg/mlrval/mlrval_get.go
View file

@ -65,7 +65,7 @@ func (mv *Mlrval) GetBoolValue() (boolValue bool, isBool bool) {
func (mv *Mlrval) GetArray() []*Mlrval {
if mv.IsArray() {
return mv.arrayval
return mv.x.arrayval
} else {
return nil
}
@ -73,7 +73,7 @@ func (mv *Mlrval) GetArray() []*Mlrval {
func (mv *Mlrval) GetMap() *Mlrmap {
if mv.IsMap() {
return mv.mapval
return mv.x.mapval
} else {
return nil
}
@ -81,7 +81,7 @@ func (mv *Mlrval) GetMap() *Mlrmap {
func (mv *Mlrval) GetFunction() interface{} {
if mv.Type() == MT_FUNC {
return mv.funcval
return mv.x.funcval
} else {
return nil
}
@ -122,12 +122,12 @@ func (mv *Mlrval) AcquireBoolValue() bool {
func (mv *Mlrval) AcquireArrayValue() []*Mlrval {
lib.InternalCodingErrorIf(mv.mvtype != MT_ARRAY)
return mv.arrayval
return mv.x.arrayval
}
func (mv *Mlrval) AcquireMapValue() *Mlrmap {
lib.InternalCodingErrorIf(mv.mvtype != MT_MAP)
return mv.mapval
return mv.x.mapval
}
func (mv *Mlrval) GetNumericToFloatValueOrDie() (floatValue float64) {

12
internal/pkg/mlrval/mlrval_json.go
View file

@ -411,7 +411,7 @@ func (mv *Mlrval) marshalJSONArray(
// TODO: libify
allTerminal := true
for _, element := range mv.arrayval {
for _, element := range mv.x.arrayval {
if element.IsArrayOrMap() {
allTerminal = false
break
@ -429,11 +429,11 @@ func (mv *Mlrval) marshalJSONArraySingleLine(
elementNestingDepth int,
outputIsStdout bool,
) (string, error) {
n := len(mv.arrayval)
n := len(mv.x.arrayval)
var buffer bytes.Buffer
buffer.WriteByte('[')
for i, element := range mv.arrayval {
for i, element := range mv.x.arrayval {
elementString, err := element.marshalJSONAux(JSON_SINGLE_LINE, elementNestingDepth+1, outputIsStdout)
if err != nil {
return "", err
@ -466,7 +466,7 @@ func (mv *Mlrval) marshalJSONArrayMultipleLines(
elementNestingDepth int,
outputIsStdout bool,
) (string, error) {
n := len(mv.arrayval)
n := len(mv.x.arrayval)
var buffer bytes.Buffer
// Write empty array as '[]'
@ -475,7 +475,7 @@ func (mv *Mlrval) marshalJSONArrayMultipleLines(
buffer.WriteByte('\n')
}
for i, element := range mv.arrayval {
for i, element := range mv.x.arrayval {
elementString, err := element.marshalJSONAux(jsonFormatting, elementNestingDepth+1, outputIsStdout)
if err != nil {
return "", err
@ -508,7 +508,7 @@ func (mv *Mlrval) marshalJSONMap(
outputIsStdout bool,
) (string, error) {
lib.InternalCodingErrorIf(mv.mvtype != MT_MAP)
s, err := mv.mapval.marshalJSONAux(jsonFormatting, elementNestingDepth, outputIsStdout)
s, err := mv.x.mapval.marshalJSONAux(jsonFormatting, elementNestingDepth, outputIsStdout)
if err != nil {
return "", err
}

12
internal/pkg/mlrval/mlrval_new.go
View file

@ -209,7 +209,9 @@ func FromFunction(funcval interface{}, name string) *Mlrval {
mvtype: MT_FUNC,
printrep: name,
printrepValid: true,
funcval: funcval,
x: &mlrvalExtended{
funcval: funcval,
},
}
}
@ -218,7 +220,9 @@ func FromArray(arrayval []*Mlrval) *Mlrval {
mvtype: MT_ARRAY,
printrep: "(bug-if-you-see-this:case-4)", // INVALID_PRINTREP,
printrepValid: false,
arrayval: CopyMlrvalArray(arrayval),
x: &mlrvalExtended{
arrayval: CopyMlrvalArray(arrayval),
},
}
}
@ -231,7 +235,9 @@ func FromMap(mapval *Mlrmap) *Mlrval {
mvtype: MT_MAP,
printrep: "(bug-if-you-see-this:case-5)", // INVALID_PRINTREP,
printrepValid: false,
mapval: mapval.Copy(),
x: &mlrvalExtended{
mapval: mapval.Copy(),
},
}
}

6
internal/pkg/mlrval/mlrval_new_test.go
View file

@ -124,17 +124,17 @@ func TestFromFunction(t *testing.T) {
mv := FromFunction("test data", "f001")
assert.Equal(t, MT_FUNC, mv.mvtype)
assert.True(t, mv.printrepValid)
assert.Equal(t, "test data", mv.funcval.(string))
assert.Equal(t, "test data", mv.x.funcval.(string))
}
func TestFromArray(t *testing.T) {
mv := FromArray([]*Mlrval{FromInt(10)})
assert.Equal(t, MT_ARRAY, mv.mvtype)
assert.Equal(t, 1, len(mv.arrayval))
assert.Equal(t, 1, len(mv.x.arrayval))
}
func TestFromMap(t *testing.T) {
mv := FromMap(NewMlrmap())
assert.Equal(t, MT_MAP, mv.mvtype)
assert.True(t, mv.mapval.IsEmpty())
assert.True(t, mv.x.mapval.IsEmpty())
}

22
internal/pkg/mlrval/mlrval_output.go
View file

@ -110,12 +110,12 @@ func (mv *Mlrval) StringifyValuesRecursively() {
switch mv.mvtype {
case MT_ARRAY:
for i, _ := range mv.arrayval {
mv.arrayval[i].StringifyValuesRecursively()
for i, _ := range mv.x.arrayval {
mv.x.arrayval[i].StringifyValuesRecursively()
}
case MT_MAP:
for pe := mv.mapval.Head; pe != nil; pe = pe.Next {
for pe := mv.x.mapval.Head; pe != nil; pe = pe.Next {
pe.Value.StringifyValuesRecursively()
}
@ -129,13 +129,17 @@ func (mv *Mlrval) ShowSizes() {
fmt.Printf("mv.intval %p %d\n", &mv.intval, reflect.TypeOf(mv.intval).Size())
fmt.Printf("mv.floatval %p %d\n", &mv.floatval, reflect.TypeOf(mv.floatval).Size())
fmt.Printf("mv.printrep %p %d\n", &mv.printrep, reflect.TypeOf(mv.printrep).Size())
fmt.Printf("mv.x %p %d\n", &mv.mvtype, reflect.TypeOf(mv.x).Size())
if mv.x != nil {
fmt.Printf("mv.x.arrayval %p %d\n", &mv.x.arrayval, reflect.TypeOf(mv.x.arrayval).Size())
fmt.Printf("mv.x.mapval %p %d\n", &mv.x.mapval, reflect.TypeOf(mv.x.mapval).Size())
if mv.x.funcval != nil {
fmt.Printf("mv.x.funcval %p %d\n", &mv.x.funcval, reflect.TypeOf(mv.x.funcval).Size())
}
}
fmt.Printf("mv.printrepValid %p %d\n", &mv.printrepValid, reflect.TypeOf(mv.printrepValid).Size())
fmt.Printf("mv.boolval %p %d\n", &mv.boolval, reflect.TypeOf(mv.boolval).Size())
fmt.Printf("mv.mvtype %p %d\n", &mv.mvtype, reflect.TypeOf(mv.mvtype).Size())
fmt.Printf("mv.arrayval %p %d\n", &mv.arrayval, reflect.TypeOf(mv.arrayval).Size())
fmt.Printf("mv.mapval %p %d\n", &mv.mapval, reflect.TypeOf(mv.mapval).Size())
if mv.funcval != nil {
fmt.Printf("mv.funcval %p %d\n", &mv.funcval, reflect.TypeOf(mv.funcval).Size())
}
}

13
internal/pkg/mlrval/mlrval_type.go
View file

@ -54,16 +54,23 @@
package mlrval
type Mlrval struct {
intval int64
floatval float64
printrep string
intval int64
floatval float64
printrep string
x *mlrvalExtended
printrepValid bool
boolval bool
// Enumeration for string / int / float / boolean / etc.
// I would call this "type" not "mvtype" but "type" is a keyword in Go.
mvtype MVType
}
// The Mlrval type is a (non-union) compound type where arrayval, mapval, and funcval are (a)
// largish, and (b) not usually used.
type mlrvalExtended struct {
arrayval []*Mlrval
mapval *Mlrmap
// First-class-function literals from internal/pkg/dsl/cst.