Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-01-22 18:06:52 +00:00
Code Issues Projects Releases Wiki Activity

iterating

Browse source
This commit is contained in:
John Kerl 2025-03-08 21:26:29 -05:00
parent 6da45fef72
commit 950b362810
9 changed files with 34 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

14
pkg/transformers/bootstrap.go
View file

@ -97,7 +97,7 @@ type TransformerBootstrap struct {
func NewTransformerBootstrap(nout int64) (*TransformerBootstrap, error) {
tr := &TransformerBootstrap{
recordsAndContexts: list.New(),
recordsAndContexts: make([]*types.RecordAndContext, 100), // XXX recordsPerBatch
nout: nout,
}
return tr, nil
@ -114,7 +114,7 @@ func (tr *TransformerBootstrap) Transform(
HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel)
// Not end of input stream: retain the record, and emit nothing until end of stream.
if !inrecAndContext.EndOfStream {
tr.recordsAndContexts.PushBack(inrecAndContext)
tr.recordsAndContexts = append(tr.recordsAndContexts, inrecAndContext)
return
}
@ -142,7 +142,7 @@ func (tr *TransformerBootstrap) Transform(
// For that reason, this transformer must copy all output.
// TODO: Go list Len() maxes at 2^31. We should track this ourselves in an int.
nin := int64(tr.recordsAndContexts.Len())
nin := int64(len(tr.recordsAndContexts))
nout := tr.nout
if nout == -1 {
nout = nin
@ -157,12 +157,12 @@ func (tr *TransformerBootstrap) Transform(
// Make an array of pointers into the input list.
recordArray := make([]*types.RecordAndContext, nin)
for i := int64(0); i < nin; i++ {
head := tr.recordsAndContexts.Front()
head := tr.recordsAndContexts[0]
if head == nil {
break
}
recordArray[i] = head.Value.(*types.RecordAndContext)
tr.recordsAndContexts.Remove(head)
recordArray[i] = head
tr.recordsAndContexts = tr.recordsAndContexts[1:]
}
// Do the sample-with-replacment, reading from random indices in the input
@ -171,7 +171,7 @@ func (tr *TransformerBootstrap) Transform(
index := lib.RandRange(0, nin)
recordAndContext := recordArray[index]
// Already emitted once; copy
outputRecordsAndContexts.PushBack(recordAndContext.Copy())
outputRecordsAndContexts = append(outputRecordsAndContexts, recordAndContext.Copy())
}
// Emit the stream-terminating null record

5
pkg/transformers/case.go
View file

@ -229,7 +229,10 @@ func (tr *TransformerCase) transformValuesOnly(
}
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(inrec, &inrecAndContext.Context))
outputRecordsAndContexts = append(
outputRecordsAndContexts,
types.NewRecordAndContext(inrec, &inrecAndContext.Context),
)
}
func (tr *TransformerCase) transformKeysAndValues(

2
pkg/transformers/count_similar.go
View file

@ -157,7 +157,7 @@ func (tr *TransformerCountSimilar) Transform(
recordAndContext := inner.Value.(*types.RecordAndContext)
recordAndContext.Record.PutCopy(tr.counterFieldName, mgroupSize)
outputRecordsAndContexts.PushBack(recordAndContext)
outputRecordsAndContexts = append(outputRecordsAndContexts, recordAndContext)
}
}

5
pkg/transformers/fraction.go
View file

@ -292,7 +292,10 @@ func (tr *TransformerFraction) Transform(
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &endOfStreamContext))
outputRecordsAndContexts = append(
outputRecordsAndContexts,
types.NewRecordAndContext(outrec, &endOfStreamContext),
)
}
outputRecordsAndContexts = append(outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}

6
pkg/transformers/group_by.go
View file

@ -136,9 +136,9 @@ func (tr *TransformerGroupBy) Transform(
} else {
for outer := tr.recordListsByGroup.Head; outer != nil; outer = outer.Next {
recordListForGroup := outer.Value.(*list.List)
for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() {
outputRecordsAndContexts.PushBack(inner.Value.(*types.RecordAndContext))
recordListForGroup := outer.Value.([]*types.RecordAndContext)
for _, record := range recordListForGroup {
outputRecordsAndContexts = append(outputRecordsAndContexts, record)
}
}
outputRecordsAndContexts = append(outputRecordsAndContexts, inrecAndContext) // end-of-stream marker

6
pkg/transformers/group_like.go
View file

@ -115,9 +115,9 @@ func (tr *TransformerGroupLike) Transform(
} else {
for outer := tr.recordListsByGroup.Head; outer != nil; outer = outer.Next {
recordListForGroup := outer.Value.(*list.List)
for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() {
outputRecordsAndContexts.PushBack(inner.Value.(*types.RecordAndContext))
recordListForGroup := outer.Value.([]*types.RecordAndContext)
for _, record := range recordListForGroup {
outputRecordsAndContexts = append(outputRecordsAndContexts, record)
}
}
outputRecordsAndContexts = append(outputRecordsAndContexts, inrecAndContext) // end-of-stream marker

10
pkg/transformers/histogram.go
View file

@ -275,7 +275,10 @@ func (tr *TransformerHistogram) emitNonAuto(
)
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, endOfStreamContext))
outputRecordsAndContexts = append(
outputRecordsAndContexts,
types.NewRecordAndContext(outrec, endOfStreamContext),
)
}
}
@ -381,6 +384,9 @@ func (tr *TransformerHistogram) emitAuto(
)
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, endOfStreamContext))
outputRecordsAndContexts = append(
outputRecordsAndContexts,
types.NewRecordAndContext(outrec, endOfStreamContext),
)
}
}

7
pkg/transformers/utils/join_bucket.go
View file

@ -5,15 +5,14 @@
package utils
import (
"container/list"
"github.com/johnkerl/miller/v6/pkg/mlrval"
"github.com/johnkerl/miller/v6/pkg/types"
)
// ----------------------------------------------------------------
type JoinBucket struct {
leftFieldValues []*mlrval.Mlrval
RecordsAndContexts *list.List
RecordsAndContexts []*types.RecordAndContext
WasPaired bool
}
@ -22,7 +21,7 @@ func NewJoinBucket(
) *JoinBucket {
return &JoinBucket{
leftFieldValues: leftFieldValues,
RecordsAndContexts: list.New(),
RecordsAndContexts: make([]*types.RecordAndContext, 0),
WasPaired: false,
}
}

2
pkg/transformers/utils/join_bucket_keeper.go
View file

@ -289,7 +289,7 @@ func (keeper *JoinBucketKeeper) FindJoinBucket(
}
// TODO: privatize more
if keeper.JoinBucket.RecordsAndContexts.Len() > 0 {
if len(keeper.JoinBucket.RecordsAndContexts) > 0 {
cmp := compareLexically(
keeper.JoinBucket.leftFieldValues,
rightFieldValues,