From 950b36281064c0f4e20d38312ea97597c07992ca Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sat, 8 Mar 2025 21:26:29 -0500 Subject: [PATCH] iterating --- pkg/transformers/bootstrap.go | 14 +++++++------- pkg/transformers/case.go | 5 ++++- pkg/transformers/count_similar.go | 2 +- pkg/transformers/fraction.go | 5 ++++- pkg/transformers/group_by.go | 6 +++--- pkg/transformers/group_like.go | 6 +++--- pkg/transformers/histogram.go | 10 ++++++++-- pkg/transformers/utils/join_bucket.go | 7 +++---- pkg/transformers/utils/join_bucket_keeper.go | 2 +- 9 files changed, 34 insertions(+), 23 deletions(-) diff --git a/pkg/transformers/bootstrap.go b/pkg/transformers/bootstrap.go index bdc06b90a..6da88e408 100644 --- a/pkg/transformers/bootstrap.go +++ b/pkg/transformers/bootstrap.go @@ -97,7 +97,7 @@ type TransformerBootstrap struct { func NewTransformerBootstrap(nout int64) (*TransformerBootstrap, error) { tr := &TransformerBootstrap{ - recordsAndContexts: list.New(), + recordsAndContexts: make([]*types.RecordAndContext, 100), // XXX recordsPerBatch nout: nout, } return tr, nil @@ -114,7 +114,7 @@ func (tr *TransformerBootstrap) Transform( HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel) // Not end of input stream: retain the record, and emit nothing until end of stream. if !inrecAndContext.EndOfStream { - tr.recordsAndContexts.PushBack(inrecAndContext) + tr.recordsAndContexts = append(tr.recordsAndContexts, inrecAndContext) return } @@ -142,7 +142,7 @@ func (tr *TransformerBootstrap) Transform( // For that reason, this transformer must copy all output. // TODO: Go list Len() maxes at 2^31. We should track this ourselves in an int. - nin := int64(tr.recordsAndContexts.Len()) + nin := int64(len(tr.recordsAndContexts)) nout := tr.nout if nout == -1 { nout = nin @@ -157,12 +157,12 @@ func (tr *TransformerBootstrap) Transform( // Make an array of pointers into the input list. recordArray := make([]*types.RecordAndContext, nin) for i := int64(0); i < nin; i++ { - head := tr.recordsAndContexts.Front() + head := tr.recordsAndContexts[0] if head == nil { break } - recordArray[i] = head.Value.(*types.RecordAndContext) - tr.recordsAndContexts.Remove(head) + recordArray[i] = head + tr.recordsAndContexts = tr.recordsAndContexts[1:] } // Do the sample-with-replacment, reading from random indices in the input @@ -171,7 +171,7 @@ func (tr *TransformerBootstrap) Transform( index := lib.RandRange(0, nin) recordAndContext := recordArray[index] // Already emitted once; copy - outputRecordsAndContexts.PushBack(recordAndContext.Copy()) + outputRecordsAndContexts = append(outputRecordsAndContexts, recordAndContext.Copy()) } // Emit the stream-terminating null record diff --git a/pkg/transformers/case.go b/pkg/transformers/case.go index a834f1d2f..ed0119ca5 100644 --- a/pkg/transformers/case.go +++ b/pkg/transformers/case.go @@ -229,7 +229,10 @@ func (tr *TransformerCase) transformValuesOnly( } } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(inrec, &inrecAndContext.Context)) + outputRecordsAndContexts = append( + outputRecordsAndContexts, + types.NewRecordAndContext(inrec, &inrecAndContext.Context), + ) } func (tr *TransformerCase) transformKeysAndValues( diff --git a/pkg/transformers/count_similar.go b/pkg/transformers/count_similar.go index 02a415dc5..9b1812340 100644 --- a/pkg/transformers/count_similar.go +++ b/pkg/transformers/count_similar.go @@ -157,7 +157,7 @@ func (tr *TransformerCountSimilar) Transform( recordAndContext := inner.Value.(*types.RecordAndContext) recordAndContext.Record.PutCopy(tr.counterFieldName, mgroupSize) - outputRecordsAndContexts.PushBack(recordAndContext) + outputRecordsAndContexts = append(outputRecordsAndContexts, recordAndContext) } } diff --git a/pkg/transformers/fraction.go b/pkg/transformers/fraction.go index 8236630fd..97aec47b3 100644 --- a/pkg/transformers/fraction.go +++ b/pkg/transformers/fraction.go @@ -292,7 +292,10 @@ func (tr *TransformerFraction) Transform( } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &endOfStreamContext)) + outputRecordsAndContexts = append( + outputRecordsAndContexts, + types.NewRecordAndContext(outrec, &endOfStreamContext), + ) } outputRecordsAndContexts = append(outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } diff --git a/pkg/transformers/group_by.go b/pkg/transformers/group_by.go index 31169d206..811460eb3 100644 --- a/pkg/transformers/group_by.go +++ b/pkg/transformers/group_by.go @@ -136,9 +136,9 @@ func (tr *TransformerGroupBy) Transform( } else { for outer := tr.recordListsByGroup.Head; outer != nil; outer = outer.Next { - recordListForGroup := outer.Value.(*list.List) - for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() { - outputRecordsAndContexts.PushBack(inner.Value.(*types.RecordAndContext)) + recordListForGroup := outer.Value.([]*types.RecordAndContext) + for _, record := range recordListForGroup { + outputRecordsAndContexts = append(outputRecordsAndContexts, record) } } outputRecordsAndContexts = append(outputRecordsAndContexts, inrecAndContext) // end-of-stream marker diff --git a/pkg/transformers/group_like.go b/pkg/transformers/group_like.go index 9211a1c3f..146dc9642 100644 --- a/pkg/transformers/group_like.go +++ b/pkg/transformers/group_like.go @@ -115,9 +115,9 @@ func (tr *TransformerGroupLike) Transform( } else { for outer := tr.recordListsByGroup.Head; outer != nil; outer = outer.Next { - recordListForGroup := outer.Value.(*list.List) - for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() { - outputRecordsAndContexts.PushBack(inner.Value.(*types.RecordAndContext)) + recordListForGroup := outer.Value.([]*types.RecordAndContext) + for _, record := range recordListForGroup { + outputRecordsAndContexts = append(outputRecordsAndContexts, record) } } outputRecordsAndContexts = append(outputRecordsAndContexts, inrecAndContext) // end-of-stream marker diff --git a/pkg/transformers/histogram.go b/pkg/transformers/histogram.go index 56fe1b4ef..664ccc92c 100644 --- a/pkg/transformers/histogram.go +++ b/pkg/transformers/histogram.go @@ -275,7 +275,10 @@ func (tr *TransformerHistogram) emitNonAuto( ) } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, endOfStreamContext)) + outputRecordsAndContexts = append( + outputRecordsAndContexts, + types.NewRecordAndContext(outrec, endOfStreamContext), + ) } } @@ -381,6 +384,9 @@ func (tr *TransformerHistogram) emitAuto( ) } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, endOfStreamContext)) + outputRecordsAndContexts = append( + outputRecordsAndContexts, + types.NewRecordAndContext(outrec, endOfStreamContext), + ) } } diff --git a/pkg/transformers/utils/join_bucket.go b/pkg/transformers/utils/join_bucket.go index f4390906f..7195da6fa 100644 --- a/pkg/transformers/utils/join_bucket.go +++ b/pkg/transformers/utils/join_bucket.go @@ -5,15 +5,14 @@ package utils import ( - "container/list" - "github.com/johnkerl/miller/v6/pkg/mlrval" + "github.com/johnkerl/miller/v6/pkg/types" ) // ---------------------------------------------------------------- type JoinBucket struct { leftFieldValues []*mlrval.Mlrval - RecordsAndContexts *list.List + RecordsAndContexts []*types.RecordAndContext WasPaired bool } @@ -22,7 +21,7 @@ func NewJoinBucket( ) *JoinBucket { return &JoinBucket{ leftFieldValues: leftFieldValues, - RecordsAndContexts: list.New(), + RecordsAndContexts: make([]*types.RecordAndContext, 0), WasPaired: false, } } diff --git a/pkg/transformers/utils/join_bucket_keeper.go b/pkg/transformers/utils/join_bucket_keeper.go index fb23758c8..91aefb468 100644 --- a/pkg/transformers/utils/join_bucket_keeper.go +++ b/pkg/transformers/utils/join_bucket_keeper.go @@ -289,7 +289,7 @@ func (keeper *JoinBucketKeeper) FindJoinBucket( } // TODO: privatize more - if keeper.JoinBucket.RecordsAndContexts.Len() > 0 { + if len(keeper.JoinBucket.RecordsAndContexts) > 0 { cmp := compareLexically( keeper.JoinBucket.leftFieldValues, rightFieldValues,