From a80c173aa4600a3a3660c4355ef69117c5bc5c6e Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sun, 9 Mar 2025 20:11:18 -0400 Subject: [PATCH] iterating --- pkg/transformers/bar.go | 2 +- pkg/transformers/bootstrap.go | 3 +-- pkg/transformers/count_similar.go | 6 +++--- pkg/transformers/fraction.go | 2 +- pkg/transformers/group_by.go | 6 +++--- pkg/transformers/group_like.go | 6 +++--- pkg/transformers/join.go | 4 ++-- pkg/transformers/nest.go | 4 ++-- pkg/transformers/remove_empty_columns.go | 2 +- pkg/transformers/shuffle.go | 2 +- pkg/transformers/sort.go | 2 +- pkg/transformers/tac.go | 2 +- pkg/transformers/unsparsify.go | 2 +- 13 files changed, 21 insertions(+), 22 deletions(-) diff --git a/pkg/transformers/bar.go b/pkg/transformers/bar.go index d2a712697..f14bdd8ca 100644 --- a/pkg/transformers/bar.go +++ b/pkg/transformers/bar.go @@ -150,7 +150,7 @@ type TransformerBar struct { oobString string blankString string bars []string - recordsForAutoMode *list.List + recordsForAutoMode *types.List[*types.RecordAndContext] recordTransformerFunc RecordTransformerFunc } diff --git a/pkg/transformers/bootstrap.go b/pkg/transformers/bootstrap.go index 5e5f9a435..dcd319d55 100644 --- a/pkg/transformers/bootstrap.go +++ b/pkg/transformers/bootstrap.go @@ -90,7 +90,7 @@ func transformerBootstrapParseCLI( // ---------------------------------------------------------------- type TransformerBootstrap struct { - recordsAndContexts *list.List + recordsAndContexts *types.List[*types.RecordAndContext] nout int64 } @@ -140,7 +140,6 @@ func (tr *TransformerBootstrap) Transform( // // For that reason, this transformer must copy all output. - // TODO: Go list Len() maxes at 2^31. We should track this ourselves in an int. nin := int64(tr.recordsAndContexts.Len()) nout := tr.nout if nout == -1 { diff --git a/pkg/transformers/count_similar.go b/pkg/transformers/count_similar.go index b4b19b29e..8610dda8c 100644 --- a/pkg/transformers/count_similar.go +++ b/pkg/transformers/count_similar.go @@ -104,7 +104,7 @@ type TransformerCountSimilar struct { counterFieldName string // State: - recordListsByGroup *lib.OrderedMap // map from string to *list.List + recordListsByGroup *lib.OrderedMap // map from string to *types.List[*types.RecordAndContext] } // ---------------------------------------------------------------- @@ -142,13 +142,13 @@ func (tr *TransformerCountSimilar) Transform( irecordListForGroup = list.New() tr.recordListsByGroup.Put(groupingKey, irecordListForGroup) } - recordListForGroup := irecordListForGroup.(*list.List) + recordListForGroup := irecordListForGroup.(*types.List[*types.RecordAndContext]) recordListForGroup.PushBack(inrecAndContext) } else { for outer := tr.recordListsByGroup.Head; outer != nil; outer = outer.Next { - recordListForGroup := outer.Value.(*list.List) + recordListForGroup := outer.Value.(*types.List[*types.RecordAndContext]) // TODO: make 64-bit friendly groupSize := recordListForGroup.Len() mgroupSize := mlrval.FromInt(int64(groupSize)) diff --git a/pkg/transformers/fraction.go b/pkg/transformers/fraction.go index c8638ff08..df76fde27 100644 --- a/pkg/transformers/fraction.go +++ b/pkg/transformers/fraction.go @@ -131,7 +131,7 @@ type TransformerFraction struct { groupByFieldNames []string doCumu bool - recordsAndContexts *list.List + recordsAndContexts *types.List[*types.RecordAndContext] // Two-level map: Group-by field names are the first keyset; // fraction field names are keys into the second. sums map[string]map[string]*mlrval.Mlrval diff --git a/pkg/transformers/group_by.go b/pkg/transformers/group_by.go index 441b2b1a1..994cb0505 100644 --- a/pkg/transformers/group_by.go +++ b/pkg/transformers/group_by.go @@ -91,7 +91,7 @@ type TransformerGroupBy struct { groupByFieldNames []string // state - // map from string to *list.List + // map from string to *types.List[*types.RecordAndContext] recordListsByGroup *lib.OrderedMap } @@ -131,11 +131,11 @@ func (tr *TransformerGroupBy) Transform( tr.recordListsByGroup.Put(groupingKey, recordListForGroup) } - recordListForGroup.(*list.List).PushBack(inrecAndContext) + recordListForGroup.(*types.List[*types.RecordAndContext]).PushBack(inrecAndContext) } else { for outer := tr.recordListsByGroup.Head; outer != nil; outer = outer.Next { - recordListForGroup := outer.Value.(*list.List) + recordListForGroup := outer.Value.(*types.List[*types.RecordAndContext]) for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() { outputRecordsAndContexts.PushBack(inner.Value.(*types.RecordAndContext)) } diff --git a/pkg/transformers/group_like.go b/pkg/transformers/group_like.go index 044e607bb..a6e6cd8c4 100644 --- a/pkg/transformers/group_like.go +++ b/pkg/transformers/group_like.go @@ -77,7 +77,7 @@ func transformerGroupLikeParseCLI( // ---------------------------------------------------------------- type TransformerGroupLike struct { - // map from string to *list.List + // map from string to *types.List[*types.RecordAndContext] recordListsByGroup *lib.OrderedMap } @@ -110,11 +110,11 @@ func (tr *TransformerGroupLike) Transform( tr.recordListsByGroup.Put(groupingKey, recordListForGroup) } - recordListForGroup.(*list.List).PushBack(inrecAndContext) + recordListForGroup.(*types.List[*types.RecordAndContext]).PushBack(inrecAndContext) } else { for outer := tr.recordListsByGroup.Head; outer != nil; outer = outer.Next { - recordListForGroup := outer.Value.(*list.List) + recordListForGroup := outer.Value.(*types.List[*types.RecordAndContext]) for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() { outputRecordsAndContexts.PushBack(inner.Value.(*types.RecordAndContext)) } diff --git a/pkg/transformers/join.go b/pkg/transformers/join.go index efcf957a0..849e4405e 100644 --- a/pkg/transformers/join.go +++ b/pkg/transformers/join.go @@ -295,7 +295,7 @@ type TransformerJoin struct { // For unsorted/half-streaming input ingested bool leftBucketsByJoinFieldValues *lib.OrderedMap - leftUnpairableRecordsAndContexts *list.List + leftUnpairableRecordsAndContexts *types.List[*types.RecordAndContext] // For sorted/doubly-streaming input joinBucketKeeper *utils.JoinBucketKeeper @@ -557,7 +557,7 @@ func (tr *TransformerJoin) ingestLeftFile() { // the doubly-streaming/sorted join. func (tr *TransformerJoin) formAndEmitPairs( - leftRecordsAndContexts *list.List, + leftRecordsAndContexts *types.List[*types.RecordAndContext], rightRecordAndContext *types.RecordAndContext, outputRecordsAndContexts *types.List[*types.RecordAndContext], ) { diff --git a/pkg/transformers/nest.go b/pkg/transformers/nest.go index 01f8ead1f..e66ab550c 100644 --- a/pkg/transformers/nest.go +++ b/pkg/transformers/nest.go @@ -601,12 +601,12 @@ func (tr *TransformerNest) implodeValueAcrossRecords( type tNestBucket struct { representative *mlrval.Mlrmap - pairs *list.List + pairs *types.List[*mlrval.Mlrmap] } func newNestBucket(representative *mlrval.Mlrmap) *tNestBucket { return &tNestBucket{ representative: representative, - pairs: list.New(), + pairs: *types.NewList[*types.RecordAndContext](100), // XXX size } } diff --git a/pkg/transformers/remove_empty_columns.go b/pkg/transformers/remove_empty_columns.go index bd67a48b6..0b37e2185 100644 --- a/pkg/transformers/remove_empty_columns.go +++ b/pkg/transformers/remove_empty_columns.go @@ -77,7 +77,7 @@ func transformerRemoveEmptyColumnsParseCLI( // ---------------------------------------------------------------- type TransformerRemoveEmptyColumns struct { - recordsAndContexts *list.List + recordsAndContexts *types.List[*types.RecordAndContext] namesWithNonEmptyValues map[string]bool } diff --git a/pkg/transformers/shuffle.go b/pkg/transformers/shuffle.go index 6ef83e4fd..a82ecf266 100644 --- a/pkg/transformers/shuffle.go +++ b/pkg/transformers/shuffle.go @@ -80,7 +80,7 @@ func transformerShuffleParseCLI( // ---------------------------------------------------------------- type TransformerShuffle struct { - recordsAndContexts *list.List + recordsAndContexts *types.List[*types.RecordAndContext] } func NewTransformerShuffle() (*TransformerShuffle, error) { diff --git a/pkg/transformers/sort.go b/pkg/transformers/sort.go index 96349505d..cba7dbfcc 100644 --- a/pkg/transformers/sort.go +++ b/pkg/transformers/sort.go @@ -305,7 +305,7 @@ type TransformerSort struct { comparatorFuncs []mlrval.CmpFuncInt // -- State - // Map from string to *list.List: + // Map from string to *types.List[*types.RecordAndContext] recordListsByGroup *lib.OrderedMap // Map from string to []*lib.Mlrval: groupHeads *lib.OrderedMap diff --git a/pkg/transformers/tac.go b/pkg/transformers/tac.go index a9462cb53..8382c9c41 100644 --- a/pkg/transformers/tac.go +++ b/pkg/transformers/tac.go @@ -76,7 +76,7 @@ func transformerTacParseCLI( // ---------------------------------------------------------------- type TransformerTac struct { - recordsAndContexts *list.List + recordsAndContexts *types.List[*types.RecordAndContext] } func NewTransformerTac() (*TransformerTac, error) { diff --git a/pkg/transformers/unsparsify.go b/pkg/transformers/unsparsify.go index 38236240c..cb82940dd 100644 --- a/pkg/transformers/unsparsify.go +++ b/pkg/transformers/unsparsify.go @@ -107,7 +107,7 @@ func transformerUnsparsifyParseCLI( // ---------------------------------------------------------------- type TransformerUnsparsify struct { fillerMlrval *mlrval.Mlrval - recordsAndContexts *list.List + recordsAndContexts *types.List[*types.RecordAndContext] fieldNamesSeen *lib.OrderedMap recordTransformerFunc RecordTransformerFunc }