Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-01-23 02:14:13 +00:00
Code Issues Projects Releases Wiki Activity

iterating

Browse source
This commit is contained in:
John Kerl 2025-03-09 14:59:16 -04:00
parent 55740c471c
commit 863c476f8d
74 changed files with 188 additions and 264 deletions
Download patch file Download diff file Expand all files Collapse all files

3
docs/src/miller-as-library/main2.go
View file

@ -3,7 +3,6 @@ package main
import (
"bufio"
"container/list"
"fmt"
"os"
@ -67,7 +66,7 @@ func run_custom_processor(
}
// Set up the channels for the record-reader.
readerChannel := make(chan *list.List, 2) // list of *types.RecordAndContext
readerChannel := make(chan *types.List[*types.RecordAndContext], 2)
inputErrorChannel := make(chan error, 1)
// Not needed in this example
readerDownstreamDoneChannel := make(chan bool, 1)

5
docs/src/miller-as-library/main3.go
View file

@ -2,7 +2,6 @@ package main
import (
"bufio"
"container/list"
"errors"
"fmt"
"os"
@ -60,8 +59,8 @@ func convert_csv_to_json(fileNames []string) error {
recordTransformers := []transformers.IRecordTransformer{cat}
// Set up the reader-to-transformer and transformer-to-writer channels.
readerChannel := make(chan *list.List, 2) // list of *types.RecordAndContext
writerChannel := make(chan *list.List, 1) // list of *types.RecordAndContext
readerChannel := make(chan *types.List[*types.RecordAndContext], 2)
writerChannel := make(chan *types.List[*types.RecordAndContext], 1)
// We're done when a fatal error is registered on input (file not found,
// etc) or when the record-writer has written all its output. We use

4
pkg/runtime/state.go
View file

@ -7,8 +7,6 @@
package runtime
import (
"container/list"
"github.com/johnkerl/miller/v6/pkg/cli"
"github.com/johnkerl/miller/v6/pkg/lib"
"github.com/johnkerl/miller/v6/pkg/mlrval"
@ -21,7 +19,7 @@ type State struct {
Oosvars *mlrval.Mlrmap
FilterExpression *mlrval.Mlrval
Stack *Stack
OutputRecordsAndContexts *list.List // list of *types.RecordAndContext
OutputRecordsAndContexts *types.List[*types.RecordAndContext]
// For holding "\0".."\9" between where they are set via things like
// '$x =~ "(..)_(...)"', and interpolated via things like '$y = "\2:\1"'.

5
pkg/stream/stream.go
View file

@ -2,7 +2,6 @@ package stream
import (
"bufio"
"container/list"
"errors"
"io"
@ -62,8 +61,8 @@ func Stream(
}
// Set up the reader-to-transformer and transformer-to-writer channels.
readerChannel := make(chan *list.List, 2) // list of *types.RecordAndContext
writerChannel := make(chan *list.List, 1) // list of *types.RecordAndContext
readerChannel := make(chan *types.List[*types.RecordAndContext], 2)
writerChannel := make(chan *types.List[*types.RecordAndContext], 1)
// We're done when a fatal error is registered on input (file not found,
// etc) or when the record-writer has written all its output. We use

4
pkg/terminals/repl/types.go
View file

@ -6,7 +6,6 @@ package repl
import (
"bufio"
"container/list"
"os"
"github.com/johnkerl/miller/v6/pkg/cli"
@ -14,6 +13,7 @@ import (
"github.com/johnkerl/miller/v6/pkg/input"
"github.com/johnkerl/miller/v6/pkg/output"
"github.com/johnkerl/miller/v6/pkg/runtime"
"github.com/johnkerl/miller/v6/pkg/types"
)
// ================================================================
@ -46,7 +46,7 @@ type Repl struct {
options *cli.TOptions
readerChannel chan *list.List // list of *types.RecordAndContext
readerChannel chan *types.List[*types.RecordAndContext]
errorChannel chan error
downstreamDoneChannel chan bool
recordReader input.IRecordReader

15
pkg/terminals/repl/verbs.go
View file

@ -5,7 +5,6 @@
package repl
import (
"container/list"
"fmt"
"os"
"strings"
@ -236,7 +235,7 @@ func (repl *Repl) openFiles(filenames []string) {
// Remember for :reopen
repl.options.FileNames = filenames
repl.readerChannel = make(chan *list.List, 2) // list of *types.RecordAndContext
repl.readerChannel = make(chan *types.List[*types.RecordAndContext], 2)
repl.errorChannel = make(chan error, 1)
repl.downstreamDoneChannel = make(chan bool, 1)
@ -288,7 +287,7 @@ func handleRead(repl *Repl, args []string) bool {
return true
}
var recordsAndContexts *list.List // list of *types.RecordAndContext
var recordsAndContexts *types.List[*types.RecordAndContext]
var err error = nil
select {
@ -308,7 +307,7 @@ func handleRead(repl *Repl, args []string) bool {
if recordsAndContexts != nil {
// TODO: comment and make very clear we've set this all up to batch by 1 for the REPL
lib.InternalCodingErrorIf(recordsAndContexts.Len() != 1)
recordAndContext := recordsAndContexts.Front().Value.(*types.RecordAndContext)
recordAndContext := recordsAndContexts.Front()
skipOrProcessRecord(
repl,
@ -436,7 +435,7 @@ func handleProcess(repl *Repl, args []string) bool {
// ----------------------------------------------------------------
func handleSkipOrProcessN(repl *Repl, n int64, processingNotSkipping bool) {
var recordsAndContexts *list.List // list of *types.RecordAndContext
var recordsAndContexts *types.List[*types.RecordAndContext]
var err error = nil
for i := int64(1); i <= n; i++ {
@ -456,7 +455,7 @@ func handleSkipOrProcessN(repl *Repl, n int64, processingNotSkipping bool) {
if recordsAndContexts != nil {
// TODO: comment and make very clear we've set this all up to batch by 1 for the REPL
lib.InternalCodingErrorIf(recordsAndContexts.Len() != 1)
recordAndContext := recordsAndContexts.Front().Value.(*types.RecordAndContext)
recordAndContext := recordsAndContexts.Front()
shouldBreak := skipOrProcessRecord(
repl,
@ -496,7 +495,7 @@ func handleSkipOrProcessUntil(repl *Repl, dslString string, processingNotSkippin
return
}
var recordsAndContexts *list.List // list of *types.RecordAndContext
var recordsAndContexts *types.List[*types.RecordAndContext]
for {
doubleBreak := false
@ -521,7 +520,7 @@ func handleSkipOrProcessUntil(repl *Repl, dslString string, processingNotSkippin
if recordsAndContexts != nil {
// TODO: comment and make very clear we've set this all up to batch by 1 for the REPL
lib.InternalCodingErrorIf(recordsAndContexts.Len() != 1)
recordAndContext := recordsAndContexts.Front().Value.(*types.RecordAndContext)
recordAndContext := recordsAndContexts.Front()
shouldBreak := skipOrProcessRecord(
repl,

7
pkg/transformers/aaa_chain_transformer.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"github.com/johnkerl/miller/v6/pkg/cli"
"github.com/johnkerl/miller/v6/pkg/types"
@ -232,12 +231,10 @@ func runSingleTransformerBatch(
outputDownstreamDoneChannel chan<- bool,
options *cli.TOptions,
) bool {
outputRecordsAndContexts := list.New()
outputRecordsAndContexts := types.NewList[*types.RecordAndContext](100) // XXX SIZint(reader.recordsPerBatch))D
done := false
for e := inputRecordsAndContexts.Front(); e != nil; e = e.Next() {
inputRecordAndContext := e.Value.(*types.RecordAndContext)
for _, inputRecordAndContext := range inputRecordsAndContexts.Items {
// --nr-progress-mod
// TODO: function-pointer this away to reduce instruction count in the
// normal case which it isn't used at all. No need to test if {static thing} != 0

1
pkg/transformers/aaa_record_transformer.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"os"
"github.com/johnkerl/miller/v6/pkg/cli"

3
pkg/transformers/altkv.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strconv"
@ -90,7 +89,7 @@ func NewTransformerAltkv() (*TransformerAltkv, error) {
func (tr *TransformerAltkv) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

7
pkg/transformers/bar.go
View file

@ -2,7 +2,6 @@ package transformers
import (
"bytes"
"container/list"
"fmt"
"os"
"strings"
@ -207,7 +206,7 @@ func NewTransformerBar(
func (tr *TransformerBar) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -218,7 +217,7 @@ func (tr *TransformerBar) Transform(
// ----------------------------------------------------------------
func (tr *TransformerBar) processNoAuto(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -253,7 +252,7 @@ func (tr *TransformerBar) processNoAuto(
// ----------------------------------------------------------------
func (tr *TransformerBar) processAuto(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/bootstrap.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -107,7 +106,7 @@ func NewTransformerBootstrap(nout int64) (*TransformerBootstrap, error) {
func (tr *TransformerBootstrap) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

9
pkg/transformers/case.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -177,7 +176,7 @@ func caseSentenceFunc(input string) string {
func (tr *TransformerCase) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -196,7 +195,7 @@ func (tr *TransformerCase) Transform(
func (tr *TransformerCase) transformKeysOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
_ <-chan bool,
__ chan<- bool,
) {
@ -216,7 +215,7 @@ func (tr *TransformerCase) transformKeysOnly(
func (tr *TransformerCase) transformValuesOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
_ <-chan bool,
__ chan<- bool,
) {
@ -234,7 +233,7 @@ func (tr *TransformerCase) transformValuesOnly(
func (tr *TransformerCase) transformKeysAndValues(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
_ <-chan bool,
__ chan<- bool,
) {

9
pkg/transformers/cat.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -165,7 +164,7 @@ func NewTransformerCat(
func (tr *TransformerCat) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -181,7 +180,7 @@ func (tr *TransformerCat) Transform(
// ----------------------------------------------------------------
func (tr *TransformerCat) simpleCat(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -199,7 +198,7 @@ func (tr *TransformerCat) simpleCat(
// ----------------------------------------------------------------
func (tr *TransformerCat) countersUngrouped(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -222,7 +221,7 @@ func (tr *TransformerCat) countersUngrouped(
// ----------------------------------------------------------------
func (tr *TransformerCat) countersGrouped(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/check.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -94,7 +93,7 @@ func NewTransformerCheck() (*TransformerCheck, error) {
func (tr *TransformerCheck) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

9
pkg/transformers/clean_whitespace.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -132,7 +131,7 @@ func NewTransformerCleanWhitespace(
func (tr *TransformerCleanWhitespace) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -143,7 +142,7 @@ func (tr *TransformerCleanWhitespace) Transform(
// ----------------------------------------------------------------
func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeysAndValues(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -168,7 +167,7 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeysAndValues(
// ----------------------------------------------------------------
func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeys(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -191,7 +190,7 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeys(
// ----------------------------------------------------------------
func (tr *TransformerCleanWhitespace) cleanWhitespaceInValues(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

7
pkg/transformers/count.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -150,7 +149,7 @@ func NewTransformerCount(
func (tr *TransformerCount) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -161,7 +160,7 @@ func (tr *TransformerCount) Transform(
// ----------------------------------------------------------------
func (tr *TransformerCount) countUngrouped(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -179,7 +178,7 @@ func (tr *TransformerCount) countUngrouped(
// ----------------------------------------------------------------
func (tr *TransformerCount) countGrouped(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/count_similar.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -125,7 +124,7 @@ func NewTransformerCountSimilar(
func (tr *TransformerCountSimilar) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

11
pkg/transformers/cut.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"regexp"
@ -181,7 +180,7 @@ func NewTransformerCut(
func (tr *TransformerCut) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -193,7 +192,7 @@ func (tr *TransformerCut) Transform(
// mlr cut -f a,b,c
func (tr *TransformerCut) includeWithInputOrder(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -218,7 +217,7 @@ func (tr *TransformerCut) includeWithInputOrder(
// mlr cut -o -f a,b,c
func (tr *TransformerCut) includeWithArgOrder(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -242,7 +241,7 @@ func (tr *TransformerCut) includeWithArgOrder(
// mlr cut -x -f a,b,c
func (tr *TransformerCut) exclude(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -260,7 +259,7 @@ func (tr *TransformerCut) exclude(
// ----------------------------------------------------------------
func (tr *TransformerCut) processWithRegexes(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/decimate.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -142,7 +141,7 @@ func NewTransformerDecimate(
func (tr *TransformerDecimate) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

7
pkg/transformers/fill_down.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -148,7 +147,7 @@ func NewTransformerFillDown(
func (tr *TransformerFillDown) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -159,7 +158,7 @@ func (tr *TransformerFillDown) Transform(
// ----------------------------------------------------------------
func (tr *TransformerFillDown) transformSpecified(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -197,7 +196,7 @@ func (tr *TransformerFillDown) transformSpecified(
// ----------------------------------------------------------------
func (tr *TransformerFillDown) transformAll(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/fill_empty.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -110,7 +109,7 @@ func NewTransformerFillEmpty(
func (tr *TransformerFillEmpty) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

7
pkg/transformers/flatten.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -134,7 +133,7 @@ func NewTransformerFlatten(
func (tr *TransformerFlatten) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -145,7 +144,7 @@ func (tr *TransformerFlatten) Transform(
// ----------------------------------------------------------------
func (tr *TransformerFlatten) flattenAll(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -165,7 +164,7 @@ func (tr *TransformerFlatten) flattenAll(
// ----------------------------------------------------------------
func (tr *TransformerFlatten) flattenSome(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/format_values.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -168,7 +167,7 @@ func NewTransformerFormatValues(
func (tr *TransformerFormatValues) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/fraction.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"errors"
"fmt"
"os"
@ -192,7 +191,7 @@ func NewTransformerFraction(
func (tr *TransformerFraction) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

7
pkg/transformers/gap.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -138,7 +137,7 @@ func NewTransformerGap(
func (tr *TransformerGap) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -148,7 +147,7 @@ func (tr *TransformerGap) Transform(
func (tr *TransformerGap) transformUnkeyed(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -168,7 +167,7 @@ func (tr *TransformerGap) transformUnkeyed(
func (tr *TransformerGap) transformKeyed(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/grep.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"regexp"
@ -155,7 +154,7 @@ func NewTransformerGrep(
func (tr *TransformerGrep) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/group_by.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -113,7 +112,7 @@ func NewTransformerGroupBy(
func (tr *TransformerGroupBy) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/group_like.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -95,7 +94,7 @@ func NewTransformerGroupLike() (*TransformerGroupLike, error) {
func (tr *TransformerGroupLike) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

15
pkg/transformers/having_fields.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"regexp"
@ -221,7 +220,7 @@ func NewTransformerHavingFields(
func (tr *TransformerHavingFields) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -232,7 +231,7 @@ func (tr *TransformerHavingFields) Transform(
// ----------------------------------------------------------------
func (tr *TransformerHavingFields) transformHavingFieldsAtLeast(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -256,7 +255,7 @@ func (tr *TransformerHavingFields) transformHavingFieldsAtLeast(
func (tr *TransformerHavingFields) transformHavingFieldsWhichAre(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -278,7 +277,7 @@ func (tr *TransformerHavingFields) transformHavingFieldsWhichAre(
func (tr *TransformerHavingFields) transformHavingFieldsAtMost(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -298,7 +297,7 @@ func (tr *TransformerHavingFields) transformHavingFieldsAtMost(
// ----------------------------------------------------------------
func (tr *TransformerHavingFields) transformHavingAllFieldsMatching(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -317,7 +316,7 @@ func (tr *TransformerHavingFields) transformHavingAllFieldsMatching(
func (tr *TransformerHavingFields) transformHavingAnyFieldsMatching(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -336,7 +335,7 @@ func (tr *TransformerHavingFields) transformHavingAnyFieldsMatching(
func (tr *TransformerHavingFields) transformHavingNoFieldsMatching(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

7
pkg/transformers/head.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -143,7 +142,7 @@ func NewTransformerHead(
func (tr *TransformerHead) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -153,7 +152,7 @@ func (tr *TransformerHead) Transform(
func (tr *TransformerHead) transformUnkeyed(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -176,7 +175,7 @@ func (tr *TransformerHead) transformUnkeyed(
func (tr *TransformerHead) transformKeyed(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

11
pkg/transformers/histogram.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -198,7 +197,7 @@ func NewTransformerHistogram(
func (tr *TransformerHistogram) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -209,7 +208,7 @@ func (tr *TransformerHistogram) Transform(
// ----------------------------------------------------------------
func (tr *TransformerHistogram) transformNonAuto(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -250,7 +249,7 @@ func (tr *TransformerHistogram) ingestNonAuto(
func (tr *TransformerHistogram) emitNonAuto(
endOfStreamContext *types.Context,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
countFieldNames := make(map[string]string)
for _, valueFieldName := range tr.valueFieldNames {
@ -282,7 +281,7 @@ func (tr *TransformerHistogram) emitNonAuto(
// ----------------------------------------------------------------
func (tr *TransformerHistogram) transformAuto(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -309,7 +308,7 @@ func (tr *TransformerHistogram) ingestAuto(
func (tr *TransformerHistogram) emitAuto(
endOfStreamContext *types.Context,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
haveLoHi := false
lo := 0.0

17
pkg/transformers/join.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -361,7 +360,7 @@ func NewTransformerJoin(
func (tr *TransformerJoin) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -375,7 +374,7 @@ func (tr *TransformerJoin) Transform(
// matching each right record against those.
func (tr *TransformerJoin) transformHalfStreaming(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -427,7 +426,7 @@ func (tr *TransformerJoin) transformHalfStreaming(
// ----------------------------------------------------------------
func (tr *TransformerJoin) transformDoublyStreaming(
rightRecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -496,7 +495,7 @@ func (tr *TransformerJoin) ingestLeftFile() {
initialContext.UpdateForStartOfFile(tr.opts.leftFileName)
// Set up channels for the record-reader.
readerChannel := make(chan *list.List, 2) // list of *types.RecordAndContext
readerChannel := make(chan *types.List[*types.RecordAndContext], 2)
errorChannel := make(chan error, 1)
downstreamDoneChannel := make(chan bool, 1)
@ -520,7 +519,7 @@ func (tr *TransformerJoin) ingestLeftFile() {
case leftrecsAndContexts := <-readerChannel:
// TODO: temp for batch-reader refactor
lib.InternalCodingErrorIf(leftrecsAndContexts.Len() != 1)
leftrecAndContext := leftrecsAndContexts.Front().Value.(*types.RecordAndContext)
leftrecAndContext := leftrecsAndContexts.Front()
leftrecAndContext.Record = utils.KeepLeftFieldNames(leftrecAndContext.Record, tr.leftKeepFieldNameSet)
if leftrecAndContext.EndOfStream {
@ -560,7 +559,7 @@ func (tr *TransformerJoin) ingestLeftFile() {
func (tr *TransformerJoin) formAndEmitPairs(
leftRecordsAndContexts *list.List,
rightRecordAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
////fmt.Println("-- pairs start") // VERBOSE
// Loop over each to-be-paired-with record from the left file.
@ -626,7 +625,7 @@ func (tr *TransformerJoin) formAndEmitPairs(
// in the second category.
func (tr *TransformerJoin) emitLeftUnpairables(
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
// Loop over each to-be-paired-with record from the left file.
for pe := tr.leftUnpairableRecordsAndContexts.Front(); pe != nil; pe = pe.Next() {
@ -636,7 +635,7 @@ func (tr *TransformerJoin) emitLeftUnpairables(
}
func (tr *TransformerJoin) emitLeftUnpairedBuckets(
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
for pe := tr.leftBucketsByJoinFieldValues.Head; pe != nil; pe = pe.Next {
bucket := pe.Value.(*utils.JoinBucket)

7
pkg/transformers/json_parse.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -131,7 +130,7 @@ func NewTransformerJSONParse(
func (tr *TransformerJSONParse) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -142,7 +141,7 @@ func (tr *TransformerJSONParse) Transform(
// ----------------------------------------------------------------
func (tr *TransformerJSONParse) jsonParseAll(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -164,7 +163,7 @@ func (tr *TransformerJSONParse) jsonParseAll(
// ----------------------------------------------------------------
func (tr *TransformerJSONParse) jsonParseSome(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

7
pkg/transformers/json_stringify.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -141,7 +140,7 @@ func NewTransformerJSONStringify(
func (tr *TransformerJSONStringify) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -152,7 +151,7 @@ func (tr *TransformerJSONStringify) Transform(
// ----------------------------------------------------------------
func (tr *TransformerJSONStringify) jsonStringifyAll(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -170,7 +169,7 @@ func (tr *TransformerJSONStringify) jsonStringifyAll(
// ----------------------------------------------------------------
func (tr *TransformerJSONStringify) jsonStringifySome(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/label.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -120,7 +119,7 @@ func NewTransformerLabel(
func (tr *TransformerLabel) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/latin1_to_utf8.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -91,7 +90,7 @@ func NewTransformerLatin1ToUTF8() (*TransformerLatin1ToUTF8, error) {
func (tr *TransformerLatin1ToUTF8) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

9
pkg/transformers/merge_fields.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"regexp"
@ -312,7 +311,7 @@ func NewTransformerMergeFields(
func (tr *TransformerMergeFields) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -323,7 +322,7 @@ func (tr *TransformerMergeFields) Transform(
// ----------------------------------------------------------------
func (tr *TransformerMergeFields) transformByNameList(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -374,7 +373,7 @@ func (tr *TransformerMergeFields) transformByNameList(
// ----------------------------------------------------------------
func (tr *TransformerMergeFields) transformByNameRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -455,7 +454,7 @@ func (tr *TransformerMergeFields) transformByNameRegex(
func (tr *TransformerMergeFields) transformByCollapsing(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/most_or_least_frequent.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"sort"
@ -205,7 +204,7 @@ func NewTransformerMostOrLeastFrequent(
func (tr *TransformerMostOrLeastFrequent) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

15
pkg/transformers/nest.go
View file

@ -2,7 +2,6 @@ package transformers
import (
"bytes"
"container/list"
"fmt"
"os"
"regexp"
@ -317,7 +316,7 @@ func NewTransformerNest(
func (tr *TransformerNest) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -328,7 +327,7 @@ func (tr *TransformerNest) Transform(
// ----------------------------------------------------------------
func (tr *TransformerNest) explodeValuesAcrossFields(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -366,7 +365,7 @@ func (tr *TransformerNest) explodeValuesAcrossFields(
// ----------------------------------------------------------------
func (tr *TransformerNest) explodeValuesAcrossRecords(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -395,7 +394,7 @@ func (tr *TransformerNest) explodeValuesAcrossRecords(
// ----------------------------------------------------------------
func (tr *TransformerNest) explodePairsAcrossFields(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -441,7 +440,7 @@ func (tr *TransformerNest) explodePairsAcrossFields(
// ----------------------------------------------------------------
func (tr *TransformerNest) explodePairsAcrossRecords(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -481,7 +480,7 @@ func (tr *TransformerNest) explodePairsAcrossRecords(
// ----------------------------------------------------------------
func (tr *TransformerNest) implodeValuesAcrossFields(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -530,7 +529,7 @@ func (tr *TransformerNest) implodeValuesAcrossFields(
// ----------------------------------------------------------------
func (tr *TransformerNest) implodeValueAcrossRecords(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/nothing.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -87,7 +86,7 @@ func NewTransformerNothing() (*TransformerNothing, error) {
func (tr *TransformerNothing) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/put_or_filter.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -494,7 +493,7 @@ func NewTransformerPut(
func (tr *TransformerPut) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/regularize.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -94,7 +93,7 @@ func NewTransformerRegularize() (*TransformerRegularize, error) {
func (tr *TransformerRegularize) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/remove_empty_columns.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -94,7 +93,7 @@ func NewTransformerRemoveEmptyColumns() (*TransformerRemoveEmptyColumns, error)
func (tr *TransformerRemoveEmptyColumns) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

7
pkg/transformers/rename.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"regexp"
@ -188,7 +187,7 @@ func NewTransformerRename(
func (tr *TransformerRename) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -199,7 +198,7 @@ func (tr *TransformerRename) Transform(
// ----------------------------------------------------------------
func (tr *TransformerRename) transformWithoutRegexes(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -220,7 +219,7 @@ func (tr *TransformerRename) transformWithoutRegexes(
// ----------------------------------------------------------------
func (tr *TransformerRename) transformWithRegexes(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

15
pkg/transformers/reorder.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"regexp"
@ -202,7 +201,7 @@ func NewTransformerReorder(
func (tr *TransformerReorder) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -219,7 +218,7 @@ func (tr *TransformerReorder) Transform(
func (tr *TransformerReorder) reorderToStartNoRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
inrec := inrecAndContext.Record
for _, fieldName := range tr.fieldNames {
@ -230,7 +229,7 @@ func (tr *TransformerReorder) reorderToStartNoRegex(
func (tr *TransformerReorder) reorderToStartWithRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
inrec := inrecAndContext.Record
@ -262,7 +261,7 @@ func (tr *TransformerReorder) reorderToStartWithRegex(
func (tr *TransformerReorder) reorderToEndNoRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
inrec := inrecAndContext.Record
for _, fieldName := range tr.fieldNames {
@ -274,7 +273,7 @@ func (tr *TransformerReorder) reorderToEndNoRegex(
func (tr *TransformerReorder) reorderToEndWithRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
inrec := inrecAndContext.Record
outrec := mlrval.NewMlrmapAsRecord()
@ -305,7 +304,7 @@ func (tr *TransformerReorder) reorderToEndWithRegex(
func (tr *TransformerReorder) reorderBeforeOrAfterNoRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
inrec := inrecAndContext.Record
if inrec.Get(tr.centerFieldName) == nil {
@ -362,7 +361,7 @@ func (tr *TransformerReorder) reorderBeforeOrAfterNoRegex(
func (tr *TransformerReorder) reorderBeforeOrAfterWithRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
inrec := inrecAndContext.Record
if inrec.Get(tr.centerFieldName) == nil {

7
pkg/transformers/repeat.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -159,7 +158,7 @@ func NewTransformerRepeat(
func (tr *TransformerRepeat) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -170,7 +169,7 @@ func (tr *TransformerRepeat) Transform(
// ----------------------------------------------------------------
func (tr *TransformerRepeat) repeatByCount(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -189,7 +188,7 @@ func (tr *TransformerRepeat) repeatByCount(
// ----------------------------------------------------------------
func (tr *TransformerRepeat) repeatByFieldName(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

9
pkg/transformers/reshape.go
View file

@ -28,7 +28,6 @@ package transformers
// 15 2009-01-05 Z 0.09719105
import (
"container/list"
"fmt"
"os"
"regexp"
@ -292,7 +291,7 @@ func NewTransformerReshape(
func (tr *TransformerReshape) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -303,7 +302,7 @@ func (tr *TransformerReshape) Transform(
// ----------------------------------------------------------------
func (tr *TransformerReshape) wideToLongNoRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -342,7 +341,7 @@ func (tr *TransformerReshape) wideToLongNoRegex(
// ----------------------------------------------------------------
func (tr *TransformerReshape) wideToLongRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -384,7 +383,7 @@ func (tr *TransformerReshape) wideToLongRegex(
// ----------------------------------------------------------------
func (tr *TransformerReshape) longToWide(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/sample.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -128,7 +127,7 @@ func NewTransformerSample(
func (tr *TransformerSample) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/sec2gmt.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
@ -149,7 +148,7 @@ func NewTransformerSec2GMT(
func (tr *TransformerSec2GMT) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/sec2gmtdate.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
@ -106,7 +105,7 @@ func NewTransformerSec2GMTDate(
func (tr *TransformerSec2GMTDate) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/seqgen.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -172,7 +171,7 @@ func NewTransformerSeqgen(
func (tr *TransformerSeqgen) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/shuffle.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -97,7 +96,7 @@ func NewTransformerShuffle() (*TransformerShuffle, error) {
func (tr *TransformerShuffle) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/skip_trivial_records.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -89,7 +88,7 @@ func NewTransformerSkipTrivialRecords() (*TransformerSkipTrivialRecords, error)
func (tr *TransformerSkipTrivialRecords) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/sort.go
View file

@ -42,7 +42,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"sort"
@ -338,7 +337,7 @@ type GroupingKeysAndMlrvals struct {
func (tr *TransformerSort) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

7
pkg/transformers/sort_within_records.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -106,7 +105,7 @@ func NewTransformerSortWithinRecords(
func (tr *TransformerSortWithinRecords) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -117,7 +116,7 @@ func (tr *TransformerSortWithinRecords) Transform(
// ----------------------------------------------------------------
func (tr *TransformerSortWithinRecords) transformNonrecursively(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -131,7 +130,7 @@ func (tr *TransformerSortWithinRecords) transformNonrecursively(
// ----------------------------------------------------------------
func (tr *TransformerSortWithinRecords) transformRecursively(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

7
pkg/transformers/sparsify.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -128,7 +127,7 @@ func NewTransformerSparsify(
func (tr *TransformerSparsify) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -148,7 +147,7 @@ func (tr *TransformerSparsify) Transform(
func (tr *TransformerSparsify) transformAll(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -169,7 +168,7 @@ func (tr *TransformerSparsify) transformAll(
// ----------------------------------------------------------------
func (tr *TransformerSparsify) transformSome(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

9
pkg/transformers/split.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"net/url"
"os"
@ -273,7 +272,7 @@ func NewTransformerSplit(
func (tr *TransformerSplit) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -284,7 +283,7 @@ func (tr *TransformerSplit) Transform(
func (tr *TransformerSplit) splitModUngrouped(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -318,7 +317,7 @@ func (tr *TransformerSplit) splitModUngrouped(
func (tr *TransformerSplit) splitSizeUngrouped(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -376,7 +375,7 @@ func (tr *TransformerSplit) splitSizeUngrouped(
func (tr *TransformerSplit) splitGrouped(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

7
pkg/transformers/stats1.go
View file

@ -2,7 +2,6 @@ package transformers
import (
"bytes"
"container/list"
"fmt"
"os"
"regexp"
@ -353,7 +352,7 @@ func NewTransformerStats1(
// the end-of-stream marker.
func (tr *TransformerStats1) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -367,7 +366,7 @@ func (tr *TransformerStats1) Transform(
func (tr *TransformerStats1) handleInputRecord(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
inrec := inrecAndContext.Record
@ -581,7 +580,7 @@ func (tr *TransformerStats1) matchValueFieldName(
func (tr *TransformerStats1) handleEndOfRecordStream(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
if tr.doIterativeStats {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker

7
pkg/transformers/stats2.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -265,7 +264,7 @@ func NewTransformerStats2(
func (tr *TransformerStats2) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -382,7 +381,7 @@ func (tr *TransformerStats2) ingest(
// ----------------------------------------------------------------
func (tr *TransformerStats2) emit(
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
context *types.Context,
) {
for pa := tr.namedAccumulators.Head; pa != nil; pa = pa.Next {
@ -433,7 +432,7 @@ func (tr *TransformerStats2) populateRecord(
}
func (tr *TransformerStats2) fit(
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
for pa := tr.namedAccumulators.Head; pa != nil; pa = pa.Next {
groupingKey := pa.Key

7
pkg/transformers/step.go
View file

@ -68,7 +68,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -350,7 +349,7 @@ func NewTransformerStep(
func (tr *TransformerStep) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -381,7 +380,7 @@ func (tr *TransformerStep) Transform(
// delayed-input records in the order in which they were received.
func (tr *TransformerStep) handleRecord(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
inrec := inrecAndContext.Record
@ -466,7 +465,7 @@ func (tr *TransformerStep) handleRecord(
// delayed-input records in the order in which they were received.
func (tr *TransformerStep) handleDrainRecord(
logEntry *tStepLogEntry,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
inrecAndContext := logEntry.recordAndContext
inrec := inrecAndContext.Record

3
pkg/transformers/subs.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"regexp"
@ -288,7 +287,7 @@ func NewTransformerSubs(
func (tr *TransformerSubs) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

11
pkg/transformers/summary.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -292,7 +291,7 @@ func NewTransformerSummary(
func (tr *TransformerSummary) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -351,7 +350,7 @@ func (tr *TransformerSummary) ingest(
func (tr *TransformerSummary) emit(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
for pe := tr.fieldSummaries.Head; pe != nil; pe = pe.Next {
@ -393,7 +392,7 @@ func (tr *TransformerSummary) emit(
func (tr *TransformerSummary) emitTransposed(
inrecAndContext *types.RecordAndContext,
oracs *list.List, // list of *types.RecordAndContext
oracs *types.List[*types.RecordAndContext],
) {
octx := &inrecAndContext.Context
@ -431,7 +430,7 @@ func (tr *TransformerSummary) emitTransposed(
// maybeEmitAccumulatorTransposed is a helper method for emitTransposed,
// for "count", "sum", "mean", etc.
func (tr *TransformerSummary) maybeEmitAccumulatorTransposed(
oracs *list.List, // list of *types.RecordAndContext
oracs *types.List[*types.RecordAndContext],
octx *types.Context,
summarizerName string,
) {
@ -449,7 +448,7 @@ func (tr *TransformerSummary) maybeEmitAccumulatorTransposed(
// maybeEmitPercentileNameTransposed is a helper method for emitTransposed,
// for "median", "iqr", "uof", etc.
func (tr *TransformerSummary) maybeEmitPercentileNameTransposed(
oracs *list.List, // list of *types.RecordAndContext
oracs *types.List[*types.RecordAndContext],
octx *types.Context,
summarizerName string,
) {

3
pkg/transformers/tac.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -88,7 +87,7 @@ func NewTransformerTac() (*TransformerTac, error) {
func (tr *TransformerTac) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/tail.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -132,7 +131,7 @@ func NewTransformerTail(
func (tr *TransformerTail) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/tee.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -166,7 +165,7 @@ func NewTransformerTee(
func (tr *TransformerTee) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/template.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -135,7 +134,7 @@ func NewTransformerTemplate(
func (tr *TransformerTemplate) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

5
pkg/transformers/top.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -174,7 +173,7 @@ func NewTransformerTop(
func (tr *TransformerTop) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -238,7 +237,7 @@ func (tr *TransformerTop) ingest(
// ----------------------------------------------------------------
func (tr *TransformerTop) emit(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
for pa := tr.groups.Head; pa != nil; pa = pa.Next {
groupingKey := pa.Key

7
pkg/transformers/unflatten.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -133,7 +132,7 @@ func NewTransformerUnflatten(
func (tr *TransformerUnflatten) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -144,7 +143,7 @@ func (tr *TransformerUnflatten) Transform(
// ----------------------------------------------------------------
func (tr *TransformerUnflatten) unflattenAll(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -164,7 +163,7 @@ func (tr *TransformerUnflatten) unflattenAll(
// ----------------------------------------------------------------
func (tr *TransformerUnflatten) unflattenSome(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

17
pkg/transformers/uniq.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -367,7 +366,7 @@ func (tr *TransformerUniq) getFieldNamesForGrouping(
func (tr *TransformerUniq) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -380,7 +379,7 @@ func (tr *TransformerUniq) Transform(
// non-streaming, with output at end of stream.
func (tr *TransformerUniq) transformUniqifyEntireRecordsShowCounts(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -416,7 +415,7 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowCounts(
// of stream.
func (tr *TransformerUniq) transformUniqifyEntireRecordsShowNumDistinctOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -443,7 +442,7 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowNumDistinctOnly(
// Print each unique record only once (on first occurrence).
func (tr *TransformerUniq) transformUniqifyEntireRecords(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -465,7 +464,7 @@ func (tr *TransformerUniq) transformUniqifyEntireRecords(
// ----------------------------------------------------------------
func (tr *TransformerUniq) transformUnlashed(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -520,7 +519,7 @@ func (tr *TransformerUniq) transformUnlashed(
// ----------------------------------------------------------------
func (tr *TransformerUniq) transformNumDistinctOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -552,7 +551,7 @@ func (tr *TransformerUniq) transformNumDistinctOnly(
// ----------------------------------------------------------------
func (tr *TransformerUniq) transformWithCounts(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -602,7 +601,7 @@ func (tr *TransformerUniq) transformWithCounts(
// ----------------------------------------------------------------
func (tr *TransformerUniq) transformWithoutCounts(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

9
pkg/transformers/unspace.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -115,7 +114,7 @@ func NewTransformerUnspace(
func (tr *TransformerUnspace) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -134,7 +133,7 @@ func (tr *TransformerUnspace) Transform(
func (tr *TransformerUnspace) transformKeysOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
_ <-chan bool,
__ chan<- bool,
) {
@ -150,7 +149,7 @@ func (tr *TransformerUnspace) transformKeysOnly(
func (tr *TransformerUnspace) transformValuesOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
_ <-chan bool,
__ chan<- bool,
) {
@ -166,7 +165,7 @@ func (tr *TransformerUnspace) transformValuesOnly(
func (tr *TransformerUnspace) transformKeysAndValues(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
_ <-chan bool,
__ chan<- bool,
) {

7
pkg/transformers/unsparsify.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -142,7 +141,7 @@ func NewTransformerUnsparsify(
func (tr *TransformerUnsparsify) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -153,7 +152,7 @@ func (tr *TransformerUnsparsify) Transform(
// ----------------------------------------------------------------
func (tr *TransformerUnsparsify) transformNonStreaming(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -191,7 +190,7 @@ func (tr *TransformerUnsparsify) transformNonStreaming(
// ----------------------------------------------------------------
func (tr *TransformerUnsparsify) transformStreaming(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

3
pkg/transformers/utf8_to_latin1.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -91,7 +90,7 @@ func NewTransformerUTF8ToLatin1() (*TransformerUTF8ToLatin1, error) {
func (tr *TransformerUTF8ToLatin1) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {

5
pkg/transformers/utils/join_bucket.go
View file

@ -5,15 +5,14 @@
package utils
import (
"container/list"
"github.com/johnkerl/miller/v6/pkg/mlrval"
"github.com/johnkerl/miller/v6/pkg/types"
)
// ----------------------------------------------------------------
type JoinBucket struct {
leftFieldValues []*mlrval.Mlrval
RecordsAndContexts *list.List
RecordsAndContexts *types.List[*types.RecordAndContext]
WasPaired bool
}

11
pkg/transformers/utils/join_bucket_keeper.go
View file

@ -108,7 +108,6 @@
package utils
import (
"container/list"
"fmt"
"os"
"strings"
@ -126,7 +125,7 @@ type JoinBucketKeeper struct {
// For streaming through the left-side file
recordReader input.IRecordReader
context *types.Context
readerChannel <-chan *list.List // list of *types.RecordAndContext
readerChannel <-chan *types.List[*types.RecordAndContext]
errorChannel chan error
// TODO: merge with leof flag
recordReaderDone bool
@ -181,7 +180,7 @@ func NewJoinBucketKeeper(
initialContext.UpdateForStartOfFile(leftFileName)
// Set up channels for the record-reader
readerChannel := make(chan *list.List, 2) // list of *types.RecordAndContext
readerChannel := make(chan *types.List[*types.RecordAndContext], 2)
errorChannel := make(chan error, 1)
downstreamDoneChannel := make(chan bool, 1)
@ -533,7 +532,7 @@ func (keeper *JoinBucketKeeper) markRemainingsAsUnpaired() {
// ----------------------------------------------------------------
// TODO: comment
func (keeper *JoinBucketKeeper) OutputAndReleaseLeftUnpaireds(
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
for {
element := keeper.leftUnpaireds.Front()
@ -547,7 +546,7 @@ func (keeper *JoinBucketKeeper) OutputAndReleaseLeftUnpaireds(
}
func (keeper *JoinBucketKeeper) ReleaseLeftUnpaireds(
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *types.List[*types.RecordAndContext],
) {
for {
element := keeper.leftUnpaireds.Front()
@ -577,7 +576,7 @@ func (keeper *JoinBucketKeeper) readRecord() *types.RecordAndContext {
case leftrecsAndContexts := <-keeper.readerChannel:
// TODO: temp
lib.InternalCodingErrorIf(leftrecsAndContexts.Len() != 1)
leftrecAndContext := leftrecsAndContexts.Front().Value.(*types.RecordAndContext)
leftrecAndContext := leftrecsAndContexts.Front()
leftrecAndContext.Record = KeepLeftFieldNames(leftrecAndContext.Record, keeper.leftKeepFieldNameSet)
if leftrecAndContext.EndOfStream { // end-of-stream marker
keeper.recordReaderDone = true