diff --git a/.gitignore b/.gitignore index 3e2ba59f1..c6985b12e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,11 @@ .sw? .*.sw? -tags *~ +/.vscode + +tags + push2 data/.gitignore diff --git a/pkg/dsl/cst/dump.go b/pkg/dsl/cst/dump.go index 1114043ff..5d79ce3f5 100644 --- a/pkg/dsl/cst/dump.go +++ b/pkg/dsl/cst/dump.go @@ -186,7 +186,7 @@ func (node *DumpStatementNode) dumpToStdout( // // The output channel is always non-nil, except for the Miller REPL. if state.OutputRecordsAndContexts != nil { - state.OutputRecordsAndContexts.PushBack(types.NewOutputString(outputString, state.Context)) + *state.OutputRecordsAndContexts = append(*state.OutputRecordsAndContexts, types.NewOutputString(outputString, state.Context)) } else { fmt.Println(outputString) } diff --git a/pkg/dsl/cst/emit1.go b/pkg/dsl/cst/emit1.go index 5ea14c8f4..af8e4fb60 100644 --- a/pkg/dsl/cst/emit1.go +++ b/pkg/dsl/cst/emit1.go @@ -66,7 +66,7 @@ func (node *Emit1StatementNode) Execute(state *runtime.State) (*BlockExitPayload } if state.OutputRecordsAndContexts != nil { - state.OutputRecordsAndContexts.PushBack(types.NewRecordAndContext(valueAsMap, state.Context)) + *state.OutputRecordsAndContexts = append(*state.OutputRecordsAndContexts, types.NewRecordAndContext(valueAsMap, state.Context)) } else { fmt.Println(valueAsMap.String()) } diff --git a/pkg/dsl/cst/emit_emitp.go b/pkg/dsl/cst/emit_emitp.go index 85b9e374a..f460080ce 100644 --- a/pkg/dsl/cst/emit_emitp.go +++ b/pkg/dsl/cst/emit_emitp.go @@ -975,7 +975,7 @@ func (node *EmitXStatementNode) emitRecordToRecordStream( ) error { // The output channel is always non-nil, except for the Miller REPL. if state.OutputRecordsAndContexts != nil { - state.OutputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, state.Context)) + *state.OutputRecordsAndContexts = append(*state.OutputRecordsAndContexts, types.NewRecordAndContext(outrec, state.Context)) } else { fmt.Println(outrec.String()) } diff --git a/pkg/dsl/cst/emitf.go b/pkg/dsl/cst/emitf.go index eb6812ef7..22017b94b 100644 --- a/pkg/dsl/cst/emitf.go +++ b/pkg/dsl/cst/emitf.go @@ -173,7 +173,7 @@ func (node *EmitFStatementNode) emitfToRecordStream( ) error { // The output channel is always non-nil, except for the Miller REPL. if state.OutputRecordsAndContexts != nil { - state.OutputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, state.Context)) + *state.OutputRecordsAndContexts = append(*state.OutputRecordsAndContexts, types.NewRecordAndContext(outrec, state.Context)) } else { fmt.Println(outrec.String()) } diff --git a/pkg/dsl/cst/print.go b/pkg/dsl/cst/print.go index 39c84f58d..7fac5cc0c 100644 --- a/pkg/dsl/cst/print.go +++ b/pkg/dsl/cst/print.go @@ -332,7 +332,7 @@ func (node *PrintStatementNode) printToStdout( // The output channel is always non-nil, except for the Miller REPL. if state.OutputRecordsAndContexts != nil { - state.OutputRecordsAndContexts.PushBack(types.NewOutputString(outputString, state.Context)) + *state.OutputRecordsAndContexts = append(*state.OutputRecordsAndContexts, types.NewOutputString(outputString, state.Context)) } else { fmt.Print(outputString) } diff --git a/pkg/input/line_reader.go b/pkg/input/line_reader.go index b1f965307..0435d3bcb 100644 --- a/pkg/input/line_reader.go +++ b/pkg/input/line_reader.go @@ -5,7 +5,6 @@ package input import ( "bufio" - "container/list" "io" "strings" @@ -172,14 +171,14 @@ func (r *MultiIRSLineReader) Read() (string, error) { // IRS) stripped off. So, callers get "a=1,b=2" rather than "a=1,b=2\n". func channelizedLineReader( lineReader ILineReader, - linesChannel chan<- *list.List, + linesChannel chan<- []string, downstreamDoneChannel <-chan bool, // for mlr head recordsPerBatch int64, ) { i := int64(0) done := false - lines := list.New() + lines := make([]string, 0, recordsPerBatch) for { line, err := lineReader.Read() @@ -194,7 +193,7 @@ func channelizedLineReader( i++ - lines.PushBack(line) + lines = append(lines, line) // See if downstream processors will be ignoring further data (e.g. mlr // head). If so, stop reading. This makes 'mlr head hugefile' exit @@ -211,7 +210,7 @@ func channelizedLineReader( break } linesChannel <- lines - lines = list.New() + lines = make([]string, 0, recordsPerBatch) } if done { diff --git a/pkg/input/pseudo_reader_gen.go b/pkg/input/pseudo_reader_gen.go index e847b59ab..d2fd8ec94 100644 --- a/pkg/input/pseudo_reader_gen.go +++ b/pkg/input/pseudo_reader_gen.go @@ -1,7 +1,6 @@ package input import ( - "container/list" "fmt" "github.com/johnkerl/miller/v6/pkg/bifs" @@ -28,7 +27,7 @@ func NewPseudoReaderGen( func (reader *PseudoReaderGen) Read( filenames []string, // ignored context types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -38,7 +37,7 @@ func (reader *PseudoReaderGen) Read( func (reader *PseudoReaderGen) process( context *types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -71,7 +70,7 @@ func (reader *PseudoReaderGen) process( key := reader.readerOptions.GeneratorOptions.FieldName value := start.Copy() - recordsAndContexts := list.New() + recordsAndContexts := make([]*types.RecordAndContext, 0, recordsPerBatch) eof := false for !eof { @@ -84,11 +83,11 @@ func (reader *PseudoReaderGen) process( record.PutCopy(key, value) context.UpdateForInputRecord() - recordsAndContexts.PushBack(types.NewRecordAndContext(record, context)) + recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context)) - if int64(recordsAndContexts.Len()) >= recordsPerBatch { + if int64(len(recordsAndContexts)) >= recordsPerBatch { readerChannel <- recordsAndContexts - recordsAndContexts = list.New() + recordsAndContexts = make([]*types.RecordAndContext, 0, recordsPerBatch) // See if downstream processors will be ignoring further data (e.g. // mlr head). If so, stop reading. This makes 'mlr head hugefile' @@ -111,7 +110,7 @@ func (reader *PseudoReaderGen) process( value = bifs.BIF_plus_binary(value, step) } - if recordsAndContexts.Len() > 0 { + if len(recordsAndContexts) > 0 { readerChannel <- recordsAndContexts } } diff --git a/pkg/input/record_reader.go b/pkg/input/record_reader.go index 3ad932f2f..bf0cd91a2 100644 --- a/pkg/input/record_reader.go +++ b/pkg/input/record_reader.go @@ -3,11 +3,7 @@ package input -import ( - "container/list" - - "github.com/johnkerl/miller/v6/pkg/types" -) +import "github.com/johnkerl/miller/v6/pkg/types" // Since Go is concurrent, the context struct (AWK-like variables such as // FILENAME, NF, NF, FNR, etc.) needs to be duplicated and passed through the @@ -19,7 +15,7 @@ type IRecordReader interface { Read( filenames []string, initialContext types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) diff --git a/pkg/input/record_reader_benchmark_test.go b/pkg/input/record_reader_benchmark_test.go index 7b79c32ad..a4584e6b1 100644 --- a/pkg/input/record_reader_benchmark_test.go +++ b/pkg/input/record_reader_benchmark_test.go @@ -57,13 +57,13 @@ func BenchmarkXTABParse(b *testing.B) { assert.Nil(b, err) stanza := newStanza() - stanza.dataLines.PushBack("color yellow") - stanza.dataLines.PushBack("shape triangle") - stanza.dataLines.PushBack("flag true") - stanza.dataLines.PushBack("k 1") - stanza.dataLines.PushBack("index 11") - stanza.dataLines.PushBack("quantity 43.6498") - stanza.dataLines.PushBack("rate 9.8870") + stanza.dataLines = append(stanza.dataLines, "color yellow") + stanza.dataLines = append(stanza.dataLines, "shape triangle") + stanza.dataLines = append(stanza.dataLines, "flag true") + stanza.dataLines = append(stanza.dataLines, "k 1") + stanza.dataLines = append(stanza.dataLines, "index 11") + stanza.dataLines = append(stanza.dataLines, "quantity 43.6498") + stanza.dataLines = append(stanza.dataLines, "rate 9.8870") for i := 0; i < b.N; i++ { _, _ = reader.recordFromXTABLines(stanza.dataLines) diff --git a/pkg/input/record_reader_csv.go b/pkg/input/record_reader_csv.go index aa7dec084..57759ea8d 100644 --- a/pkg/input/record_reader_csv.go +++ b/pkg/input/record_reader_csv.go @@ -1,7 +1,6 @@ package input import ( - "container/list" "fmt" "io" "strconv" @@ -56,7 +55,7 @@ func NewRecordReaderCSV( func (reader *RecordReaderCSV) Read( filenames []string, context types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -96,7 +95,7 @@ func (reader *RecordReaderCSV) processHandle( handle io.Reader, filename string, context *types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -121,13 +120,13 @@ func (reader *RecordReaderCSV) processHandle( } } - csvRecordsChannel := make(chan *list.List, recordsPerBatch) + csvRecordsChannel := make(chan [][]string, recordsPerBatch) go channelizedCSVRecordScanner(csvReader, csvRecordsChannel, downstreamDoneChannel, errorChannel, recordsPerBatch) for { recordsAndContexts, eof := reader.getRecordBatch(csvRecordsChannel, errorChannel, context) - if recordsAndContexts.Len() > 0 { + if len(recordsAndContexts) > 0 { readerChannel <- recordsAndContexts } if eof { @@ -139,7 +138,7 @@ func (reader *RecordReaderCSV) processHandle( // TODO: comment func channelizedCSVRecordScanner( csvReader *csv.Reader, - csvRecordsChannel chan<- *list.List, + csvRecordsChannel chan<- [][]string, downstreamDoneChannel <-chan bool, // for mlr head errorChannel chan error, recordsPerBatch int64, @@ -147,7 +146,7 @@ func channelizedCSVRecordScanner( i := int64(0) done := false - csvRecords := list.New() + csvRecords := make([][]string, 0, recordsPerBatch) for { i++ @@ -163,7 +162,7 @@ func channelizedCSVRecordScanner( break } - csvRecords.PushBack(csvRecord) + csvRecords = append(csvRecords, csvRecord) // See if downstream processors will be ignoring further data (e.g. mlr // head). If so, stop reading. This makes 'mlr head hugefile' exit @@ -180,7 +179,7 @@ func channelizedCSVRecordScanner( break } csvRecordsChannel <- csvRecords - csvRecords = list.New() + csvRecords = make([][]string, 0, recordsPerBatch) } if done { @@ -193,14 +192,14 @@ func channelizedCSVRecordScanner( // TODO: comment copiously we're trying to handle slow/fast/short/long reads: tail -f, smallfile, bigfile. func (reader *RecordReaderCSV) getRecordBatch( - csvRecordsChannel <-chan *list.List, + csvRecordsChannel <-chan [][]string, errorChannel chan error, context *types.Context, ) ( - recordsAndContexts *list.List, + recordsAndContexts []*types.RecordAndContext, eof bool, ) { - recordsAndContexts = list.New() + recordsAndContexts = make([]*types.RecordAndContext, 0) dedupeFieldNames := reader.readerOptions.DedupeFieldNames csvRecords, more := <-csvRecordsChannel @@ -208,11 +207,10 @@ func (reader *RecordReaderCSV) getRecordBatch( return recordsAndContexts, true } - for e := csvRecords.Front(); e != nil; e = e.Next() { - csvRecord := e.Value.([]string) + for _, csvRecord := range csvRecords { if reader.needHeader { - isData := reader.maybeConsumeComment(csvRecord, context, recordsAndContexts) + isData := reader.maybeConsumeComment(csvRecord, context, &recordsAndContexts) if !isData { continue } @@ -223,7 +221,7 @@ func (reader *RecordReaderCSV) getRecordBatch( continue } - isData := reader.maybeConsumeComment(csvRecord, context, recordsAndContexts) + isData := reader.maybeConsumeComment(csvRecord, context, &recordsAndContexts) if !isData { continue } @@ -291,7 +289,7 @@ func (reader *RecordReaderCSV) getRecordBatch( context.UpdateForInputRecord() - recordsAndContexts.PushBack(types.NewRecordAndContext(record, context)) + recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context)) } return recordsAndContexts, false @@ -302,7 +300,7 @@ func (reader *RecordReaderCSV) getRecordBatch( func (reader *RecordReaderCSV) maybeConsumeComment( csvRecord []string, context *types.Context, - recordsAndContexts *list.List, // list of *types.RecordAndContext + recordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) bool { if reader.readerOptions.CommentHandling == cli.CommentsAreData { // Nothing is to be construed as a comment @@ -333,7 +331,7 @@ func (reader *RecordReaderCSV) maybeConsumeComment( // Contract with our fork of the go-csv CSV Reader, and, our own constructor. lib.InternalCodingErrorIf(len(csvRecord) != 1) - recordsAndContexts.PushBack(types.NewOutputString(csvRecord[0], context)) + *recordsAndContexts = append(*recordsAndContexts, types.NewOutputString(csvRecord[0], context)) } else /* reader.readerOptions.CommentHandling == cli.SkipComments */ { // discard entirely diff --git a/pkg/input/record_reader_csvlite.go b/pkg/input/record_reader_csvlite.go index d658a4f99..d9a28ce5c 100644 --- a/pkg/input/record_reader_csvlite.go +++ b/pkg/input/record_reader_csvlite.go @@ -19,7 +19,6 @@ package input // 3,4,5,6 3,4,5 import ( - "container/list" "fmt" "io" "strconv" @@ -35,12 +34,12 @@ import ( // implicit-CSV-header record-batch getter. type recordBatchGetterCSV func( reader *RecordReaderCSVLite, - linesChannel <-chan *list.List, + linesChannel <-chan []string, filename string, context *types.Context, errorChannel chan error, ) ( - recordsAndContexts *list.List, + recordsAndContexts []*types.RecordAndContext, eof bool, ) @@ -81,7 +80,7 @@ func NewRecordReaderCSVLite( func (reader *RecordReaderCSVLite) Read( filenames []string, context types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -135,7 +134,7 @@ func (reader *RecordReaderCSVLite) processHandle( handle io.Reader, filename string, context *types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -145,12 +144,12 @@ func (reader *RecordReaderCSVLite) processHandle( recordsPerBatch := reader.recordsPerBatch lineReader := NewLineReader(handle, reader.readerOptions.IRS) - linesChannel := make(chan *list.List, recordsPerBatch) + linesChannel := make(chan []string, recordsPerBatch) go channelizedLineReader(lineReader, linesChannel, downstreamDoneChannel, recordsPerBatch) for { recordsAndContexts, eof := reader.recordBatchGetter(reader, linesChannel, filename, context, errorChannel) - if recordsAndContexts.Len() > 0 { + if len(recordsAndContexts) > 0 { readerChannel <- recordsAndContexts } if eof { @@ -161,15 +160,15 @@ func (reader *RecordReaderCSVLite) processHandle( func getRecordBatchExplicitCSVHeader( reader *RecordReaderCSVLite, - linesChannel <-chan *list.List, + linesChannel <-chan []string, filename string, context *types.Context, errorChannel chan error, ) ( - recordsAndContexts *list.List, + recordsAndContexts []*types.RecordAndContext, eof bool, ) { - recordsAndContexts = list.New() + recordsAndContexts = make([]*types.RecordAndContext, 0) dedupeFieldNames := reader.readerOptions.DedupeFieldNames lines, more := <-linesChannel @@ -177,8 +176,7 @@ func getRecordBatchExplicitCSVHeader( return recordsAndContexts, true } - for e := lines.Front(); e != nil; e = e.Next() { - line := e.Value.(string) + for _, line := range lines { reader.inputLineNumber++ @@ -194,7 +192,7 @@ func getRecordBatchExplicitCSVHeader( if reader.readerOptions.CommentHandling != cli.CommentsAreData { if strings.HasPrefix(line, reader.readerOptions.CommentString) { if reader.readerOptions.CommentHandling == cli.PassComments { - recordsAndContexts.PushBack(types.NewOutputString(line+"\n", context)) + recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+"\n", context)) continue } else if reader.readerOptions.CommentHandling == cli.SkipComments { continue @@ -275,7 +273,7 @@ func getRecordBatchExplicitCSVHeader( } context.UpdateForInputRecord() - recordsAndContexts.PushBack(types.NewRecordAndContext(record, context)) + recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context)) } } @@ -284,15 +282,15 @@ func getRecordBatchExplicitCSVHeader( func getRecordBatchImplicitCSVHeader( reader *RecordReaderCSVLite, - linesChannel <-chan *list.List, + linesChannel <-chan []string, filename string, context *types.Context, errorChannel chan error, ) ( - recordsAndContexts *list.List, + recordsAndContexts []*types.RecordAndContext, eof bool, ) { - recordsAndContexts = list.New() + recordsAndContexts = make([]*types.RecordAndContext, 0) dedupeFieldNames := reader.readerOptions.DedupeFieldNames lines, more := <-linesChannel @@ -300,8 +298,7 @@ func getRecordBatchImplicitCSVHeader( return recordsAndContexts, true } - for e := lines.Front(); e != nil; e = e.Next() { - line := e.Value.(string) + for _, line := range lines { reader.inputLineNumber++ @@ -310,7 +307,7 @@ func getRecordBatchImplicitCSVHeader( if reader.readerOptions.CommentHandling != cli.CommentsAreData { if strings.HasPrefix(line, reader.readerOptions.CommentString) { if reader.readerOptions.CommentHandling == cli.PassComments { - recordsAndContexts.PushBack(types.NewOutputString(line+"\n", context)) + recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+"\n", context)) continue } else if reader.readerOptions.CommentHandling == cli.SkipComments { continue @@ -402,7 +399,7 @@ func getRecordBatchImplicitCSVHeader( } context.UpdateForInputRecord() - recordsAndContexts.PushBack(types.NewRecordAndContext(record, context)) + recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context)) } return recordsAndContexts, false diff --git a/pkg/input/record_reader_dkvp_nidx.go b/pkg/input/record_reader_dkvp_nidx.go index 6a53c8c26..ec010606b 100644 --- a/pkg/input/record_reader_dkvp_nidx.go +++ b/pkg/input/record_reader_dkvp_nidx.go @@ -3,7 +3,6 @@ package input import ( - "container/list" "io" "strconv" "strings" @@ -55,7 +54,7 @@ func NewRecordReaderNIDX( func (reader *RecordReaderDKVPNIDX) Read( filenames []string, context types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -95,7 +94,7 @@ func (reader *RecordReaderDKVPNIDX) processHandle( handle io.Reader, filename string, context *types.Context, - readerChannel chan<- *list.List, + readerChannel chan<- []*types.RecordAndContext, errorChannel chan<- error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -103,12 +102,12 @@ func (reader *RecordReaderDKVPNIDX) processHandle( recordsPerBatch := reader.recordsPerBatch lineReader := NewLineReader(handle, reader.readerOptions.IRS) - linesChannel := make(chan *list.List, recordsPerBatch) + linesChannel := make(chan []string, recordsPerBatch) go channelizedLineReader(lineReader, linesChannel, downstreamDoneChannel, recordsPerBatch) for { recordsAndContexts, eof := reader.getRecordBatch(linesChannel, errorChannel, context) - if recordsAndContexts.Len() > 0 { + if len(recordsAndContexts) > 0 { readerChannel <- recordsAndContexts } if eof { @@ -119,29 +118,28 @@ func (reader *RecordReaderDKVPNIDX) processHandle( // TODO: comment copiously we're trying to handle slow/fast/short/long reads: tail -f, smallfile, bigfile. func (reader *RecordReaderDKVPNIDX) getRecordBatch( - linesChannel <-chan *list.List, + linesChannel <-chan []string, errorChannel chan<- error, context *types.Context, ) ( - recordsAndContexts *list.List, + recordsAndContexts []*types.RecordAndContext, eof bool, ) { - recordsAndContexts = list.New() + recordsAndContexts = make([]*types.RecordAndContext, 0) lines, more := <-linesChannel if !more { return recordsAndContexts, true } - for e := lines.Front(); e != nil; e = e.Next() { - line := e.Value.(string) + for _, line := range lines { // Check for comments-in-data feature // TODO: function-pointer this away if reader.readerOptions.CommentHandling != cli.CommentsAreData { if strings.HasPrefix(line, reader.readerOptions.CommentString) { if reader.readerOptions.CommentHandling == cli.PassComments { - recordsAndContexts.PushBack(types.NewOutputString(line+"\n", context)) + recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+"\n", context)) continue } else if reader.readerOptions.CommentHandling == cli.SkipComments { continue @@ -157,7 +155,7 @@ func (reader *RecordReaderDKVPNIDX) getRecordBatch( } context.UpdateForInputRecord() recordAndContext := types.NewRecordAndContext(record, context) - recordsAndContexts.PushBack(recordAndContext) + recordsAndContexts = append(recordsAndContexts, recordAndContext) } return recordsAndContexts, false diff --git a/pkg/input/record_reader_json.go b/pkg/input/record_reader_json.go index 63d9f7368..8f0814895 100644 --- a/pkg/input/record_reader_json.go +++ b/pkg/input/record_reader_json.go @@ -1,7 +1,6 @@ package input import ( - "container/list" "fmt" "io" "strings" @@ -34,7 +33,7 @@ func NewRecordReaderJSON( func (reader *RecordReaderJSON) Read( filenames []string, context types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -75,7 +74,7 @@ func (reader *RecordReaderJSON) processHandle( handle io.Reader, filename string, context *types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -87,7 +86,7 @@ func (reader *RecordReaderJSON) processHandle( handle = NewJSONCommentEnabledReader(handle, reader.readerOptions, readerChannel) } decoder := json.NewDecoder(handle) - recordsAndContexts := list.New() + recordsAndContexts := make([]*types.RecordAndContext, 0, recordsPerBatch) eof := false i := int64(0) @@ -132,11 +131,11 @@ func (reader *RecordReaderJSON) processHandle( return } context.UpdateForInputRecord() - recordsAndContexts.PushBack(types.NewRecordAndContext(record, context)) + recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context)) - if int64(recordsAndContexts.Len()) >= recordsPerBatch { + if int64(len(recordsAndContexts)) >= recordsPerBatch { readerChannel <- recordsAndContexts - recordsAndContexts = list.New() + recordsAndContexts = make([]*types.RecordAndContext, 0, recordsPerBatch) } } else if mlrval.IsArray() { @@ -164,11 +163,11 @@ func (reader *RecordReaderJSON) processHandle( return } context.UpdateForInputRecord() - recordsAndContexts.PushBack(types.NewRecordAndContext(record, context)) + recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context)) - if int64(recordsAndContexts.Len()) >= recordsPerBatch { + if int64(len(recordsAndContexts)) >= recordsPerBatch { readerChannel <- recordsAndContexts - recordsAndContexts = list.New() + recordsAndContexts = make([]*types.RecordAndContext, 0, recordsPerBatch) } } @@ -181,7 +180,7 @@ func (reader *RecordReaderJSON) processHandle( } } - if recordsAndContexts.Len() > 0 { + if len(recordsAndContexts) > 0 { readerChannel <- recordsAndContexts } } @@ -211,8 +210,8 @@ func (reader *RecordReaderJSON) processHandle( type JSONCommentEnabledReader struct { lineReader ILineReader readerOptions *cli.TReaderOptions - context *types.Context // Needed for channelized stdout-printing logic - readerChannel chan<- *list.List // list of *types.RecordAndContext + context *types.Context // Needed for channelized stdout-printing logic + readerChannel chan<- []*types.RecordAndContext // list of *types.RecordAndContext // In case a line was ingested which was longer than the read-buffer passed // to us, in which case we need to split up that line and return it over @@ -223,7 +222,7 @@ type JSONCommentEnabledReader struct { func NewJSONCommentEnabledReader( underlying io.Reader, readerOptions *cli.TReaderOptions, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext ) *JSONCommentEnabledReader { return &JSONCommentEnabledReader{ lineReader: NewLineReader(underlying, "\n"), @@ -260,9 +259,10 @@ func (bsr *JSONCommentEnabledReader) Read(p []byte) (n int, err error) { if bsr.readerOptions.CommentHandling == cli.PassComments { // Insert the string into the record-output stream, so that goroutine can // print it, resulting in deterministic output-ordering. - ell := list.New() - ell.PushBack(types.NewOutputString(line+"\n", bsr.context)) - bsr.readerChannel <- ell + recordsAndContexts := []*types.RecordAndContext{ + types.NewOutputString(line+"\n", bsr.context), + } + bsr.readerChannel <- recordsAndContexts } if done { diff --git a/pkg/input/record_reader_pprint.go b/pkg/input/record_reader_pprint.go index aad87769c..45f0b3c54 100644 --- a/pkg/input/record_reader_pprint.go +++ b/pkg/input/record_reader_pprint.go @@ -1,7 +1,6 @@ package input import ( - "container/list" "fmt" "io" "regexp" @@ -73,19 +72,19 @@ type RecordReaderPprintBarredOrMarkdown struct { // implicit-PPRINT-header record-batch getter. type recordBatchGetterPprint func( reader *RecordReaderPprintBarredOrMarkdown, - linesChannel <-chan *list.List, + linesChannel <-chan []string, filename string, context *types.Context, errorChannel chan error, ) ( - recordsAndContexts *list.List, + recordsAndContexts []*types.RecordAndContext, eof bool, ) func (reader *RecordReaderPprintBarredOrMarkdown) Read( filenames []string, context types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -139,7 +138,7 @@ func (reader *RecordReaderPprintBarredOrMarkdown) processHandle( handle io.Reader, filename string, context *types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -149,12 +148,12 @@ func (reader *RecordReaderPprintBarredOrMarkdown) processHandle( recordsPerBatch := reader.recordsPerBatch lineReader := NewLineReader(handle, reader.readerOptions.IRS) - linesChannel := make(chan *list.List, recordsPerBatch) + linesChannel := make(chan []string, recordsPerBatch) go channelizedLineReader(lineReader, linesChannel, downstreamDoneChannel, recordsPerBatch) for { recordsAndContexts, eof := reader.recordBatchGetter(reader, linesChannel, filename, context, errorChannel) - if recordsAndContexts.Len() > 0 { + if len(recordsAndContexts) > 0 { readerChannel <- recordsAndContexts } if eof { @@ -165,15 +164,15 @@ func (reader *RecordReaderPprintBarredOrMarkdown) processHandle( func getRecordBatchExplicitPprintHeader( reader *RecordReaderPprintBarredOrMarkdown, - linesChannel <-chan *list.List, + linesChannel <-chan []string, filename string, context *types.Context, errorChannel chan error, ) ( - recordsAndContexts *list.List, + recordsAndContexts []*types.RecordAndContext, eof bool, ) { - recordsAndContexts = list.New() + recordsAndContexts = make([]*types.RecordAndContext, 0) dedupeFieldNames := reader.readerOptions.DedupeFieldNames lines, more := <-linesChannel @@ -181,8 +180,7 @@ func getRecordBatchExplicitPprintHeader( return recordsAndContexts, true } - for e := lines.Front(); e != nil; e = e.Next() { - line := e.Value.(string) + for _, line := range lines { reader.inputLineNumber++ @@ -191,7 +189,7 @@ func getRecordBatchExplicitPprintHeader( if reader.readerOptions.CommentHandling != cli.CommentsAreData { if strings.HasPrefix(line, reader.readerOptions.CommentString) { if reader.readerOptions.CommentHandling == cli.PassComments { - recordsAndContexts.PushBack(types.NewOutputString(line+"\n", context)) + recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+"\n", context)) continue } else if reader.readerOptions.CommentHandling == cli.SkipComments { continue @@ -292,7 +290,7 @@ func getRecordBatchExplicitPprintHeader( } context.UpdateForInputRecord() - recordsAndContexts.PushBack(types.NewRecordAndContext(record, context)) + recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context)) } } @@ -302,15 +300,15 @@ func getRecordBatchExplicitPprintHeader( func getRecordBatchImplicitPprintHeader( reader *RecordReaderPprintBarredOrMarkdown, - linesChannel <-chan *list.List, + linesChannel <-chan []string, filename string, context *types.Context, errorChannel chan error, ) ( - recordsAndContexts *list.List, + recordsAndContexts []*types.RecordAndContext, eof bool, ) { - recordsAndContexts = list.New() + recordsAndContexts = make([]*types.RecordAndContext, 0) dedupeFieldNames := reader.readerOptions.DedupeFieldNames lines, more := <-linesChannel @@ -318,8 +316,7 @@ func getRecordBatchImplicitPprintHeader( return recordsAndContexts, true } - for e := lines.Front(); e != nil; e = e.Next() { - line := e.Value.(string) + for _, line := range lines { reader.inputLineNumber++ @@ -328,7 +325,7 @@ func getRecordBatchImplicitPprintHeader( if reader.readerOptions.CommentHandling != cli.CommentsAreData { if strings.HasPrefix(line, reader.readerOptions.CommentString) { if reader.readerOptions.CommentHandling == cli.PassComments { - recordsAndContexts.PushBack(types.NewOutputString(line+"\n", context)) + recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+"\n", context)) continue } else if reader.readerOptions.CommentHandling == cli.SkipComments { continue @@ -436,7 +433,7 @@ func getRecordBatchImplicitPprintHeader( } context.UpdateForInputRecord() - recordsAndContexts.PushBack(types.NewRecordAndContext(record, context)) + recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context)) } return recordsAndContexts, false diff --git a/pkg/input/record_reader_tsv.go b/pkg/input/record_reader_tsv.go index f70042bbe..fac255199 100644 --- a/pkg/input/record_reader_tsv.go +++ b/pkg/input/record_reader_tsv.go @@ -1,7 +1,6 @@ package input import ( - "container/list" "fmt" "io" "strconv" @@ -17,12 +16,12 @@ import ( // implicit-TSV-header record-batch getter. type recordBatchGetterTSV func( reader *RecordReaderTSV, - linesChannel <-chan *list.List, + linesChannel <-chan []string, filename string, context *types.Context, errorChannel chan error, ) ( - recordsAndContexts *list.List, + recordsAndContexts []*types.RecordAndContext, eof bool, ) @@ -63,7 +62,7 @@ func NewRecordReaderTSV( func (reader *RecordReaderTSV) Read( filenames []string, context types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -117,7 +116,7 @@ func (reader *RecordReaderTSV) processHandle( handle io.Reader, filename string, context *types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -127,12 +126,12 @@ func (reader *RecordReaderTSV) processHandle( recordsPerBatch := reader.recordsPerBatch lineReader := NewLineReader(handle, reader.readerOptions.IRS) - linesChannel := make(chan *list.List, recordsPerBatch) + linesChannel := make(chan []string, recordsPerBatch) go channelizedLineReader(lineReader, linesChannel, downstreamDoneChannel, recordsPerBatch) for { recordsAndContexts, eof := reader.recordBatchGetter(reader, linesChannel, filename, context, errorChannel) - if recordsAndContexts.Len() > 0 { + if len(recordsAndContexts) > 0 { readerChannel <- recordsAndContexts } if eof { @@ -143,15 +142,15 @@ func (reader *RecordReaderTSV) processHandle( func getRecordBatchExplicitTSVHeader( reader *RecordReaderTSV, - linesChannel <-chan *list.List, + linesChannel <-chan []string, filename string, context *types.Context, errorChannel chan error, ) ( - recordsAndContexts *list.List, + recordsAndContexts []*types.RecordAndContext, eof bool, ) { - recordsAndContexts = list.New() + recordsAndContexts = make([]*types.RecordAndContext, 0) dedupeFieldNames := reader.readerOptions.DedupeFieldNames lines, more := <-linesChannel @@ -159,8 +158,7 @@ func getRecordBatchExplicitTSVHeader( return recordsAndContexts, true } - for e := lines.Front(); e != nil; e = e.Next() { - line := e.Value.(string) + for _, line := range lines { reader.inputLineNumber++ @@ -169,7 +167,7 @@ func getRecordBatchExplicitTSVHeader( if reader.readerOptions.CommentHandling != cli.CommentsAreData { if strings.HasPrefix(line, reader.readerOptions.CommentString) { if reader.readerOptions.CommentHandling == cli.PassComments { - recordsAndContexts.PushBack(types.NewOutputString(line+"\n", context)) + recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+"\n", context)) continue } else if reader.readerOptions.CommentHandling == cli.SkipComments { continue @@ -240,7 +238,7 @@ func getRecordBatchExplicitTSVHeader( } context.UpdateForInputRecord() - recordsAndContexts.PushBack(types.NewRecordAndContext(record, context)) + recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context)) } } @@ -249,15 +247,15 @@ func getRecordBatchExplicitTSVHeader( func getRecordBatchImplicitTSVHeader( reader *RecordReaderTSV, - linesChannel <-chan *list.List, + linesChannel <-chan []string, filename string, context *types.Context, errorChannel chan error, ) ( - recordsAndContexts *list.List, + recordsAndContexts []*types.RecordAndContext, eof bool, ) { - recordsAndContexts = list.New() + recordsAndContexts = make([]*types.RecordAndContext, 0) dedupeFieldNames := reader.readerOptions.DedupeFieldNames lines, more := <-linesChannel @@ -265,8 +263,7 @@ func getRecordBatchImplicitTSVHeader( return recordsAndContexts, true } - for e := lines.Front(); e != nil; e = e.Next() { - line := e.Value.(string) + for _, line := range lines { reader.inputLineNumber++ @@ -275,7 +272,7 @@ func getRecordBatchImplicitTSVHeader( if reader.readerOptions.CommentHandling != cli.CommentsAreData { if strings.HasPrefix(line, reader.readerOptions.CommentString) { if reader.readerOptions.CommentHandling == cli.PassComments { - recordsAndContexts.PushBack(types.NewOutputString(line+"\n", context)) + recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+"\n", context)) continue } else if reader.readerOptions.CommentHandling == cli.SkipComments { continue @@ -363,7 +360,7 @@ func getRecordBatchImplicitTSVHeader( } context.UpdateForInputRecord() - recordsAndContexts.PushBack(types.NewRecordAndContext(record, context)) + recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context)) } return recordsAndContexts, false diff --git a/pkg/input/record_reader_xtab.go b/pkg/input/record_reader_xtab.go index 5d1530007..679e7458c 100644 --- a/pkg/input/record_reader_xtab.go +++ b/pkg/input/record_reader_xtab.go @@ -1,7 +1,6 @@ package input import ( - "container/list" "fmt" "io" "os" @@ -33,14 +32,14 @@ type RecordReaderXTAB struct { // 500 or so). This struct helps us keep each stanza's comment lines along with // the stanza they originated in. type tStanza struct { - dataLines *list.List - commentLines *list.List + dataLines []string + commentLines []string } func newStanza() *tStanza { return &tStanza{ - dataLines: list.New(), - commentLines: list.New(), + dataLines: make([]string, 0), + commentLines: make([]string, 0), } } @@ -58,7 +57,7 @@ func NewRecordReaderXTAB( func (reader *RecordReaderXTAB) Read( filenames []string, context types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -98,7 +97,7 @@ func (reader *RecordReaderXTAB) processHandle( handle io.Reader, filename string, context *types.Context, - readerChannel chan<- *list.List, // list of *types.RecordAndContext + readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext errorChannel chan error, downstreamDoneChannel <-chan bool, // for mlr head ) { @@ -108,13 +107,13 @@ func (reader *RecordReaderXTAB) processHandle( // XTAB uses repeated IFS, rather than IRS, to delimit records lineReader := NewLineReader(handle, reader.readerOptions.IFS) - stanzasChannel := make(chan *list.List, recordsPerBatch) + stanzasChannel := make(chan []*tStanza, recordsPerBatch) go channelizedStanzaScanner(lineReader, reader.readerOptions, stanzasChannel, downstreamDoneChannel, recordsPerBatch) for { recordsAndContexts, eof := reader.getRecordBatch(stanzasChannel, context, errorChannel) - if recordsAndContexts.Len() > 0 { + if len(recordsAndContexts) > 0 { readerChannel <- recordsAndContexts } if eof { @@ -140,7 +139,7 @@ func (reader *RecordReaderXTAB) processHandle( func channelizedStanzaScanner( lineReader ILineReader, readerOptions *cli.TReaderOptions, - stanzasChannel chan<- *list.List, // list of list of string + stanzasChannel chan<- []*tStanza, // list of list of string downstreamDoneChannel <-chan bool, // for mlr head recordsPerBatch int64, ) { @@ -148,7 +147,7 @@ func channelizedStanzaScanner( inStanza := false done := false - stanzas := list.New() + stanzas := make([]*tStanza, 0, recordsPerBatch) stanza := newStanza() for { @@ -168,7 +167,7 @@ func channelizedStanzaScanner( if readerOptions.CommentHandling != cli.CommentsAreData { if strings.HasPrefix(line, readerOptions.CommentString) { if readerOptions.CommentHandling == cli.PassComments { - stanza.commentLines.PushBack(line) + stanza.commentLines = append(stanza.commentLines, line) continue } else if readerOptions.CommentHandling == cli.SkipComments { continue @@ -184,7 +183,7 @@ func channelizedStanzaScanner( // 3. At end of file, multiple empty lines are ignored. if inStanza { inStanza = false - stanzas.PushBack(stanza) + stanzas = append(stanzas, stanza) numStanzasSeen++ stanza = newStanza() } else { @@ -194,7 +193,7 @@ func channelizedStanzaScanner( if !inStanza { inStanza = true } - stanza.dataLines.PushBack(line) + stanza.dataLines = append(stanza.dataLines, line) } // See if downstream processors will be ignoring further data (e.g. mlr @@ -212,7 +211,7 @@ func channelizedStanzaScanner( break } stanzasChannel <- stanzas - stanzas = list.New() + stanzas = make([]*tStanza, 0, recordsPerBatch) } if done { @@ -222,8 +221,8 @@ func channelizedStanzaScanner( // The last stanza may not have a trailing newline after it. Any lines in the stanza // at this point will form the final record in the stream. - if stanza.dataLines.Len() > 0 || stanza.commentLines.Len() > 0 { - stanzas.PushBack(stanza) + if len(stanza.dataLines) > 0 || len(stanza.commentLines) > 0 { + stanzas = append(stanzas, stanza) } stanzasChannel <- stanzas @@ -232,38 +231,36 @@ func channelizedStanzaScanner( // TODO: comment copiously we're trying to handle slow/fast/short/long reads: tail -f, smallfile, bigfile. func (reader *RecordReaderXTAB) getRecordBatch( - stanzasChannel <-chan *list.List, + stanzasChannel <-chan []*tStanza, context *types.Context, errorChannel chan error, ) ( - recordsAndContexts *list.List, + recordsAndContexts []*types.RecordAndContext, eof bool, ) { - recordsAndContexts = list.New() + recordsAndContexts = make([]*types.RecordAndContext, 0) stanzas, more := <-stanzasChannel if !more { return recordsAndContexts, true } - for e := stanzas.Front(); e != nil; e = e.Next() { - stanza := e.Value.(*tStanza) + for _, stanza := range stanzas { - if stanza.commentLines.Len() > 0 { - for f := stanza.commentLines.Front(); f != nil; f = f.Next() { - line := f.Value.(string) - recordsAndContexts.PushBack(types.NewOutputString(line+reader.readerOptions.IFS, context)) + if len(stanza.commentLines) > 0 { + for _, line := range stanza.commentLines { + recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+reader.readerOptions.IFS, context)) } } - if stanza.dataLines.Len() > 0 { + if len(stanza.dataLines) > 0 { record, err := reader.recordFromXTABLines(stanza.dataLines) if err != nil { errorChannel <- err return } context.UpdateForInputRecord() - recordsAndContexts.PushBack(types.NewRecordAndContext(record, context)) + recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context)) } } @@ -271,13 +268,12 @@ func (reader *RecordReaderXTAB) getRecordBatch( } func (reader *RecordReaderXTAB) recordFromXTABLines( - stanza *list.List, + stanza []string, ) (*mlrval.Mlrmap, error) { record := mlrval.NewMlrmapAsRecord() dedupeFieldNames := reader.readerOptions.DedupeFieldNames - for e := stanza.Front(); e != nil; e = e.Next() { - line := e.Value.(string) + for _, line := range stanza { key, value, err := reader.pairSplitter.Split(line) if err != nil { diff --git a/pkg/output/channel_writer.go b/pkg/output/channel_writer.go index ac025398b..e5824fae0 100644 --- a/pkg/output/channel_writer.go +++ b/pkg/output/channel_writer.go @@ -2,7 +2,6 @@ package output import ( "bufio" - "container/list" "fmt" "os" @@ -11,7 +10,7 @@ import ( ) func ChannelWriter( - writerChannel <-chan *list.List, // list of *types.RecordAndContext + writerChannel <-chan []*types.RecordAndContext, // list of *types.RecordAndContext recordWriter IRecordWriter, writerOptions *cli.TWriterOptions, doneChannel chan<- bool, @@ -45,16 +44,14 @@ func ChannelWriter( // TODO: comment // Returns true on end of record stream func channelWriterHandleBatch( - recordsAndContexts *list.List, + recordsAndContexts []*types.RecordAndContext, recordWriter IRecordWriter, writerOptions *cli.TWriterOptions, dataProcessingErrorChannel chan<- bool, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) (done bool, errored bool) { - for e := recordsAndContexts.Front(); e != nil; e = e.Next() { - recordAndContext := e.Value.(*types.RecordAndContext) - + for _, recordAndContext := range recordsAndContexts { // Three things can come through: // * End-of-stream marker // * Non-nil records to be printed diff --git a/pkg/output/file_output_handlers.go b/pkg/output/file_output_handlers.go index 31f6b89a0..cce7bfde0 100644 --- a/pkg/output/file_output_handlers.go +++ b/pkg/output/file_output_handlers.go @@ -14,7 +14,6 @@ package output import ( "bufio" - "container/list" "errors" "fmt" "io" @@ -216,7 +215,7 @@ type FileOutputHandler struct { // print and dump variants call WriteString. recordWriterOptions *cli.TWriterOptions recordWriter IRecordWriter - recordOutputChannel chan *list.List // list of *types.RecordAndContext + recordOutputChannel chan []*types.RecordAndContext // list of *types.RecordAndContext recordDoneChannel chan bool recordErroredChannel chan bool } @@ -352,9 +351,7 @@ func (handler *FileOutputHandler) WriteRecordAndContext( } // TODO: myybe refactor to batch better - ell := list.New() - ell.PushBack(outrecAndContext) - handler.recordOutputChannel <- ell + handler.recordOutputChannel <- []*types.RecordAndContext{outrecAndContext} return nil } @@ -369,7 +366,7 @@ func (handler *FileOutputHandler) setUpRecordWriter() error { } handler.recordWriter = recordWriter - handler.recordOutputChannel = make(chan *list.List, 1) // list of *types.RecordAndContext + handler.recordOutputChannel = make(chan []*types.RecordAndContext, 1) // list of *types.RecordAndContext handler.recordDoneChannel = make(chan bool, 1) handler.recordErroredChannel = make(chan bool, 1) diff --git a/pkg/output/record_writer_pprint.go b/pkg/output/record_writer_pprint.go index acb3366e2..bd552eb8a 100644 --- a/pkg/output/record_writer_pprint.go +++ b/pkg/output/record_writer_pprint.go @@ -2,7 +2,6 @@ package output import ( "bufio" - "container/list" "fmt" "strings" "unicode/utf8" @@ -16,20 +15,20 @@ import ( type RecordWriterPPRINT struct { writerOptions *cli.TWriterOptions // Input: - records *list.List + records []*mlrval.Mlrmap // State: lastJoinedHeader *string - batch *list.List + batch []*mlrval.Mlrmap } func NewRecordWriterPPRINT(writerOptions *cli.TWriterOptions) (*RecordWriterPPRINT, error) { return &RecordWriterPPRINT{ writerOptions: writerOptions, - records: list.New(), + records: make([]*mlrval.Mlrmap, 0), lastJoinedHeader: nil, - batch: list.New(), + batch: make([]*mlrval.Mlrmap, 0), }, nil } @@ -51,7 +50,7 @@ func (writer *RecordWriterPPRINT) Write( // First output record: // * New batch // * No old batch to print - writer.batch.PushBack(outrec) + writer.batch = append(writer.batch, outrec) temp := strings.Join(outrec.GetKeys(), ",") writer.lastJoinedHeader = &temp } else { @@ -70,17 +69,16 @@ func (writer *RecordWriterPPRINT) Write( bufferedOutputStream.WriteString(writer.writerOptions.ORS) } // Start a new batch - writer.batch = list.New() - writer.batch.PushBack(outrec) + writer.batch = []*mlrval.Mlrmap{outrec} writer.lastJoinedHeader = &joinedHeader } else { // Continue the batch - writer.batch.PushBack(outrec) + writer.batch = append(writer.batch, outrec) } } } else { // End of record stream - if writer.batch.Front() != nil { + if len(writer.batch) > 0 { writer.writeHeterogenousList(writer.batch, writer.writerOptions.BarredPprintOutput, bufferedOutputStream, outputIsStdout) } @@ -92,7 +90,7 @@ func (writer *RecordWriterPPRINT) Write( // ---------------------------------------------------------------- // Returns false if there was nothing but empty record(s), e.g. 'mlr gap -n 10'. func (writer *RecordWriterPPRINT) writeHeterogenousList( - records *list.List, + records []*mlrval.Mlrmap, barred bool, bufferedOutputStream *bufio.Writer, outputIsStdout bool, @@ -100,8 +98,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousList( maxWidths := make(map[string]int) var maxNR int64 = 0 - for e := records.Front(); e != nil; e = e.Next() { - outrec := e.Value.(*mlrval.Mlrmap) + for _, outrec := range records { nr := outrec.FieldCount if maxNR < nr { maxNR = nr @@ -148,15 +145,14 @@ func (writer *RecordWriterPPRINT) writeHeterogenousList( // wye pan 5 0.5732889198020006 0.8636244699032729 func (writer *RecordWriterPPRINT) writeHeterogenousListNonBarred( - records *list.List, + records []*mlrval.Mlrmap, maxWidths map[string]int, bufferedOutputStream *bufio.Writer, outputIsStdout bool, ) { onFirst := true - for e := records.Front(); e != nil; e = e.Next() { - outrec := e.Value.(*mlrval.Mlrmap) + for _, outrec := range records { // Print header line if onFirst && !writer.writerOptions.HeaderlessOutput { @@ -238,7 +234,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousListNonBarred( // +-----+-----+----+----------------------+---------------------+ func (writer *RecordWriterPPRINT) writeHeterogenousListBarred( - records *list.List, + records []*mlrval.Mlrmap, maxWidths map[string]int, bufferedOutputStream *bufio.Writer, outputIsStdout bool, @@ -257,8 +253,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousListBarred( verticalEnd := ofs + "|" onFirst := true - for e := records.Front(); e != nil; e = e.Next() { - outrec := e.Value.(*mlrval.Mlrmap) + for i, outrec := range records { // Print header line if onFirst && !writer.writerOptions.HeaderlessOutput { @@ -322,7 +317,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousListBarred( } } - if e.Next() == nil { + if i == len(records)-1 { bufferedOutputStream.WriteString(horizontalStart) for pe := outrec.Head; pe != nil; pe = pe.Next { bufferedOutputStream.WriteString(horizontalBars[pe.Key]) diff --git a/pkg/runtime/state.go b/pkg/runtime/state.go index 30e836f31..987ee9d42 100644 --- a/pkg/runtime/state.go +++ b/pkg/runtime/state.go @@ -7,8 +7,6 @@ package runtime import ( - "container/list" - "github.com/johnkerl/miller/v6/pkg/cli" "github.com/johnkerl/miller/v6/pkg/lib" "github.com/johnkerl/miller/v6/pkg/mlrval" @@ -21,7 +19,7 @@ type State struct { Oosvars *mlrval.Mlrmap FilterExpression *mlrval.Mlrval Stack *Stack - OutputRecordsAndContexts *list.List // list of *types.RecordAndContext + OutputRecordsAndContexts *[]*types.RecordAndContext // list of *types.RecordAndContext // For holding "\0".."\9" between where they are set via things like // '$x =~ "(..)_(...)"', and interpolated via things like '$y = "\2:\1"'. diff --git a/pkg/stream/stream.go b/pkg/stream/stream.go index 1aafe95c9..c207a70c9 100644 --- a/pkg/stream/stream.go +++ b/pkg/stream/stream.go @@ -2,7 +2,6 @@ package stream import ( "bufio" - "container/list" "errors" "io" @@ -62,8 +61,8 @@ func Stream( } // Set up the reader-to-transformer and transformer-to-writer channels. - readerChannel := make(chan *list.List, 2) // list of *types.RecordAndContext - writerChannel := make(chan *list.List, 1) // list of *types.RecordAndContext + readerChannel := make(chan []*types.RecordAndContext, 2) // list of *types.RecordAndContext + writerChannel := make(chan []*types.RecordAndContext, 1) // list of *types.RecordAndContext // We're done when a fatal error is registered on input (file not found, // etc) or when the record-writer has written all its output. We use diff --git a/pkg/terminals/repl/types.go b/pkg/terminals/repl/types.go index 76f8507f0..360a434a0 100644 --- a/pkg/terminals/repl/types.go +++ b/pkg/terminals/repl/types.go @@ -6,7 +6,6 @@ package repl import ( "bufio" - "container/list" "os" "github.com/johnkerl/miller/v6/pkg/cli" @@ -14,6 +13,7 @@ import ( "github.com/johnkerl/miller/v6/pkg/input" "github.com/johnkerl/miller/v6/pkg/output" "github.com/johnkerl/miller/v6/pkg/runtime" + "github.com/johnkerl/miller/v6/pkg/types" ) // ================================================================ @@ -46,7 +46,7 @@ type Repl struct { options *cli.TOptions - readerChannel chan *list.List // list of *types.RecordAndContext + readerChannel chan []*types.RecordAndContext // list of *types.RecordAndContext errorChannel chan error downstreamDoneChannel chan bool recordReader input.IRecordReader diff --git a/pkg/terminals/repl/verbs.go b/pkg/terminals/repl/verbs.go index c2b992746..22e65d04d 100644 --- a/pkg/terminals/repl/verbs.go +++ b/pkg/terminals/repl/verbs.go @@ -5,7 +5,6 @@ package repl import ( - "container/list" "fmt" "os" "strings" @@ -236,7 +235,7 @@ func (repl *Repl) openFiles(filenames []string) { // Remember for :reopen repl.options.FileNames = filenames - repl.readerChannel = make(chan *list.List, 2) // list of *types.RecordAndContext + repl.readerChannel = make(chan []*types.RecordAndContext, 2) // list of *types.RecordAndContext repl.errorChannel = make(chan error, 1) repl.downstreamDoneChannel = make(chan bool, 1) @@ -288,7 +287,7 @@ func handleRead(repl *Repl, args []string) bool { return true } - var recordsAndContexts *list.List // list of *types.RecordAndContext + var recordsAndContexts []*types.RecordAndContext // list of *types.RecordAndContext var err error = nil select { @@ -307,8 +306,8 @@ func handleRead(repl *Repl, args []string) bool { if recordsAndContexts != nil { // TODO: comment and make very clear we've set this all up to batch by 1 for the REPL - lib.InternalCodingErrorIf(recordsAndContexts.Len() != 1) - recordAndContext := recordsAndContexts.Front().Value.(*types.RecordAndContext) + lib.InternalCodingErrorIf(len(recordsAndContexts) != 1) + recordAndContext := recordsAndContexts[0] skipOrProcessRecord( repl, @@ -436,7 +435,7 @@ func handleProcess(repl *Repl, args []string) bool { // ---------------------------------------------------------------- func handleSkipOrProcessN(repl *Repl, n int64, processingNotSkipping bool) { - var recordsAndContexts *list.List // list of *types.RecordAndContext + var recordsAndContexts []*types.RecordAndContext // list of *types.RecordAndContext var err error = nil for i := int64(1); i <= n; i++ { @@ -455,8 +454,8 @@ func handleSkipOrProcessN(repl *Repl, n int64, processingNotSkipping bool) { if recordsAndContexts != nil { // TODO: comment and make very clear we've set this all up to batch by 1 for the REPL - lib.InternalCodingErrorIf(recordsAndContexts.Len() != 1) - recordAndContext := recordsAndContexts.Front().Value.(*types.RecordAndContext) + lib.InternalCodingErrorIf(len(recordsAndContexts) != 1) + recordAndContext := recordsAndContexts[0] shouldBreak := skipOrProcessRecord( repl, @@ -496,7 +495,7 @@ func handleSkipOrProcessUntil(repl *Repl, dslString string, processingNotSkippin return } - var recordsAndContexts *list.List // list of *types.RecordAndContext + var recordsAndContexts []*types.RecordAndContext // list of *types.RecordAndContext for { doubleBreak := false @@ -520,8 +519,8 @@ func handleSkipOrProcessUntil(repl *Repl, dslString string, processingNotSkippin if recordsAndContexts != nil { // TODO: comment and make very clear we've set this all up to batch by 1 for the REPL - lib.InternalCodingErrorIf(recordsAndContexts.Len() != 1) - recordAndContext := recordsAndContexts.Front().Value.(*types.RecordAndContext) + lib.InternalCodingErrorIf(len(recordsAndContexts) != 1) + recordAndContext := recordsAndContexts[0] shouldBreak := skipOrProcessRecord( repl, diff --git a/pkg/transformers/aaa_chain_transformer.go b/pkg/transformers/aaa_chain_transformer.go index a4137b763..e5c98824c 100644 --- a/pkg/transformers/aaa_chain_transformer.go +++ b/pkg/transformers/aaa_chain_transformer.go @@ -1,11 +1,11 @@ package transformers import ( - "container/list" "fmt" + "os" + "github.com/johnkerl/miller/v6/pkg/cli" "github.com/johnkerl/miller/v6/pkg/types" - "os" ) // ================================================================ @@ -143,18 +143,18 @@ import ( // subdivides goroutines for each transformer in the chain, with intermediary // channels between them. func ChainTransformer( - readerRecordChannel <-chan *list.List, // list of *types.RecordAndContext + readerRecordChannel <-chan []*types.RecordAndContext, // list of *types.RecordAndContext readerDownstreamDoneChannel chan<- bool, // for mlr head -- see also stream.go recordTransformers []IRecordTransformer, // not *recordTransformer since this is an interface - writerRecordChannel chan<- *list.List, // list of *types.RecordAndContext + writerRecordChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext options *cli.TOptions, ) { i := 0 n := len(recordTransformers) - intermediateRecordChannels := make([]chan *list.List, n-1) // list of *types.RecordAndContext + intermediateRecordChannels := make([]chan []*types.RecordAndContext, n-1) // list of *types.RecordAndContext for i = 0; i < n-1; i++ { - intermediateRecordChannels[i] = make(chan *list.List, 1) // list of *types.RecordAndContext + intermediateRecordChannels[i] = make(chan []*types.RecordAndContext, 1) // list of *types.RecordAndContext } intermediateDownstreamDoneChannels := make([]chan bool, n) @@ -199,8 +199,8 @@ func ChainTransformer( func runSingleTransformer( recordTransformer IRecordTransformer, isFirstInChain bool, - inputRecordChannel <-chan *list.List, // list of *types.RecordAndContext - outputRecordChannel chan<- *list.List, // list of *types.RecordAndContext + inputRecordChannel <-chan []*types.RecordAndContext, // list of *types.RecordAndContext + outputRecordChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, options *cli.TOptions, @@ -224,20 +224,18 @@ func runSingleTransformer( // TODO: comment // Returns true on end of record stream func runSingleTransformerBatch( - inputRecordsAndContexts *list.List, // list of types.RecordAndContext + inputRecordsAndContexts []*types.RecordAndContext, // list of types.RecordAndContext recordTransformer IRecordTransformer, isFirstInChain bool, - outputRecordChannel chan<- *list.List, // list of *types.RecordAndContext + outputRecordChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, options *cli.TOptions, ) bool { - outputRecordsAndContexts := list.New() + outputRecordsAndContexts := make([]*types.RecordAndContext, 0, len(inputRecordsAndContexts)) done := false - for e := inputRecordsAndContexts.Front(); e != nil; e = e.Next() { - inputRecordAndContext := e.Value.(*types.RecordAndContext) - + for _, inputRecordAndContext := range inputRecordsAndContexts { // --nr-progress-mod // TODO: function-pointer this away to reduce instruction count in the // normal case which it isn't used at all. No need to test if {static thing} != 0 @@ -268,14 +266,14 @@ func runSingleTransformerBatch( if inputRecordAndContext.EndOfStream || inputRecordAndContext.Record != nil { recordTransformer.Transform( inputRecordAndContext, - outputRecordsAndContexts, + &outputRecordsAndContexts, // TODO: maybe refactor these out of each transformer. // And/or maybe poll them once per batch not once per record. inputDownstreamDoneChannel, outputDownstreamDoneChannel, ) } else { - outputRecordsAndContexts.PushBack(inputRecordAndContext) + outputRecordsAndContexts = append(outputRecordsAndContexts, inputRecordAndContext) } if inputRecordAndContext.EndOfStream { diff --git a/pkg/transformers/aaa_record_transformer.go b/pkg/transformers/aaa_record_transformer.go index 516a11a31..009d0d2cf 100644 --- a/pkg/transformers/aaa_record_transformer.go +++ b/pkg/transformers/aaa_record_transformer.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "os" "github.com/johnkerl/miller/v6/pkg/cli" @@ -14,7 +13,7 @@ import ( type IRecordTransformer interface { Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) @@ -22,7 +21,7 @@ type IRecordTransformer interface { type RecordTransformerFunc func( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) @@ -30,7 +29,7 @@ type RecordTransformerFunc func( // Used within some verbs type RecordTransformerHelperFunc func( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) type TransformerUsageFunc func( diff --git a/pkg/transformers/altkv.go b/pkg/transformers/altkv.go index cb1d5c8d2..add095e50 100644 --- a/pkg/transformers/altkv.go +++ b/pkg/transformers/altkv.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strconv" @@ -90,7 +89,7 @@ func NewTransformerAltkv() (*TransformerAltkv, error) { func (tr *TransformerAltkv) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -121,9 +120,9 @@ func (tr *TransformerAltkv) Transform( pe = pe.Next } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } else { // end of record stream - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/bar.go b/pkg/transformers/bar.go index 09713c272..6aa5830c6 100644 --- a/pkg/transformers/bar.go +++ b/pkg/transformers/bar.go @@ -2,7 +2,6 @@ package transformers import ( "bytes" - "container/list" "fmt" "os" "strings" @@ -151,7 +150,7 @@ type TransformerBar struct { oobString string blankString string bars []string - recordsForAutoMode *list.List + recordsForAutoMode []*types.RecordAndContext recordTransformerFunc RecordTransformerFunc } @@ -194,7 +193,7 @@ func NewTransformerBar( if doAuto { tr.recordTransformerFunc = tr.processAuto - tr.recordsForAutoMode = list.New() + tr.recordsForAutoMode = make([]*types.RecordAndContext, 0) } else { tr.recordTransformerFunc = tr.processNoAuto tr.recordsForAutoMode = nil @@ -207,7 +206,7 @@ func NewTransformerBar( func (tr *TransformerBar) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -218,7 +217,7 @@ func (tr *TransformerBar) Transform( // ---------------------------------------------------------------- func (tr *TransformerBar) processNoAuto( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -244,21 +243,21 @@ func (tr *TransformerBar) processNoAuto( inrec.PutReference(fieldName, mlrval.FromString(tr.bars[idx])) } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerBar) processAuto( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { if !inrecAndContext.EndOfStream { - tr.recordsForAutoMode.PushBack(inrecAndContext.Copy()) + tr.recordsForAutoMode = append(tr.recordsForAutoMode, inrecAndContext.Copy()) return } @@ -271,9 +270,8 @@ func (tr *TransformerBar) processAuto( // The first pass computes lo and hi from the data onFirst := true - for e := tr.recordsForAutoMode.Front(); e != nil; e = e.Next() { - recordAndContexts := e.Value.(*types.RecordAndContext) - record := recordAndContexts.Record + for _, recordAndContext := range tr.recordsForAutoMode { + record := recordAndContext.Record mvalue := record.Get(fieldName) if mvalue == nil { continue @@ -301,8 +299,7 @@ func (tr *TransformerBar) processAuto( slo := fmt.Sprintf("%g", lo) shi := fmt.Sprintf("%g", hi) - for e := tr.recordsForAutoMode.Front(); e != nil; e = e.Next() { - recordAndContext := e.Value.(*types.RecordAndContext) + for _, recordAndContext := range tr.recordsForAutoMode { record := recordAndContext.Record mvalue := record.Get(fieldName) if mvalue == nil { @@ -333,10 +330,9 @@ func (tr *TransformerBar) processAuto( } } - for e := tr.recordsForAutoMode.Front(); e != nil; e = e.Next() { - recordAndContext := e.Value.(*types.RecordAndContext) - outputRecordsAndContexts.PushBack(recordAndContext) + for _, recordAndContext := range tr.recordsForAutoMode { + *outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext) } - outputRecordsAndContexts.PushBack(inrecAndContext) // Emit the end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // Emit the end-of-stream marker } diff --git a/pkg/transformers/bootstrap.go b/pkg/transformers/bootstrap.go index 47a200499..8b8a11d51 100644 --- a/pkg/transformers/bootstrap.go +++ b/pkg/transformers/bootstrap.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -91,13 +90,13 @@ func transformerBootstrapParseCLI( // ---------------------------------------------------------------- type TransformerBootstrap struct { - recordsAndContexts *list.List + recordsAndContexts []*types.RecordAndContext nout int64 } func NewTransformerBootstrap(nout int64) (*TransformerBootstrap, error) { tr := &TransformerBootstrap{ - recordsAndContexts: list.New(), + recordsAndContexts: make([]*types.RecordAndContext, 0), nout: nout, } return tr, nil @@ -107,14 +106,14 @@ func NewTransformerBootstrap(nout int64) (*TransformerBootstrap, error) { func (tr *TransformerBootstrap) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel) // Not end of input stream: retain the record, and emit nothing until end of stream. if !inrecAndContext.EndOfStream { - tr.recordsAndContexts.PushBack(inrecAndContext) + tr.recordsAndContexts = append(tr.recordsAndContexts, inrecAndContext) return } @@ -141,8 +140,7 @@ func (tr *TransformerBootstrap) Transform( // // For that reason, this transformer must copy all output. - // TODO: Go list Len() maxes at 2^31. We should track this ourselves in an int. - nin := int64(tr.recordsAndContexts.Len()) + nin := int64(len(tr.recordsAndContexts)) nout := tr.nout if nout == -1 { nout = nin @@ -150,20 +148,12 @@ func (tr *TransformerBootstrap) Transform( if nout == 0 { // Emit the stream-terminating null record - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } // Make an array of pointers into the input list. - recordArray := make([]*types.RecordAndContext, nin) - for i := int64(0); i < nin; i++ { - head := tr.recordsAndContexts.Front() - if head == nil { - break - } - recordArray[i] = head.Value.(*types.RecordAndContext) - tr.recordsAndContexts.Remove(head) - } + recordArray := tr.recordsAndContexts // Do the sample-with-replacment, reading from random indices in the input // array and emitting output. @@ -171,9 +161,9 @@ func (tr *TransformerBootstrap) Transform( index := lib.RandRange(0, nin) recordAndContext := recordArray[index] // Already emitted once; copy - outputRecordsAndContexts.PushBack(recordAndContext.Copy()) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext.Copy()) } // Emit the stream-terminating null record - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } diff --git a/pkg/transformers/case.go b/pkg/transformers/case.go index 1020876e4..9c9b5e827 100644 --- a/pkg/transformers/case.go +++ b/pkg/transformers/case.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -177,7 +176,7 @@ func caseSentenceFunc(input string) string { func (tr *TransformerCase) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -190,13 +189,13 @@ func (tr *TransformerCase) Transform( outputDownstreamDoneChannel, ) } else { // end of record stream - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } func (tr *TransformerCase) transformKeysOnly( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext _ <-chan bool, __ chan<- bool, ) { @@ -211,12 +210,12 @@ func (tr *TransformerCase) transformKeysOnly( newrec.PutReference(pe.Key, pe.Value) } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } func (tr *TransformerCase) transformValuesOnly( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext _ <-chan bool, __ chan<- bool, ) { @@ -229,12 +228,12 @@ func (tr *TransformerCase) transformValuesOnly( } } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(inrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(inrec, &inrecAndContext.Context)) } func (tr *TransformerCase) transformKeysAndValues( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext _ <-chan bool, __ chan<- bool, ) { @@ -254,5 +253,5 @@ func (tr *TransformerCase) transformKeysAndValues( newrec.PutReference(pe.Key, pe.Value) } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } diff --git a/pkg/transformers/cat.go b/pkg/transformers/cat.go index 74df80eca..39423d77a 100644 --- a/pkg/transformers/cat.go +++ b/pkg/transformers/cat.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -165,7 +164,7 @@ func NewTransformerCat( func (tr *TransformerCat) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -181,7 +180,7 @@ func (tr *TransformerCat) Transform( // ---------------------------------------------------------------- func (tr *TransformerCat) simpleCat( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -193,13 +192,13 @@ func (tr *TransformerCat) simpleCat( inrecAndContext.Record.PrependCopy("filenum", mlrval.FromInt(inrecAndContext.Context.FILENUM)) } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } // ---------------------------------------------------------------- func (tr *TransformerCat) countersUngrouped( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -216,13 +215,13 @@ func (tr *TransformerCat) countersUngrouped( inrec.PrependCopy("filenum", mlrval.FromInt(inrecAndContext.Context.FILENUM)) } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } // ---------------------------------------------------------------- func (tr *TransformerCat) countersGrouped( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -255,5 +254,5 @@ func (tr *TransformerCat) countersGrouped( inrec.PrependCopy("filenum", mlrval.FromInt(inrecAndContext.Context.FILENUM)) } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } diff --git a/pkg/transformers/check.go b/pkg/transformers/check.go index ed68d0afa..ca842ee23 100644 --- a/pkg/transformers/check.go +++ b/pkg/transformers/check.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -94,7 +93,7 @@ func NewTransformerCheck() (*TransformerCheck, error) { func (tr *TransformerCheck) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -120,6 +119,6 @@ func (tr *TransformerCheck) Transform( } } } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/clean_whitespace.go b/pkg/transformers/clean_whitespace.go index bdb032f66..a6e14e6aa 100644 --- a/pkg/transformers/clean_whitespace.go +++ b/pkg/transformers/clean_whitespace.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -132,7 +131,7 @@ func NewTransformerCleanWhitespace( func (tr *TransformerCleanWhitespace) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -143,7 +142,7 @@ func (tr *TransformerCleanWhitespace) Transform( // ---------------------------------------------------------------- func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeysAndValues( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -159,16 +158,16 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeysAndValues( newrec.PutReference(newKey.String(), newValue) } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } // ---------------------------------------------------------------- func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeys( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -182,16 +181,16 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeys( newrec.PutReference(newKey.String(), pe.Value) } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } // ---------------------------------------------------------------- func (tr *TransformerCleanWhitespace) cleanWhitespaceInValues( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -199,8 +198,8 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInValues( for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next { pe.Value = bifs.BIF_clean_whitespace(pe.Value) } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/count.go b/pkg/transformers/count.go index bb018661e..a911a2004 100644 --- a/pkg/transformers/count.go +++ b/pkg/transformers/count.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -150,7 +149,7 @@ func NewTransformerCount( func (tr *TransformerCount) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -161,7 +160,7 @@ func (tr *TransformerCount) Transform( // ---------------------------------------------------------------- func (tr *TransformerCount) countUngrouped( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -170,16 +169,16 @@ func (tr *TransformerCount) countUngrouped( } else { newrec := mlrval.NewMlrmapAsRecord() newrec.PutCopy(tr.outputFieldName, mlrval.FromInt(tr.ungroupedCount)) - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerCount) countGrouped( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -210,7 +209,7 @@ func (tr *TransformerCount) countGrouped( newrec.PutCopy(tr.outputFieldName, mlrval.FromInt(tr.groupedCounts.FieldCount)) outrecAndContext := types.NewRecordAndContext(newrec, &inrecAndContext.Context) - outputRecordsAndContexts.PushBack(outrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) } else { for outer := tr.groupedCounts.Head; outer != nil; outer = outer.Next { @@ -235,10 +234,10 @@ func (tr *TransformerCount) countGrouped( newrec.PutCopy(tr.outputFieldName, mlrval.FromInt(countForGroup)) outrecAndContext := types.NewRecordAndContext(newrec, &inrecAndContext.Context) - outputRecordsAndContexts.PushBack(outrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) } } - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/count_similar.go b/pkg/transformers/count_similar.go index be9970265..ff573edc2 100644 --- a/pkg/transformers/count_similar.go +++ b/pkg/transformers/count_similar.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -105,7 +104,7 @@ type TransformerCountSimilar struct { counterFieldName string // State: - recordListsByGroup *lib.OrderedMap[*list.List] // map from string to *list.List + recordListsByGroup *lib.OrderedMap[*[]*types.RecordAndContext] // map from string to records } // ---------------------------------------------------------------- @@ -116,7 +115,7 @@ func NewTransformerCountSimilar( tr := &TransformerCountSimilar{ groupByFieldNames: groupByFieldNames, counterFieldName: counterFieldName, - recordListsByGroup: lib.NewOrderedMap[*list.List](), + recordListsByGroup: lib.NewOrderedMap[*[]*types.RecordAndContext](), } return tr, nil } @@ -125,7 +124,7 @@ func NewTransformerCountSimilar( func (tr *TransformerCountSimilar) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -140,26 +139,26 @@ func (tr *TransformerCountSimilar) Transform( recordListForGroup := tr.recordListsByGroup.Get(groupingKey) if recordListForGroup == nil { // first time - recordListForGroup = list.New() + records := make([]*types.RecordAndContext, 0) + recordListForGroup = &records tr.recordListsByGroup.Put(groupingKey, recordListForGroup) } - recordListForGroup.PushBack(inrecAndContext) + *recordListForGroup = append(*recordListForGroup, inrecAndContext) } else { for outer := tr.recordListsByGroup.Head; outer != nil; outer = outer.Next { recordListForGroup := outer.Value // TODO: make 64-bit friendly - groupSize := recordListForGroup.Len() + groupSize := len(*recordListForGroup) mgroupSize := mlrval.FromInt(int64(groupSize)) - for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() { - recordAndContext := inner.Value.(*types.RecordAndContext) + for _, recordAndContext := range *recordListForGroup { recordAndContext.Record.PutCopy(tr.counterFieldName, mgroupSize) - outputRecordsAndContexts.PushBack(recordAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext) } } - outputRecordsAndContexts.PushBack(inrecAndContext) // Emit the stream-terminating null record + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // Emit the stream-terminating null record } } diff --git a/pkg/transformers/cut.go b/pkg/transformers/cut.go index e9e57d438..bdf3521e2 100644 --- a/pkg/transformers/cut.go +++ b/pkg/transformers/cut.go @@ -2,7 +2,6 @@ package transformers import ( "cmp" - "container/list" "fmt" "os" "regexp" @@ -186,7 +185,7 @@ func NewTransformerCut( func (tr *TransformerCut) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -198,7 +197,7 @@ func (tr *TransformerCut) Transform( // mlr cut -f a,b,c func (tr *TransformerCut) includeWithInputOrder( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -213,9 +212,9 @@ func (tr *TransformerCut) includeWithInputOrder( } } outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context) - outputRecordsAndContexts.PushBack(outrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } @@ -223,7 +222,7 @@ func (tr *TransformerCut) includeWithInputOrder( // mlr cut -o -f a,b,c func (tr *TransformerCut) includeWithArgOrder( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -237,9 +236,9 @@ func (tr *TransformerCut) includeWithArgOrder( } } outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context) - outputRecordsAndContexts.PushBack(outrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } @@ -247,7 +246,7 @@ func (tr *TransformerCut) includeWithArgOrder( // mlr cut -x -f a,b,c func (tr *TransformerCut) exclude( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -259,7 +258,7 @@ func (tr *TransformerCut) exclude( } } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } type entryIndex struct { @@ -270,7 +269,7 @@ type entryIndex struct { // ---------------------------------------------------------------- func (tr *TransformerCut) processWithRegexes( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -307,8 +306,8 @@ func (tr *TransformerCut) processWithRegexes( newrec.PutReference(ei.entry.Key, ei.entry.Value) } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/decimate.go b/pkg/transformers/decimate.go index 112d9395e..336e7d06e 100644 --- a/pkg/transformers/decimate.go +++ b/pkg/transformers/decimate.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -142,7 +141,7 @@ func NewTransformerDecimate( func (tr *TransformerDecimate) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -163,13 +162,13 @@ func (tr *TransformerDecimate) Transform( remainder := countForGroup % tr.decimateCount if remainder == tr.remainderToKeep { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } countForGroup++ tr.countsByGroup[groupingKey] = countForGroup } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // Emit the stream-terminating null record + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // Emit the stream-terminating null record } } diff --git a/pkg/transformers/fill_down.go b/pkg/transformers/fill_down.go index cf779e336..bbe101f51 100644 --- a/pkg/transformers/fill_down.go +++ b/pkg/transformers/fill_down.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -148,7 +147,7 @@ func NewTransformerFillDown( func (tr *TransformerFillDown) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -159,7 +158,7 @@ func (tr *TransformerFillDown) Transform( // ---------------------------------------------------------------- func (tr *TransformerFillDown) transformSpecified( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -187,17 +186,17 @@ func (tr *TransformerFillDown) transformSpecified( } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerFillDown) transformAll( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -226,9 +225,9 @@ func (tr *TransformerFillDown) transformAll( } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/fill_empty.go b/pkg/transformers/fill_empty.go index 833ed5b12..af8caee68 100644 --- a/pkg/transformers/fill_empty.go +++ b/pkg/transformers/fill_empty.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -110,7 +109,7 @@ func NewTransformerFillEmpty( func (tr *TransformerFillEmpty) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -124,9 +123,9 @@ func (tr *TransformerFillEmpty) Transform( } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { // end of record stream - outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker } } diff --git a/pkg/transformers/flatten.go b/pkg/transformers/flatten.go index 86c4c0fd1..f385a226b 100644 --- a/pkg/transformers/flatten.go +++ b/pkg/transformers/flatten.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -134,7 +133,7 @@ func NewTransformerFlatten( func (tr *TransformerFlatten) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -145,7 +144,7 @@ func (tr *TransformerFlatten) Transform( // ---------------------------------------------------------------- func (tr *TransformerFlatten) flattenAll( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -156,16 +155,16 @@ func (tr *TransformerFlatten) flattenAll( oFlatSep = tr.options.WriterOptions.FLATSEP } inrec.Flatten(oFlatSep) - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerFlatten) flattenSome( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -176,8 +175,8 @@ func (tr *TransformerFlatten) flattenSome( oFlatSep = tr.options.WriterOptions.FLATSEP } inrec.FlattenFields(tr.fieldNameSet, oFlatSep) - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/format_values.go b/pkg/transformers/format_values.go index edaf0389f..6ff83cf1d 100644 --- a/pkg/transformers/format_values.go +++ b/pkg/transformers/format_values.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -168,13 +167,13 @@ func NewTransformerFormatValues( func (tr *TransformerFormatValues) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel) if inrecAndContext.EndOfStream { - outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker return } @@ -199,5 +198,5 @@ func (tr *TransformerFormatValues) Transform( } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } diff --git a/pkg/transformers/fraction.go b/pkg/transformers/fraction.go index 432d600cf..91eb09866 100644 --- a/pkg/transformers/fraction.go +++ b/pkg/transformers/fraction.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "errors" "fmt" "os" @@ -132,7 +131,7 @@ type TransformerFraction struct { groupByFieldNames []string doCumu bool - recordsAndContexts *list.List + recordsAndContexts []*types.RecordAndContext // Two-level map: Group-by field names are the first keyset; // fraction field names are keys into the second. sums map[string]map[string]*mlrval.Mlrval @@ -151,7 +150,7 @@ func NewTransformerFraction( doCumu bool, ) (*TransformerFraction, error) { - recordsAndContexts := list.New() + recordsAndContexts := make([]*types.RecordAndContext, 0) sums := make(map[string]map[string]*mlrval.Mlrval) cumus := make(map[string]map[string]*mlrval.Mlrval) @@ -192,7 +191,7 @@ func NewTransformerFraction( func (tr *TransformerFraction) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -201,7 +200,7 @@ func (tr *TransformerFraction) Transform( inrec := inrecAndContext.Record // Append records into a single output list (so that this verb is order-preserving). - tr.recordsAndContexts.PushBack(inrecAndContext) + tr.recordsAndContexts = append(tr.recordsAndContexts, inrecAndContext) // Accumulate sums of fraction-field values grouped by group-by field names groupingKey, hasAll := inrec.GetSelectedValuesJoined(tr.groupByFieldNames) @@ -234,13 +233,7 @@ func (tr *TransformerFraction) Transform( // Iterate over the retained records, decorating them with fraction fields. endOfStreamContext := inrecAndContext.Context - for { - element := tr.recordsAndContexts.Front() - if element == nil { - break - } - tr.recordsAndContexts.Remove(element) - recordAndContext := element.Value.(*types.RecordAndContext) + for _, recordAndContext := range tr.recordsAndContexts { outrec := recordAndContext.Record groupingKey, hasAll := outrec.GetSelectedValuesJoined(tr.groupByFieldNames) @@ -292,8 +285,9 @@ func (tr *TransformerFraction) Transform( } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &endOfStreamContext)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &endOfStreamContext)) } - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + tr.recordsAndContexts = tr.recordsAndContexts[:0] + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/gap.go b/pkg/transformers/gap.go index 9bec0b2ab..fbe70cc86 100644 --- a/pkg/transformers/gap.go +++ b/pkg/transformers/gap.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -138,7 +137,7 @@ func NewTransformerGap( func (tr *TransformerGap) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -148,27 +147,27 @@ func (tr *TransformerGap) Transform( func (tr *TransformerGap) transformUnkeyed( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { if !inrecAndContext.EndOfStream { if tr.recordCount > 0 && tr.recordCount%tr.gapCount == 0 { newrec := mlrval.NewMlrmapAsRecord() - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) tr.recordCount++ } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } func (tr *TransformerGap) transformKeyed( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -182,15 +181,15 @@ func (tr *TransformerGap) transformKeyed( if groupingKey != tr.previousGroupingKey && tr.recordCount > 0 { newrec := mlrval.NewMlrmapAsRecord() - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) tr.previousGroupingKey = groupingKey tr.recordCount++ } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/grep.go b/pkg/transformers/grep.go index 38942d650..efdbd8429 100644 --- a/pkg/transformers/grep.go +++ b/pkg/transformers/grep.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "regexp" @@ -155,7 +154,7 @@ func NewTransformerGrep( func (tr *TransformerGrep) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -171,14 +170,14 @@ func (tr *TransformerGrep) Transform( matches := tr.regexp.MatchString(inrecAsString) if tr.invert { if !matches { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } else { if matches { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/group_by.go b/pkg/transformers/group_by.go index d25e21ee5..c9916be46 100644 --- a/pkg/transformers/group_by.go +++ b/pkg/transformers/group_by.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -92,8 +91,8 @@ type TransformerGroupBy struct { groupByFieldNames []string // state - // map from string to *list.List - recordListsByGroup *lib.OrderedMap[*list.List] + // map from string to record slices + recordListsByGroup *lib.OrderedMap[*[]*types.RecordAndContext] } func NewTransformerGroupBy( @@ -103,7 +102,7 @@ func NewTransformerGroupBy( tr := &TransformerGroupBy{ groupByFieldNames: groupByFieldNames, - recordListsByGroup: lib.NewOrderedMap[*list.List](), + recordListsByGroup: lib.NewOrderedMap[*[]*types.RecordAndContext](), } return tr, nil @@ -113,7 +112,7 @@ func NewTransformerGroupBy( func (tr *TransformerGroupBy) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -128,19 +127,20 @@ func (tr *TransformerGroupBy) Transform( recordListForGroup := tr.recordListsByGroup.Get(groupingKey) if recordListForGroup == nil { - recordListForGroup = list.New() + records := make([]*types.RecordAndContext, 0) + recordListForGroup = &records tr.recordListsByGroup.Put(groupingKey, recordListForGroup) } - recordListForGroup.PushBack(inrecAndContext) + *recordListForGroup = append(*recordListForGroup, inrecAndContext) } else { for outer := tr.recordListsByGroup.Head; outer != nil; outer = outer.Next { recordListForGroup := outer.Value - for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() { - outputRecordsAndContexts.PushBack(inner.Value.(*types.RecordAndContext)) + for _, recordAndContext := range *recordListForGroup { + *outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext) } } - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/group_like.go b/pkg/transformers/group_like.go index 664bb72b9..aa18da3aa 100644 --- a/pkg/transformers/group_like.go +++ b/pkg/transformers/group_like.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -78,14 +77,14 @@ func transformerGroupLikeParseCLI( // ---------------------------------------------------------------- type TransformerGroupLike struct { - // map from string to *list.List - recordListsByGroup *lib.OrderedMap[*list.List] + // map from string to record slices + recordListsByGroup *lib.OrderedMap[*[]*types.RecordAndContext] } func NewTransformerGroupLike() (*TransformerGroupLike, error) { tr := &TransformerGroupLike{ - recordListsByGroup: lib.NewOrderedMap[*list.List](), + recordListsByGroup: lib.NewOrderedMap[*[]*types.RecordAndContext](), } return tr, nil @@ -95,7 +94,7 @@ func NewTransformerGroupLike() (*TransformerGroupLike, error) { func (tr *TransformerGroupLike) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -107,19 +106,20 @@ func (tr *TransformerGroupLike) Transform( recordListForGroup := tr.recordListsByGroup.Get(groupingKey) if recordListForGroup == nil { // first time - recordListForGroup = list.New() + records := make([]*types.RecordAndContext, 0) + recordListForGroup = &records tr.recordListsByGroup.Put(groupingKey, recordListForGroup) } - recordListForGroup.PushBack(inrecAndContext) + *recordListForGroup = append(*recordListForGroup, inrecAndContext) } else { for outer := tr.recordListsByGroup.Head; outer != nil; outer = outer.Next { recordListForGroup := outer.Value - for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() { - outputRecordsAndContexts.PushBack(inner.Value.(*types.RecordAndContext)) + for _, recordAndContext := range *recordListForGroup { + *outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext) } } - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/having_fields.go b/pkg/transformers/having_fields.go index 467c0a7d3..bd6ef9ae5 100644 --- a/pkg/transformers/having_fields.go +++ b/pkg/transformers/having_fields.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "regexp" @@ -221,7 +220,7 @@ func NewTransformerHavingFields( func (tr *TransformerHavingFields) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -232,7 +231,7 @@ func (tr *TransformerHavingFields) Transform( // ---------------------------------------------------------------- func (tr *TransformerHavingFields) transformHavingFieldsAtLeast( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -243,20 +242,20 @@ func (tr *TransformerHavingFields) transformHavingFieldsAtLeast( if tr.fieldNameSet[pe.Key] { numFound++ if numFound == tr.numFieldNames { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } } } } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } func (tr *TransformerHavingFields) transformHavingFieldsWhichAre( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -270,15 +269,15 @@ func (tr *TransformerHavingFields) transformHavingFieldsWhichAre( return } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } func (tr *TransformerHavingFields) transformHavingFieldsAtMost( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -289,16 +288,16 @@ func (tr *TransformerHavingFields) transformHavingFieldsAtMost( return } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } // ---------------------------------------------------------------- func (tr *TransformerHavingFields) transformHavingAllFieldsMatching( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -309,15 +308,15 @@ func (tr *TransformerHavingFields) transformHavingAllFieldsMatching( return } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } func (tr *TransformerHavingFields) transformHavingAnyFieldsMatching( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -325,18 +324,18 @@ func (tr *TransformerHavingFields) transformHavingAnyFieldsMatching( inrec := inrecAndContext.Record for pe := inrec.Head; pe != nil; pe = pe.Next { if tr.regex.MatchString(pe.Key) { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } } } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } func (tr *TransformerHavingFields) transformHavingNoFieldsMatching( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -347,8 +346,8 @@ func (tr *TransformerHavingFields) transformHavingNoFieldsMatching( return } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/head.go b/pkg/transformers/head.go index 6f7ff5a19..c62abca38 100644 --- a/pkg/transformers/head.go +++ b/pkg/transformers/head.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -143,7 +142,7 @@ func NewTransformerHead( func (tr *TransformerHead) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -153,30 +152,30 @@ func (tr *TransformerHead) Transform( func (tr *TransformerHead) transformUnkeyed( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { if !inrecAndContext.EndOfStream { tr.unkeyedRecordCount++ if tr.unkeyedRecordCount <= tr.headCount { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else if !tr.wroteDownstreamDone { // Signify to data producers upstream that we'll ignore further // data, so as far as we're concerned they can stop sending it. See // ChainTransformer. - //TODO: maybe remove: outputRecordsAndContexts.PushBack(types.NewEndOfStreamMarker(&inrecAndContext.Context)) + //TODO: maybe remove: *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewEndOfStreamMarker(&inrecAndContext.Context)) outputDownstreamDoneChannel <- true tr.wroteDownstreamDone = true } } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } func (tr *TransformerHead) transformKeyed( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -198,10 +197,10 @@ func (tr *TransformerHead) transformKeyed( } if count <= tr.headCount { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/histogram.go b/pkg/transformers/histogram.go index 32f581107..d6d4c5ec0 100644 --- a/pkg/transformers/histogram.go +++ b/pkg/transformers/histogram.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -198,7 +197,7 @@ func NewTransformerHistogram( func (tr *TransformerHistogram) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -209,7 +208,7 @@ func (tr *TransformerHistogram) Transform( // ---------------------------------------------------------------- func (tr *TransformerHistogram) transformNonAuto( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -217,7 +216,7 @@ func (tr *TransformerHistogram) transformNonAuto( tr.ingestNonAuto(inrecAndContext) } else { tr.emitNonAuto(&inrecAndContext.Context, outputRecordsAndContexts) - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } @@ -250,7 +249,7 @@ func (tr *TransformerHistogram) ingestNonAuto( func (tr *TransformerHistogram) emitNonAuto( endOfStreamContext *types.Context, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { countFieldNames := make(map[string]string) for _, valueFieldName := range tr.valueFieldNames { @@ -275,14 +274,14 @@ func (tr *TransformerHistogram) emitNonAuto( ) } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, endOfStreamContext)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, endOfStreamContext)) } } // ---------------------------------------------------------------- func (tr *TransformerHistogram) transformAuto( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -290,7 +289,7 @@ func (tr *TransformerHistogram) transformAuto( tr.ingestAuto(inrecAndContext) } else { tr.emitAuto(&inrecAndContext.Context, outputRecordsAndContexts) - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } @@ -309,7 +308,7 @@ func (tr *TransformerHistogram) ingestAuto( func (tr *TransformerHistogram) emitAuto( endOfStreamContext *types.Context, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { haveLoHi := false lo := 0.0 @@ -381,6 +380,6 @@ func (tr *TransformerHistogram) emitAuto( ) } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, endOfStreamContext)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, endOfStreamContext)) } } diff --git a/pkg/transformers/join.go b/pkg/transformers/join.go index 4c9eccefd..290875b37 100644 --- a/pkg/transformers/join.go +++ b/pkg/transformers/join.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -296,7 +295,7 @@ type TransformerJoin struct { // For unsorted/half-streaming input ingested bool leftBucketsByJoinFieldValues *lib.OrderedMap[*utils.JoinBucket] - leftUnpairableRecordsAndContexts *list.List + leftUnpairableRecordsAndContexts []*types.RecordAndContext // For sorted/doubly-streaming input joinBucketKeeper *utils.JoinBucketKeeper @@ -333,7 +332,7 @@ func NewTransformerJoin( if opts.allowUnsortedInput { // Half-streaming (default) case: ingest entire left file first. - tr.leftUnpairableRecordsAndContexts = list.New() + tr.leftUnpairableRecordsAndContexts = make([]*types.RecordAndContext, 0) tr.leftBucketsByJoinFieldValues = lib.NewOrderedMap[*utils.JoinBucket]() tr.recordTransformerFunc = tr.transformHalfStreaming @@ -361,7 +360,7 @@ func NewTransformerJoin( func (tr *TransformerJoin) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -375,7 +374,7 @@ func (tr *TransformerJoin) Transform( // matching each right record against those. func (tr *TransformerJoin) transformHalfStreaming( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -398,7 +397,7 @@ func (tr *TransformerJoin) transformHalfStreaming( leftBucket := tr.leftBucketsByJoinFieldValues.Get(groupingKey) if leftBucket == nil { if tr.opts.emitRightUnpairables { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } else { leftBucket.WasPaired = true @@ -411,7 +410,7 @@ func (tr *TransformerJoin) transformHalfStreaming( } } } else if tr.opts.emitRightUnpairables { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } else { // end of record stream @@ -419,14 +418,14 @@ func (tr *TransformerJoin) transformHalfStreaming( tr.emitLeftUnpairedBuckets(outputRecordsAndContexts) tr.emitLeftUnpairables(outputRecordsAndContexts) } - outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerJoin) transformDoublyStreaming( rightRecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -451,7 +450,7 @@ func (tr *TransformerJoin) transformDoublyStreaming( lefts := keeper.JoinBucket.RecordsAndContexts // keystroke-saver if !isPaired && tr.opts.emitRightUnpairables { - outputRecordsAndContexts.PushBack(rightRecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, rightRecAndContext) } if isPaired && tr.opts.emitPairables && lefts != nil { @@ -465,7 +464,7 @@ func (tr *TransformerJoin) transformDoublyStreaming( keeper.OutputAndReleaseLeftUnpaireds(outputRecordsAndContexts) } - outputRecordsAndContexts.PushBack(rightRecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, rightRecAndContext) // emit end-of-stream marker } } @@ -495,7 +494,7 @@ func (tr *TransformerJoin) ingestLeftFile() { initialContext.UpdateForStartOfFile(tr.opts.leftFileName) // Set up channels for the record-reader. - readerChannel := make(chan *list.List, 2) // list of *types.RecordAndContext + readerChannel := make(chan []*types.RecordAndContext, 2) // list of *types.RecordAndContext errorChannel := make(chan error, 1) downstreamDoneChannel := make(chan bool, 1) @@ -518,8 +517,8 @@ func (tr *TransformerJoin) ingestLeftFile() { case leftrecsAndContexts := <-readerChannel: // TODO: temp for batch-reader refactor - lib.InternalCodingErrorIf(leftrecsAndContexts.Len() != 1) - leftrecAndContext := leftrecsAndContexts.Front().Value.(*types.RecordAndContext) + lib.InternalCodingErrorIf(len(leftrecsAndContexts) != 1) + leftrecAndContext := leftrecsAndContexts[0] leftrecAndContext.Record = utils.KeepLeftFieldNames(leftrecAndContext.Record, tr.leftKeepFieldNameSet) if leftrecAndContext.EndOfStream { @@ -539,13 +538,13 @@ func (tr *TransformerJoin) ingestLeftFile() { bucket := tr.leftBucketsByJoinFieldValues.Get(groupingKey) if bucket == nil { // New key-field-value: new bucket and hash-map entry bucket := utils.NewJoinBucket(leftFieldValues) - bucket.RecordsAndContexts.PushBack(leftrecAndContext) + bucket.RecordsAndContexts = append(bucket.RecordsAndContexts, leftrecAndContext) tr.leftBucketsByJoinFieldValues.Put(groupingKey, bucket) } else { // Previously seen key-field-value: append record to bucket - bucket.RecordsAndContexts.PushBack(leftrecAndContext) + bucket.RecordsAndContexts = append(bucket.RecordsAndContexts, leftrecAndContext) } } else { - tr.leftUnpairableRecordsAndContexts.PushBack(leftrecAndContext) + tr.leftUnpairableRecordsAndContexts = append(tr.leftUnpairableRecordsAndContexts, leftrecAndContext) } } } @@ -556,15 +555,14 @@ func (tr *TransformerJoin) ingestLeftFile() { // the doubly-streaming/sorted join. func (tr *TransformerJoin) formAndEmitPairs( - leftRecordsAndContexts *list.List, + leftRecordsAndContexts []*types.RecordAndContext, rightRecordAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { ////fmt.Println("-- pairs start") // VERBOSE // Loop over each to-be-paired-with record from the left file. - for pe := leftRecordsAndContexts.Front(); pe != nil; pe = pe.Next() { + for _, leftRecordAndContext := range leftRecordsAndContexts { ////fmt.Println("-- pairs pe") // VERBOSE - leftRecordAndContext := pe.Value.(*types.RecordAndContext) leftrec := leftRecordAndContext.Record rightrec := rightRecordAndContext.Record @@ -608,7 +606,7 @@ func (tr *TransformerJoin) formAndEmitPairs( outrecAndContext := types.NewRecordAndContext(outrec, &context) // Emit the new joined record on the downstream channel - outputRecordsAndContexts.PushBack(outrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) } ////fmt.Println("-- pairs end") // VERBOSE } @@ -624,24 +622,22 @@ func (tr *TransformerJoin) formAndEmitPairs( // in the second category. func (tr *TransformerJoin) emitLeftUnpairables( - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { // Loop over each to-be-paired-with record from the left file. - for pe := tr.leftUnpairableRecordsAndContexts.Front(); pe != nil; pe = pe.Next() { - leftRecordAndContext := pe.Value.(*types.RecordAndContext) - outputRecordsAndContexts.PushBack(leftRecordAndContext) + for _, leftRecordAndContext := range tr.leftUnpairableRecordsAndContexts { + *outputRecordsAndContexts = append(*outputRecordsAndContexts, leftRecordAndContext) } } func (tr *TransformerJoin) emitLeftUnpairedBuckets( - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { for pe := tr.leftBucketsByJoinFieldValues.Head; pe != nil; pe = pe.Next { bucket := pe.Value if !bucket.WasPaired { - for pf := bucket.RecordsAndContexts.Front(); pf != nil; pf = pf.Next() { - recordAndContext := pf.Value.(*types.RecordAndContext) - outputRecordsAndContexts.PushBack(recordAndContext) + for _, recordAndContext := range bucket.RecordsAndContexts { + *outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext) } } } diff --git a/pkg/transformers/json_parse.go b/pkg/transformers/json_parse.go index 0690b410e..324e4bc5f 100644 --- a/pkg/transformers/json_parse.go +++ b/pkg/transformers/json_parse.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -131,7 +130,7 @@ func NewTransformerJSONParse( func (tr *TransformerJSONParse) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -142,7 +141,7 @@ func (tr *TransformerJSONParse) Transform( // ---------------------------------------------------------------- func (tr *TransformerJSONParse) jsonParseAll( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -155,16 +154,16 @@ func (tr *TransformerJSONParse) jsonParseAll( pe.JSONParseInPlace() } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerJSONParse) jsonParseSome( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -179,8 +178,8 @@ func (tr *TransformerJSONParse) jsonParseSome( } } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/json_stringify.go b/pkg/transformers/json_stringify.go index c6b5642a8..6c88523f7 100644 --- a/pkg/transformers/json_stringify.go +++ b/pkg/transformers/json_stringify.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -141,7 +140,7 @@ func NewTransformerJSONStringify( func (tr *TransformerJSONStringify) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -152,7 +151,7 @@ func (tr *TransformerJSONStringify) Transform( // ---------------------------------------------------------------- func (tr *TransformerJSONStringify) jsonStringifyAll( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -161,16 +160,16 @@ func (tr *TransformerJSONStringify) jsonStringifyAll( for pe := inrec.Head; pe != nil; pe = pe.Next { pe.JSONStringifyInPlace(tr.jsonFormatting) } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerJSONStringify) jsonStringifySome( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -181,8 +180,8 @@ func (tr *TransformerJSONStringify) jsonStringifySome( pe.JSONStringifyInPlace(tr.jsonFormatting) } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/label.go b/pkg/transformers/label.go index 3ad653595..ac7d748be 100644 --- a/pkg/transformers/label.go +++ b/pkg/transformers/label.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -120,7 +119,7 @@ func NewTransformerLabel( func (tr *TransformerLabel) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -129,5 +128,5 @@ func (tr *TransformerLabel) Transform( inrec := inrecAndContext.Record inrec.Label(tr.newNames) } - outputRecordsAndContexts.PushBack(inrecAndContext) // including end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker } diff --git a/pkg/transformers/latin1_to_utf8.go b/pkg/transformers/latin1_to_utf8.go index 0dd509f39..8aca9885d 100644 --- a/pkg/transformers/latin1_to_utf8.go +++ b/pkg/transformers/latin1_to_utf8.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -91,7 +90,7 @@ func NewTransformerLatin1ToUTF8() (*TransformerLatin1ToUTF8, error) { func (tr *TransformerLatin1ToUTF8) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -111,9 +110,9 @@ func (tr *TransformerLatin1ToUTF8) Transform( } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(inrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(inrec, &inrecAndContext.Context)) } else { // end of record stream - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/merge_fields.go b/pkg/transformers/merge_fields.go index e2a729afa..7ff5a39f8 100644 --- a/pkg/transformers/merge_fields.go +++ b/pkg/transformers/merge_fields.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "regexp" @@ -312,7 +311,7 @@ func NewTransformerMergeFields( func (tr *TransformerMergeFields) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -323,12 +322,12 @@ func (tr *TransformerMergeFields) Transform( // ---------------------------------------------------------------- func (tr *TransformerMergeFields) transformByNameList( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { if inrecAndContext.EndOfStream { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker return } @@ -368,18 +367,18 @@ func (tr *TransformerMergeFields) transformByNameList( inrec.PutReference(key, value) } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } // ---------------------------------------------------------------- func (tr *TransformerMergeFields) transformByNameRegex( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { if inrecAndContext.EndOfStream { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker return } @@ -443,7 +442,7 @@ func (tr *TransformerMergeFields) transformByNameRegex( inrec.PutReference(key, value) } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } // ---------------------------------------------------------------- @@ -455,12 +454,12 @@ func (tr *TransformerMergeFields) transformByNameRegex( func (tr *TransformerMergeFields) transformByCollapsing( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { if inrecAndContext.EndOfStream { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker return } @@ -549,5 +548,5 @@ func (tr *TransformerMergeFields) transformByCollapsing( } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } diff --git a/pkg/transformers/most_or_least_frequent.go b/pkg/transformers/most_or_least_frequent.go index 78addd921..321c93296 100644 --- a/pkg/transformers/most_or_least_frequent.go +++ b/pkg/transformers/most_or_least_frequent.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "sort" @@ -205,7 +204,7 @@ func NewTransformerMostOrLeastFrequent( func (tr *TransformerMostOrLeastFrequent) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -280,9 +279,9 @@ func (tr *TransformerMostOrLeastFrequent) Transform( if tr.showCounts { outrec.PutReference(tr.outputFieldName, mlrval.FromInt(sortPairs[i].count)) } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } - outputRecordsAndContexts.PushBack(inrecAndContext) // End-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // End-of-stream marker } } diff --git a/pkg/transformers/nest.go b/pkg/transformers/nest.go index b49f36e48..2cefda50d 100644 --- a/pkg/transformers/nest.go +++ b/pkg/transformers/nest.go @@ -2,7 +2,6 @@ package transformers import ( "bytes" - "container/list" "fmt" "os" "regexp" @@ -317,7 +316,7 @@ func NewTransformerNest( func (tr *TransformerNest) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -328,7 +327,7 @@ func (tr *TransformerNest) Transform( // ---------------------------------------------------------------- func (tr *TransformerNest) explodeValuesAcrossFields( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -337,7 +336,7 @@ func (tr *TransformerNest) explodeValuesAcrossFields( inrec := inrecAndContext.Record originalEntry := inrec.GetEntry(tr.fieldName) if originalEntry == nil { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } @@ -356,17 +355,17 @@ func (tr *TransformerNest) explodeValuesAcrossFields( } inrec.Unlink(originalEntry) - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerNest) explodeValuesAcrossRecords( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -374,7 +373,7 @@ func (tr *TransformerNest) explodeValuesAcrossRecords( inrec := inrecAndContext.Record mvalue := inrec.Get(tr.fieldName) if mvalue == nil { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } svalue := mvalue.String() @@ -384,18 +383,18 @@ func (tr *TransformerNest) explodeValuesAcrossRecords( for _, piece := range pieces { outrec := inrec.Copy() outrec.PutReference(tr.fieldName, mlrval.FromString(piece)) - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerNest) explodePairsAcrossFields( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -404,7 +403,7 @@ func (tr *TransformerNest) explodePairsAcrossFields( inrec := inrecAndContext.Record originalEntry := inrec.GetEntry(tr.fieldName) if originalEntry == nil { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } @@ -431,17 +430,17 @@ func (tr *TransformerNest) explodePairsAcrossFields( } inrec.Unlink(originalEntry) - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerNest) explodePairsAcrossRecords( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -449,7 +448,7 @@ func (tr *TransformerNest) explodePairsAcrossRecords( inrec := inrecAndContext.Record mvalue := inrec.Get(tr.fieldName) if mvalue == nil { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } @@ -470,18 +469,18 @@ func (tr *TransformerNest) explodePairsAcrossRecords( } outrec.Unlink(originalEntry) - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerNest) implodeValuesAcrossFields( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -520,17 +519,17 @@ func (tr *TransformerNest) implodeValuesAcrossFields( } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerNest) implodeValueAcrossRecords( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -539,7 +538,7 @@ func (tr *TransformerNest) implodeValueAcrossRecords( originalEntry := inrec.GetEntry(tr.fieldName) if originalEntry == nil { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } @@ -566,7 +565,7 @@ func (tr *TransformerNest) implodeValueAcrossRecords( pair := mlrval.NewMlrmapAsRecord() pair.PutReference(tr.fieldName, fieldValueCopy) - bucket.pairs.PushBack(pair) + bucket.pairs = append(bucket.pairs, pair) } else { // end of input stream @@ -579,8 +578,7 @@ func (tr *TransformerNest) implodeValueAcrossRecords( bucket.representative = nil // ownership transfer i := 0 - for pg := bucket.pairs.Front(); pg != nil; pg = pg.Next() { - pr := pg.Value.(*mlrval.Mlrmap) + for _, pr := range bucket.pairs { if i > 0 { buffer.WriteString(tr.nestedFS) } @@ -590,22 +588,22 @@ func (tr *TransformerNest) implodeValueAcrossRecords( // tr.fieldName was already present so we'll overwrite it in-place here. outrec.PutReference(tr.fieldName, mlrval.FromString(buffer.String())) - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } } - outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker } } type tNestBucket struct { representative *mlrval.Mlrmap - pairs *list.List + pairs []*mlrval.Mlrmap } func newNestBucket(representative *mlrval.Mlrmap) *tNestBucket { return &tNestBucket{ representative: representative, - pairs: list.New(), + pairs: make([]*mlrval.Mlrmap, 0), } } diff --git a/pkg/transformers/nothing.go b/pkg/transformers/nothing.go index b64688352..fa90f2349 100644 --- a/pkg/transformers/nothing.go +++ b/pkg/transformers/nothing.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -87,12 +86,12 @@ func NewTransformerNothing() (*TransformerNothing, error) { func (tr *TransformerNothing) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel) if inrecAndContext.EndOfStream { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/put_or_filter.go b/pkg/transformers/put_or_filter.go index 0b2f05ead..32a40d25d 100644 --- a/pkg/transformers/put_or_filter.go +++ b/pkg/transformers/put_or_filter.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -500,7 +499,7 @@ func NewTransformerPut( func (tr *TransformerPut) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -566,7 +565,7 @@ func (tr *TransformerPut) Transform( wantToEmit := lib.BooleanXOR(filterBool, tr.invertFilter) if wantToEmit { - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &context)) } } @@ -594,6 +593,6 @@ func (tr *TransformerPut) Transform( // indicator. tr.cstRootNode.ProcessEndOfStream() - outputRecordsAndContexts.PushBack(types.NewEndOfStreamMarker(&context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewEndOfStreamMarker(&context)) } } diff --git a/pkg/transformers/regularize.go b/pkg/transformers/regularize.go index d39ffbad8..bf571106f 100644 --- a/pkg/transformers/regularize.go +++ b/pkg/transformers/regularize.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -94,7 +93,7 @@ func NewTransformerRegularize() (*TransformerRegularize, error) { func (tr *TransformerRegularize) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -107,16 +106,16 @@ func (tr *TransformerRegularize) Transform( previousSortedFieldNames := tr.sortedToOriginal[currentSortedFieldNamesJoined] if previousSortedFieldNames == nil { tr.sortedToOriginal[currentSortedFieldNamesJoined] = currentFieldNames - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { outrec := mlrval.NewMlrmapAsRecord() for _, fieldName := range previousSortedFieldNames { outrec.PutReference(fieldName, inrec.Get(fieldName)) // inrec will be GC'ed } outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context) - outputRecordsAndContexts.PushBack(outrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) } } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/remove_empty_columns.go b/pkg/transformers/remove_empty_columns.go index 3b6b74263..33f9eb83b 100644 --- a/pkg/transformers/remove_empty_columns.go +++ b/pkg/transformers/remove_empty_columns.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -78,13 +77,13 @@ func transformerRemoveEmptyColumnsParseCLI( // ---------------------------------------------------------------- type TransformerRemoveEmptyColumns struct { - recordsAndContexts *list.List + recordsAndContexts []*types.RecordAndContext namesWithNonEmptyValues map[string]bool } func NewTransformerRemoveEmptyColumns() (*TransformerRemoveEmptyColumns, error) { tr := &TransformerRemoveEmptyColumns{ - recordsAndContexts: list.New(), + recordsAndContexts: make([]*types.RecordAndContext, 0), namesWithNonEmptyValues: make(map[string]bool), } return tr, nil @@ -94,14 +93,14 @@ func NewTransformerRemoveEmptyColumns() (*TransformerRemoveEmptyColumns, error) func (tr *TransformerRemoveEmptyColumns) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel) if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - tr.recordsAndContexts.PushBack(inrecAndContext) + tr.recordsAndContexts = append(tr.recordsAndContexts, inrecAndContext) for pe := inrec.Head; pe != nil; pe = pe.Next { if !pe.Value.IsVoid() { @@ -111,8 +110,7 @@ func (tr *TransformerRemoveEmptyColumns) Transform( } else { // end of record stream - for e := tr.recordsAndContexts.Front(); e != nil; e = e.Next() { - outrecAndContext := e.Value.(*types.RecordAndContext) + for _, outrecAndContext := range tr.recordsAndContexts { outrec := outrecAndContext.Record newrec := mlrval.NewMlrmapAsRecord() @@ -125,9 +123,9 @@ func (tr *TransformerRemoveEmptyColumns) Transform( } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &outrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &outrecAndContext.Context)) } - outputRecordsAndContexts.PushBack(inrecAndContext) // Emit the stream-terminating null record + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // Emit the stream-terminating null record } } diff --git a/pkg/transformers/rename.go b/pkg/transformers/rename.go index abba22dfd..c6d928d86 100644 --- a/pkg/transformers/rename.go +++ b/pkg/transformers/rename.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "regexp" @@ -135,7 +134,7 @@ type tRegexAndReplacement struct { type TransformerRename struct { oldToNewNames *lib.OrderedMap[string] - regexesAndReplacements *list.List + regexesAndReplacements []*tRegexAndReplacement doGsub bool recordTransformerFunc RecordTransformerFunc } @@ -164,7 +163,7 @@ func NewTransformerRename( tr.doGsub = false tr.recordTransformerFunc = tr.transformWithoutRegexes } else { - tr.regexesAndReplacements = list.New() + tr.regexesAndReplacements = make([]*tRegexAndReplacement, 0) for pe := oldToNewNames.Head; pe != nil; pe = pe.Next { regexString := pe.Key regex := lib.CompileMillerRegexOrDie(regexString) @@ -175,7 +174,7 @@ func NewTransformerRename( replacement: replacement, replacementCaptureMatrix: replacementCaptureMatrix, } - tr.regexesAndReplacements.PushBack(®exAndReplacement) + tr.regexesAndReplacements = append(tr.regexesAndReplacements, ®exAndReplacement) } tr.doGsub = doGsub tr.recordTransformerFunc = tr.transformWithRegexes @@ -188,7 +187,7 @@ func NewTransformerRename( func (tr *TransformerRename) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -199,7 +198,7 @@ func (tr *TransformerRename) Transform( // ---------------------------------------------------------------- func (tr *TransformerRename) transformWithoutRegexes( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -214,21 +213,20 @@ func (tr *TransformerRename) transformWithoutRegexes( } } - outputRecordsAndContexts.PushBack(inrecAndContext) // including end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker } // ---------------------------------------------------------------- func (tr *TransformerRename) transformWithRegexes( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { if !inrecAndContext.EndOfStream { inrec := inrecAndContext.Record - for pr := tr.regexesAndReplacements.Front(); pr != nil; pr = pr.Next() { - regexAndReplacement := pr.Value.(*tRegexAndReplacement) + for _, regexAndReplacement := range tr.regexesAndReplacements { regex := regexAndReplacement.regex replacement := regexAndReplacement.replacement replacementCaptureMatrix := regexAndReplacement.replacementCaptureMatrix @@ -249,8 +247,8 @@ func (tr *TransformerRename) transformWithRegexes( } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // including end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker } } diff --git a/pkg/transformers/reorder.go b/pkg/transformers/reorder.go index d6afd2a51..87ace191d 100644 --- a/pkg/transformers/reorder.go +++ b/pkg/transformers/reorder.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "regexp" @@ -202,7 +201,7 @@ func NewTransformerReorder( func (tr *TransformerReorder) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -213,29 +212,29 @@ func (tr *TransformerReorder) Transform( outputRecordsAndContexts, ) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } func (tr *TransformerReorder) reorderToStartNoRegex( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { inrec := inrecAndContext.Record for _, fieldName := range tr.fieldNames { inrec.MoveToHead(fieldName) } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } func (tr *TransformerReorder) reorderToStartWithRegex( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { inrec := inrecAndContext.Record outrec := mlrval.NewMlrmapAsRecord() - atEnds := list.New() + atEnds := make([]*mlrval.MlrmapEntry, 0) for pe := inrec.Head; pe != nil; pe = pe.Next { found := false for _, regex := range tr.regexes { @@ -246,44 +245,43 @@ func (tr *TransformerReorder) reorderToStartWithRegex( } } if !found { - atEnds.PushBack(pe) + atEnds = append(atEnds, pe) } } - for atEnd := atEnds.Front(); atEnd != nil; atEnd = atEnd.Next() { + for _, pe := range atEnds { // Ownership transfer; no copy needed - pe := atEnd.Value.(*mlrval.MlrmapEntry) outrec.PutReference(pe.Key, pe.Value) } outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context) - outputRecordsAndContexts.PushBack(outrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) } func (tr *TransformerReorder) reorderToEndNoRegex( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { inrec := inrecAndContext.Record for _, fieldName := range tr.fieldNames { inrec.MoveToTail(fieldName) } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } func (tr *TransformerReorder) reorderToEndWithRegex( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { inrec := inrecAndContext.Record outrec := mlrval.NewMlrmapAsRecord() - atEnds := list.New() + atEnds := make([]*mlrval.MlrmapEntry, 0) for pe := inrec.Head; pe != nil; pe = pe.Next { found := false for _, regex := range tr.regexes { if regex.MatchString(pe.Key) { - atEnds.PushBack(pe) + atEnds = append(atEnds, pe) found = true break } @@ -293,23 +291,22 @@ func (tr *TransformerReorder) reorderToEndWithRegex( } } - for atEnd := atEnds.Front(); atEnd != nil; atEnd = atEnd.Next() { + for _, pe := range atEnds { // Ownership transfer; no copy needed - pe := atEnd.Value.(*mlrval.MlrmapEntry) outrec.PutReference(pe.Key, pe.Value) } outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context) - outputRecordsAndContexts.PushBack(outrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) } func (tr *TransformerReorder) reorderBeforeOrAfterNoRegex( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { inrec := inrecAndContext.Record if inrec.Get(tr.centerFieldName) == nil { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } @@ -356,17 +353,17 @@ func (tr *TransformerReorder) reorderBeforeOrAfterNoRegex( } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } func (tr *TransformerReorder) reorderBeforeOrAfterWithRegex( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { inrec := inrecAndContext.Record if inrec.Get(tr.centerFieldName) == nil { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } @@ -400,5 +397,5 @@ func (tr *TransformerReorder) reorderBeforeOrAfterWithRegex( } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } diff --git a/pkg/transformers/repeat.go b/pkg/transformers/repeat.go index a9761415a..cc9656c3d 100644 --- a/pkg/transformers/repeat.go +++ b/pkg/transformers/repeat.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -159,7 +158,7 @@ func NewTransformerRepeat( func (tr *TransformerRepeat) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -170,26 +169,26 @@ func (tr *TransformerRepeat) Transform( // ---------------------------------------------------------------- func (tr *TransformerRepeat) repeatByCount( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { if !inrecAndContext.EndOfStream { for i := int64(0); i < tr.repeatCount; i++ { - outputRecordsAndContexts.PushBack(types.NewRecordAndContext( + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext( inrecAndContext.Record.Copy(), &inrecAndContext.Context, )) } } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } // ---------------------------------------------------------------- func (tr *TransformerRepeat) repeatByFieldName( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -203,13 +202,13 @@ func (tr *TransformerRepeat) repeatByFieldName( return } for i := 0; i < int(repeatCount); i++ { - outputRecordsAndContexts.PushBack(types.NewRecordAndContext( + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext( inrecAndContext.Record.Copy(), &inrecAndContext.Context, )) } } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/reshape.go b/pkg/transformers/reshape.go index 3ab88347b..12f8b4723 100644 --- a/pkg/transformers/reshape.go +++ b/pkg/transformers/reshape.go @@ -28,7 +28,6 @@ package transformers // 15 2009-01-05 Z 0.09719105 import ( - "container/list" "fmt" "os" "regexp" @@ -292,7 +291,7 @@ func NewTransformerReshape( func (tr *TransformerReshape) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -303,7 +302,7 @@ func (tr *TransformerReshape) Transform( // ---------------------------------------------------------------- func (tr *TransformerReshape) wideToLongNoRegex( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -324,25 +323,25 @@ func (tr *TransformerReshape) wideToLongNoRegex( } if pairs.IsEmpty() { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { for pf := pairs.Head; pf != nil; pf = pf.Next { outrec := inrec.Copy() outrec.PutReference(tr.outputKeyFieldName, mlrval.FromString(pf.Key)) outrec.PutReference(tr.outputValueFieldName, pf.Value) - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } } } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerReshape) wideToLongRegex( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -366,25 +365,25 @@ func (tr *TransformerReshape) wideToLongRegex( } if pairs.IsEmpty() { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { for pf := pairs.Head; pf != nil; pf = pf.Next { outrec := inrec.Copy() outrec.PutReference(tr.outputKeyFieldName, mlrval.FromString(pf.Key)) outrec.PutReference(tr.outputValueFieldName, pf.Value) - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } } } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerReshape) longToWide( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -394,7 +393,7 @@ func (tr *TransformerReshape) longToWide( splitOutKeyFieldValue := inrec.Get(tr.splitOutKeyFieldName) splitOutValueFieldValue := inrec.Get(tr.splitOutValueFieldName) if splitOutKeyFieldValue == nil || splitOutValueFieldValue == nil { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } @@ -436,11 +435,11 @@ func (tr *TransformerReshape) longToWide( outrec.PutReference(pg.Key, pg.Value) } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } } - outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker } } diff --git a/pkg/transformers/sample.go b/pkg/transformers/sample.go index 9c32172ce..55e8db7e0 100644 --- a/pkg/transformers/sample.go +++ b/pkg/transformers/sample.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -128,7 +127,7 @@ func NewTransformerSample( func (tr *TransformerSample) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -151,13 +150,13 @@ func (tr *TransformerSample) Transform( for pe := tr.bucketsByGroup.Head; pe != nil; pe = pe.Next { sampleBucket := pe.Value for i := int64(0); i < sampleBucket.nused; i++ { - outputRecordsAndContexts.PushBack(sampleBucket.recordsAndContexts[i]) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, sampleBucket.recordsAndContexts[i]) } } // Emit the stream-terminating null record - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/sec2gmt.go b/pkg/transformers/sec2gmt.go index 33cbad387..e0d63f77f 100644 --- a/pkg/transformers/sec2gmt.go +++ b/pkg/transformers/sec2gmt.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" @@ -149,7 +148,7 @@ func NewTransformerSec2GMT( func (tr *TransformerSec2GMT) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -169,9 +168,9 @@ func (tr *TransformerSec2GMT) Transform( } } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { // End of record stream - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/sec2gmtdate.go b/pkg/transformers/sec2gmtdate.go index ee440f607..73fcbda4d 100644 --- a/pkg/transformers/sec2gmtdate.go +++ b/pkg/transformers/sec2gmtdate.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" @@ -106,7 +105,7 @@ func NewTransformerSec2GMTDate( func (tr *TransformerSec2GMTDate) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -119,9 +118,9 @@ func (tr *TransformerSec2GMTDate) Transform( inrec.PutReference(fieldName, bifs.BIF_sec2gmtdate(value)) } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { // End of record stream - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/seqgen.go b/pkg/transformers/seqgen.go index 9d77e7173..608a50f63 100644 --- a/pkg/transformers/seqgen.go +++ b/pkg/transformers/seqgen.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -172,7 +171,7 @@ func NewTransformerSeqgen( func (tr *TransformerSeqgen) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -210,10 +209,10 @@ func (tr *TransformerSeqgen) Transform( context.UpdateForInputRecord() outrecAndContext := types.NewRecordAndContext(outrec, context) - outputRecordsAndContexts.PushBack(outrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) counter = bifs.BIF_plus_binary(counter, tr.step) } - outputRecordsAndContexts.PushBack(types.NewEndOfStreamMarker(context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewEndOfStreamMarker(context)) } diff --git a/pkg/transformers/shuffle.go b/pkg/transformers/shuffle.go index 77659e5a8..26e92deeb 100644 --- a/pkg/transformers/shuffle.go +++ b/pkg/transformers/shuffle.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -81,13 +80,13 @@ func transformerShuffleParseCLI( // ---------------------------------------------------------------- type TransformerShuffle struct { - recordsAndContexts *list.List + recordsAndContexts []*types.RecordAndContext } func NewTransformerShuffle() (*TransformerShuffle, error) { tr := &TransformerShuffle{ - recordsAndContexts: list.New(), + recordsAndContexts: make([]*types.RecordAndContext, 0), } return tr, nil @@ -97,21 +96,20 @@ func NewTransformerShuffle() (*TransformerShuffle, error) { func (tr *TransformerShuffle) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel) // Not end of input stream: retain the record, and emit nothing until end of stream. if !inrecAndContext.EndOfStream { - tr.recordsAndContexts.PushBack(inrecAndContext) + tr.recordsAndContexts = append(tr.recordsAndContexts, inrecAndContext) } else { // end of record stream // Knuth shuffle: // * Initial permutation is identity. // * Make a pseudorandom permutation using pseudorandom swaps in the image map. - // TODO: Go list Len() maxes at 2^31. We should track this ourselves in an int64. - n := int64(tr.recordsAndContexts.Len()) + n := int64(len(tr.recordsAndContexts)) images := make([]int64, n) for i := int64(0); i < n; i++ { images[i] = i @@ -130,25 +128,17 @@ func (tr *TransformerShuffle) Transform( numUnused-- } - // Move the record-pointers from linked list to array. - array := make([]*types.RecordAndContext, n) - for i := int64(0); i < n; i++ { - head := tr.recordsAndContexts.Front() - if head == nil { - break - } - array[i] = head.Value.(*types.RecordAndContext) - tr.recordsAndContexts.Remove(head) - } + // Move the record-pointers from slice to array. + array := tr.recordsAndContexts // Transfer from input array to output list. Because permutations are one-to-one maps, // all input records have ownership transferred exactly once. So, there are no // records to copy here. for i := int64(0); i < n; i++ { - outputRecordsAndContexts.PushBack(array[images[i]]) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, array[images[i]]) } // Emit the stream-terminating null record - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/skip_trivial_records.go b/pkg/transformers/skip_trivial_records.go index 4a0245edb..fa5b4d7e8 100644 --- a/pkg/transformers/skip_trivial_records.go +++ b/pkg/transformers/skip_trivial_records.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -89,7 +88,7 @@ func NewTransformerSkipTrivialRecords() (*TransformerSkipTrivialRecords, error) func (tr *TransformerSkipTrivialRecords) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -105,10 +104,10 @@ func (tr *TransformerSkipTrivialRecords) Transform( } if hasAny { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/sort.go b/pkg/transformers/sort.go index 6be45005a..24f7c0f44 100644 --- a/pkg/transformers/sort.go +++ b/pkg/transformers/sort.go @@ -42,7 +42,6 @@ package transformers import ( - "container/list" "fmt" "os" "sort" @@ -313,11 +312,11 @@ type TransformerSort struct { doMoveToHead bool // -- State - // Map from string to *list.List: - recordListsByGroup *lib.OrderedMap[*list.List] + // Map from string to record slices: + recordListsByGroup *lib.OrderedMap[*[]*types.RecordAndContext] // Map from string to []*lib.Mlrval: groupHeads *lib.OrderedMap[[]*mlrval.Mlrval] - spillGroup *list.List // e.g. sort by field "a" -- this is for records lacking a field named "a" + spillGroup []*types.RecordAndContext // e.g. sort by field "a" -- this is for records lacking a field named "a" } func NewTransformerSort( @@ -331,9 +330,9 @@ func NewTransformerSort( comparatorFuncs: comparatorFuncs, doMoveToHead: doMoveToHead, - recordListsByGroup: lib.NewOrderedMap[*list.List](), + recordListsByGroup: lib.NewOrderedMap[*[]*types.RecordAndContext](), groupHeads: lib.NewOrderedMap[[]*mlrval.Mlrval](), - spillGroup: list.New(), + spillGroup: make([]*types.RecordAndContext, 0), } return tr, nil @@ -347,7 +346,7 @@ type GroupingKeysAndMlrvals struct { func (tr *TransformerSort) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -366,18 +365,19 @@ func (tr *TransformerSort) Transform( tr.groupByFieldNames, ) if !ok { - tr.spillGroup.PushBack(inrecAndContext) + tr.spillGroup = append(tr.spillGroup, inrecAndContext) return } recordListForGroup := tr.recordListsByGroup.Get(groupingKey) if recordListForGroup == nil { - recordListForGroup = list.New() + records := make([]*types.RecordAndContext, 0) + recordListForGroup = &records tr.recordListsByGroup.Put(groupingKey, recordListForGroup) tr.groupHeads.Put(groupingKey, selectedValues) } - recordListForGroup.PushBack(inrecAndContext) + *recordListForGroup = append(*recordListForGroup, inrecAndContext) } else { // End of record stream @@ -416,16 +416,16 @@ func (tr *TransformerSort) Transform( // Now output the groups for _, groupingKeyAndMlrvals := range groupingKeysAndMlrvals { recordsInGroup := tr.recordListsByGroup.Get(groupingKeyAndMlrvals.groupingKey) - for iRecord := recordsInGroup.Front(); iRecord != nil; iRecord = iRecord.Next() { - outputRecordsAndContexts.PushBack(iRecord.Value.(*types.RecordAndContext)) + for _, recordAndContext := range *recordsInGroup { + *outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext) } } - for iRecord := tr.spillGroup.Front(); iRecord != nil; iRecord = iRecord.Next() { - outputRecordsAndContexts.PushBack(iRecord.Value.(*types.RecordAndContext)) + for _, recordAndContext := range tr.spillGroup { + *outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext) } - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/sort_within_records.go b/pkg/transformers/sort_within_records.go index 50ce51bc2..0c38fbd25 100644 --- a/pkg/transformers/sort_within_records.go +++ b/pkg/transformers/sort_within_records.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -106,7 +105,7 @@ func NewTransformerSortWithinRecords( func (tr *TransformerSortWithinRecords) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -117,7 +116,7 @@ func (tr *TransformerSortWithinRecords) Transform( // ---------------------------------------------------------------- func (tr *TransformerSortWithinRecords) transformNonrecursively( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -125,13 +124,13 @@ func (tr *TransformerSortWithinRecords) transformNonrecursively( inrec := inrecAndContext.Record inrec.SortByKey() } - outputRecordsAndContexts.PushBack(inrecAndContext) // including end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker } // ---------------------------------------------------------------- func (tr *TransformerSortWithinRecords) transformRecursively( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -139,5 +138,5 @@ func (tr *TransformerSortWithinRecords) transformRecursively( inrec := inrecAndContext.Record inrec.SortByKeyRecursively() } - outputRecordsAndContexts.PushBack(inrecAndContext) // including end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker } diff --git a/pkg/transformers/sparsify.go b/pkg/transformers/sparsify.go index 6d6212a33..1f5076e83 100644 --- a/pkg/transformers/sparsify.go +++ b/pkg/transformers/sparsify.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -128,7 +127,7 @@ func NewTransformerSparsify( func (tr *TransformerSparsify) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -142,13 +141,13 @@ func (tr *TransformerSparsify) Transform( outputDownstreamDoneChannel, ) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } func (tr *TransformerSparsify) transformAll( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -163,13 +162,13 @@ func (tr *TransformerSparsify) transformAll( } outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context) - outputRecordsAndContexts.PushBack(outrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) } // ---------------------------------------------------------------- func (tr *TransformerSparsify) transformSome( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -188,5 +187,5 @@ func (tr *TransformerSparsify) transformSome( } outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context) - outputRecordsAndContexts.PushBack(outrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) } diff --git a/pkg/transformers/split.go b/pkg/transformers/split.go index 7295fa174..ad93c69a4 100644 --- a/pkg/transformers/split.go +++ b/pkg/transformers/split.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "net/url" "os" @@ -273,7 +272,7 @@ func NewTransformerSplit( func (tr *TransformerSplit) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -284,7 +283,7 @@ func (tr *TransformerSplit) Transform( func (tr *TransformerSplit) splitModUngrouped( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -299,13 +298,13 @@ func (tr *TransformerSplit) splitModUngrouped( } if tr.emitDownstream { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } tr.ungroupedCounter++ } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker errs := tr.outputHandlerManager.Close() if len(errs) > 0 { for _, err := range errs { @@ -318,7 +317,7 @@ func (tr *TransformerSplit) splitModUngrouped( func (tr *TransformerSplit) splitSizeUngrouped( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -356,13 +355,13 @@ func (tr *TransformerSplit) splitSizeUngrouped( } if tr.emitDownstream { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } tr.ungroupedCounter++ } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker if tr.outputHandler != nil { err := tr.outputHandler.Close() @@ -376,7 +375,7 @@ func (tr *TransformerSplit) splitSizeUngrouped( func (tr *TransformerSplit) splitGrouped( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -395,11 +394,11 @@ func (tr *TransformerSplit) splitGrouped( } if tr.emitDownstream { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker errs := tr.outputHandlerManager.Close() if len(errs) > 0 { diff --git a/pkg/transformers/stats1.go b/pkg/transformers/stats1.go index ff11f0354..83cddbe4f 100644 --- a/pkg/transformers/stats1.go +++ b/pkg/transformers/stats1.go @@ -2,7 +2,6 @@ package transformers import ( "bytes" - "container/list" "fmt" "os" "regexp" @@ -353,7 +352,7 @@ func NewTransformerStats1( // the end-of-stream marker. func (tr *TransformerStats1) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -367,7 +366,7 @@ func (tr *TransformerStats1) Transform( func (tr *TransformerStats1) handleInputRecord( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { inrec := inrecAndContext.Record @@ -416,7 +415,7 @@ func (tr *TransformerStats1) handleInputRecord( level2, inrec, ) - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } @@ -596,10 +595,10 @@ func (tr *TransformerStats1) matchValueFieldName( func (tr *TransformerStats1) handleEndOfRecordStream( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { if tr.doIterativeStats { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker return } @@ -617,10 +616,10 @@ func (tr *TransformerStats1) handleEndOfRecordStream( newrec, ) - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } func (tr *TransformerStats1) emitIntoOutputRecord( diff --git a/pkg/transformers/stats2.go b/pkg/transformers/stats2.go index f14c98ac0..280401a70 100644 --- a/pkg/transformers/stats2.go +++ b/pkg/transformers/stats2.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -194,7 +193,7 @@ type TransformerStats2 struct { // For hold-and-fit: // ordered map from grouping-key to list of RecordAndContext - recordGroups *lib.OrderedMap[*list.List] + recordGroups *lib.OrderedMap[*[]*types.RecordAndContext] } func NewTransformerStats2( @@ -221,7 +220,7 @@ func NewTransformerStats2( accumulatorFactory: utils.NewStats2AccumulatorFactory(), namedAccumulators: lib.NewOrderedMap[*lib.OrderedMap[*lib.OrderedMap[utils.IStats2Accumulator]]](), groupingKeysToGroupByFieldValues: lib.NewOrderedMap[[]*mlrval.Mlrval](), - recordGroups: lib.NewOrderedMap[*list.List](), + recordGroups: lib.NewOrderedMap[*[]*types.RecordAndContext](), } return tr, nil } @@ -265,7 +264,7 @@ func NewTransformerStats2( func (tr *TransformerStats2) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -276,7 +275,7 @@ func (tr *TransformerStats2) Transform( if tr.doIterativeStats { // The input record is modified in this case, with new fields appended - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } // if tr.doHoldAndFit, the input record is held by the ingestor @@ -288,7 +287,7 @@ func (tr *TransformerStats2) Transform( tr.emit(outputRecordsAndContexts, &inrecAndContext.Context) } } - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } @@ -316,10 +315,11 @@ func (tr *TransformerStats2) ingest( if tr.doHoldAndFit { // Retain the input record in memory, for fitting and delivery at end of stream groupToRecords := tr.recordGroups.Get(groupingKey) if groupToRecords == nil { - groupToRecords = list.New() + records := make([]*types.RecordAndContext, 0) + groupToRecords = &records tr.recordGroups.Put(groupingKey, groupToRecords) } - groupToRecords.PushBack(inrecAndContext) + *groupToRecords = append(*groupToRecords, inrecAndContext) } // for [["x","y"]] @@ -382,7 +382,7 @@ func (tr *TransformerStats2) ingest( // ---------------------------------------------------------------- func (tr *TransformerStats2) emit( - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext context *types.Context, ) { for pa := tr.namedAccumulators.Head; pa != nil; pa = pa.Next { @@ -415,7 +415,7 @@ func (tr *TransformerStats2) emit( } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, context)) } } @@ -433,15 +433,16 @@ func (tr *TransformerStats2) populateRecord( } func (tr *TransformerStats2) fit( - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { for pa := tr.namedAccumulators.Head; pa != nil; pa = pa.Next { groupingKey := pa.Key groupToValueFields := pa.Value recordsAndContexts := tr.recordGroups.Get(groupingKey) - - for recordsAndContexts.Front() != nil { - recordAndContext := recordsAndContexts.Remove(recordsAndContexts.Front()).(*types.RecordAndContext) + if recordsAndContexts == nil { + continue + } + for _, recordAndContext := range *recordsAndContexts { record := recordAndContext.Record // For "x","y" @@ -468,7 +469,8 @@ func (tr *TransformerStats2) fit( } } - outputRecordsAndContexts.PushBack(recordAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext) } + *recordsAndContexts = (*recordsAndContexts)[:0] } } diff --git a/pkg/transformers/step.go b/pkg/transformers/step.go index 77382f9eb..2ea727ed3 100644 --- a/pkg/transformers/step.go +++ b/pkg/transformers/step.go @@ -68,7 +68,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -350,7 +349,7 @@ func NewTransformerStep( func (tr *TransformerStep) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -370,7 +369,7 @@ func (tr *TransformerStep) Transform( tr.handleDrainRecord(logEntry, outputRecordsAndContexts) } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } } @@ -381,7 +380,7 @@ func (tr *TransformerStep) Transform( // delayed-input records in the order in which they were received. func (tr *TransformerStep) handleRecord( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { inrec := inrecAndContext.Record @@ -390,7 +389,7 @@ func (tr *TransformerStep) handleRecord( // Grouping key is "s,t" groupingKey, gok := inrec.GetSelectedValuesJoined(tr.groupByFieldNames) if !gok { // current record doesn't have fields to be stepped; pass it along - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } @@ -455,7 +454,7 @@ func (tr *TransformerStep) handleRecord( if windowKeeper.Get(0) != nil { outrecAndContext := windowKeeper.Get(0).(*types.RecordAndContext) - outputRecordsAndContexts.PushBack(outrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) tr.removeFromLog(outrecAndContext) } } @@ -466,7 +465,7 @@ func (tr *TransformerStep) handleRecord( // delayed-input records in the order in which they were received. func (tr *TransformerStep) handleDrainRecord( logEntry *tStepLogEntry, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { inrecAndContext := logEntry.recordAndContext inrec := inrecAndContext.Record @@ -496,7 +495,7 @@ func (tr *TransformerStep) handleDrainRecord( if windowKeeper.Get(0) != nil { outrecAndContext := windowKeeper.Get(0).(*types.RecordAndContext) - outputRecordsAndContexts.PushBack(outrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) } } diff --git a/pkg/transformers/subs.go b/pkg/transformers/subs.go index b5530bb17..85daa5169 100644 --- a/pkg/transformers/subs.go +++ b/pkg/transformers/subs.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "regexp" @@ -288,7 +287,7 @@ func NewTransformerSubs( func (tr *TransformerSubs) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -304,7 +303,7 @@ func (tr *TransformerSubs) Transform( } } // Including emit of end-of-stream marker - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } // fieldAcceptorByNames implements -f diff --git a/pkg/transformers/summary.go b/pkg/transformers/summary.go index 151886f76..27a443fc6 100644 --- a/pkg/transformers/summary.go +++ b/pkg/transformers/summary.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -292,7 +291,7 @@ func NewTransformerSummary( func (tr *TransformerSummary) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -346,7 +345,7 @@ func (tr *TransformerSummary) ingest( func (tr *TransformerSummary) emit( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { for pe := tr.fieldSummaries.Head; pe != nil; pe = pe.Next { @@ -380,15 +379,15 @@ func (tr *TransformerSummary) emit( } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } func (tr *TransformerSummary) emitTransposed( inrecAndContext *types.RecordAndContext, - oracs *list.List, // list of *types.RecordAndContext + oracs *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { octx := &inrecAndContext.Context @@ -407,7 +406,7 @@ func (tr *TransformerSummary) emitTransposed( } newrec.PutCopy(pe.Key, mlrval.FromString(strings.Join(fieldTypesList, "-"))) } - oracs.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *oracs = append(*oracs, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } for _, info := range allSummarizerInfos { @@ -418,7 +417,7 @@ func (tr *TransformerSummary) emitTransposed( } } - oracs.PushBack(inrecAndContext) // end-of-stream marker + *oracs = append(*oracs, inrecAndContext) // end-of-stream marker } // ---------------------------------------------------------------- @@ -426,7 +425,7 @@ func (tr *TransformerSummary) emitTransposed( // maybeEmitAccumulatorTransposed is a helper method for emitTransposed, // for "count", "sum", "mean", etc. func (tr *TransformerSummary) maybeEmitAccumulatorTransposed( - oracs *list.List, // list of *types.RecordAndContext + oracs *[]*types.RecordAndContext, // list of *types.RecordAndContext octx *types.Context, summarizerName string, ) { @@ -437,14 +436,14 @@ func (tr *TransformerSummary) maybeEmitAccumulatorTransposed( fieldSummary := pe.Value newrec.PutCopy(pe.Key, fieldSummary.accumulators[summarizerName].Emit()) } - oracs.PushBack(types.NewRecordAndContext(newrec, octx)) + *oracs = append(*oracs, types.NewRecordAndContext(newrec, octx)) } } // maybeEmitPercentileNameTransposed is a helper method for emitTransposed, // for "median", "iqr", "uof", etc. func (tr *TransformerSummary) maybeEmitPercentileNameTransposed( - oracs *list.List, // list of *types.RecordAndContext + oracs *[]*types.RecordAndContext, // list of *types.RecordAndContext octx *types.Context, summarizerName string, ) { @@ -455,6 +454,6 @@ func (tr *TransformerSummary) maybeEmitPercentileNameTransposed( fieldSummary := pe.Value newrec.PutCopy(pe.Key, fieldSummary.percentileKeeper.EmitNamed(summarizerName)) } - oracs.PushBack(types.NewRecordAndContext(newrec, octx)) + *oracs = append(*oracs, types.NewRecordAndContext(newrec, octx)) } } diff --git a/pkg/transformers/surv.go b/pkg/transformers/surv.go index 6d4b38f09..d2cf289c6 100644 --- a/pkg/transformers/surv.go +++ b/pkg/transformers/surv.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -114,7 +113,7 @@ func NewTransformerSurv(durationField, statusField string) IRecordTransformer { // Transform processes each record or emits results at end-of-stream. func (tr *TransformerSurv) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, + outputRecordsAndContexts *[]*types.RecordAndContext, inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -139,7 +138,7 @@ func (tr *TransformerSurv) Transform( // Compute survival using kshedden/statmodel n := len(tr.times) if n == 0 { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) return } durations := tr.times @@ -166,8 +165,8 @@ func (tr *TransformerSurv) Transform( newrec := mlrval.NewMlrmapAsRecord() newrec.PutCopy("time", mlrval.FromFloat(t)) newrec.PutCopy("survival", mlrval.FromFloat(survProbs[i])) - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/tac.go b/pkg/transformers/tac.go index ba25195cb..58bcad121 100644 --- a/pkg/transformers/tac.go +++ b/pkg/transformers/tac.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -77,29 +76,29 @@ func transformerTacParseCLI( // ---------------------------------------------------------------- type TransformerTac struct { - recordsAndContexts *list.List + recordsAndContexts []*types.RecordAndContext } func NewTransformerTac() (*TransformerTac, error) { return &TransformerTac{ - recordsAndContexts: list.New(), + recordsAndContexts: make([]*types.RecordAndContext, 0), }, nil } func (tr *TransformerTac) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel) if !inrecAndContext.EndOfStream { - tr.recordsAndContexts.PushFront(inrecAndContext) + tr.recordsAndContexts = append(tr.recordsAndContexts, inrecAndContext) } else { // end of stream - for e := tr.recordsAndContexts.Front(); e != nil; e = e.Next() { - outputRecordsAndContexts.PushBack(e.Value.(*types.RecordAndContext)) + for i := len(tr.recordsAndContexts) - 1; i >= 0; i-- { + *outputRecordsAndContexts = append(*outputRecordsAndContexts, tr.recordsAndContexts[i]) } - outputRecordsAndContexts.PushBack(types.NewEndOfStreamMarker(&inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewEndOfStreamMarker(&inrecAndContext.Context)) } } diff --git a/pkg/transformers/tail.go b/pkg/transformers/tail.go index 4d1979b67..a6c56257f 100644 --- a/pkg/transformers/tail.go +++ b/pkg/transformers/tail.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -109,8 +108,8 @@ type TransformerTail struct { groupByFieldNames []string // state - // map from string to *list.List - recordListsByGroup *lib.OrderedMap[*list.List] + // map from string to record slices + recordListsByGroup *lib.OrderedMap[*[]*types.RecordAndContext] } func NewTransformerTail( @@ -122,7 +121,7 @@ func NewTransformerTail( tailCount: tailCount, groupByFieldNames: groupByFieldNames, - recordListsByGroup: lib.NewOrderedMap[*list.List](), + recordListsByGroup: lib.NewOrderedMap[*[]*types.RecordAndContext](), } return tr, nil @@ -132,7 +131,7 @@ func NewTransformerTail( func (tr *TransformerTail) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -147,22 +146,23 @@ func (tr *TransformerTail) Transform( recordListForGroup := tr.recordListsByGroup.Get(groupingKey) if recordListForGroup == nil { // first time - recordListForGroup = list.New() + records := make([]*types.RecordAndContext, 0) + recordListForGroup = &records tr.recordListsByGroup.Put(groupingKey, recordListForGroup) } - recordListForGroup.PushBack(inrecAndContext) - for int64(recordListForGroup.Len()) > tr.tailCount { - recordListForGroup.Remove(recordListForGroup.Front()) + *recordListForGroup = append(*recordListForGroup, inrecAndContext) + for int64(len(*recordListForGroup)) > tr.tailCount { + *recordListForGroup = (*recordListForGroup)[1:] } } else { for outer := tr.recordListsByGroup.Head; outer != nil; outer = outer.Next { recordListForGroup := outer.Value - for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() { - outputRecordsAndContexts.PushBack(inner.Value.(*types.RecordAndContext)) + for _, recordAndContext := range *recordListForGroup { + *outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext) } } - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/tee.go b/pkg/transformers/tee.go index e5f5413ca..5f5f21943 100644 --- a/pkg/transformers/tee.go +++ b/pkg/transformers/tee.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -166,7 +165,7 @@ func NewTransformerTee( func (tr *TransformerTee) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -202,7 +201,7 @@ func (tr *TransformerTee) Transform( os.Exit(1) } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { err := tr.fileOutputHandler.Close() if err != nil { @@ -214,6 +213,6 @@ func (tr *TransformerTee) Transform( fmt.Fprintln(os.Stderr, err) os.Exit(1) } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/template.go b/pkg/transformers/template.go index a30bf7dae..656c995d0 100644 --- a/pkg/transformers/template.go +++ b/pkg/transformers/template.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -135,7 +134,7 @@ func NewTransformerTemplate( func (tr *TransformerTemplate) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -152,8 +151,8 @@ func (tr *TransformerTemplate) Transform( } } outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context) - outputRecordsAndContexts.PushBack(outrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/top.go b/pkg/transformers/top.go index 6074065dd..53c18d6d5 100644 --- a/pkg/transformers/top.go +++ b/pkg/transformers/top.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -174,7 +173,7 @@ func NewTransformerTop( func (tr *TransformerTop) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -238,7 +237,7 @@ func (tr *TransformerTop) ingest( // ---------------------------------------------------------------- func (tr *TransformerTop) emit( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { for pa := tr.groups.Head; pa != nil; pa = pa.Next { groupingKey := pa.Key @@ -252,7 +251,7 @@ func (tr *TransformerTop) emit( for pb := secondLevel.Head; pb != nil; pb = pb.Next { topKeeper := pb.Value for i := int64(0); i < topKeeper.GetSize(); i++ { - outputRecordsAndContexts.PushBack(topKeeper.TopRecordsAndContexts[i].Copy()) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, topKeeper.TopRecordsAndContexts[i].Copy()) } } @@ -281,10 +280,10 @@ func (tr *TransformerTop) emit( } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } } } - outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker } diff --git a/pkg/transformers/unflatten.go b/pkg/transformers/unflatten.go index dcf8014af..f3b2ca184 100644 --- a/pkg/transformers/unflatten.go +++ b/pkg/transformers/unflatten.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -133,7 +132,7 @@ func NewTransformerUnflatten( func (tr *TransformerUnflatten) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -144,7 +143,7 @@ func (tr *TransformerUnflatten) Transform( // ---------------------------------------------------------------- func (tr *TransformerUnflatten) unflattenAll( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -155,16 +154,16 @@ func (tr *TransformerUnflatten) unflattenAll( oFlatSep = tr.options.WriterOptions.FLATSEP } inrec.Unflatten(oFlatSep) - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerUnflatten) unflattenSome( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -175,8 +174,8 @@ func (tr *TransformerUnflatten) unflattenSome( oFlatSep = tr.options.WriterOptions.FLATSEP } inrec.UnflattenFields(tr.fieldNameSet, oFlatSep) - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/uniq.go b/pkg/transformers/uniq.go index 24e794c1b..aabc88ce7 100644 --- a/pkg/transformers/uniq.go +++ b/pkg/transformers/uniq.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -367,7 +366,7 @@ func (tr *TransformerUniq) getFieldNamesForGrouping( func (tr *TransformerUniq) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -380,7 +379,7 @@ func (tr *TransformerUniq) Transform( // non-streaming, with output at end of stream. func (tr *TransformerUniq) transformUniqifyEntireRecordsShowCounts( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -403,10 +402,10 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowCounts( icount := tr.uniqifiedRecordCounts.Get(pe.Key) mcount := mlrval.FromInt(icount) outrecAndContext.Record.PrependReference(tr.outputFieldName, mcount) - outputRecordsAndContexts.PushBack(outrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext) } - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } @@ -416,7 +415,7 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowCounts( // of stream. func (tr *TransformerUniq) transformUniqifyEntireRecordsShowNumDistinctOnly( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -433,9 +432,9 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowNumDistinctOnly( tr.outputFieldName, mlrval.FromInt(tr.uniqifiedRecordCounts.FieldCount), ) - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context)) - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } @@ -443,7 +442,7 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowNumDistinctOnly( // Print each unique record only once (on first occurrence). func (tr *TransformerUniq) transformUniqifyEntireRecords( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -453,19 +452,19 @@ func (tr *TransformerUniq) transformUniqifyEntireRecords( recordAsString := inrec.String() if !tr.uniqifiedRecordCounts.Has(recordAsString) { tr.uniqifiedRecordCounts.Put(recordAsString, int64(1)) - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } else { // end of record stream - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerUniq) transformUnlashed( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -509,18 +508,18 @@ func (tr *TransformerUniq) transformUnlashed( tr.unlashedCountValues.Get(fieldName).Get(fieldValueString), ) outrec.PutReference("count", mlrval.FromInt(pf.Value)) - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } } - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerUniq) transformNumDistinctOnly( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -543,16 +542,16 @@ func (tr *TransformerUniq) transformNumDistinctOnly( "count", mlrval.FromInt(tr.countsByGroup.FieldCount), ) - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context)) - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerUniq) transformWithCounts( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -592,17 +591,17 @@ func (tr *TransformerUniq) transformWithCounts( mlrval.FromInt(pa.Value), ) } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerUniq) transformWithoutCounts( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -627,13 +626,13 @@ func (tr *TransformerUniq) transformWithoutCounts( ) } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context)) } else { tr.countsByGroup.Put(groupingKey, iCount+1) } } else { // end of record stream - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/unspace.go b/pkg/transformers/unspace.go index 274f28fbd..8c7933ad4 100644 --- a/pkg/transformers/unspace.go +++ b/pkg/transformers/unspace.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -115,7 +114,7 @@ func NewTransformerUnspace( func (tr *TransformerUnspace) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -128,13 +127,13 @@ func (tr *TransformerUnspace) Transform( outputDownstreamDoneChannel, ) } else { // end of record stream - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } func (tr *TransformerUnspace) transformKeysOnly( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext _ <-chan bool, __ chan<- bool, ) { @@ -145,12 +144,12 @@ func (tr *TransformerUnspace) transformKeysOnly( // Reference not copy since this is ownership transfer of the value from the now-abandoned inrec newrec.PutReference(newkey, pe.Value) } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } func (tr *TransformerUnspace) transformValuesOnly( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext _ <-chan bool, __ chan<- bool, ) { @@ -161,12 +160,12 @@ func (tr *TransformerUnspace) transformValuesOnly( pe.Value = mlrval.FromString(tr.unspace(stringval)) } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(inrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(inrec, &inrecAndContext.Context)) } func (tr *TransformerUnspace) transformKeysAndValues( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext _ <-chan bool, __ chan<- bool, ) { @@ -182,7 +181,7 @@ func (tr *TransformerUnspace) transformKeysAndValues( newrec.PutReference(newkey, pe.Value) } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context)) } func (tr *TransformerUnspace) unspace(input string) string { diff --git a/pkg/transformers/unsparsify.go b/pkg/transformers/unsparsify.go index 8cbd9d41a..bf7c0afd6 100644 --- a/pkg/transformers/unsparsify.go +++ b/pkg/transformers/unsparsify.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -108,7 +107,7 @@ func transformerUnsparsifyParseCLI( // ---------------------------------------------------------------- type TransformerUnsparsify struct { fillerMlrval *mlrval.Mlrval - recordsAndContexts *list.List + recordsAndContexts []*types.RecordAndContext fieldNamesSeen *lib.OrderedMap[string] recordTransformerFunc RecordTransformerFunc } @@ -125,7 +124,7 @@ func NewTransformerUnsparsify( tr := &TransformerUnsparsify{ fillerMlrval: mlrval.FromString(fillerString), - recordsAndContexts: list.New(), + recordsAndContexts: make([]*types.RecordAndContext, 0), fieldNamesSeen: fieldNamesSeen, } @@ -142,7 +141,7 @@ func NewTransformerUnsparsify( func (tr *TransformerUnsparsify) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -153,7 +152,7 @@ func (tr *TransformerUnsparsify) Transform( // ---------------------------------------------------------------- func (tr *TransformerUnsparsify) transformNonStreaming( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -165,10 +164,9 @@ func (tr *TransformerUnsparsify) transformNonStreaming( tr.fieldNamesSeen.Put(key, key) } } - tr.recordsAndContexts.PushBack(inrecAndContext) + tr.recordsAndContexts = append(tr.recordsAndContexts, inrecAndContext) } else { - for e := tr.recordsAndContexts.Front(); e != nil; e = e.Next() { - outrecAndContext := e.Value.(*types.RecordAndContext) + for _, outrecAndContext := range tr.recordsAndContexts { outrec := outrecAndContext.Record newrec := mlrval.NewMlrmapAsRecord() @@ -181,17 +179,17 @@ func (tr *TransformerUnsparsify) transformNonStreaming( } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &outrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &outrecAndContext.Context)) } - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } // ---------------------------------------------------------------- func (tr *TransformerUnsparsify) transformStreaming( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -204,9 +202,9 @@ func (tr *TransformerUnsparsify) transformStreaming( } } - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } else { - outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker } } diff --git a/pkg/transformers/utf8_to_latin1.go b/pkg/transformers/utf8_to_latin1.go index bc744c8fa..60edeeddf 100644 --- a/pkg/transformers/utf8_to_latin1.go +++ b/pkg/transformers/utf8_to_latin1.go @@ -1,7 +1,6 @@ package transformers import ( - "container/list" "fmt" "os" "strings" @@ -91,7 +90,7 @@ func NewTransformerUTF8ToLatin1() (*TransformerUTF8ToLatin1, error) { func (tr *TransformerUTF8ToLatin1) Transform( inrecAndContext *types.RecordAndContext, - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext inputDownstreamDoneChannel <-chan bool, outputDownstreamDoneChannel chan<- bool, ) { @@ -111,9 +110,9 @@ func (tr *TransformerUTF8ToLatin1) Transform( } } - outputRecordsAndContexts.PushBack(types.NewRecordAndContext(inrec, &inrecAndContext.Context)) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(inrec, &inrecAndContext.Context)) } else { // end of record stream - outputRecordsAndContexts.PushBack(inrecAndContext) + *outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) } } diff --git a/pkg/transformers/utils/join_bucket.go b/pkg/transformers/utils/join_bucket.go index f4390906f..7195da6fa 100644 --- a/pkg/transformers/utils/join_bucket.go +++ b/pkg/transformers/utils/join_bucket.go @@ -5,15 +5,14 @@ package utils import ( - "container/list" - "github.com/johnkerl/miller/v6/pkg/mlrval" + "github.com/johnkerl/miller/v6/pkg/types" ) // ---------------------------------------------------------------- type JoinBucket struct { leftFieldValues []*mlrval.Mlrval - RecordsAndContexts *list.List + RecordsAndContexts []*types.RecordAndContext WasPaired bool } @@ -22,7 +21,7 @@ func NewJoinBucket( ) *JoinBucket { return &JoinBucket{ leftFieldValues: leftFieldValues, - RecordsAndContexts: list.New(), + RecordsAndContexts: make([]*types.RecordAndContext, 0), WasPaired: false, } } diff --git a/pkg/transformers/utils/join_bucket_keeper.go b/pkg/transformers/utils/join_bucket_keeper.go index 237f2648b..a3ae5c6bb 100644 --- a/pkg/transformers/utils/join_bucket_keeper.go +++ b/pkg/transformers/utils/join_bucket_keeper.go @@ -108,7 +108,6 @@ package utils import ( - "container/list" "fmt" "os" "strings" @@ -126,7 +125,7 @@ type JoinBucketKeeper struct { // For streaming through the left-side file recordReader input.IRecordReader context *types.Context - readerChannel <-chan *list.List // list of *types.RecordAndContext + readerChannel <-chan []*types.RecordAndContext // list of *types.RecordAndContext errorChannel chan error // TODO: merge with leof flag recordReaderDone bool @@ -152,7 +151,7 @@ type JoinBucketKeeper struct { peekRecordAndContext *types.RecordAndContext JoinBucket *JoinBucket - leftUnpaireds *list.List + leftUnpaireds []*types.RecordAndContext leof bool state tJoinBucketKeeperState @@ -181,7 +180,7 @@ func NewJoinBucketKeeper( initialContext.UpdateForStartOfFile(leftFileName) // Set up channels for the record-reader - readerChannel := make(chan *list.List, 2) // list of *types.RecordAndContext + readerChannel := make(chan []*types.RecordAndContext, 2) // list of *types.RecordAndContext errorChannel := make(chan error, 1) downstreamDoneChannel := make(chan bool, 1) @@ -201,7 +200,7 @@ func NewJoinBucketKeeper( JoinBucket: NewJoinBucket(nil), peekRecordAndContext: nil, - leftUnpaireds: list.New(), + leftUnpaireds: make([]*types.RecordAndContext, 0), leof: false, state: LEFT_STATE_0_PREFILL, @@ -289,7 +288,7 @@ func (keeper *JoinBucketKeeper) FindJoinBucket( } // TODO: privatize more - if keeper.JoinBucket.RecordsAndContexts.Len() > 0 { + if len(keeper.JoinBucket.RecordsAndContexts) > 0 { cmp := compareLexically( keeper.JoinBucket.leftFieldValues, rightFieldValues, @@ -344,7 +343,7 @@ func (keeper *JoinBucketKeeper) prepareForFirstJoinBucket() { if keeper.peekRecordAndContext.Record.HasSelectedKeys(keeper.leftJoinFieldNames) { break } - keeper.leftUnpaireds.PushBack(keeper.peekRecordAndContext) + keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext) } if keeper.peekRecordAndContext == nil { @@ -374,7 +373,7 @@ func (keeper *JoinBucketKeeper) prepareForNewJoinBucket( rightFieldValues []*mlrval.Mlrval, ) { if !keeper.JoinBucket.WasPaired { - moveRecordsAndContexts(keeper.leftUnpaireds, keeper.JoinBucket.RecordsAndContexts) + moveRecordsAndContexts(&keeper.leftUnpaireds, &keeper.JoinBucket.RecordsAndContexts) } keeper.JoinBucket = NewJoinBucket(nil) @@ -401,7 +400,7 @@ func (keeper *JoinBucketKeeper) prepareForNewJoinBucket( // Keep seeking and filling the bucket until = or >; this may or may not // end up being a match. for { - keeper.leftUnpaireds.PushBack(keeper.peekRecordAndContext) + keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext) keeper.peekRecordAndContext = nil for { @@ -416,7 +415,7 @@ func (keeper *JoinBucketKeeper) prepareForNewJoinBucket( if peekRec.HasSelectedKeys(keeper.leftJoinFieldNames) { break } - keeper.leftUnpaireds.PushBack(keeper.peekRecordAndContext) + keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext) } // Double break from double for-loop @@ -470,7 +469,7 @@ func (keeper *JoinBucketKeeper) fillNextJoinBucket() { } keeper.JoinBucket.leftFieldValues = mlrval.CopyMlrvalArray(peekFieldValues) - keeper.JoinBucket.RecordsAndContexts.PushBack(keeper.peekRecordAndContext) + keeper.JoinBucket.RecordsAndContexts = append(keeper.JoinBucket.RecordsAndContexts, keeper.peekRecordAndContext) keeper.JoinBucket.WasPaired = false keeper.peekRecordAndContext = nil @@ -497,9 +496,9 @@ func (keeper *JoinBucketKeeper) fillNextJoinBucket() { if cmp != 0 { break } - keeper.JoinBucket.RecordsAndContexts.PushBack(keeper.peekRecordAndContext) + keeper.JoinBucket.RecordsAndContexts = append(keeper.JoinBucket.RecordsAndContexts, keeper.peekRecordAndContext) } else { - keeper.leftUnpaireds.PushBack(keeper.peekRecordAndContext) + keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext) } keeper.peekRecordAndContext = nil } @@ -510,13 +509,13 @@ func (keeper *JoinBucketKeeper) fillNextJoinBucket() { func (keeper *JoinBucketKeeper) markRemainingsAsUnpaired() { // 1. Any records already in keeper.JoinBucket.records (current bucket) if !keeper.JoinBucket.WasPaired { - moveRecordsAndContexts(keeper.leftUnpaireds, keeper.JoinBucket.RecordsAndContexts) + moveRecordsAndContexts(&keeper.leftUnpaireds, &keeper.JoinBucket.RecordsAndContexts) } keeper.JoinBucket.RecordsAndContexts = nil // 2. Peek-record, if any if keeper.peekRecordAndContext != nil { - keeper.leftUnpaireds.PushBack(keeper.peekRecordAndContext) + keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext) keeper.peekRecordAndContext = nil } @@ -526,36 +525,26 @@ func (keeper *JoinBucketKeeper) markRemainingsAsUnpaired() { if keeper.peekRecordAndContext == nil { break } - keeper.leftUnpaireds.PushBack(keeper.peekRecordAndContext) + keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext) } } // ---------------------------------------------------------------- // TODO: comment func (keeper *JoinBucketKeeper) OutputAndReleaseLeftUnpaireds( - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { - for { - element := keeper.leftUnpaireds.Front() - if element == nil { - break - } - recordAndContext := element.Value.(*types.RecordAndContext) - outputRecordsAndContexts.PushBack(recordAndContext) - keeper.leftUnpaireds.Remove(element) + for len(keeper.leftUnpaireds) > 0 { + recordAndContext := keeper.leftUnpaireds[0] + *outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext) + keeper.leftUnpaireds = keeper.leftUnpaireds[1:] } } func (keeper *JoinBucketKeeper) ReleaseLeftUnpaireds( - outputRecordsAndContexts *list.List, // list of *types.RecordAndContext + outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext ) { - for { - element := keeper.leftUnpaireds.Front() - if element == nil { - break - } - keeper.leftUnpaireds.Remove(element) - } + keeper.leftUnpaireds = keeper.leftUnpaireds[:0] } // ================================================================ @@ -576,8 +565,8 @@ func (keeper *JoinBucketKeeper) readRecord() *types.RecordAndContext { os.Exit(1) case leftrecsAndContexts := <-keeper.readerChannel: // TODO: temp - lib.InternalCodingErrorIf(leftrecsAndContexts.Len() != 1) - leftrecAndContext := leftrecsAndContexts.Front().Value.(*types.RecordAndContext) + lib.InternalCodingErrorIf(len(leftrecsAndContexts) != 1) + leftrecAndContext := leftrecsAndContexts[0] leftrecAndContext.Record = KeepLeftFieldNames(leftrecAndContext.Record, keeper.leftKeepFieldNameSet) if leftrecAndContext.EndOfStream { // end-of-stream marker keeper.recordReaderDone = true @@ -594,17 +583,11 @@ func (keeper *JoinBucketKeeper) readRecord() *types.RecordAndContext { // Pops everything off second-argument list and push to first-argument list. func moveRecordsAndContexts( - destination *list.List, - source *list.List, + destination *[]*types.RecordAndContext, + source *[]*types.RecordAndContext, ) { - for { - element := source.Front() - if element == nil { - break - } - destination.PushBack(element.Value.(*types.RecordAndContext)) - source.Remove(element) - } + *destination = append(*destination, (*source)...) + *source = (*source)[:0] } // ---------------------------------------------------------------- diff --git a/pkg/types/context.go b/pkg/types/context.go index 3ecf9ebb8..6eaf2c1f4 100644 --- a/pkg/types/context.go +++ b/pkg/types/context.go @@ -2,7 +2,6 @@ package types import ( "bytes" - "container/list" "strconv" "github.com/johnkerl/miller/v6/pkg/mlrval" @@ -84,10 +83,9 @@ func NewEndOfStreamMarker(context *Context) *RecordAndContext { // TODO: comment // For the record-readers to update their initial context as each new record is read. -func NewEndOfStreamMarkerList(context *Context) *list.List { - ell := list.New() - ell.PushBack(NewEndOfStreamMarker(context)) - return ell +func NewEndOfStreamMarkerList(context *Context) []*RecordAndContext { + recordsAndContexts := []*RecordAndContext{NewEndOfStreamMarker(context)} + return recordsAndContexts } // ----------------------------------------------------------------