Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-17 16:38:54 +00:00
Code Issues Projects Releases Wiki Activity

Replace list.List with Go slices (#1950)

Browse source 
* Add .vscode to .gitignore

* Replace `list.List` with slices
This commit is contained in:
John Kerl 2026-02-01 17:22:28 -05:00 • committed by GitHub
parent 0f7f415974
commit 7da6f1453d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
94 changed files with 767 additions and 918 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.gitignore vendored
View file

@ -2,8 +2,11 @@
.sw?
.*.sw?
tags
*~
/.vscode
tags
push2
data/.gitignore

2
pkg/dsl/cst/dump.go
View file

@ -186,7 +186,7 @@ func (node *DumpStatementNode) dumpToStdout(
//
// The output channel is always non-nil, except for the Miller REPL.
if state.OutputRecordsAndContexts != nil {
state.OutputRecordsAndContexts.PushBack(types.NewOutputString(outputString, state.Context))
*state.OutputRecordsAndContexts = append(*state.OutputRecordsAndContexts, types.NewOutputString(outputString, state.Context))
} else {
fmt.Println(outputString)
}

2
pkg/dsl/cst/emit1.go
View file

@ -66,7 +66,7 @@ func (node *Emit1StatementNode) Execute(state *runtime.State) (*BlockExitPayload
}
if state.OutputRecordsAndContexts != nil {
state.OutputRecordsAndContexts.PushBack(types.NewRecordAndContext(valueAsMap, state.Context))
*state.OutputRecordsAndContexts = append(*state.OutputRecordsAndContexts, types.NewRecordAndContext(valueAsMap, state.Context))
} else {
fmt.Println(valueAsMap.String())
}

2
pkg/dsl/cst/emit_emitp.go
View file

@ -975,7 +975,7 @@ func (node *EmitXStatementNode) emitRecordToRecordStream(
) error {
// The output channel is always non-nil, except for the Miller REPL.
if state.OutputRecordsAndContexts != nil {
state.OutputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, state.Context))
*state.OutputRecordsAndContexts = append(*state.OutputRecordsAndContexts, types.NewRecordAndContext(outrec, state.Context))
} else {
fmt.Println(outrec.String())
}

2
pkg/dsl/cst/emitf.go
View file

@ -173,7 +173,7 @@ func (node *EmitFStatementNode) emitfToRecordStream(
) error {
// The output channel is always non-nil, except for the Miller REPL.
if state.OutputRecordsAndContexts != nil {
state.OutputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, state.Context))
*state.OutputRecordsAndContexts = append(*state.OutputRecordsAndContexts, types.NewRecordAndContext(outrec, state.Context))
} else {
fmt.Println(outrec.String())
}

2
pkg/dsl/cst/print.go
View file

@ -332,7 +332,7 @@ func (node *PrintStatementNode) printToStdout(
// The output channel is always non-nil, except for the Miller REPL.
if state.OutputRecordsAndContexts != nil {
state.OutputRecordsAndContexts.PushBack(types.NewOutputString(outputString, state.Context))
*state.OutputRecordsAndContexts = append(*state.OutputRecordsAndContexts, types.NewOutputString(outputString, state.Context))
} else {
fmt.Print(outputString)
}

9
pkg/input/line_reader.go
View file

@ -5,7 +5,6 @@ package input
import (
"bufio"
"container/list"
"io"
"strings"
@ -172,14 +171,14 @@ func (r *MultiIRSLineReader) Read() (string, error) {
// IRS) stripped off. So, callers get "a=1,b=2" rather than "a=1,b=2\n".
func channelizedLineReader(
lineReader ILineReader,
linesChannel chan<- *list.List,
linesChannel chan<- []string,
downstreamDoneChannel <-chan bool, // for mlr head
recordsPerBatch int64,
) {
i := int64(0)
done := false
lines := list.New()
lines := make([]string, 0, recordsPerBatch)
for {
line, err := lineReader.Read()
@ -194,7 +193,7 @@ func channelizedLineReader(
i++
lines.PushBack(line)
lines = append(lines, line)
// See if downstream processors will be ignoring further data (e.g. mlr
// head). If so, stop reading. This makes 'mlr head hugefile' exit
@ -211,7 +210,7 @@ func channelizedLineReader(
break
}
linesChannel <- lines
lines = list.New()
lines = make([]string, 0, recordsPerBatch)
}
if done {

15
pkg/input/pseudo_reader_gen.go
View file

@ -1,7 +1,6 @@
package input
import (
"container/list"
"fmt"
"github.com/johnkerl/miller/v6/pkg/bifs"
@ -28,7 +27,7 @@ func NewPseudoReaderGen(
func (reader *PseudoReaderGen) Read(
filenames []string, // ignored
context types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -38,7 +37,7 @@ func (reader *PseudoReaderGen) Read(
func (reader *PseudoReaderGen) process(
context *types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -71,7 +70,7 @@ func (reader *PseudoReaderGen) process(
key := reader.readerOptions.GeneratorOptions.FieldName
value := start.Copy()
recordsAndContexts := list.New()
recordsAndContexts := make([]*types.RecordAndContext, 0, recordsPerBatch)
eof := false
for !eof {
@ -84,11 +83,11 @@ func (reader *PseudoReaderGen) process(
record.PutCopy(key, value)
context.UpdateForInputRecord()
recordsAndContexts.PushBack(types.NewRecordAndContext(record, context))
recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context))
if int64(recordsAndContexts.Len()) >= recordsPerBatch {
if int64(len(recordsAndContexts)) >= recordsPerBatch {
readerChannel <- recordsAndContexts
recordsAndContexts = list.New()
recordsAndContexts = make([]*types.RecordAndContext, 0, recordsPerBatch)
// See if downstream processors will be ignoring further data (e.g.
// mlr head). If so, stop reading. This makes 'mlr head hugefile'
@ -111,7 +110,7 @@ func (reader *PseudoReaderGen) process(
value = bifs.BIF_plus_binary(value, step)
}
if recordsAndContexts.Len() > 0 {
if len(recordsAndContexts) > 0 {
readerChannel <- recordsAndContexts
}
}

8
pkg/input/record_reader.go
View file

@ -3,11 +3,7 @@
package input
import (
"container/list"
"github.com/johnkerl/miller/v6/pkg/types"
)
import "github.com/johnkerl/miller/v6/pkg/types"
// Since Go is concurrent, the context struct (AWK-like variables such as
// FILENAME, NF, NF, FNR, etc.) needs to be duplicated and passed through the
@ -19,7 +15,7 @@ type IRecordReader interface {
Read(
filenames []string,
initialContext types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
)

14
pkg/input/record_reader_benchmark_test.go
View file

@ -57,13 +57,13 @@ func BenchmarkXTABParse(b *testing.B) {
assert.Nil(b, err)
stanza := newStanza()
stanza.dataLines.PushBack("color yellow")
stanza.dataLines.PushBack("shape triangle")
stanza.dataLines.PushBack("flag true")
stanza.dataLines.PushBack("k 1")
stanza.dataLines.PushBack("index 11")
stanza.dataLines.PushBack("quantity 43.6498")
stanza.dataLines.PushBack("rate 9.8870")
stanza.dataLines = append(stanza.dataLines, "color yellow")
stanza.dataLines = append(stanza.dataLines, "shape triangle")
stanza.dataLines = append(stanza.dataLines, "flag true")
stanza.dataLines = append(stanza.dataLines, "k 1")
stanza.dataLines = append(stanza.dataLines, "index 11")
stanza.dataLines = append(stanza.dataLines, "quantity 43.6498")
stanza.dataLines = append(stanza.dataLines, "rate 9.8870")
for i := 0; i < b.N; i++ {
_, _ = reader.recordFromXTABLines(stanza.dataLines)

36
pkg/input/record_reader_csv.go
View file

@ -1,7 +1,6 @@
package input
import (
"container/list"
"fmt"
"io"
"strconv"
@ -56,7 +55,7 @@ func NewRecordReaderCSV(
func (reader *RecordReaderCSV) Read(
filenames []string,
context types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -96,7 +95,7 @@ func (reader *RecordReaderCSV) processHandle(
handle io.Reader,
filename string,
context *types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -121,13 +120,13 @@ func (reader *RecordReaderCSV) processHandle(
}
}
csvRecordsChannel := make(chan *list.List, recordsPerBatch)
csvRecordsChannel := make(chan [][]string, recordsPerBatch)
go channelizedCSVRecordScanner(csvReader, csvRecordsChannel, downstreamDoneChannel, errorChannel,
recordsPerBatch)
for {
recordsAndContexts, eof := reader.getRecordBatch(csvRecordsChannel, errorChannel, context)
if recordsAndContexts.Len() > 0 {
if len(recordsAndContexts) > 0 {
readerChannel <- recordsAndContexts
}
if eof {
@ -139,7 +138,7 @@ func (reader *RecordReaderCSV) processHandle(
// TODO: comment
func channelizedCSVRecordScanner(
csvReader *csv.Reader,
csvRecordsChannel chan<- *list.List,
csvRecordsChannel chan<- [][]string,
downstreamDoneChannel <-chan bool, // for mlr head
errorChannel chan error,
recordsPerBatch int64,
@ -147,7 +146,7 @@ func channelizedCSVRecordScanner(
i := int64(0)
done := false
csvRecords := list.New()
csvRecords := make([][]string, 0, recordsPerBatch)
for {
i++
@ -163,7 +162,7 @@ func channelizedCSVRecordScanner(
break
}
csvRecords.PushBack(csvRecord)
csvRecords = append(csvRecords, csvRecord)
// See if downstream processors will be ignoring further data (e.g. mlr
// head). If so, stop reading. This makes 'mlr head hugefile' exit
@ -180,7 +179,7 @@ func channelizedCSVRecordScanner(
break
}
csvRecordsChannel <- csvRecords
csvRecords = list.New()
csvRecords = make([][]string, 0, recordsPerBatch)
}
if done {
@ -193,14 +192,14 @@ func channelizedCSVRecordScanner(
// TODO: comment copiously we're trying to handle slow/fast/short/long reads: tail -f, smallfile, bigfile.
func (reader *RecordReaderCSV) getRecordBatch(
csvRecordsChannel <-chan *list.List,
csvRecordsChannel <-chan [][]string,
errorChannel chan error,
context *types.Context,
) (
recordsAndContexts *list.List,
recordsAndContexts []*types.RecordAndContext,
eof bool,
) {
recordsAndContexts = list.New()
recordsAndContexts = make([]*types.RecordAndContext, 0)
dedupeFieldNames := reader.readerOptions.DedupeFieldNames
csvRecords, more := <-csvRecordsChannel
@ -208,11 +207,10 @@ func (reader *RecordReaderCSV) getRecordBatch(
return recordsAndContexts, true
}
for e := csvRecords.Front(); e != nil; e = e.Next() {
csvRecord := e.Value.([]string)
for _, csvRecord := range csvRecords {
if reader.needHeader {
isData := reader.maybeConsumeComment(csvRecord, context, recordsAndContexts)
isData := reader.maybeConsumeComment(csvRecord, context, &recordsAndContexts)
if !isData {
continue
}
@ -223,7 +221,7 @@ func (reader *RecordReaderCSV) getRecordBatch(
continue
}
isData := reader.maybeConsumeComment(csvRecord, context, recordsAndContexts)
isData := reader.maybeConsumeComment(csvRecord, context, &recordsAndContexts)
if !isData {
continue
}
@ -291,7 +289,7 @@ func (reader *RecordReaderCSV) getRecordBatch(
context.UpdateForInputRecord()
recordsAndContexts.PushBack(types.NewRecordAndContext(record, context))
recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context))
}
return recordsAndContexts, false
@ -302,7 +300,7 @@ func (reader *RecordReaderCSV) getRecordBatch(
func (reader *RecordReaderCSV) maybeConsumeComment(
csvRecord []string,
context *types.Context,
recordsAndContexts *list.List, // list of *types.RecordAndContext
recordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) bool {
if reader.readerOptions.CommentHandling == cli.CommentsAreData {
// Nothing is to be construed as a comment
@ -333,7 +331,7 @@ func (reader *RecordReaderCSV) maybeConsumeComment(
// Contract with our fork of the go-csv CSV Reader, and, our own constructor.
lib.InternalCodingErrorIf(len(csvRecord) != 1)
recordsAndContexts.PushBack(types.NewOutputString(csvRecord[0], context))
*recordsAndContexts = append(*recordsAndContexts, types.NewOutputString(csvRecord[0], context))
} else /* reader.readerOptions.CommentHandling == cli.SkipComments */ {
// discard entirely

39
pkg/input/record_reader_csvlite.go
View file

@ -19,7 +19,6 @@ package input
// 3,4,5,6 3,4,5
import (
"container/list"
"fmt"
"io"
"strconv"
@ -35,12 +34,12 @@ import (
// implicit-CSV-header record-batch getter.
type recordBatchGetterCSV func(
reader *RecordReaderCSVLite,
linesChannel <-chan *list.List,
linesChannel <-chan []string,
filename string,
context *types.Context,
errorChannel chan error,
) (
recordsAndContexts *list.List,
recordsAndContexts []*types.RecordAndContext,
eof bool,
)
@ -81,7 +80,7 @@ func NewRecordReaderCSVLite(
func (reader *RecordReaderCSVLite) Read(
filenames []string,
context types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -135,7 +134,7 @@ func (reader *RecordReaderCSVLite) processHandle(
handle io.Reader,
filename string,
context *types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -145,12 +144,12 @@ func (reader *RecordReaderCSVLite) processHandle(
recordsPerBatch := reader.recordsPerBatch
lineReader := NewLineReader(handle, reader.readerOptions.IRS)
linesChannel := make(chan *list.List, recordsPerBatch)
linesChannel := make(chan []string, recordsPerBatch)
go channelizedLineReader(lineReader, linesChannel, downstreamDoneChannel, recordsPerBatch)
for {
recordsAndContexts, eof := reader.recordBatchGetter(reader, linesChannel, filename, context, errorChannel)
if recordsAndContexts.Len() > 0 {
if len(recordsAndContexts) > 0 {
readerChannel <- recordsAndContexts
}
if eof {
@ -161,15 +160,15 @@ func (reader *RecordReaderCSVLite) processHandle(
func getRecordBatchExplicitCSVHeader(
reader *RecordReaderCSVLite,
linesChannel <-chan *list.List,
linesChannel <-chan []string,
filename string,
context *types.Context,
errorChannel chan error,
) (
recordsAndContexts *list.List,
recordsAndContexts []*types.RecordAndContext,
eof bool,
) {
recordsAndContexts = list.New()
recordsAndContexts = make([]*types.RecordAndContext, 0)
dedupeFieldNames := reader.readerOptions.DedupeFieldNames
lines, more := <-linesChannel
@ -177,8 +176,7 @@ func getRecordBatchExplicitCSVHeader(
return recordsAndContexts, true
}
for e := lines.Front(); e != nil; e = e.Next() {
line := e.Value.(string)
for _, line := range lines {
reader.inputLineNumber++
@ -194,7 +192,7 @@ func getRecordBatchExplicitCSVHeader(
if reader.readerOptions.CommentHandling != cli.CommentsAreData {
if strings.HasPrefix(line, reader.readerOptions.CommentString) {
if reader.readerOptions.CommentHandling == cli.PassComments {
recordsAndContexts.PushBack(types.NewOutputString(line+"\n", context))
recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+"\n", context))
continue
} else if reader.readerOptions.CommentHandling == cli.SkipComments {
continue
@ -275,7 +273,7 @@ func getRecordBatchExplicitCSVHeader(
}
context.UpdateForInputRecord()
recordsAndContexts.PushBack(types.NewRecordAndContext(record, context))
recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context))
}
}
@ -284,15 +282,15 @@ func getRecordBatchExplicitCSVHeader(
func getRecordBatchImplicitCSVHeader(
reader *RecordReaderCSVLite,
linesChannel <-chan *list.List,
linesChannel <-chan []string,
filename string,
context *types.Context,
errorChannel chan error,
) (
recordsAndContexts *list.List,
recordsAndContexts []*types.RecordAndContext,
eof bool,
) {
recordsAndContexts = list.New()
recordsAndContexts = make([]*types.RecordAndContext, 0)
dedupeFieldNames := reader.readerOptions.DedupeFieldNames
lines, more := <-linesChannel
@ -300,8 +298,7 @@ func getRecordBatchImplicitCSVHeader(
return recordsAndContexts, true
}
for e := lines.Front(); e != nil; e = e.Next() {
line := e.Value.(string)
for _, line := range lines {
reader.inputLineNumber++
@ -310,7 +307,7 @@ func getRecordBatchImplicitCSVHeader(
if reader.readerOptions.CommentHandling != cli.CommentsAreData {
if strings.HasPrefix(line, reader.readerOptions.CommentString) {
if reader.readerOptions.CommentHandling == cli.PassComments {
recordsAndContexts.PushBack(types.NewOutputString(line+"\n", context))
recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+"\n", context))
continue
} else if reader.readerOptions.CommentHandling == cli.SkipComments {
continue
@ -402,7 +399,7 @@ func getRecordBatchImplicitCSVHeader(
}
context.UpdateForInputRecord()
recordsAndContexts.PushBack(types.NewRecordAndContext(record, context))
recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context))
}
return recordsAndContexts, false

22
pkg/input/record_reader_dkvp_nidx.go
View file

@ -3,7 +3,6 @@
package input
import (
"container/list"
"io"
"strconv"
"strings"
@ -55,7 +54,7 @@ func NewRecordReaderNIDX(
func (reader *RecordReaderDKVPNIDX) Read(
filenames []string,
context types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -95,7 +94,7 @@ func (reader *RecordReaderDKVPNIDX) processHandle(
handle io.Reader,
filename string,
context *types.Context,
readerChannel chan<- *list.List,
readerChannel chan<- []*types.RecordAndContext,
errorChannel chan<- error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -103,12 +102,12 @@ func (reader *RecordReaderDKVPNIDX) processHandle(
recordsPerBatch := reader.recordsPerBatch
lineReader := NewLineReader(handle, reader.readerOptions.IRS)
linesChannel := make(chan *list.List, recordsPerBatch)
linesChannel := make(chan []string, recordsPerBatch)
go channelizedLineReader(lineReader, linesChannel, downstreamDoneChannel, recordsPerBatch)
for {
recordsAndContexts, eof := reader.getRecordBatch(linesChannel, errorChannel, context)
if recordsAndContexts.Len() > 0 {
if len(recordsAndContexts) > 0 {
readerChannel <- recordsAndContexts
}
if eof {
@ -119,29 +118,28 @@ func (reader *RecordReaderDKVPNIDX) processHandle(
// TODO: comment copiously we're trying to handle slow/fast/short/long reads: tail -f, smallfile, bigfile.
func (reader *RecordReaderDKVPNIDX) getRecordBatch(
linesChannel <-chan *list.List,
linesChannel <-chan []string,
errorChannel chan<- error,
context *types.Context,
) (
recordsAndContexts *list.List,
recordsAndContexts []*types.RecordAndContext,
eof bool,
) {
recordsAndContexts = list.New()
recordsAndContexts = make([]*types.RecordAndContext, 0)
lines, more := <-linesChannel
if !more {
return recordsAndContexts, true
}
for e := lines.Front(); e != nil; e = e.Next() {
line := e.Value.(string)
for _, line := range lines {
// Check for comments-in-data feature
// TODO: function-pointer this away
if reader.readerOptions.CommentHandling != cli.CommentsAreData {
if strings.HasPrefix(line, reader.readerOptions.CommentString) {
if reader.readerOptions.CommentHandling == cli.PassComments {
recordsAndContexts.PushBack(types.NewOutputString(line+"\n", context))
recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+"\n", context))
continue
} else if reader.readerOptions.CommentHandling == cli.SkipComments {
continue
@ -157,7 +155,7 @@ func (reader *RecordReaderDKVPNIDX) getRecordBatch(
}
context.UpdateForInputRecord()
recordAndContext := types.NewRecordAndContext(record, context)
recordsAndContexts.PushBack(recordAndContext)
recordsAndContexts = append(recordsAndContexts, recordAndContext)
}
return recordsAndContexts, false

34
pkg/input/record_reader_json.go
View file

@ -1,7 +1,6 @@
package input
import (
"container/list"
"fmt"
"io"
"strings"
@ -34,7 +33,7 @@ func NewRecordReaderJSON(
func (reader *RecordReaderJSON) Read(
filenames []string,
context types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -75,7 +74,7 @@ func (reader *RecordReaderJSON) processHandle(
handle io.Reader,
filename string,
context *types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -87,7 +86,7 @@ func (reader *RecordReaderJSON) processHandle(
handle = NewJSONCommentEnabledReader(handle, reader.readerOptions, readerChannel)
}
decoder := json.NewDecoder(handle)
recordsAndContexts := list.New()
recordsAndContexts := make([]*types.RecordAndContext, 0, recordsPerBatch)
eof := false
i := int64(0)
@ -132,11 +131,11 @@ func (reader *RecordReaderJSON) processHandle(
return
}
context.UpdateForInputRecord()
recordsAndContexts.PushBack(types.NewRecordAndContext(record, context))
recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context))
if int64(recordsAndContexts.Len()) >= recordsPerBatch {
if int64(len(recordsAndContexts)) >= recordsPerBatch {
readerChannel <- recordsAndContexts
recordsAndContexts = list.New()
recordsAndContexts = make([]*types.RecordAndContext, 0, recordsPerBatch)
}
} else if mlrval.IsArray() {
@ -164,11 +163,11 @@ func (reader *RecordReaderJSON) processHandle(
return
}
context.UpdateForInputRecord()
recordsAndContexts.PushBack(types.NewRecordAndContext(record, context))
recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context))
if int64(recordsAndContexts.Len()) >= recordsPerBatch {
if int64(len(recordsAndContexts)) >= recordsPerBatch {
readerChannel <- recordsAndContexts
recordsAndContexts = list.New()
recordsAndContexts = make([]*types.RecordAndContext, 0, recordsPerBatch)
}
}
@ -181,7 +180,7 @@ func (reader *RecordReaderJSON) processHandle(
}
}
if recordsAndContexts.Len() > 0 {
if len(recordsAndContexts) > 0 {
readerChannel <- recordsAndContexts
}
}
@ -211,8 +210,8 @@ func (reader *RecordReaderJSON) processHandle(
type JSONCommentEnabledReader struct {
lineReader ILineReader
readerOptions *cli.TReaderOptions
context *types.Context // Needed for channelized stdout-printing logic
readerChannel chan<- *list.List // list of *types.RecordAndContext
context *types.Context // Needed for channelized stdout-printing logic
readerChannel chan<- []*types.RecordAndContext // list of *types.RecordAndContext
// In case a line was ingested which was longer than the read-buffer passed
// to us, in which case we need to split up that line and return it over
@ -223,7 +222,7 @@ type JSONCommentEnabledReader struct {
func NewJSONCommentEnabledReader(
underlying io.Reader,
readerOptions *cli.TReaderOptions,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
) *JSONCommentEnabledReader {
return &JSONCommentEnabledReader{
lineReader: NewLineReader(underlying, "\n"),
@ -260,9 +259,10 @@ func (bsr *JSONCommentEnabledReader) Read(p []byte) (n int, err error) {
if bsr.readerOptions.CommentHandling == cli.PassComments {
// Insert the string into the record-output stream, so that goroutine can
// print it, resulting in deterministic output-ordering.
ell := list.New()
ell.PushBack(types.NewOutputString(line+"\n", bsr.context))
bsr.readerChannel <- ell
recordsAndContexts := []*types.RecordAndContext{
types.NewOutputString(line+"\n", bsr.context),
}
bsr.readerChannel <- recordsAndContexts
}
if done {

39
pkg/input/record_reader_pprint.go
View file

@ -1,7 +1,6 @@
package input
import (
"container/list"
"fmt"
"io"
"regexp"
@ -73,19 +72,19 @@ type RecordReaderPprintBarredOrMarkdown struct {
// implicit-PPRINT-header record-batch getter.
type recordBatchGetterPprint func(
reader *RecordReaderPprintBarredOrMarkdown,
linesChannel <-chan *list.List,
linesChannel <-chan []string,
filename string,
context *types.Context,
errorChannel chan error,
) (
recordsAndContexts *list.List,
recordsAndContexts []*types.RecordAndContext,
eof bool,
)
func (reader *RecordReaderPprintBarredOrMarkdown) Read(
filenames []string,
context types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -139,7 +138,7 @@ func (reader *RecordReaderPprintBarredOrMarkdown) processHandle(
handle io.Reader,
filename string,
context *types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -149,12 +148,12 @@ func (reader *RecordReaderPprintBarredOrMarkdown) processHandle(
recordsPerBatch := reader.recordsPerBatch
lineReader := NewLineReader(handle, reader.readerOptions.IRS)
linesChannel := make(chan *list.List, recordsPerBatch)
linesChannel := make(chan []string, recordsPerBatch)
go channelizedLineReader(lineReader, linesChannel, downstreamDoneChannel, recordsPerBatch)
for {
recordsAndContexts, eof := reader.recordBatchGetter(reader, linesChannel, filename, context, errorChannel)
if recordsAndContexts.Len() > 0 {
if len(recordsAndContexts) > 0 {
readerChannel <- recordsAndContexts
}
if eof {
@ -165,15 +164,15 @@ func (reader *RecordReaderPprintBarredOrMarkdown) processHandle(
func getRecordBatchExplicitPprintHeader(
reader *RecordReaderPprintBarredOrMarkdown,
linesChannel <-chan *list.List,
linesChannel <-chan []string,
filename string,
context *types.Context,
errorChannel chan error,
) (
recordsAndContexts *list.List,
recordsAndContexts []*types.RecordAndContext,
eof bool,
) {
recordsAndContexts = list.New()
recordsAndContexts = make([]*types.RecordAndContext, 0)
dedupeFieldNames := reader.readerOptions.DedupeFieldNames
lines, more := <-linesChannel
@ -181,8 +180,7 @@ func getRecordBatchExplicitPprintHeader(
return recordsAndContexts, true
}
for e := lines.Front(); e != nil; e = e.Next() {
line := e.Value.(string)
for _, line := range lines {
reader.inputLineNumber++
@ -191,7 +189,7 @@ func getRecordBatchExplicitPprintHeader(
if reader.readerOptions.CommentHandling != cli.CommentsAreData {
if strings.HasPrefix(line, reader.readerOptions.CommentString) {
if reader.readerOptions.CommentHandling == cli.PassComments {
recordsAndContexts.PushBack(types.NewOutputString(line+"\n", context))
recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+"\n", context))
continue
} else if reader.readerOptions.CommentHandling == cli.SkipComments {
continue
@ -292,7 +290,7 @@ func getRecordBatchExplicitPprintHeader(
}
context.UpdateForInputRecord()
recordsAndContexts.PushBack(types.NewRecordAndContext(record, context))
recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context))
}
}
@ -302,15 +300,15 @@ func getRecordBatchExplicitPprintHeader(
func getRecordBatchImplicitPprintHeader(
reader *RecordReaderPprintBarredOrMarkdown,
linesChannel <-chan *list.List,
linesChannel <-chan []string,
filename string,
context *types.Context,
errorChannel chan error,
) (
recordsAndContexts *list.List,
recordsAndContexts []*types.RecordAndContext,
eof bool,
) {
recordsAndContexts = list.New()
recordsAndContexts = make([]*types.RecordAndContext, 0)
dedupeFieldNames := reader.readerOptions.DedupeFieldNames
lines, more := <-linesChannel
@ -318,8 +316,7 @@ func getRecordBatchImplicitPprintHeader(
return recordsAndContexts, true
}
for e := lines.Front(); e != nil; e = e.Next() {
line := e.Value.(string)
for _, line := range lines {
reader.inputLineNumber++
@ -328,7 +325,7 @@ func getRecordBatchImplicitPprintHeader(
if reader.readerOptions.CommentHandling != cli.CommentsAreData {
if strings.HasPrefix(line, reader.readerOptions.CommentString) {
if reader.readerOptions.CommentHandling == cli.PassComments {
recordsAndContexts.PushBack(types.NewOutputString(line+"\n", context))
recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+"\n", context))
continue
} else if reader.readerOptions.CommentHandling == cli.SkipComments {
continue
@ -436,7 +433,7 @@ func getRecordBatchImplicitPprintHeader(
}
context.UpdateForInputRecord()
recordsAndContexts.PushBack(types.NewRecordAndContext(record, context))
recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context))
}
return recordsAndContexts, false

39
pkg/input/record_reader_tsv.go
View file

@ -1,7 +1,6 @@
package input
import (
"container/list"
"fmt"
"io"
"strconv"
@ -17,12 +16,12 @@ import (
// implicit-TSV-header record-batch getter.
type recordBatchGetterTSV func(
reader *RecordReaderTSV,
linesChannel <-chan *list.List,
linesChannel <-chan []string,
filename string,
context *types.Context,
errorChannel chan error,
) (
recordsAndContexts *list.List,
recordsAndContexts []*types.RecordAndContext,
eof bool,
)
@ -63,7 +62,7 @@ func NewRecordReaderTSV(
func (reader *RecordReaderTSV) Read(
filenames []string,
context types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -117,7 +116,7 @@ func (reader *RecordReaderTSV) processHandle(
handle io.Reader,
filename string,
context *types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -127,12 +126,12 @@ func (reader *RecordReaderTSV) processHandle(
recordsPerBatch := reader.recordsPerBatch
lineReader := NewLineReader(handle, reader.readerOptions.IRS)
linesChannel := make(chan *list.List, recordsPerBatch)
linesChannel := make(chan []string, recordsPerBatch)
go channelizedLineReader(lineReader, linesChannel, downstreamDoneChannel, recordsPerBatch)
for {
recordsAndContexts, eof := reader.recordBatchGetter(reader, linesChannel, filename, context, errorChannel)
if recordsAndContexts.Len() > 0 {
if len(recordsAndContexts) > 0 {
readerChannel <- recordsAndContexts
}
if eof {
@ -143,15 +142,15 @@ func (reader *RecordReaderTSV) processHandle(
func getRecordBatchExplicitTSVHeader(
reader *RecordReaderTSV,
linesChannel <-chan *list.List,
linesChannel <-chan []string,
filename string,
context *types.Context,
errorChannel chan error,
) (
recordsAndContexts *list.List,
recordsAndContexts []*types.RecordAndContext,
eof bool,
) {
recordsAndContexts = list.New()
recordsAndContexts = make([]*types.RecordAndContext, 0)
dedupeFieldNames := reader.readerOptions.DedupeFieldNames
lines, more := <-linesChannel
@ -159,8 +158,7 @@ func getRecordBatchExplicitTSVHeader(
return recordsAndContexts, true
}
for e := lines.Front(); e != nil; e = e.Next() {
line := e.Value.(string)
for _, line := range lines {
reader.inputLineNumber++
@ -169,7 +167,7 @@ func getRecordBatchExplicitTSVHeader(
if reader.readerOptions.CommentHandling != cli.CommentsAreData {
if strings.HasPrefix(line, reader.readerOptions.CommentString) {
if reader.readerOptions.CommentHandling == cli.PassComments {
recordsAndContexts.PushBack(types.NewOutputString(line+"\n", context))
recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+"\n", context))
continue
} else if reader.readerOptions.CommentHandling == cli.SkipComments {
continue
@ -240,7 +238,7 @@ func getRecordBatchExplicitTSVHeader(
}
context.UpdateForInputRecord()
recordsAndContexts.PushBack(types.NewRecordAndContext(record, context))
recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context))
}
}
@ -249,15 +247,15 @@ func getRecordBatchExplicitTSVHeader(
func getRecordBatchImplicitTSVHeader(
reader *RecordReaderTSV,
linesChannel <-chan *list.List,
linesChannel <-chan []string,
filename string,
context *types.Context,
errorChannel chan error,
) (
recordsAndContexts *list.List,
recordsAndContexts []*types.RecordAndContext,
eof bool,
) {
recordsAndContexts = list.New()
recordsAndContexts = make([]*types.RecordAndContext, 0)
dedupeFieldNames := reader.readerOptions.DedupeFieldNames
lines, more := <-linesChannel
@ -265,8 +263,7 @@ func getRecordBatchImplicitTSVHeader(
return recordsAndContexts, true
}
for e := lines.Front(); e != nil; e = e.Next() {
line := e.Value.(string)
for _, line := range lines {
reader.inputLineNumber++
@ -275,7 +272,7 @@ func getRecordBatchImplicitTSVHeader(
if reader.readerOptions.CommentHandling != cli.CommentsAreData {
if strings.HasPrefix(line, reader.readerOptions.CommentString) {
if reader.readerOptions.CommentHandling == cli.PassComments {
recordsAndContexts.PushBack(types.NewOutputString(line+"\n", context))
recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+"\n", context))
continue
} else if reader.readerOptions.CommentHandling == cli.SkipComments {
continue
@ -363,7 +360,7 @@ func getRecordBatchImplicitTSVHeader(
}
context.UpdateForInputRecord()
recordsAndContexts.PushBack(types.NewRecordAndContext(record, context))
recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context))
}
return recordsAndContexts, false

58
pkg/input/record_reader_xtab.go
View file

@ -1,7 +1,6 @@
package input
import (
"container/list"
"fmt"
"io"
"os"
@ -33,14 +32,14 @@ type RecordReaderXTAB struct {
// 500 or so). This struct helps us keep each stanza's comment lines along with
// the stanza they originated in.
type tStanza struct {
dataLines *list.List
commentLines *list.List
dataLines []string
commentLines []string
}
func newStanza() *tStanza {
return &tStanza{
dataLines: list.New(),
commentLines: list.New(),
dataLines: make([]string, 0),
commentLines: make([]string, 0),
}
}
@ -58,7 +57,7 @@ func NewRecordReaderXTAB(
func (reader *RecordReaderXTAB) Read(
filenames []string,
context types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -98,7 +97,7 @@ func (reader *RecordReaderXTAB) processHandle(
handle io.Reader,
filename string,
context *types.Context,
readerChannel chan<- *list.List, // list of *types.RecordAndContext
readerChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
errorChannel chan error,
downstreamDoneChannel <-chan bool, // for mlr head
) {
@ -108,13 +107,13 @@ func (reader *RecordReaderXTAB) processHandle(
// XTAB uses repeated IFS, rather than IRS, to delimit records
lineReader := NewLineReader(handle, reader.readerOptions.IFS)
stanzasChannel := make(chan *list.List, recordsPerBatch)
stanzasChannel := make(chan []*tStanza, recordsPerBatch)
go channelizedStanzaScanner(lineReader, reader.readerOptions, stanzasChannel, downstreamDoneChannel,
recordsPerBatch)
for {
recordsAndContexts, eof := reader.getRecordBatch(stanzasChannel, context, errorChannel)
if recordsAndContexts.Len() > 0 {
if len(recordsAndContexts) > 0 {
readerChannel <- recordsAndContexts
}
if eof {
@ -140,7 +139,7 @@ func (reader *RecordReaderXTAB) processHandle(
func channelizedStanzaScanner(
lineReader ILineReader,
readerOptions *cli.TReaderOptions,
stanzasChannel chan<- *list.List, // list of list of string
stanzasChannel chan<- []*tStanza, // list of list of string
downstreamDoneChannel <-chan bool, // for mlr head
recordsPerBatch int64,
) {
@ -148,7 +147,7 @@ func channelizedStanzaScanner(
inStanza := false
done := false
stanzas := list.New()
stanzas := make([]*tStanza, 0, recordsPerBatch)
stanza := newStanza()
for {
@ -168,7 +167,7 @@ func channelizedStanzaScanner(
if readerOptions.CommentHandling != cli.CommentsAreData {
if strings.HasPrefix(line, readerOptions.CommentString) {
if readerOptions.CommentHandling == cli.PassComments {
stanza.commentLines.PushBack(line)
stanza.commentLines = append(stanza.commentLines, line)
continue
} else if readerOptions.CommentHandling == cli.SkipComments {
continue
@ -184,7 +183,7 @@ func channelizedStanzaScanner(
// 3. At end of file, multiple empty lines are ignored.
if inStanza {
inStanza = false
stanzas.PushBack(stanza)
stanzas = append(stanzas, stanza)
numStanzasSeen++
stanza = newStanza()
} else {
@ -194,7 +193,7 @@ func channelizedStanzaScanner(
if !inStanza {
inStanza = true
}
stanza.dataLines.PushBack(line)
stanza.dataLines = append(stanza.dataLines, line)
}
// See if downstream processors will be ignoring further data (e.g. mlr
@ -212,7 +211,7 @@ func channelizedStanzaScanner(
break
}
stanzasChannel <- stanzas
stanzas = list.New()
stanzas = make([]*tStanza, 0, recordsPerBatch)
}
if done {
@ -222,8 +221,8 @@ func channelizedStanzaScanner(
// The last stanza may not have a trailing newline after it. Any lines in the stanza
// at this point will form the final record in the stream.
if stanza.dataLines.Len() > 0 || stanza.commentLines.Len() > 0 {
stanzas.PushBack(stanza)
if len(stanza.dataLines) > 0 || len(stanza.commentLines) > 0 {
stanzas = append(stanzas, stanza)
}
stanzasChannel <- stanzas
@ -232,38 +231,36 @@ func channelizedStanzaScanner(
// TODO: comment copiously we're trying to handle slow/fast/short/long reads: tail -f, smallfile, bigfile.
func (reader *RecordReaderXTAB) getRecordBatch(
stanzasChannel <-chan *list.List,
stanzasChannel <-chan []*tStanza,
context *types.Context,
errorChannel chan error,
) (
recordsAndContexts *list.List,
recordsAndContexts []*types.RecordAndContext,
eof bool,
) {
recordsAndContexts = list.New()
recordsAndContexts = make([]*types.RecordAndContext, 0)
stanzas, more := <-stanzasChannel
if !more {
return recordsAndContexts, true
}
for e := stanzas.Front(); e != nil; e = e.Next() {
stanza := e.Value.(*tStanza)
for _, stanza := range stanzas {
if stanza.commentLines.Len() > 0 {
for f := stanza.commentLines.Front(); f != nil; f = f.Next() {
line := f.Value.(string)
recordsAndContexts.PushBack(types.NewOutputString(line+reader.readerOptions.IFS, context))
if len(stanza.commentLines) > 0 {
for _, line := range stanza.commentLines {
recordsAndContexts = append(recordsAndContexts, types.NewOutputString(line+reader.readerOptions.IFS, context))
}
}
if stanza.dataLines.Len() > 0 {
if len(stanza.dataLines) > 0 {
record, err := reader.recordFromXTABLines(stanza.dataLines)
if err != nil {
errorChannel <- err
return
}
context.UpdateForInputRecord()
recordsAndContexts.PushBack(types.NewRecordAndContext(record, context))
recordsAndContexts = append(recordsAndContexts, types.NewRecordAndContext(record, context))
}
}
@ -271,13 +268,12 @@ func (reader *RecordReaderXTAB) getRecordBatch(
}
func (reader *RecordReaderXTAB) recordFromXTABLines(
stanza *list.List,
stanza []string,
) (*mlrval.Mlrmap, error) {
record := mlrval.NewMlrmapAsRecord()
dedupeFieldNames := reader.readerOptions.DedupeFieldNames
for e := stanza.Front(); e != nil; e = e.Next() {
line := e.Value.(string)
for _, line := range stanza {
key, value, err := reader.pairSplitter.Split(line)
if err != nil {

9
pkg/output/channel_writer.go
View file

@ -2,7 +2,6 @@ package output
import (
"bufio"
"container/list"
"fmt"
"os"
@ -11,7 +10,7 @@ import (
)
func ChannelWriter(
writerChannel <-chan *list.List, // list of *types.RecordAndContext
writerChannel <-chan []*types.RecordAndContext, // list of *types.RecordAndContext
recordWriter IRecordWriter,
writerOptions *cli.TWriterOptions,
doneChannel chan<- bool,
@ -45,16 +44,14 @@ func ChannelWriter(
// TODO: comment
// Returns true on end of record stream
func channelWriterHandleBatch(
recordsAndContexts *list.List,
recordsAndContexts []*types.RecordAndContext,
recordWriter IRecordWriter,
writerOptions *cli.TWriterOptions,
dataProcessingErrorChannel chan<- bool,
bufferedOutputStream *bufio.Writer,
outputIsStdout bool,
) (done bool, errored bool) {
for e := recordsAndContexts.Front(); e != nil; e = e.Next() {
recordAndContext := e.Value.(*types.RecordAndContext)
for _, recordAndContext := range recordsAndContexts {
// Three things can come through:
// * End-of-stream marker
// * Non-nil records to be printed

9
pkg/output/file_output_handlers.go
View file

@ -14,7 +14,6 @@ package output
import (
"bufio"
"container/list"
"errors"
"fmt"
"io"
@ -216,7 +215,7 @@ type FileOutputHandler struct {
// print and dump variants call WriteString.
recordWriterOptions *cli.TWriterOptions
recordWriter IRecordWriter
recordOutputChannel chan *list.List // list of *types.RecordAndContext
recordOutputChannel chan []*types.RecordAndContext // list of *types.RecordAndContext
recordDoneChannel chan bool
recordErroredChannel chan bool
}
@ -352,9 +351,7 @@ func (handler *FileOutputHandler) WriteRecordAndContext(
}
// TODO: myybe refactor to batch better
ell := list.New()
ell.PushBack(outrecAndContext)
handler.recordOutputChannel <- ell
handler.recordOutputChannel <- []*types.RecordAndContext{outrecAndContext}
return nil
}
@ -369,7 +366,7 @@ func (handler *FileOutputHandler) setUpRecordWriter() error {
}
handler.recordWriter = recordWriter
handler.recordOutputChannel = make(chan *list.List, 1) // list of *types.RecordAndContext
handler.recordOutputChannel = make(chan []*types.RecordAndContext, 1) // list of *types.RecordAndContext
handler.recordDoneChannel = make(chan bool, 1)
handler.recordErroredChannel = make(chan bool, 1)

35
pkg/output/record_writer_pprint.go
View file

@ -2,7 +2,6 @@ package output
import (
"bufio"
"container/list"
"fmt"
"strings"
"unicode/utf8"
@ -16,20 +15,20 @@ import (
type RecordWriterPPRINT struct {
writerOptions *cli.TWriterOptions
// Input:
records *list.List
records []*mlrval.Mlrmap
// State:
lastJoinedHeader *string
batch *list.List
batch []*mlrval.Mlrmap
}
func NewRecordWriterPPRINT(writerOptions *cli.TWriterOptions) (*RecordWriterPPRINT, error) {
return &RecordWriterPPRINT{
writerOptions: writerOptions,
records: list.New(),
records: make([]*mlrval.Mlrmap, 0),
lastJoinedHeader: nil,
batch: list.New(),
batch: make([]*mlrval.Mlrmap, 0),
}, nil
}
@ -51,7 +50,7 @@ func (writer *RecordWriterPPRINT) Write(
// First output record:
// * New batch
// * No old batch to print
writer.batch.PushBack(outrec)
writer.batch = append(writer.batch, outrec)
temp := strings.Join(outrec.GetKeys(), ",")
writer.lastJoinedHeader = &temp
} else {
@ -70,17 +69,16 @@ func (writer *RecordWriterPPRINT) Write(
bufferedOutputStream.WriteString(writer.writerOptions.ORS)
}
// Start a new batch
writer.batch = list.New()
writer.batch.PushBack(outrec)
writer.batch = []*mlrval.Mlrmap{outrec}
writer.lastJoinedHeader = &joinedHeader
} else {
// Continue the batch
writer.batch.PushBack(outrec)
writer.batch = append(writer.batch, outrec)
}
}
} else { // End of record stream
if writer.batch.Front() != nil {
if len(writer.batch) > 0 {
writer.writeHeterogenousList(writer.batch, writer.writerOptions.BarredPprintOutput,
bufferedOutputStream, outputIsStdout)
}
@ -92,7 +90,7 @@ func (writer *RecordWriterPPRINT) Write(
// ----------------------------------------------------------------
// Returns false if there was nothing but empty record(s), e.g. 'mlr gap -n 10'.
func (writer *RecordWriterPPRINT) writeHeterogenousList(
records *list.List,
records []*mlrval.Mlrmap,
barred bool,
bufferedOutputStream *bufio.Writer,
outputIsStdout bool,
@ -100,8 +98,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousList(
maxWidths := make(map[string]int)
var maxNR int64 = 0
for e := records.Front(); e != nil; e = e.Next() {
outrec := e.Value.(*mlrval.Mlrmap)
for _, outrec := range records {
nr := outrec.FieldCount
if maxNR < nr {
maxNR = nr
@ -148,15 +145,14 @@ func (writer *RecordWriterPPRINT) writeHeterogenousList(
// wye pan 5 0.5732889198020006 0.8636244699032729
func (writer *RecordWriterPPRINT) writeHeterogenousListNonBarred(
records *list.List,
records []*mlrval.Mlrmap,
maxWidths map[string]int,
bufferedOutputStream *bufio.Writer,
outputIsStdout bool,
) {
onFirst := true
for e := records.Front(); e != nil; e = e.Next() {
outrec := e.Value.(*mlrval.Mlrmap)
for _, outrec := range records {
// Print header line
if onFirst && !writer.writerOptions.HeaderlessOutput {
@ -238,7 +234,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousListNonBarred(
// +-----+-----+----+----------------------+---------------------+
func (writer *RecordWriterPPRINT) writeHeterogenousListBarred(
records *list.List,
records []*mlrval.Mlrmap,
maxWidths map[string]int,
bufferedOutputStream *bufio.Writer,
outputIsStdout bool,
@ -257,8 +253,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousListBarred(
verticalEnd := ofs + "|"
onFirst := true
for e := records.Front(); e != nil; e = e.Next() {
outrec := e.Value.(*mlrval.Mlrmap)
for i, outrec := range records {
// Print header line
if onFirst && !writer.writerOptions.HeaderlessOutput {
@ -322,7 +317,7 @@ func (writer *RecordWriterPPRINT) writeHeterogenousListBarred(
}
}
if e.Next() == nil {
if i == len(records)-1 {
bufferedOutputStream.WriteString(horizontalStart)
for pe := outrec.Head; pe != nil; pe = pe.Next {
bufferedOutputStream.WriteString(horizontalBars[pe.Key])

4
pkg/runtime/state.go
View file

@ -7,8 +7,6 @@
package runtime
import (
"container/list"
"github.com/johnkerl/miller/v6/pkg/cli"
"github.com/johnkerl/miller/v6/pkg/lib"
"github.com/johnkerl/miller/v6/pkg/mlrval"
@ -21,7 +19,7 @@ type State struct {
Oosvars *mlrval.Mlrmap
FilterExpression *mlrval.Mlrval
Stack *Stack
OutputRecordsAndContexts *list.List // list of *types.RecordAndContext
OutputRecordsAndContexts *[]*types.RecordAndContext // list of *types.RecordAndContext
// For holding "\0".."\9" between where they are set via things like
// '$x =~ "(..)_(...)"', and interpolated via things like '$y = "\2:\1"'.

5
pkg/stream/stream.go
View file

@ -2,7 +2,6 @@ package stream
import (
"bufio"
"container/list"
"errors"
"io"
@ -62,8 +61,8 @@ func Stream(
}
// Set up the reader-to-transformer and transformer-to-writer channels.
readerChannel := make(chan *list.List, 2) // list of *types.RecordAndContext
writerChannel := make(chan *list.List, 1) // list of *types.RecordAndContext
readerChannel := make(chan []*types.RecordAndContext, 2) // list of *types.RecordAndContext
writerChannel := make(chan []*types.RecordAndContext, 1) // list of *types.RecordAndContext
// We're done when a fatal error is registered on input (file not found,
// etc) or when the record-writer has written all its output. We use

4
pkg/terminals/repl/types.go
View file

@ -6,7 +6,6 @@ package repl
import (
"bufio"
"container/list"
"os"
"github.com/johnkerl/miller/v6/pkg/cli"
@ -14,6 +13,7 @@ import (
"github.com/johnkerl/miller/v6/pkg/input"
"github.com/johnkerl/miller/v6/pkg/output"
"github.com/johnkerl/miller/v6/pkg/runtime"
"github.com/johnkerl/miller/v6/pkg/types"
)
// ================================================================
@ -46,7 +46,7 @@ type Repl struct {
options *cli.TOptions
readerChannel chan *list.List // list of *types.RecordAndContext
readerChannel chan []*types.RecordAndContext // list of *types.RecordAndContext
errorChannel chan error
downstreamDoneChannel chan bool
recordReader input.IRecordReader

21
pkg/terminals/repl/verbs.go
View file

@ -5,7 +5,6 @@
package repl
import (
"container/list"
"fmt"
"os"
"strings"
@ -236,7 +235,7 @@ func (repl *Repl) openFiles(filenames []string) {
// Remember for :reopen
repl.options.FileNames = filenames
repl.readerChannel = make(chan *list.List, 2) // list of *types.RecordAndContext
repl.readerChannel = make(chan []*types.RecordAndContext, 2) // list of *types.RecordAndContext
repl.errorChannel = make(chan error, 1)
repl.downstreamDoneChannel = make(chan bool, 1)
@ -288,7 +287,7 @@ func handleRead(repl *Repl, args []string) bool {
return true
}
var recordsAndContexts *list.List // list of *types.RecordAndContext
var recordsAndContexts []*types.RecordAndContext // list of *types.RecordAndContext
var err error = nil
select {
@ -307,8 +306,8 @@ func handleRead(repl *Repl, args []string) bool {
if recordsAndContexts != nil {
// TODO: comment and make very clear we've set this all up to batch by 1 for the REPL
lib.InternalCodingErrorIf(recordsAndContexts.Len() != 1)
recordAndContext := recordsAndContexts.Front().Value.(*types.RecordAndContext)
lib.InternalCodingErrorIf(len(recordsAndContexts) != 1)
recordAndContext := recordsAndContexts[0]
skipOrProcessRecord(
repl,
@ -436,7 +435,7 @@ func handleProcess(repl *Repl, args []string) bool {
// ----------------------------------------------------------------
func handleSkipOrProcessN(repl *Repl, n int64, processingNotSkipping bool) {
var recordsAndContexts *list.List // list of *types.RecordAndContext
var recordsAndContexts []*types.RecordAndContext // list of *types.RecordAndContext
var err error = nil
for i := int64(1); i <= n; i++ {
@ -455,8 +454,8 @@ func handleSkipOrProcessN(repl *Repl, n int64, processingNotSkipping bool) {
if recordsAndContexts != nil {
// TODO: comment and make very clear we've set this all up to batch by 1 for the REPL
lib.InternalCodingErrorIf(recordsAndContexts.Len() != 1)
recordAndContext := recordsAndContexts.Front().Value.(*types.RecordAndContext)
lib.InternalCodingErrorIf(len(recordsAndContexts) != 1)
recordAndContext := recordsAndContexts[0]
shouldBreak := skipOrProcessRecord(
repl,
@ -496,7 +495,7 @@ func handleSkipOrProcessUntil(repl *Repl, dslString string, processingNotSkippin
return
}
var recordsAndContexts *list.List // list of *types.RecordAndContext
var recordsAndContexts []*types.RecordAndContext // list of *types.RecordAndContext
for {
doubleBreak := false
@ -520,8 +519,8 @@ func handleSkipOrProcessUntil(repl *Repl, dslString string, processingNotSkippin
if recordsAndContexts != nil {
// TODO: comment and make very clear we've set this all up to batch by 1 for the REPL
lib.InternalCodingErrorIf(recordsAndContexts.Len() != 1)
recordAndContext := recordsAndContexts.Front().Value.(*types.RecordAndContext)
lib.InternalCodingErrorIf(len(recordsAndContexts) != 1)
recordAndContext := recordsAndContexts[0]
shouldBreak := skipOrProcessRecord(
repl,

30
pkg/transformers/aaa_chain_transformer.go
View file

@ -1,11 +1,11 @@
package transformers
import (
"container/list"
"fmt"
"os"
"github.com/johnkerl/miller/v6/pkg/cli"
"github.com/johnkerl/miller/v6/pkg/types"
"os"
)
// ================================================================
@ -143,18 +143,18 @@ import (
// subdivides goroutines for each transformer in the chain, with intermediary
// channels between them.
func ChainTransformer(
readerRecordChannel <-chan *list.List, // list of *types.RecordAndContext
readerRecordChannel <-chan []*types.RecordAndContext, // list of *types.RecordAndContext
readerDownstreamDoneChannel chan<- bool, // for mlr head -- see also stream.go
recordTransformers []IRecordTransformer, // not *recordTransformer since this is an interface
writerRecordChannel chan<- *list.List, // list of *types.RecordAndContext
writerRecordChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
options *cli.TOptions,
) {
i := 0
n := len(recordTransformers)
intermediateRecordChannels := make([]chan *list.List, n-1) // list of *types.RecordAndContext
intermediateRecordChannels := make([]chan []*types.RecordAndContext, n-1) // list of *types.RecordAndContext
for i = 0; i < n-1; i++ {
intermediateRecordChannels[i] = make(chan *list.List, 1) // list of *types.RecordAndContext
intermediateRecordChannels[i] = make(chan []*types.RecordAndContext, 1) // list of *types.RecordAndContext
}
intermediateDownstreamDoneChannels := make([]chan bool, n)
@ -199,8 +199,8 @@ func ChainTransformer(
func runSingleTransformer(
recordTransformer IRecordTransformer,
isFirstInChain bool,
inputRecordChannel <-chan *list.List, // list of *types.RecordAndContext
outputRecordChannel chan<- *list.List, // list of *types.RecordAndContext
inputRecordChannel <-chan []*types.RecordAndContext, // list of *types.RecordAndContext
outputRecordChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
options *cli.TOptions,
@ -224,20 +224,18 @@ func runSingleTransformer(
// TODO: comment
// Returns true on end of record stream
func runSingleTransformerBatch(
inputRecordsAndContexts *list.List, // list of types.RecordAndContext
inputRecordsAndContexts []*types.RecordAndContext, // list of types.RecordAndContext
recordTransformer IRecordTransformer,
isFirstInChain bool,
outputRecordChannel chan<- *list.List, // list of *types.RecordAndContext
outputRecordChannel chan<- []*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
options *cli.TOptions,
) bool {
outputRecordsAndContexts := list.New()
outputRecordsAndContexts := make([]*types.RecordAndContext, 0, len(inputRecordsAndContexts))
done := false
for e := inputRecordsAndContexts.Front(); e != nil; e = e.Next() {
inputRecordAndContext := e.Value.(*types.RecordAndContext)
for _, inputRecordAndContext := range inputRecordsAndContexts {
// --nr-progress-mod
// TODO: function-pointer this away to reduce instruction count in the
// normal case which it isn't used at all. No need to test if {static thing} != 0
@ -268,14 +266,14 @@ func runSingleTransformerBatch(
if inputRecordAndContext.EndOfStream || inputRecordAndContext.Record != nil {
recordTransformer.Transform(
inputRecordAndContext,
outputRecordsAndContexts,
&outputRecordsAndContexts,
// TODO: maybe refactor these out of each transformer.
// And/or maybe poll them once per batch not once per record.
inputDownstreamDoneChannel,
outputDownstreamDoneChannel,
)
} else {
outputRecordsAndContexts.PushBack(inputRecordAndContext)
outputRecordsAndContexts = append(outputRecordsAndContexts, inputRecordAndContext)
}
if inputRecordAndContext.EndOfStream {

7
pkg/transformers/aaa_record_transformer.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"os"
"github.com/johnkerl/miller/v6/pkg/cli"
@ -14,7 +13,7 @@ import (
type IRecordTransformer interface {
Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
)
@ -22,7 +21,7 @@ type IRecordTransformer interface {
type RecordTransformerFunc func(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
)
@ -30,7 +29,7 @@ type RecordTransformerFunc func(
// Used within some verbs
type RecordTransformerHelperFunc func(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
)
type TransformerUsageFunc func(

7
pkg/transformers/altkv.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strconv"
@ -90,7 +89,7 @@ func NewTransformerAltkv() (*TransformerAltkv, error) {
func (tr *TransformerAltkv) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -121,9 +120,9 @@ func (tr *TransformerAltkv) Transform(
pe = pe.Next
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
} else { // end of record stream
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

32
pkg/transformers/bar.go
View file

@ -2,7 +2,6 @@ package transformers
import (
"bytes"
"container/list"
"fmt"
"os"
"strings"
@ -151,7 +150,7 @@ type TransformerBar struct {
oobString string
blankString string
bars []string
recordsForAutoMode *list.List
recordsForAutoMode []*types.RecordAndContext
recordTransformerFunc RecordTransformerFunc
}
@ -194,7 +193,7 @@ func NewTransformerBar(
if doAuto {
tr.recordTransformerFunc = tr.processAuto
tr.recordsForAutoMode = list.New()
tr.recordsForAutoMode = make([]*types.RecordAndContext, 0)
} else {
tr.recordTransformerFunc = tr.processNoAuto
tr.recordsForAutoMode = nil
@ -207,7 +206,7 @@ func NewTransformerBar(
func (tr *TransformerBar) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -218,7 +217,7 @@ func (tr *TransformerBar) Transform(
// ----------------------------------------------------------------
func (tr *TransformerBar) processNoAuto(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -244,21 +243,21 @@ func (tr *TransformerBar) processNoAuto(
inrec.PutReference(fieldName, mlrval.FromString(tr.bars[idx]))
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerBar) processAuto(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
if !inrecAndContext.EndOfStream {
tr.recordsForAutoMode.PushBack(inrecAndContext.Copy())
tr.recordsForAutoMode = append(tr.recordsForAutoMode, inrecAndContext.Copy())
return
}
@ -271,9 +270,8 @@ func (tr *TransformerBar) processAuto(
// The first pass computes lo and hi from the data
onFirst := true
for e := tr.recordsForAutoMode.Front(); e != nil; e = e.Next() {
recordAndContexts := e.Value.(*types.RecordAndContext)
record := recordAndContexts.Record
for _, recordAndContext := range tr.recordsForAutoMode {
record := recordAndContext.Record
mvalue := record.Get(fieldName)
if mvalue == nil {
continue
@ -301,8 +299,7 @@ func (tr *TransformerBar) processAuto(
slo := fmt.Sprintf("%g", lo)
shi := fmt.Sprintf("%g", hi)
for e := tr.recordsForAutoMode.Front(); e != nil; e = e.Next() {
recordAndContext := e.Value.(*types.RecordAndContext)
for _, recordAndContext := range tr.recordsForAutoMode {
record := recordAndContext.Record
mvalue := record.Get(fieldName)
if mvalue == nil {
@ -333,10 +330,9 @@ func (tr *TransformerBar) processAuto(
}
}
for e := tr.recordsForAutoMode.Front(); e != nil; e = e.Next() {
recordAndContext := e.Value.(*types.RecordAndContext)
outputRecordsAndContexts.PushBack(recordAndContext)
for _, recordAndContext := range tr.recordsForAutoMode {
*outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext)
}
outputRecordsAndContexts.PushBack(inrecAndContext) // Emit the end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // Emit the end-of-stream marker
}

28
pkg/transformers/bootstrap.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -91,13 +90,13 @@ func transformerBootstrapParseCLI(
// ----------------------------------------------------------------
type TransformerBootstrap struct {
recordsAndContexts *list.List
recordsAndContexts []*types.RecordAndContext
nout int64
}
func NewTransformerBootstrap(nout int64) (*TransformerBootstrap, error) {
tr := &TransformerBootstrap{
recordsAndContexts: list.New(),
recordsAndContexts: make([]*types.RecordAndContext, 0),
nout: nout,
}
return tr, nil
@ -107,14 +106,14 @@ func NewTransformerBootstrap(nout int64) (*TransformerBootstrap, error) {
func (tr *TransformerBootstrap) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel)
// Not end of input stream: retain the record, and emit nothing until end of stream.
if !inrecAndContext.EndOfStream {
tr.recordsAndContexts.PushBack(inrecAndContext)
tr.recordsAndContexts = append(tr.recordsAndContexts, inrecAndContext)
return
}
@ -141,8 +140,7 @@ func (tr *TransformerBootstrap) Transform(
//
// For that reason, this transformer must copy all output.
// TODO: Go list Len() maxes at 2^31. We should track this ourselves in an int.
nin := int64(tr.recordsAndContexts.Len())
nin := int64(len(tr.recordsAndContexts))
nout := tr.nout
if nout == -1 {
nout = nin
@ -150,20 +148,12 @@ func (tr *TransformerBootstrap) Transform(
if nout == 0 {
// Emit the stream-terminating null record
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
return
}
// Make an array of pointers into the input list.
recordArray := make([]*types.RecordAndContext, nin)
for i := int64(0); i < nin; i++ {
head := tr.recordsAndContexts.Front()
if head == nil {
break
}
recordArray[i] = head.Value.(*types.RecordAndContext)
tr.recordsAndContexts.Remove(head)
}
recordArray := tr.recordsAndContexts
// Do the sample-with-replacment, reading from random indices in the input
// array and emitting output.
@ -171,9 +161,9 @@ func (tr *TransformerBootstrap) Transform(
index := lib.RandRange(0, nin)
recordAndContext := recordArray[index]
// Already emitted once; copy
outputRecordsAndContexts.PushBack(recordAndContext.Copy())
*outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext.Copy())
}
// Emit the stream-terminating null record
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}

17
pkg/transformers/case.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -177,7 +176,7 @@ func caseSentenceFunc(input string) string {
func (tr *TransformerCase) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -190,13 +189,13 @@ func (tr *TransformerCase) Transform(
outputDownstreamDoneChannel,
)
} else { // end of record stream
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
func (tr *TransformerCase) transformKeysOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
_ <-chan bool,
__ chan<- bool,
) {
@ -211,12 +210,12 @@ func (tr *TransformerCase) transformKeysOnly(
newrec.PutReference(pe.Key, pe.Value)
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
}
func (tr *TransformerCase) transformValuesOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
_ <-chan bool,
__ chan<- bool,
) {
@ -229,12 +228,12 @@ func (tr *TransformerCase) transformValuesOnly(
}
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(inrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(inrec, &inrecAndContext.Context))
}
func (tr *TransformerCase) transformKeysAndValues(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
_ <-chan bool,
__ chan<- bool,
) {
@ -254,5 +253,5 @@ func (tr *TransformerCase) transformKeysAndValues(
newrec.PutReference(pe.Key, pe.Value)
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
}

15
pkg/transformers/cat.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -165,7 +164,7 @@ func NewTransformerCat(
func (tr *TransformerCat) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -181,7 +180,7 @@ func (tr *TransformerCat) Transform(
// ----------------------------------------------------------------
func (tr *TransformerCat) simpleCat(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -193,13 +192,13 @@ func (tr *TransformerCat) simpleCat(
inrecAndContext.Record.PrependCopy("filenum", mlrval.FromInt(inrecAndContext.Context.FILENUM))
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
// ----------------------------------------------------------------
func (tr *TransformerCat) countersUngrouped(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -216,13 +215,13 @@ func (tr *TransformerCat) countersUngrouped(
inrec.PrependCopy("filenum", mlrval.FromInt(inrecAndContext.Context.FILENUM))
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
// ----------------------------------------------------------------
func (tr *TransformerCat) countersGrouped(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -255,5 +254,5 @@ func (tr *TransformerCat) countersGrouped(
inrec.PrependCopy("filenum", mlrval.FromInt(inrecAndContext.Context.FILENUM))
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}

5
pkg/transformers/check.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -94,7 +93,7 @@ func NewTransformerCheck() (*TransformerCheck, error) {
func (tr *TransformerCheck) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -120,6 +119,6 @@ func (tr *TransformerCheck) Transform(
}
}
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

21
pkg/transformers/clean_whitespace.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -132,7 +131,7 @@ func NewTransformerCleanWhitespace(
func (tr *TransformerCleanWhitespace) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -143,7 +142,7 @@ func (tr *TransformerCleanWhitespace) Transform(
// ----------------------------------------------------------------
func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeysAndValues(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -159,16 +158,16 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeysAndValues(
newrec.PutReference(newKey.String(), newValue)
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
// ----------------------------------------------------------------
func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeys(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -182,16 +181,16 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInKeys(
newrec.PutReference(newKey.String(), pe.Value)
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
// ----------------------------------------------------------------
func (tr *TransformerCleanWhitespace) cleanWhitespaceInValues(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -199,8 +198,8 @@ func (tr *TransformerCleanWhitespace) cleanWhitespaceInValues(
for pe := inrecAndContext.Record.Head; pe != nil; pe = pe.Next {
pe.Value = bifs.BIF_clean_whitespace(pe.Value)
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

17
pkg/transformers/count.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -150,7 +149,7 @@ func NewTransformerCount(
func (tr *TransformerCount) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -161,7 +160,7 @@ func (tr *TransformerCount) Transform(
// ----------------------------------------------------------------
func (tr *TransformerCount) countUngrouped(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -170,16 +169,16 @@ func (tr *TransformerCount) countUngrouped(
} else {
newrec := mlrval.NewMlrmapAsRecord()
newrec.PutCopy(tr.outputFieldName, mlrval.FromInt(tr.ungroupedCount))
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerCount) countGrouped(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -210,7 +209,7 @@ func (tr *TransformerCount) countGrouped(
newrec.PutCopy(tr.outputFieldName, mlrval.FromInt(tr.groupedCounts.FieldCount))
outrecAndContext := types.NewRecordAndContext(newrec, &inrecAndContext.Context)
outputRecordsAndContexts.PushBack(outrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
} else {
for outer := tr.groupedCounts.Head; outer != nil; outer = outer.Next {
@ -235,10 +234,10 @@ func (tr *TransformerCount) countGrouped(
newrec.PutCopy(tr.outputFieldName, mlrval.FromInt(countForGroup))
outrecAndContext := types.NewRecordAndContext(newrec, &inrecAndContext.Context)
outputRecordsAndContexts.PushBack(outrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
}
}
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

21
pkg/transformers/count_similar.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -105,7 +104,7 @@ type TransformerCountSimilar struct {
counterFieldName string
// State:
recordListsByGroup *lib.OrderedMap[*list.List] // map from string to *list.List
recordListsByGroup *lib.OrderedMap[*[]*types.RecordAndContext] // map from string to records
}
// ----------------------------------------------------------------
@ -116,7 +115,7 @@ func NewTransformerCountSimilar(
tr := &TransformerCountSimilar{
groupByFieldNames: groupByFieldNames,
counterFieldName: counterFieldName,
recordListsByGroup: lib.NewOrderedMap[*list.List](),
recordListsByGroup: lib.NewOrderedMap[*[]*types.RecordAndContext](),
}
return tr, nil
}
@ -125,7 +124,7 @@ func NewTransformerCountSimilar(
func (tr *TransformerCountSimilar) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -140,26 +139,26 @@ func (tr *TransformerCountSimilar) Transform(
recordListForGroup := tr.recordListsByGroup.Get(groupingKey)
if recordListForGroup == nil { // first time
recordListForGroup = list.New()
records := make([]*types.RecordAndContext, 0)
recordListForGroup = &records
tr.recordListsByGroup.Put(groupingKey, recordListForGroup)
}
recordListForGroup.PushBack(inrecAndContext)
*recordListForGroup = append(*recordListForGroup, inrecAndContext)
} else {
for outer := tr.recordListsByGroup.Head; outer != nil; outer = outer.Next {
recordListForGroup := outer.Value
// TODO: make 64-bit friendly
groupSize := recordListForGroup.Len()
groupSize := len(*recordListForGroup)
mgroupSize := mlrval.FromInt(int64(groupSize))
for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() {
recordAndContext := inner.Value.(*types.RecordAndContext)
for _, recordAndContext := range *recordListForGroup {
recordAndContext.Record.PutCopy(tr.counterFieldName, mgroupSize)
outputRecordsAndContexts.PushBack(recordAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext)
}
}
outputRecordsAndContexts.PushBack(inrecAndContext) // Emit the stream-terminating null record
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // Emit the stream-terminating null record
}
}

25
pkg/transformers/cut.go
View file

@ -2,7 +2,6 @@ package transformers
import (
"cmp"
"container/list"
"fmt"
"os"
"regexp"
@ -186,7 +185,7 @@ func NewTransformerCut(
func (tr *TransformerCut) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -198,7 +197,7 @@ func (tr *TransformerCut) Transform(
// mlr cut -f a,b,c
func (tr *TransformerCut) includeWithInputOrder(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -213,9 +212,9 @@ func (tr *TransformerCut) includeWithInputOrder(
}
}
outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context)
outputRecordsAndContexts.PushBack(outrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
@ -223,7 +222,7 @@ func (tr *TransformerCut) includeWithInputOrder(
// mlr cut -o -f a,b,c
func (tr *TransformerCut) includeWithArgOrder(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -237,9 +236,9 @@ func (tr *TransformerCut) includeWithArgOrder(
}
}
outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context)
outputRecordsAndContexts.PushBack(outrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
@ -247,7 +246,7 @@ func (tr *TransformerCut) includeWithArgOrder(
// mlr cut -x -f a,b,c
func (tr *TransformerCut) exclude(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -259,7 +258,7 @@ func (tr *TransformerCut) exclude(
}
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
type entryIndex struct {
@ -270,7 +269,7 @@ type entryIndex struct {
// ----------------------------------------------------------------
func (tr *TransformerCut) processWithRegexes(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -307,8 +306,8 @@ func (tr *TransformerCut) processWithRegexes(
newrec.PutReference(ei.entry.Key, ei.entry.Value)
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

7
pkg/transformers/decimate.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -142,7 +141,7 @@ func NewTransformerDecimate(
func (tr *TransformerDecimate) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -163,13 +162,13 @@ func (tr *TransformerDecimate) Transform(
remainder := countForGroup % tr.decimateCount
if remainder == tr.remainderToKeep {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
countForGroup++
tr.countsByGroup[groupingKey] = countForGroup
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // Emit the stream-terminating null record
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // Emit the stream-terminating null record
}
}

15
pkg/transformers/fill_down.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -148,7 +147,7 @@ func NewTransformerFillDown(
func (tr *TransformerFillDown) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -159,7 +158,7 @@ func (tr *TransformerFillDown) Transform(
// ----------------------------------------------------------------
func (tr *TransformerFillDown) transformSpecified(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -187,17 +186,17 @@ func (tr *TransformerFillDown) transformSpecified(
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerFillDown) transformAll(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -226,9 +225,9 @@ func (tr *TransformerFillDown) transformAll(
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

7
pkg/transformers/fill_empty.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -110,7 +109,7 @@ func NewTransformerFillEmpty(
func (tr *TransformerFillEmpty) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -124,9 +123,9 @@ func (tr *TransformerFillEmpty) Transform(
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else { // end of record stream
outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker
}
}

15
pkg/transformers/flatten.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -134,7 +133,7 @@ func NewTransformerFlatten(
func (tr *TransformerFlatten) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -145,7 +144,7 @@ func (tr *TransformerFlatten) Transform(
// ----------------------------------------------------------------
func (tr *TransformerFlatten) flattenAll(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -156,16 +155,16 @@ func (tr *TransformerFlatten) flattenAll(
oFlatSep = tr.options.WriterOptions.FLATSEP
}
inrec.Flatten(oFlatSep)
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerFlatten) flattenSome(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -176,8 +175,8 @@ func (tr *TransformerFlatten) flattenSome(
oFlatSep = tr.options.WriterOptions.FLATSEP
}
inrec.FlattenFields(tr.fieldNameSet, oFlatSep)
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

7
pkg/transformers/format_values.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -168,13 +167,13 @@ func NewTransformerFormatValues(
func (tr *TransformerFormatValues) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel)
if inrecAndContext.EndOfStream {
outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker
return
}
@ -199,5 +198,5 @@ func (tr *TransformerFormatValues) Transform(
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}

22
pkg/transformers/fraction.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"errors"
"fmt"
"os"
@ -132,7 +131,7 @@ type TransformerFraction struct {
groupByFieldNames []string
doCumu bool
recordsAndContexts *list.List
recordsAndContexts []*types.RecordAndContext
// Two-level map: Group-by field names are the first keyset;
// fraction field names are keys into the second.
sums map[string]map[string]*mlrval.Mlrval
@ -151,7 +150,7 @@ func NewTransformerFraction(
doCumu bool,
) (*TransformerFraction, error) {
recordsAndContexts := list.New()
recordsAndContexts := make([]*types.RecordAndContext, 0)
sums := make(map[string]map[string]*mlrval.Mlrval)
cumus := make(map[string]map[string]*mlrval.Mlrval)
@ -192,7 +191,7 @@ func NewTransformerFraction(
func (tr *TransformerFraction) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -201,7 +200,7 @@ func (tr *TransformerFraction) Transform(
inrec := inrecAndContext.Record
// Append records into a single output list (so that this verb is order-preserving).
tr.recordsAndContexts.PushBack(inrecAndContext)
tr.recordsAndContexts = append(tr.recordsAndContexts, inrecAndContext)
// Accumulate sums of fraction-field values grouped by group-by field names
groupingKey, hasAll := inrec.GetSelectedValuesJoined(tr.groupByFieldNames)
@ -234,13 +233,7 @@ func (tr *TransformerFraction) Transform(
// Iterate over the retained records, decorating them with fraction fields.
endOfStreamContext := inrecAndContext.Context
for {
element := tr.recordsAndContexts.Front()
if element == nil {
break
}
tr.recordsAndContexts.Remove(element)
recordAndContext := element.Value.(*types.RecordAndContext)
for _, recordAndContext := range tr.recordsAndContexts {
outrec := recordAndContext.Record
groupingKey, hasAll := outrec.GetSelectedValuesJoined(tr.groupByFieldNames)
@ -292,8 +285,9 @@ func (tr *TransformerFraction) Transform(
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &endOfStreamContext))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &endOfStreamContext))
}
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
tr.recordsAndContexts = tr.recordsAndContexts[:0]
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

19
pkg/transformers/gap.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -138,7 +137,7 @@ func NewTransformerGap(
func (tr *TransformerGap) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -148,27 +147,27 @@ func (tr *TransformerGap) Transform(
func (tr *TransformerGap) transformUnkeyed(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
if !inrecAndContext.EndOfStream {
if tr.recordCount > 0 && tr.recordCount%tr.gapCount == 0 {
newrec := mlrval.NewMlrmapAsRecord()
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
tr.recordCount++
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
func (tr *TransformerGap) transformKeyed(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -182,15 +181,15 @@ func (tr *TransformerGap) transformKeyed(
if groupingKey != tr.previousGroupingKey && tr.recordCount > 0 {
newrec := mlrval.NewMlrmapAsRecord()
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
tr.previousGroupingKey = groupingKey
tr.recordCount++
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

9
pkg/transformers/grep.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"regexp"
@ -155,7 +154,7 @@ func NewTransformerGrep(
func (tr *TransformerGrep) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -171,14 +170,14 @@ func (tr *TransformerGrep) Transform(
matches := tr.regexp.MatchString(inrecAsString)
if tr.invert {
if !matches {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
} else {
if matches {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

20
pkg/transformers/group_by.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -92,8 +91,8 @@ type TransformerGroupBy struct {
groupByFieldNames []string
// state
// map from string to *list.List
recordListsByGroup *lib.OrderedMap[*list.List]
// map from string to record slices
recordListsByGroup *lib.OrderedMap[*[]*types.RecordAndContext]
}
func NewTransformerGroupBy(
@ -103,7 +102,7 @@ func NewTransformerGroupBy(
tr := &TransformerGroupBy{
groupByFieldNames: groupByFieldNames,
recordListsByGroup: lib.NewOrderedMap[*list.List](),
recordListsByGroup: lib.NewOrderedMap[*[]*types.RecordAndContext](),
}
return tr, nil
@ -113,7 +112,7 @@ func NewTransformerGroupBy(
func (tr *TransformerGroupBy) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -128,19 +127,20 @@ func (tr *TransformerGroupBy) Transform(
recordListForGroup := tr.recordListsByGroup.Get(groupingKey)
if recordListForGroup == nil {
recordListForGroup = list.New()
records := make([]*types.RecordAndContext, 0)
recordListForGroup = &records
tr.recordListsByGroup.Put(groupingKey, recordListForGroup)
}
recordListForGroup.PushBack(inrecAndContext)
*recordListForGroup = append(*recordListForGroup, inrecAndContext)
} else {
for outer := tr.recordListsByGroup.Head; outer != nil; outer = outer.Next {
recordListForGroup := outer.Value
for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() {
outputRecordsAndContexts.PushBack(inner.Value.(*types.RecordAndContext))
for _, recordAndContext := range *recordListForGroup {
*outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext)
}
}
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

20
pkg/transformers/group_like.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -78,14 +77,14 @@ func transformerGroupLikeParseCLI(
// ----------------------------------------------------------------
type TransformerGroupLike struct {
// map from string to *list.List
recordListsByGroup *lib.OrderedMap[*list.List]
// map from string to record slices
recordListsByGroup *lib.OrderedMap[*[]*types.RecordAndContext]
}
func NewTransformerGroupLike() (*TransformerGroupLike, error) {
tr := &TransformerGroupLike{
recordListsByGroup: lib.NewOrderedMap[*list.List](),
recordListsByGroup: lib.NewOrderedMap[*[]*types.RecordAndContext](),
}
return tr, nil
@ -95,7 +94,7 @@ func NewTransformerGroupLike() (*TransformerGroupLike, error) {
func (tr *TransformerGroupLike) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -107,19 +106,20 @@ func (tr *TransformerGroupLike) Transform(
recordListForGroup := tr.recordListsByGroup.Get(groupingKey)
if recordListForGroup == nil { // first time
recordListForGroup = list.New()
records := make([]*types.RecordAndContext, 0)
recordListForGroup = &records
tr.recordListsByGroup.Put(groupingKey, recordListForGroup)
}
recordListForGroup.PushBack(inrecAndContext)
*recordListForGroup = append(*recordListForGroup, inrecAndContext)
} else {
for outer := tr.recordListsByGroup.Head; outer != nil; outer = outer.Next {
recordListForGroup := outer.Value
for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() {
outputRecordsAndContexts.PushBack(inner.Value.(*types.RecordAndContext))
for _, recordAndContext := range *recordListForGroup {
*outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext)
}
}
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

39
pkg/transformers/having_fields.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"regexp"
@ -221,7 +220,7 @@ func NewTransformerHavingFields(
func (tr *TransformerHavingFields) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -232,7 +231,7 @@ func (tr *TransformerHavingFields) Transform(
// ----------------------------------------------------------------
func (tr *TransformerHavingFields) transformHavingFieldsAtLeast(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -243,20 +242,20 @@ func (tr *TransformerHavingFields) transformHavingFieldsAtLeast(
if tr.fieldNameSet[pe.Key] {
numFound++
if numFound == tr.numFieldNames {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
return
}
}
}
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
func (tr *TransformerHavingFields) transformHavingFieldsWhichAre(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -270,15 +269,15 @@ func (tr *TransformerHavingFields) transformHavingFieldsWhichAre(
return
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
func (tr *TransformerHavingFields) transformHavingFieldsAtMost(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -289,16 +288,16 @@ func (tr *TransformerHavingFields) transformHavingFieldsAtMost(
return
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
// ----------------------------------------------------------------
func (tr *TransformerHavingFields) transformHavingAllFieldsMatching(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -309,15 +308,15 @@ func (tr *TransformerHavingFields) transformHavingAllFieldsMatching(
return
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
func (tr *TransformerHavingFields) transformHavingAnyFieldsMatching(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -325,18 +324,18 @@ func (tr *TransformerHavingFields) transformHavingAnyFieldsMatching(
inrec := inrecAndContext.Record
for pe := inrec.Head; pe != nil; pe = pe.Next {
if tr.regex.MatchString(pe.Key) {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
return
}
}
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
func (tr *TransformerHavingFields) transformHavingNoFieldsMatching(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -347,8 +346,8 @@ func (tr *TransformerHavingFields) transformHavingNoFieldsMatching(
return
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

17
pkg/transformers/head.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -143,7 +142,7 @@ func NewTransformerHead(
func (tr *TransformerHead) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -153,30 +152,30 @@ func (tr *TransformerHead) Transform(
func (tr *TransformerHead) transformUnkeyed(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
if !inrecAndContext.EndOfStream {
tr.unkeyedRecordCount++
if tr.unkeyedRecordCount <= tr.headCount {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else if !tr.wroteDownstreamDone {
// Signify to data producers upstream that we'll ignore further
// data, so as far as we're concerned they can stop sending it. See
// ChainTransformer.
//TODO: maybe remove: outputRecordsAndContexts.PushBack(types.NewEndOfStreamMarker(&inrecAndContext.Context))
//TODO: maybe remove: *outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewEndOfStreamMarker(&inrecAndContext.Context))
outputDownstreamDoneChannel <- true
tr.wroteDownstreamDone = true
}
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
func (tr *TransformerHead) transformKeyed(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -198,10 +197,10 @@ func (tr *TransformerHead) transformKeyed(
}
if count <= tr.headCount {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

19
pkg/transformers/histogram.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -198,7 +197,7 @@ func NewTransformerHistogram(
func (tr *TransformerHistogram) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -209,7 +208,7 @@ func (tr *TransformerHistogram) Transform(
// ----------------------------------------------------------------
func (tr *TransformerHistogram) transformNonAuto(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -217,7 +216,7 @@ func (tr *TransformerHistogram) transformNonAuto(
tr.ingestNonAuto(inrecAndContext)
} else {
tr.emitNonAuto(&inrecAndContext.Context, outputRecordsAndContexts)
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
@ -250,7 +249,7 @@ func (tr *TransformerHistogram) ingestNonAuto(
func (tr *TransformerHistogram) emitNonAuto(
endOfStreamContext *types.Context,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
countFieldNames := make(map[string]string)
for _, valueFieldName := range tr.valueFieldNames {
@ -275,14 +274,14 @@ func (tr *TransformerHistogram) emitNonAuto(
)
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, endOfStreamContext))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, endOfStreamContext))
}
}
// ----------------------------------------------------------------
func (tr *TransformerHistogram) transformAuto(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -290,7 +289,7 @@ func (tr *TransformerHistogram) transformAuto(
tr.ingestAuto(inrecAndContext)
} else {
tr.emitAuto(&inrecAndContext.Context, outputRecordsAndContexts)
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
@ -309,7 +308,7 @@ func (tr *TransformerHistogram) ingestAuto(
func (tr *TransformerHistogram) emitAuto(
endOfStreamContext *types.Context,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
haveLoHi := false
lo := 0.0
@ -381,6 +380,6 @@ func (tr *TransformerHistogram) emitAuto(
)
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, endOfStreamContext))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, endOfStreamContext))
}
}

56
pkg/transformers/join.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -296,7 +295,7 @@ type TransformerJoin struct {
// For unsorted/half-streaming input
ingested bool
leftBucketsByJoinFieldValues *lib.OrderedMap[*utils.JoinBucket]
leftUnpairableRecordsAndContexts *list.List
leftUnpairableRecordsAndContexts []*types.RecordAndContext
// For sorted/doubly-streaming input
joinBucketKeeper *utils.JoinBucketKeeper
@ -333,7 +332,7 @@ func NewTransformerJoin(
if opts.allowUnsortedInput {
// Half-streaming (default) case: ingest entire left file first.
tr.leftUnpairableRecordsAndContexts = list.New()
tr.leftUnpairableRecordsAndContexts = make([]*types.RecordAndContext, 0)
tr.leftBucketsByJoinFieldValues = lib.NewOrderedMap[*utils.JoinBucket]()
tr.recordTransformerFunc = tr.transformHalfStreaming
@ -361,7 +360,7 @@ func NewTransformerJoin(
func (tr *TransformerJoin) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -375,7 +374,7 @@ func (tr *TransformerJoin) Transform(
// matching each right record against those.
func (tr *TransformerJoin) transformHalfStreaming(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -398,7 +397,7 @@ func (tr *TransformerJoin) transformHalfStreaming(
leftBucket := tr.leftBucketsByJoinFieldValues.Get(groupingKey)
if leftBucket == nil {
if tr.opts.emitRightUnpairables {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
} else {
leftBucket.WasPaired = true
@ -411,7 +410,7 @@ func (tr *TransformerJoin) transformHalfStreaming(
}
}
} else if tr.opts.emitRightUnpairables {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
} else { // end of record stream
@ -419,14 +418,14 @@ func (tr *TransformerJoin) transformHalfStreaming(
tr.emitLeftUnpairedBuckets(outputRecordsAndContexts)
tr.emitLeftUnpairables(outputRecordsAndContexts)
}
outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerJoin) transformDoublyStreaming(
rightRecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -451,7 +450,7 @@ func (tr *TransformerJoin) transformDoublyStreaming(
lefts := keeper.JoinBucket.RecordsAndContexts // keystroke-saver
if !isPaired && tr.opts.emitRightUnpairables {
outputRecordsAndContexts.PushBack(rightRecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, rightRecAndContext)
}
if isPaired && tr.opts.emitPairables && lefts != nil {
@ -465,7 +464,7 @@ func (tr *TransformerJoin) transformDoublyStreaming(
keeper.OutputAndReleaseLeftUnpaireds(outputRecordsAndContexts)
}
outputRecordsAndContexts.PushBack(rightRecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, rightRecAndContext) // emit end-of-stream marker
}
}
@ -495,7 +494,7 @@ func (tr *TransformerJoin) ingestLeftFile() {
initialContext.UpdateForStartOfFile(tr.opts.leftFileName)
// Set up channels for the record-reader.
readerChannel := make(chan *list.List, 2) // list of *types.RecordAndContext
readerChannel := make(chan []*types.RecordAndContext, 2) // list of *types.RecordAndContext
errorChannel := make(chan error, 1)
downstreamDoneChannel := make(chan bool, 1)
@ -518,8 +517,8 @@ func (tr *TransformerJoin) ingestLeftFile() {
case leftrecsAndContexts := <-readerChannel:
// TODO: temp for batch-reader refactor
lib.InternalCodingErrorIf(leftrecsAndContexts.Len() != 1)
leftrecAndContext := leftrecsAndContexts.Front().Value.(*types.RecordAndContext)
lib.InternalCodingErrorIf(len(leftrecsAndContexts) != 1)
leftrecAndContext := leftrecsAndContexts[0]
leftrecAndContext.Record = utils.KeepLeftFieldNames(leftrecAndContext.Record, tr.leftKeepFieldNameSet)
if leftrecAndContext.EndOfStream {
@ -539,13 +538,13 @@ func (tr *TransformerJoin) ingestLeftFile() {
bucket := tr.leftBucketsByJoinFieldValues.Get(groupingKey)
if bucket == nil { // New key-field-value: new bucket and hash-map entry
bucket := utils.NewJoinBucket(leftFieldValues)
bucket.RecordsAndContexts.PushBack(leftrecAndContext)
bucket.RecordsAndContexts = append(bucket.RecordsAndContexts, leftrecAndContext)
tr.leftBucketsByJoinFieldValues.Put(groupingKey, bucket)
} else { // Previously seen key-field-value: append record to bucket
bucket.RecordsAndContexts.PushBack(leftrecAndContext)
bucket.RecordsAndContexts = append(bucket.RecordsAndContexts, leftrecAndContext)
}
} else {
tr.leftUnpairableRecordsAndContexts.PushBack(leftrecAndContext)
tr.leftUnpairableRecordsAndContexts = append(tr.leftUnpairableRecordsAndContexts, leftrecAndContext)
}
}
}
@ -556,15 +555,14 @@ func (tr *TransformerJoin) ingestLeftFile() {
// the doubly-streaming/sorted join.
func (tr *TransformerJoin) formAndEmitPairs(
leftRecordsAndContexts *list.List,
leftRecordsAndContexts []*types.RecordAndContext,
rightRecordAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
////fmt.Println("-- pairs start") // VERBOSE
// Loop over each to-be-paired-with record from the left file.
for pe := leftRecordsAndContexts.Front(); pe != nil; pe = pe.Next() {
for _, leftRecordAndContext := range leftRecordsAndContexts {
////fmt.Println("-- pairs pe") // VERBOSE
leftRecordAndContext := pe.Value.(*types.RecordAndContext)
leftrec := leftRecordAndContext.Record
rightrec := rightRecordAndContext.Record
@ -608,7 +606,7 @@ func (tr *TransformerJoin) formAndEmitPairs(
outrecAndContext := types.NewRecordAndContext(outrec, &context)
// Emit the new joined record on the downstream channel
outputRecordsAndContexts.PushBack(outrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
}
////fmt.Println("-- pairs end") // VERBOSE
}
@ -624,24 +622,22 @@ func (tr *TransformerJoin) formAndEmitPairs(
// in the second category.
func (tr *TransformerJoin) emitLeftUnpairables(
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
// Loop over each to-be-paired-with record from the left file.
for pe := tr.leftUnpairableRecordsAndContexts.Front(); pe != nil; pe = pe.Next() {
leftRecordAndContext := pe.Value.(*types.RecordAndContext)
outputRecordsAndContexts.PushBack(leftRecordAndContext)
for _, leftRecordAndContext := range tr.leftUnpairableRecordsAndContexts {
*outputRecordsAndContexts = append(*outputRecordsAndContexts, leftRecordAndContext)
}
}
func (tr *TransformerJoin) emitLeftUnpairedBuckets(
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
for pe := tr.leftBucketsByJoinFieldValues.Head; pe != nil; pe = pe.Next {
bucket := pe.Value
if !bucket.WasPaired {
for pf := bucket.RecordsAndContexts.Front(); pf != nil; pf = pf.Next() {
recordAndContext := pf.Value.(*types.RecordAndContext)
outputRecordsAndContexts.PushBack(recordAndContext)
for _, recordAndContext := range bucket.RecordsAndContexts {
*outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext)
}
}
}

15
pkg/transformers/json_parse.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -131,7 +130,7 @@ func NewTransformerJSONParse(
func (tr *TransformerJSONParse) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -142,7 +141,7 @@ func (tr *TransformerJSONParse) Transform(
// ----------------------------------------------------------------
func (tr *TransformerJSONParse) jsonParseAll(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -155,16 +154,16 @@ func (tr *TransformerJSONParse) jsonParseAll(
pe.JSONParseInPlace()
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerJSONParse) jsonParseSome(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -179,8 +178,8 @@ func (tr *TransformerJSONParse) jsonParseSome(
}
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

15
pkg/transformers/json_stringify.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -141,7 +140,7 @@ func NewTransformerJSONStringify(
func (tr *TransformerJSONStringify) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -152,7 +151,7 @@ func (tr *TransformerJSONStringify) Transform(
// ----------------------------------------------------------------
func (tr *TransformerJSONStringify) jsonStringifyAll(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -161,16 +160,16 @@ func (tr *TransformerJSONStringify) jsonStringifyAll(
for pe := inrec.Head; pe != nil; pe = pe.Next {
pe.JSONStringifyInPlace(tr.jsonFormatting)
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerJSONStringify) jsonStringifySome(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -181,8 +180,8 @@ func (tr *TransformerJSONStringify) jsonStringifySome(
pe.JSONStringifyInPlace(tr.jsonFormatting)
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

5
pkg/transformers/label.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -120,7 +119,7 @@ func NewTransformerLabel(
func (tr *TransformerLabel) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -129,5 +128,5 @@ func (tr *TransformerLabel) Transform(
inrec := inrecAndContext.Record
inrec.Label(tr.newNames)
}
outputRecordsAndContexts.PushBack(inrecAndContext) // including end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker
}

7
pkg/transformers/latin1_to_utf8.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -91,7 +90,7 @@ func NewTransformerLatin1ToUTF8() (*TransformerLatin1ToUTF8, error) {
func (tr *TransformerLatin1ToUTF8) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -111,9 +110,9 @@ func (tr *TransformerLatin1ToUTF8) Transform(
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(inrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(inrec, &inrecAndContext.Context))
} else { // end of record stream
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

21
pkg/transformers/merge_fields.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"regexp"
@ -312,7 +311,7 @@ func NewTransformerMergeFields(
func (tr *TransformerMergeFields) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -323,12 +322,12 @@ func (tr *TransformerMergeFields) Transform(
// ----------------------------------------------------------------
func (tr *TransformerMergeFields) transformByNameList(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
if inrecAndContext.EndOfStream {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
return
}
@ -368,18 +367,18 @@ func (tr *TransformerMergeFields) transformByNameList(
inrec.PutReference(key, value)
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
// ----------------------------------------------------------------
func (tr *TransformerMergeFields) transformByNameRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
if inrecAndContext.EndOfStream {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
return
}
@ -443,7 +442,7 @@ func (tr *TransformerMergeFields) transformByNameRegex(
inrec.PutReference(key, value)
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
// ----------------------------------------------------------------
@ -455,12 +454,12 @@ func (tr *TransformerMergeFields) transformByNameRegex(
func (tr *TransformerMergeFields) transformByCollapsing(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
if inrecAndContext.EndOfStream {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
return
}
@ -549,5 +548,5 @@ func (tr *TransformerMergeFields) transformByCollapsing(
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}

7
pkg/transformers/most_or_least_frequent.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"sort"
@ -205,7 +204,7 @@ func NewTransformerMostOrLeastFrequent(
func (tr *TransformerMostOrLeastFrequent) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -280,9 +279,9 @@ func (tr *TransformerMostOrLeastFrequent) Transform(
if tr.showCounts {
outrec.PutReference(tr.outputFieldName, mlrval.FromInt(sortPairs[i].count))
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
}
outputRecordsAndContexts.PushBack(inrecAndContext) // End-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // End-of-stream marker
}
}

58
pkg/transformers/nest.go
View file

@ -2,7 +2,6 @@ package transformers
import (
"bytes"
"container/list"
"fmt"
"os"
"regexp"
@ -317,7 +316,7 @@ func NewTransformerNest(
func (tr *TransformerNest) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -328,7 +327,7 @@ func (tr *TransformerNest) Transform(
// ----------------------------------------------------------------
func (tr *TransformerNest) explodeValuesAcrossFields(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -337,7 +336,7 @@ func (tr *TransformerNest) explodeValuesAcrossFields(
inrec := inrecAndContext.Record
originalEntry := inrec.GetEntry(tr.fieldName)
if originalEntry == nil {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
return
}
@ -356,17 +355,17 @@ func (tr *TransformerNest) explodeValuesAcrossFields(
}
inrec.Unlink(originalEntry)
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerNest) explodeValuesAcrossRecords(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -374,7 +373,7 @@ func (tr *TransformerNest) explodeValuesAcrossRecords(
inrec := inrecAndContext.Record
mvalue := inrec.Get(tr.fieldName)
if mvalue == nil {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
return
}
svalue := mvalue.String()
@ -384,18 +383,18 @@ func (tr *TransformerNest) explodeValuesAcrossRecords(
for _, piece := range pieces {
outrec := inrec.Copy()
outrec.PutReference(tr.fieldName, mlrval.FromString(piece))
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
}
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerNest) explodePairsAcrossFields(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -404,7 +403,7 @@ func (tr *TransformerNest) explodePairsAcrossFields(
inrec := inrecAndContext.Record
originalEntry := inrec.GetEntry(tr.fieldName)
if originalEntry == nil {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
return
}
@ -431,17 +430,17 @@ func (tr *TransformerNest) explodePairsAcrossFields(
}
inrec.Unlink(originalEntry)
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerNest) explodePairsAcrossRecords(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -449,7 +448,7 @@ func (tr *TransformerNest) explodePairsAcrossRecords(
inrec := inrecAndContext.Record
mvalue := inrec.Get(tr.fieldName)
if mvalue == nil {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
return
}
@ -470,18 +469,18 @@ func (tr *TransformerNest) explodePairsAcrossRecords(
}
outrec.Unlink(originalEntry)
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
}
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerNest) implodeValuesAcrossFields(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -520,17 +519,17 @@ func (tr *TransformerNest) implodeValuesAcrossFields(
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerNest) implodeValueAcrossRecords(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -539,7 +538,7 @@ func (tr *TransformerNest) implodeValueAcrossRecords(
originalEntry := inrec.GetEntry(tr.fieldName)
if originalEntry == nil {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
return
}
@ -566,7 +565,7 @@ func (tr *TransformerNest) implodeValueAcrossRecords(
pair := mlrval.NewMlrmapAsRecord()
pair.PutReference(tr.fieldName, fieldValueCopy)
bucket.pairs.PushBack(pair)
bucket.pairs = append(bucket.pairs, pair)
} else { // end of input stream
@ -579,8 +578,7 @@ func (tr *TransformerNest) implodeValueAcrossRecords(
bucket.representative = nil // ownership transfer
i := 0
for pg := bucket.pairs.Front(); pg != nil; pg = pg.Next() {
pr := pg.Value.(*mlrval.Mlrmap)
for _, pr := range bucket.pairs {
if i > 0 {
buffer.WriteString(tr.nestedFS)
}
@ -590,22 +588,22 @@ func (tr *TransformerNest) implodeValueAcrossRecords(
// tr.fieldName was already present so we'll overwrite it in-place here.
outrec.PutReference(tr.fieldName, mlrval.FromString(buffer.String()))
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
}
}
outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker
}
}
type tNestBucket struct {
representative *mlrval.Mlrmap
pairs *list.List
pairs []*mlrval.Mlrmap
}
func newNestBucket(representative *mlrval.Mlrmap) *tNestBucket {
return &tNestBucket{
representative: representative,
pairs: list.New(),
pairs: make([]*mlrval.Mlrmap, 0),
}
}

5
pkg/transformers/nothing.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -87,12 +86,12 @@ func NewTransformerNothing() (*TransformerNothing, error) {
func (tr *TransformerNothing) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel)
if inrecAndContext.EndOfStream {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

7
pkg/transformers/put_or_filter.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -500,7 +499,7 @@ func NewTransformerPut(
func (tr *TransformerPut) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -566,7 +565,7 @@ func (tr *TransformerPut) Transform(
wantToEmit := lib.BooleanXOR(filterBool, tr.invertFilter)
if wantToEmit {
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &context))
}
}
@ -594,6 +593,6 @@ func (tr *TransformerPut) Transform(
// indicator.
tr.cstRootNode.ProcessEndOfStream()
outputRecordsAndContexts.PushBack(types.NewEndOfStreamMarker(&context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewEndOfStreamMarker(&context))
}
}

9
pkg/transformers/regularize.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -94,7 +93,7 @@ func NewTransformerRegularize() (*TransformerRegularize, error) {
func (tr *TransformerRegularize) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -107,16 +106,16 @@ func (tr *TransformerRegularize) Transform(
previousSortedFieldNames := tr.sortedToOriginal[currentSortedFieldNamesJoined]
if previousSortedFieldNames == nil {
tr.sortedToOriginal[currentSortedFieldNamesJoined] = currentFieldNames
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outrec := mlrval.NewMlrmapAsRecord()
for _, fieldName := range previousSortedFieldNames {
outrec.PutReference(fieldName, inrec.Get(fieldName)) // inrec will be GC'ed
}
outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context)
outputRecordsAndContexts.PushBack(outrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
}
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

16
pkg/transformers/remove_empty_columns.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -78,13 +77,13 @@ func transformerRemoveEmptyColumnsParseCLI(
// ----------------------------------------------------------------
type TransformerRemoveEmptyColumns struct {
recordsAndContexts *list.List
recordsAndContexts []*types.RecordAndContext
namesWithNonEmptyValues map[string]bool
}
func NewTransformerRemoveEmptyColumns() (*TransformerRemoveEmptyColumns, error) {
tr := &TransformerRemoveEmptyColumns{
recordsAndContexts: list.New(),
recordsAndContexts: make([]*types.RecordAndContext, 0),
namesWithNonEmptyValues: make(map[string]bool),
}
return tr, nil
@ -94,14 +93,14 @@ func NewTransformerRemoveEmptyColumns() (*TransformerRemoveEmptyColumns, error)
func (tr *TransformerRemoveEmptyColumns) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel)
if !inrecAndContext.EndOfStream {
inrec := inrecAndContext.Record
tr.recordsAndContexts.PushBack(inrecAndContext)
tr.recordsAndContexts = append(tr.recordsAndContexts, inrecAndContext)
for pe := inrec.Head; pe != nil; pe = pe.Next {
if !pe.Value.IsVoid() {
@ -111,8 +110,7 @@ func (tr *TransformerRemoveEmptyColumns) Transform(
} else { // end of record stream
for e := tr.recordsAndContexts.Front(); e != nil; e = e.Next() {
outrecAndContext := e.Value.(*types.RecordAndContext)
for _, outrecAndContext := range tr.recordsAndContexts {
outrec := outrecAndContext.Record
newrec := mlrval.NewMlrmapAsRecord()
@ -125,9 +123,9 @@ func (tr *TransformerRemoveEmptyColumns) Transform(
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &outrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &outrecAndContext.Context))
}
outputRecordsAndContexts.PushBack(inrecAndContext) // Emit the stream-terminating null record
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // Emit the stream-terminating null record
}
}

22
pkg/transformers/rename.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"regexp"
@ -135,7 +134,7 @@ type tRegexAndReplacement struct {
type TransformerRename struct {
oldToNewNames *lib.OrderedMap[string]
regexesAndReplacements *list.List
regexesAndReplacements []*tRegexAndReplacement
doGsub bool
recordTransformerFunc RecordTransformerFunc
}
@ -164,7 +163,7 @@ func NewTransformerRename(
tr.doGsub = false
tr.recordTransformerFunc = tr.transformWithoutRegexes
} else {
tr.regexesAndReplacements = list.New()
tr.regexesAndReplacements = make([]*tRegexAndReplacement, 0)
for pe := oldToNewNames.Head; pe != nil; pe = pe.Next {
regexString := pe.Key
regex := lib.CompileMillerRegexOrDie(regexString)
@ -175,7 +174,7 @@ func NewTransformerRename(
replacement: replacement,
replacementCaptureMatrix: replacementCaptureMatrix,
}
tr.regexesAndReplacements.PushBack(&regexAndReplacement)
tr.regexesAndReplacements = append(tr.regexesAndReplacements, &regexAndReplacement)
}
tr.doGsub = doGsub
tr.recordTransformerFunc = tr.transformWithRegexes
@ -188,7 +187,7 @@ func NewTransformerRename(
func (tr *TransformerRename) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -199,7 +198,7 @@ func (tr *TransformerRename) Transform(
// ----------------------------------------------------------------
func (tr *TransformerRename) transformWithoutRegexes(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -214,21 +213,20 @@ func (tr *TransformerRename) transformWithoutRegexes(
}
}
outputRecordsAndContexts.PushBack(inrecAndContext) // including end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker
}
// ----------------------------------------------------------------
func (tr *TransformerRename) transformWithRegexes(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
if !inrecAndContext.EndOfStream {
inrec := inrecAndContext.Record
for pr := tr.regexesAndReplacements.Front(); pr != nil; pr = pr.Next() {
regexAndReplacement := pr.Value.(*tRegexAndReplacement)
for _, regexAndReplacement := range tr.regexesAndReplacements {
regex := regexAndReplacement.regex
replacement := regexAndReplacement.replacement
replacementCaptureMatrix := regexAndReplacement.replacementCaptureMatrix
@ -249,8 +247,8 @@ func (tr *TransformerRename) transformWithRegexes(
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // including end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker
}
}

47
pkg/transformers/reorder.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"regexp"
@ -202,7 +201,7 @@ func NewTransformerReorder(
func (tr *TransformerReorder) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -213,29 +212,29 @@ func (tr *TransformerReorder) Transform(
outputRecordsAndContexts,
)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
func (tr *TransformerReorder) reorderToStartNoRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
inrec := inrecAndContext.Record
for _, fieldName := range tr.fieldNames {
inrec.MoveToHead(fieldName)
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
func (tr *TransformerReorder) reorderToStartWithRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
inrec := inrecAndContext.Record
outrec := mlrval.NewMlrmapAsRecord()
atEnds := list.New()
atEnds := make([]*mlrval.MlrmapEntry, 0)
for pe := inrec.Head; pe != nil; pe = pe.Next {
found := false
for _, regex := range tr.regexes {
@ -246,44 +245,43 @@ func (tr *TransformerReorder) reorderToStartWithRegex(
}
}
if !found {
atEnds.PushBack(pe)
atEnds = append(atEnds, pe)
}
}
for atEnd := atEnds.Front(); atEnd != nil; atEnd = atEnd.Next() {
for _, pe := range atEnds {
// Ownership transfer; no copy needed
pe := atEnd.Value.(*mlrval.MlrmapEntry)
outrec.PutReference(pe.Key, pe.Value)
}
outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context)
outputRecordsAndContexts.PushBack(outrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
}
func (tr *TransformerReorder) reorderToEndNoRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
inrec := inrecAndContext.Record
for _, fieldName := range tr.fieldNames {
inrec.MoveToTail(fieldName)
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
func (tr *TransformerReorder) reorderToEndWithRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
inrec := inrecAndContext.Record
outrec := mlrval.NewMlrmapAsRecord()
atEnds := list.New()
atEnds := make([]*mlrval.MlrmapEntry, 0)
for pe := inrec.Head; pe != nil; pe = pe.Next {
found := false
for _, regex := range tr.regexes {
if regex.MatchString(pe.Key) {
atEnds.PushBack(pe)
atEnds = append(atEnds, pe)
found = true
break
}
@ -293,23 +291,22 @@ func (tr *TransformerReorder) reorderToEndWithRegex(
}
}
for atEnd := atEnds.Front(); atEnd != nil; atEnd = atEnd.Next() {
for _, pe := range atEnds {
// Ownership transfer; no copy needed
pe := atEnd.Value.(*mlrval.MlrmapEntry)
outrec.PutReference(pe.Key, pe.Value)
}
outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context)
outputRecordsAndContexts.PushBack(outrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
}
func (tr *TransformerReorder) reorderBeforeOrAfterNoRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
inrec := inrecAndContext.Record
if inrec.Get(tr.centerFieldName) == nil {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
return
}
@ -356,17 +353,17 @@ func (tr *TransformerReorder) reorderBeforeOrAfterNoRegex(
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
}
func (tr *TransformerReorder) reorderBeforeOrAfterWithRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
inrec := inrecAndContext.Record
if inrec.Get(tr.centerFieldName) == nil {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
return
}
@ -400,5 +397,5 @@ func (tr *TransformerReorder) reorderBeforeOrAfterWithRegex(
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
}

15
pkg/transformers/repeat.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -159,7 +158,7 @@ func NewTransformerRepeat(
func (tr *TransformerRepeat) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -170,26 +169,26 @@ func (tr *TransformerRepeat) Transform(
// ----------------------------------------------------------------
func (tr *TransformerRepeat) repeatByCount(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
if !inrecAndContext.EndOfStream {
for i := int64(0); i < tr.repeatCount; i++ {
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(
inrecAndContext.Record.Copy(),
&inrecAndContext.Context,
))
}
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
// ----------------------------------------------------------------
func (tr *TransformerRepeat) repeatByFieldName(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -203,13 +202,13 @@ func (tr *TransformerRepeat) repeatByFieldName(
return
}
for i := 0; i < int(repeatCount); i++ {
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(
inrecAndContext.Record.Copy(),
&inrecAndContext.Context,
))
}
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

27
pkg/transformers/reshape.go
View file

@ -28,7 +28,6 @@ package transformers
// 15 2009-01-05 Z 0.09719105
import (
"container/list"
"fmt"
"os"
"regexp"
@ -292,7 +291,7 @@ func NewTransformerReshape(
func (tr *TransformerReshape) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -303,7 +302,7 @@ func (tr *TransformerReshape) Transform(
// ----------------------------------------------------------------
func (tr *TransformerReshape) wideToLongNoRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -324,25 +323,25 @@ func (tr *TransformerReshape) wideToLongNoRegex(
}
if pairs.IsEmpty() {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
for pf := pairs.Head; pf != nil; pf = pf.Next {
outrec := inrec.Copy()
outrec.PutReference(tr.outputKeyFieldName, mlrval.FromString(pf.Key))
outrec.PutReference(tr.outputValueFieldName, pf.Value)
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
}
}
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerReshape) wideToLongRegex(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -366,25 +365,25 @@ func (tr *TransformerReshape) wideToLongRegex(
}
if pairs.IsEmpty() {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
for pf := pairs.Head; pf != nil; pf = pf.Next {
outrec := inrec.Copy()
outrec.PutReference(tr.outputKeyFieldName, mlrval.FromString(pf.Key))
outrec.PutReference(tr.outputValueFieldName, pf.Value)
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
}
}
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerReshape) longToWide(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -394,7 +393,7 @@ func (tr *TransformerReshape) longToWide(
splitOutKeyFieldValue := inrec.Get(tr.splitOutKeyFieldName)
splitOutValueFieldValue := inrec.Get(tr.splitOutValueFieldName)
if splitOutKeyFieldValue == nil || splitOutValueFieldValue == nil {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
return
}
@ -436,11 +435,11 @@ func (tr *TransformerReshape) longToWide(
outrec.PutReference(pg.Key, pg.Value)
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
}
}
outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker
}
}

7
pkg/transformers/sample.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -128,7 +127,7 @@ func NewTransformerSample(
func (tr *TransformerSample) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -151,13 +150,13 @@ func (tr *TransformerSample) Transform(
for pe := tr.bucketsByGroup.Head; pe != nil; pe = pe.Next {
sampleBucket := pe.Value
for i := int64(0); i < sampleBucket.nused; i++ {
outputRecordsAndContexts.PushBack(sampleBucket.recordsAndContexts[i])
*outputRecordsAndContexts = append(*outputRecordsAndContexts, sampleBucket.recordsAndContexts[i])
}
}
// Emit the stream-terminating null record
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

7
pkg/transformers/sec2gmt.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
@ -149,7 +148,7 @@ func NewTransformerSec2GMT(
func (tr *TransformerSec2GMT) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -169,9 +168,9 @@ func (tr *TransformerSec2GMT) Transform(
}
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else { // End of record stream
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

7
pkg/transformers/sec2gmtdate.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
@ -106,7 +105,7 @@ func NewTransformerSec2GMTDate(
func (tr *TransformerSec2GMTDate) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -119,9 +118,9 @@ func (tr *TransformerSec2GMTDate) Transform(
inrec.PutReference(fieldName, bifs.BIF_sec2gmtdate(value))
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else { // End of record stream
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

7
pkg/transformers/seqgen.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -172,7 +171,7 @@ func NewTransformerSeqgen(
func (tr *TransformerSeqgen) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -210,10 +209,10 @@ func (tr *TransformerSeqgen) Transform(
context.UpdateForInputRecord()
outrecAndContext := types.NewRecordAndContext(outrec, context)
outputRecordsAndContexts.PushBack(outrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
counter = bifs.BIF_plus_binary(counter, tr.step)
}
outputRecordsAndContexts.PushBack(types.NewEndOfStreamMarker(context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewEndOfStreamMarker(context))
}

28
pkg/transformers/shuffle.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -81,13 +80,13 @@ func transformerShuffleParseCLI(
// ----------------------------------------------------------------
type TransformerShuffle struct {
recordsAndContexts *list.List
recordsAndContexts []*types.RecordAndContext
}
func NewTransformerShuffle() (*TransformerShuffle, error) {
tr := &TransformerShuffle{
recordsAndContexts: list.New(),
recordsAndContexts: make([]*types.RecordAndContext, 0),
}
return tr, nil
@ -97,21 +96,20 @@ func NewTransformerShuffle() (*TransformerShuffle, error) {
func (tr *TransformerShuffle) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel)
// Not end of input stream: retain the record, and emit nothing until end of stream.
if !inrecAndContext.EndOfStream {
tr.recordsAndContexts.PushBack(inrecAndContext)
tr.recordsAndContexts = append(tr.recordsAndContexts, inrecAndContext)
} else { // end of record stream
// Knuth shuffle:
// * Initial permutation is identity.
// * Make a pseudorandom permutation using pseudorandom swaps in the image map.
// TODO: Go list Len() maxes at 2^31. We should track this ourselves in an int64.
n := int64(tr.recordsAndContexts.Len())
n := int64(len(tr.recordsAndContexts))
images := make([]int64, n)
for i := int64(0); i < n; i++ {
images[i] = i
@ -130,25 +128,17 @@ func (tr *TransformerShuffle) Transform(
numUnused--
}
// Move the record-pointers from linked list to array.
array := make([]*types.RecordAndContext, n)
for i := int64(0); i < n; i++ {
head := tr.recordsAndContexts.Front()
if head == nil {
break
}
array[i] = head.Value.(*types.RecordAndContext)
tr.recordsAndContexts.Remove(head)
}
// Move the record-pointers from slice to array.
array := tr.recordsAndContexts
// Transfer from input array to output list. Because permutations are one-to-one maps,
// all input records have ownership transferred exactly once. So, there are no
// records to copy here.
for i := int64(0); i < n; i++ {
outputRecordsAndContexts.PushBack(array[images[i]])
*outputRecordsAndContexts = append(*outputRecordsAndContexts, array[images[i]])
}
// Emit the stream-terminating null record
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

7
pkg/transformers/skip_trivial_records.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -89,7 +88,7 @@ func NewTransformerSkipTrivialRecords() (*TransformerSkipTrivialRecords, error)
func (tr *TransformerSkipTrivialRecords) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -105,10 +104,10 @@ func (tr *TransformerSkipTrivialRecords) Transform(
}
if hasAny {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

30
pkg/transformers/sort.go
View file

@ -42,7 +42,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"sort"
@ -313,11 +312,11 @@ type TransformerSort struct {
doMoveToHead bool
// -- State
// Map from string to *list.List:
recordListsByGroup *lib.OrderedMap[*list.List]
// Map from string to record slices:
recordListsByGroup *lib.OrderedMap[*[]*types.RecordAndContext]
// Map from string to []*lib.Mlrval:
groupHeads *lib.OrderedMap[[]*mlrval.Mlrval]
spillGroup *list.List // e.g. sort by field "a" -- this is for records lacking a field named "a"
spillGroup []*types.RecordAndContext // e.g. sort by field "a" -- this is for records lacking a field named "a"
}
func NewTransformerSort(
@ -331,9 +330,9 @@ func NewTransformerSort(
comparatorFuncs: comparatorFuncs,
doMoveToHead: doMoveToHead,
recordListsByGroup: lib.NewOrderedMap[*list.List](),
recordListsByGroup: lib.NewOrderedMap[*[]*types.RecordAndContext](),
groupHeads: lib.NewOrderedMap[[]*mlrval.Mlrval](),
spillGroup: list.New(),
spillGroup: make([]*types.RecordAndContext, 0),
}
return tr, nil
@ -347,7 +346,7 @@ type GroupingKeysAndMlrvals struct {
func (tr *TransformerSort) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -366,18 +365,19 @@ func (tr *TransformerSort) Transform(
tr.groupByFieldNames,
)
if !ok {
tr.spillGroup.PushBack(inrecAndContext)
tr.spillGroup = append(tr.spillGroup, inrecAndContext)
return
}
recordListForGroup := tr.recordListsByGroup.Get(groupingKey)
if recordListForGroup == nil {
recordListForGroup = list.New()
records := make([]*types.RecordAndContext, 0)
recordListForGroup = &records
tr.recordListsByGroup.Put(groupingKey, recordListForGroup)
tr.groupHeads.Put(groupingKey, selectedValues)
}
recordListForGroup.PushBack(inrecAndContext)
*recordListForGroup = append(*recordListForGroup, inrecAndContext)
} else { // End of record stream
@ -416,16 +416,16 @@ func (tr *TransformerSort) Transform(
// Now output the groups
for _, groupingKeyAndMlrvals := range groupingKeysAndMlrvals {
recordsInGroup := tr.recordListsByGroup.Get(groupingKeyAndMlrvals.groupingKey)
for iRecord := recordsInGroup.Front(); iRecord != nil; iRecord = iRecord.Next() {
outputRecordsAndContexts.PushBack(iRecord.Value.(*types.RecordAndContext))
for _, recordAndContext := range *recordsInGroup {
*outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext)
}
}
for iRecord := tr.spillGroup.Front(); iRecord != nil; iRecord = iRecord.Next() {
outputRecordsAndContexts.PushBack(iRecord.Value.(*types.RecordAndContext))
for _, recordAndContext := range tr.spillGroup {
*outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext)
}
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

11
pkg/transformers/sort_within_records.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -106,7 +105,7 @@ func NewTransformerSortWithinRecords(
func (tr *TransformerSortWithinRecords) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -117,7 +116,7 @@ func (tr *TransformerSortWithinRecords) Transform(
// ----------------------------------------------------------------
func (tr *TransformerSortWithinRecords) transformNonrecursively(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -125,13 +124,13 @@ func (tr *TransformerSortWithinRecords) transformNonrecursively(
inrec := inrecAndContext.Record
inrec.SortByKey()
}
outputRecordsAndContexts.PushBack(inrecAndContext) // including end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker
}
// ----------------------------------------------------------------
func (tr *TransformerSortWithinRecords) transformRecursively(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -139,5 +138,5 @@ func (tr *TransformerSortWithinRecords) transformRecursively(
inrec := inrecAndContext.Record
inrec.SortByKeyRecursively()
}
outputRecordsAndContexts.PushBack(inrecAndContext) // including end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // including end-of-stream marker
}

13
pkg/transformers/sparsify.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -128,7 +127,7 @@ func NewTransformerSparsify(
func (tr *TransformerSparsify) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -142,13 +141,13 @@ func (tr *TransformerSparsify) Transform(
outputDownstreamDoneChannel,
)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
func (tr *TransformerSparsify) transformAll(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -163,13 +162,13 @@ func (tr *TransformerSparsify) transformAll(
}
outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context)
outputRecordsAndContexts.PushBack(outrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
}
// ----------------------------------------------------------------
func (tr *TransformerSparsify) transformSome(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -188,5 +187,5 @@ func (tr *TransformerSparsify) transformSome(
}
outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context)
outputRecordsAndContexts.PushBack(outrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
}

21
pkg/transformers/split.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"net/url"
"os"
@ -273,7 +272,7 @@ func NewTransformerSplit(
func (tr *TransformerSplit) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -284,7 +283,7 @@ func (tr *TransformerSplit) Transform(
func (tr *TransformerSplit) splitModUngrouped(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -299,13 +298,13 @@ func (tr *TransformerSplit) splitModUngrouped(
}
if tr.emitDownstream {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
tr.ungroupedCounter++
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
errs := tr.outputHandlerManager.Close()
if len(errs) > 0 {
for _, err := range errs {
@ -318,7 +317,7 @@ func (tr *TransformerSplit) splitModUngrouped(
func (tr *TransformerSplit) splitSizeUngrouped(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -356,13 +355,13 @@ func (tr *TransformerSplit) splitSizeUngrouped(
}
if tr.emitDownstream {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
tr.ungroupedCounter++
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
if tr.outputHandler != nil {
err := tr.outputHandler.Close()
@ -376,7 +375,7 @@ func (tr *TransformerSplit) splitSizeUngrouped(
func (tr *TransformerSplit) splitGrouped(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -395,11 +394,11 @@ func (tr *TransformerSplit) splitGrouped(
}
if tr.emitDownstream {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker
errs := tr.outputHandlerManager.Close()
if len(errs) > 0 {

15
pkg/transformers/stats1.go
View file

@ -2,7 +2,6 @@ package transformers
import (
"bytes"
"container/list"
"fmt"
"os"
"regexp"
@ -353,7 +352,7 @@ func NewTransformerStats1(
// the end-of-stream marker.
func (tr *TransformerStats1) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -367,7 +366,7 @@ func (tr *TransformerStats1) Transform(
func (tr *TransformerStats1) handleInputRecord(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
inrec := inrecAndContext.Record
@ -416,7 +415,7 @@ func (tr *TransformerStats1) handleInputRecord(
level2,
inrec,
)
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
@ -596,10 +595,10 @@ func (tr *TransformerStats1) matchValueFieldName(
func (tr *TransformerStats1) handleEndOfRecordStream(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
if tr.doIterativeStats {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
return
}
@ -617,10 +616,10 @@ func (tr *TransformerStats1) handleEndOfRecordStream(
newrec,
)
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
}
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
func (tr *TransformerStats1) emitIntoOutputRecord(

32
pkg/transformers/stats2.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -194,7 +193,7 @@ type TransformerStats2 struct {
// For hold-and-fit:
// ordered map from grouping-key to list of RecordAndContext
recordGroups *lib.OrderedMap[*list.List]
recordGroups *lib.OrderedMap[*[]*types.RecordAndContext]
}
func NewTransformerStats2(
@ -221,7 +220,7 @@ func NewTransformerStats2(
accumulatorFactory: utils.NewStats2AccumulatorFactory(),
namedAccumulators: lib.NewOrderedMap[*lib.OrderedMap[*lib.OrderedMap[utils.IStats2Accumulator]]](),
groupingKeysToGroupByFieldValues: lib.NewOrderedMap[[]*mlrval.Mlrval](),
recordGroups: lib.NewOrderedMap[*list.List](),
recordGroups: lib.NewOrderedMap[*[]*types.RecordAndContext](),
}
return tr, nil
}
@ -265,7 +264,7 @@ func NewTransformerStats2(
func (tr *TransformerStats2) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -276,7 +275,7 @@ func (tr *TransformerStats2) Transform(
if tr.doIterativeStats {
// The input record is modified in this case, with new fields appended
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
// if tr.doHoldAndFit, the input record is held by the ingestor
@ -288,7 +287,7 @@ func (tr *TransformerStats2) Transform(
tr.emit(outputRecordsAndContexts, &inrecAndContext.Context)
}
}
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
@ -316,10 +315,11 @@ func (tr *TransformerStats2) ingest(
if tr.doHoldAndFit { // Retain the input record in memory, for fitting and delivery at end of stream
groupToRecords := tr.recordGroups.Get(groupingKey)
if groupToRecords == nil {
groupToRecords = list.New()
records := make([]*types.RecordAndContext, 0)
groupToRecords = &records
tr.recordGroups.Put(groupingKey, groupToRecords)
}
groupToRecords.PushBack(inrecAndContext)
*groupToRecords = append(*groupToRecords, inrecAndContext)
}
// for [["x","y"]]
@ -382,7 +382,7 @@ func (tr *TransformerStats2) ingest(
// ----------------------------------------------------------------
func (tr *TransformerStats2) emit(
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
context *types.Context,
) {
for pa := tr.namedAccumulators.Head; pa != nil; pa = pa.Next {
@ -415,7 +415,7 @@ func (tr *TransformerStats2) emit(
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, context))
}
}
@ -433,15 +433,16 @@ func (tr *TransformerStats2) populateRecord(
}
func (tr *TransformerStats2) fit(
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
for pa := tr.namedAccumulators.Head; pa != nil; pa = pa.Next {
groupingKey := pa.Key
groupToValueFields := pa.Value
recordsAndContexts := tr.recordGroups.Get(groupingKey)
for recordsAndContexts.Front() != nil {
recordAndContext := recordsAndContexts.Remove(recordsAndContexts.Front()).(*types.RecordAndContext)
if recordsAndContexts == nil {
continue
}
for _, recordAndContext := range *recordsAndContexts {
record := recordAndContext.Record
// For "x","y"
@ -468,7 +469,8 @@ func (tr *TransformerStats2) fit(
}
}
outputRecordsAndContexts.PushBack(recordAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext)
}
*recordsAndContexts = (*recordsAndContexts)[:0]
}
}

15
pkg/transformers/step.go
View file

@ -68,7 +68,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -350,7 +349,7 @@ func NewTransformerStep(
func (tr *TransformerStep) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -370,7 +369,7 @@ func (tr *TransformerStep) Transform(
tr.handleDrainRecord(logEntry, outputRecordsAndContexts)
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
return
}
}
@ -381,7 +380,7 @@ func (tr *TransformerStep) Transform(
// delayed-input records in the order in which they were received.
func (tr *TransformerStep) handleRecord(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
inrec := inrecAndContext.Record
@ -390,7 +389,7 @@ func (tr *TransformerStep) handleRecord(
// Grouping key is "s,t"
groupingKey, gok := inrec.GetSelectedValuesJoined(tr.groupByFieldNames)
if !gok { // current record doesn't have fields to be stepped; pass it along
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
return
}
@ -455,7 +454,7 @@ func (tr *TransformerStep) handleRecord(
if windowKeeper.Get(0) != nil {
outrecAndContext := windowKeeper.Get(0).(*types.RecordAndContext)
outputRecordsAndContexts.PushBack(outrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
tr.removeFromLog(outrecAndContext)
}
}
@ -466,7 +465,7 @@ func (tr *TransformerStep) handleRecord(
// delayed-input records in the order in which they were received.
func (tr *TransformerStep) handleDrainRecord(
logEntry *tStepLogEntry,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
inrecAndContext := logEntry.recordAndContext
inrec := inrecAndContext.Record
@ -496,7 +495,7 @@ func (tr *TransformerStep) handleDrainRecord(
if windowKeeper.Get(0) != nil {
outrecAndContext := windowKeeper.Get(0).(*types.RecordAndContext)
outputRecordsAndContexts.PushBack(outrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
}
}

5
pkg/transformers/subs.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"regexp"
@ -288,7 +287,7 @@ func NewTransformerSubs(
func (tr *TransformerSubs) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -304,7 +303,7 @@ func (tr *TransformerSubs) Transform(
}
}
// Including emit of end-of-stream marker
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
// fieldAcceptorByNames implements -f

23
pkg/transformers/summary.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -292,7 +291,7 @@ func NewTransformerSummary(
func (tr *TransformerSummary) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -346,7 +345,7 @@ func (tr *TransformerSummary) ingest(
func (tr *TransformerSummary) emit(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
for pe := tr.fieldSummaries.Head; pe != nil; pe = pe.Next {
@ -380,15 +379,15 @@ func (tr *TransformerSummary) emit(
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
}
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
func (tr *TransformerSummary) emitTransposed(
inrecAndContext *types.RecordAndContext,
oracs *list.List, // list of *types.RecordAndContext
oracs *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
octx := &inrecAndContext.Context
@ -407,7 +406,7 @@ func (tr *TransformerSummary) emitTransposed(
}
newrec.PutCopy(pe.Key, mlrval.FromString(strings.Join(fieldTypesList, "-")))
}
oracs.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*oracs = append(*oracs, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
}
for _, info := range allSummarizerInfos {
@ -418,7 +417,7 @@ func (tr *TransformerSummary) emitTransposed(
}
}
oracs.PushBack(inrecAndContext) // end-of-stream marker
*oracs = append(*oracs, inrecAndContext) // end-of-stream marker
}
// ----------------------------------------------------------------
@ -426,7 +425,7 @@ func (tr *TransformerSummary) emitTransposed(
// maybeEmitAccumulatorTransposed is a helper method for emitTransposed,
// for "count", "sum", "mean", etc.
func (tr *TransformerSummary) maybeEmitAccumulatorTransposed(
oracs *list.List, // list of *types.RecordAndContext
oracs *[]*types.RecordAndContext, // list of *types.RecordAndContext
octx *types.Context,
summarizerName string,
) {
@ -437,14 +436,14 @@ func (tr *TransformerSummary) maybeEmitAccumulatorTransposed(
fieldSummary := pe.Value
newrec.PutCopy(pe.Key, fieldSummary.accumulators[summarizerName].Emit())
}
oracs.PushBack(types.NewRecordAndContext(newrec, octx))
*oracs = append(*oracs, types.NewRecordAndContext(newrec, octx))
}
}
// maybeEmitPercentileNameTransposed is a helper method for emitTransposed,
// for "median", "iqr", "uof", etc.
func (tr *TransformerSummary) maybeEmitPercentileNameTransposed(
oracs *list.List, // list of *types.RecordAndContext
oracs *[]*types.RecordAndContext, // list of *types.RecordAndContext
octx *types.Context,
summarizerName string,
) {
@ -455,6 +454,6 @@ func (tr *TransformerSummary) maybeEmitPercentileNameTransposed(
fieldSummary := pe.Value
newrec.PutCopy(pe.Key, fieldSummary.percentileKeeper.EmitNamed(summarizerName))
}
oracs.PushBack(types.NewRecordAndContext(newrec, octx))
*oracs = append(*oracs, types.NewRecordAndContext(newrec, octx))
}
}

9
pkg/transformers/surv.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -114,7 +113,7 @@ func NewTransformerSurv(durationField, statusField string) IRecordTransformer {
// Transform processes each record or emits results at end-of-stream.
func (tr *TransformerSurv) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List,
outputRecordsAndContexts *[]*types.RecordAndContext,
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -139,7 +138,7 @@ func (tr *TransformerSurv) Transform(
// Compute survival using kshedden/statmodel
n := len(tr.times)
if n == 0 {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
return
}
durations := tr.times
@ -166,8 +165,8 @@ func (tr *TransformerSurv) Transform(
newrec := mlrval.NewMlrmapAsRecord()
newrec.PutCopy("time", mlrval.FromFloat(t))
newrec.PutCopy("survival", mlrval.FromFloat(survProbs[i]))
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

15
pkg/transformers/tac.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -77,29 +76,29 @@ func transformerTacParseCLI(
// ----------------------------------------------------------------
type TransformerTac struct {
recordsAndContexts *list.List
recordsAndContexts []*types.RecordAndContext
}
func NewTransformerTac() (*TransformerTac, error) {
return &TransformerTac{
recordsAndContexts: list.New(),
recordsAndContexts: make([]*types.RecordAndContext, 0),
}, nil
}
func (tr *TransformerTac) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel)
if !inrecAndContext.EndOfStream {
tr.recordsAndContexts.PushFront(inrecAndContext)
tr.recordsAndContexts = append(tr.recordsAndContexts, inrecAndContext)
} else {
// end of stream
for e := tr.recordsAndContexts.Front(); e != nil; e = e.Next() {
outputRecordsAndContexts.PushBack(e.Value.(*types.RecordAndContext))
for i := len(tr.recordsAndContexts) - 1; i >= 0; i-- {
*outputRecordsAndContexts = append(*outputRecordsAndContexts, tr.recordsAndContexts[i])
}
outputRecordsAndContexts.PushBack(types.NewEndOfStreamMarker(&inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewEndOfStreamMarker(&inrecAndContext.Context))
}
}

24
pkg/transformers/tail.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -109,8 +108,8 @@ type TransformerTail struct {
groupByFieldNames []string
// state
// map from string to *list.List
recordListsByGroup *lib.OrderedMap[*list.List]
// map from string to record slices
recordListsByGroup *lib.OrderedMap[*[]*types.RecordAndContext]
}
func NewTransformerTail(
@ -122,7 +121,7 @@ func NewTransformerTail(
tailCount: tailCount,
groupByFieldNames: groupByFieldNames,
recordListsByGroup: lib.NewOrderedMap[*list.List](),
recordListsByGroup: lib.NewOrderedMap[*[]*types.RecordAndContext](),
}
return tr, nil
@ -132,7 +131,7 @@ func NewTransformerTail(
func (tr *TransformerTail) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -147,22 +146,23 @@ func (tr *TransformerTail) Transform(
recordListForGroup := tr.recordListsByGroup.Get(groupingKey)
if recordListForGroup == nil { // first time
recordListForGroup = list.New()
records := make([]*types.RecordAndContext, 0)
recordListForGroup = &records
tr.recordListsByGroup.Put(groupingKey, recordListForGroup)
}
recordListForGroup.PushBack(inrecAndContext)
for int64(recordListForGroup.Len()) > tr.tailCount {
recordListForGroup.Remove(recordListForGroup.Front())
*recordListForGroup = append(*recordListForGroup, inrecAndContext)
for int64(len(*recordListForGroup)) > tr.tailCount {
*recordListForGroup = (*recordListForGroup)[1:]
}
} else {
for outer := tr.recordListsByGroup.Head; outer != nil; outer = outer.Next {
recordListForGroup := outer.Value
for inner := recordListForGroup.Front(); inner != nil; inner = inner.Next() {
outputRecordsAndContexts.PushBack(inner.Value.(*types.RecordAndContext))
for _, recordAndContext := range *recordListForGroup {
*outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext)
}
}
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

7
pkg/transformers/tee.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -166,7 +165,7 @@ func NewTransformerTee(
func (tr *TransformerTee) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -202,7 +201,7 @@ func (tr *TransformerTee) Transform(
os.Exit(1)
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
err := tr.fileOutputHandler.Close()
if err != nil {
@ -214,6 +213,6 @@ func (tr *TransformerTee) Transform(
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

7
pkg/transformers/template.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -135,7 +134,7 @@ func NewTransformerTemplate(
func (tr *TransformerTemplate) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -152,8 +151,8 @@ func (tr *TransformerTemplate) Transform(
}
}
outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context)
outputRecordsAndContexts.PushBack(outrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

11
pkg/transformers/top.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -174,7 +173,7 @@ func NewTransformerTop(
func (tr *TransformerTop) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -238,7 +237,7 @@ func (tr *TransformerTop) ingest(
// ----------------------------------------------------------------
func (tr *TransformerTop) emit(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
for pa := tr.groups.Head; pa != nil; pa = pa.Next {
groupingKey := pa.Key
@ -252,7 +251,7 @@ func (tr *TransformerTop) emit(
for pb := secondLevel.Head; pb != nil; pb = pb.Next {
topKeeper := pb.Value
for i := int64(0); i < topKeeper.GetSize(); i++ {
outputRecordsAndContexts.PushBack(topKeeper.TopRecordsAndContexts[i].Copy())
*outputRecordsAndContexts = append(*outputRecordsAndContexts, topKeeper.TopRecordsAndContexts[i].Copy())
}
}
@ -281,10 +280,10 @@ func (tr *TransformerTop) emit(
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
}
}
}
outputRecordsAndContexts.PushBack(inrecAndContext) // emit end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // emit end-of-stream marker
}

15
pkg/transformers/unflatten.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -133,7 +132,7 @@ func NewTransformerUnflatten(
func (tr *TransformerUnflatten) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -144,7 +143,7 @@ func (tr *TransformerUnflatten) Transform(
// ----------------------------------------------------------------
func (tr *TransformerUnflatten) unflattenAll(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -155,16 +154,16 @@ func (tr *TransformerUnflatten) unflattenAll(
oFlatSep = tr.options.WriterOptions.FLATSEP
}
inrec.Unflatten(oFlatSep)
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerUnflatten) unflattenSome(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -175,8 +174,8 @@ func (tr *TransformerUnflatten) unflattenSome(
oFlatSep = tr.options.WriterOptions.FLATSEP
}
inrec.UnflattenFields(tr.fieldNameSet, oFlatSep)
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

45
pkg/transformers/uniq.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -367,7 +366,7 @@ func (tr *TransformerUniq) getFieldNamesForGrouping(
func (tr *TransformerUniq) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -380,7 +379,7 @@ func (tr *TransformerUniq) Transform(
// non-streaming, with output at end of stream.
func (tr *TransformerUniq) transformUniqifyEntireRecordsShowCounts(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -403,10 +402,10 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowCounts(
icount := tr.uniqifiedRecordCounts.Get(pe.Key)
mcount := mlrval.FromInt(icount)
outrecAndContext.Record.PrependReference(tr.outputFieldName, mcount)
outputRecordsAndContexts.PushBack(outrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
}
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
@ -416,7 +415,7 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowCounts(
// of stream.
func (tr *TransformerUniq) transformUniqifyEntireRecordsShowNumDistinctOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -433,9 +432,9 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowNumDistinctOnly(
tr.outputFieldName,
mlrval.FromInt(tr.uniqifiedRecordCounts.FieldCount),
)
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
@ -443,7 +442,7 @@ func (tr *TransformerUniq) transformUniqifyEntireRecordsShowNumDistinctOnly(
// Print each unique record only once (on first occurrence).
func (tr *TransformerUniq) transformUniqifyEntireRecords(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -453,19 +452,19 @@ func (tr *TransformerUniq) transformUniqifyEntireRecords(
recordAsString := inrec.String()
if !tr.uniqifiedRecordCounts.Has(recordAsString) {
tr.uniqifiedRecordCounts.Put(recordAsString, int64(1))
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
} else { // end of record stream
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerUniq) transformUnlashed(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -509,18 +508,18 @@ func (tr *TransformerUniq) transformUnlashed(
tr.unlashedCountValues.Get(fieldName).Get(fieldValueString),
)
outrec.PutReference("count", mlrval.FromInt(pf.Value))
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
}
}
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerUniq) transformNumDistinctOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -543,16 +542,16 @@ func (tr *TransformerUniq) transformNumDistinctOnly(
"count",
mlrval.FromInt(tr.countsByGroup.FieldCount),
)
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerUniq) transformWithCounts(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -592,17 +591,17 @@ func (tr *TransformerUniq) transformWithCounts(
mlrval.FromInt(pa.Value),
)
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
}
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerUniq) transformWithoutCounts(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -627,13 +626,13 @@ func (tr *TransformerUniq) transformWithoutCounts(
)
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(outrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
} else {
tr.countsByGroup.Put(groupingKey, iCount+1)
}
} else { // end of record stream
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

17
pkg/transformers/unspace.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -115,7 +114,7 @@ func NewTransformerUnspace(
func (tr *TransformerUnspace) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -128,13 +127,13 @@ func (tr *TransformerUnspace) Transform(
outputDownstreamDoneChannel,
)
} else { // end of record stream
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}
func (tr *TransformerUnspace) transformKeysOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
_ <-chan bool,
__ chan<- bool,
) {
@ -145,12 +144,12 @@ func (tr *TransformerUnspace) transformKeysOnly(
// Reference not copy since this is ownership transfer of the value from the now-abandoned inrec
newrec.PutReference(newkey, pe.Value)
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
}
func (tr *TransformerUnspace) transformValuesOnly(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
_ <-chan bool,
__ chan<- bool,
) {
@ -161,12 +160,12 @@ func (tr *TransformerUnspace) transformValuesOnly(
pe.Value = mlrval.FromString(tr.unspace(stringval))
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(inrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(inrec, &inrecAndContext.Context))
}
func (tr *TransformerUnspace) transformKeysAndValues(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
_ <-chan bool,
__ chan<- bool,
) {
@ -182,7 +181,7 @@ func (tr *TransformerUnspace) transformKeysAndValues(
newrec.PutReference(newkey, pe.Value)
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &inrecAndContext.Context))
}
func (tr *TransformerUnspace) unspace(input string) string {

24
pkg/transformers/unsparsify.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -108,7 +107,7 @@ func transformerUnsparsifyParseCLI(
// ----------------------------------------------------------------
type TransformerUnsparsify struct {
fillerMlrval *mlrval.Mlrval
recordsAndContexts *list.List
recordsAndContexts []*types.RecordAndContext
fieldNamesSeen *lib.OrderedMap[string]
recordTransformerFunc RecordTransformerFunc
}
@ -125,7 +124,7 @@ func NewTransformerUnsparsify(
tr := &TransformerUnsparsify{
fillerMlrval: mlrval.FromString(fillerString),
recordsAndContexts: list.New(),
recordsAndContexts: make([]*types.RecordAndContext, 0),
fieldNamesSeen: fieldNamesSeen,
}
@ -142,7 +141,7 @@ func NewTransformerUnsparsify(
func (tr *TransformerUnsparsify) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -153,7 +152,7 @@ func (tr *TransformerUnsparsify) Transform(
// ----------------------------------------------------------------
func (tr *TransformerUnsparsify) transformNonStreaming(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -165,10 +164,9 @@ func (tr *TransformerUnsparsify) transformNonStreaming(
tr.fieldNamesSeen.Put(key, key)
}
}
tr.recordsAndContexts.PushBack(inrecAndContext)
tr.recordsAndContexts = append(tr.recordsAndContexts, inrecAndContext)
} else {
for e := tr.recordsAndContexts.Front(); e != nil; e = e.Next() {
outrecAndContext := e.Value.(*types.RecordAndContext)
for _, outrecAndContext := range tr.recordsAndContexts {
outrec := outrecAndContext.Record
newrec := mlrval.NewMlrmapAsRecord()
@ -181,17 +179,17 @@ func (tr *TransformerUnsparsify) transformNonStreaming(
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(newrec, &outrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(newrec, &outrecAndContext.Context))
}
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}
// ----------------------------------------------------------------
func (tr *TransformerUnsparsify) transformStreaming(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -204,9 +202,9 @@ func (tr *TransformerUnsparsify) transformStreaming(
}
}
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
} else {
outputRecordsAndContexts.PushBack(inrecAndContext) // end-of-stream marker
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
}
}

7
pkg/transformers/utf8_to_latin1.go
View file

@ -1,7 +1,6 @@
package transformers
import (
"container/list"
"fmt"
"os"
"strings"
@ -91,7 +90,7 @@ func NewTransformerUTF8ToLatin1() (*TransformerUTF8ToLatin1, error) {
func (tr *TransformerUTF8ToLatin1) Transform(
inrecAndContext *types.RecordAndContext,
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
inputDownstreamDoneChannel <-chan bool,
outputDownstreamDoneChannel chan<- bool,
) {
@ -111,9 +110,9 @@ func (tr *TransformerUTF8ToLatin1) Transform(
}
}
outputRecordsAndContexts.PushBack(types.NewRecordAndContext(inrec, &inrecAndContext.Context))
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(inrec, &inrecAndContext.Context))
} else { // end of record stream
outputRecordsAndContexts.PushBack(inrecAndContext)
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
}
}

7
pkg/transformers/utils/join_bucket.go
View file

@ -5,15 +5,14 @@
package utils
import (
"container/list"
"github.com/johnkerl/miller/v6/pkg/mlrval"
"github.com/johnkerl/miller/v6/pkg/types"
)
// ----------------------------------------------------------------
type JoinBucket struct {
leftFieldValues []*mlrval.Mlrval
RecordsAndContexts *list.List
RecordsAndContexts []*types.RecordAndContext
WasPaired bool
}
@ -22,7 +21,7 @@ func NewJoinBucket(
) *JoinBucket {
return &JoinBucket{
leftFieldValues: leftFieldValues,
RecordsAndContexts: list.New(),
RecordsAndContexts: make([]*types.RecordAndContext, 0),
WasPaired: false,
}
}

73
pkg/transformers/utils/join_bucket_keeper.go
View file

@ -108,7 +108,6 @@
package utils
import (
"container/list"
"fmt"
"os"
"strings"
@ -126,7 +125,7 @@ type JoinBucketKeeper struct {
// For streaming through the left-side file
recordReader input.IRecordReader
context *types.Context
readerChannel <-chan *list.List // list of *types.RecordAndContext
readerChannel <-chan []*types.RecordAndContext // list of *types.RecordAndContext
errorChannel chan error
// TODO: merge with leof flag
recordReaderDone bool
@ -152,7 +151,7 @@ type JoinBucketKeeper struct {
peekRecordAndContext *types.RecordAndContext
JoinBucket *JoinBucket
leftUnpaireds *list.List
leftUnpaireds []*types.RecordAndContext
leof bool
state tJoinBucketKeeperState
@ -181,7 +180,7 @@ func NewJoinBucketKeeper(
initialContext.UpdateForStartOfFile(leftFileName)
// Set up channels for the record-reader
readerChannel := make(chan *list.List, 2) // list of *types.RecordAndContext
readerChannel := make(chan []*types.RecordAndContext, 2) // list of *types.RecordAndContext
errorChannel := make(chan error, 1)
downstreamDoneChannel := make(chan bool, 1)
@ -201,7 +200,7 @@ func NewJoinBucketKeeper(
JoinBucket: NewJoinBucket(nil),
peekRecordAndContext: nil,
leftUnpaireds: list.New(),
leftUnpaireds: make([]*types.RecordAndContext, 0),
leof: false,
state: LEFT_STATE_0_PREFILL,
@ -289,7 +288,7 @@ func (keeper *JoinBucketKeeper) FindJoinBucket(
}
// TODO: privatize more
if keeper.JoinBucket.RecordsAndContexts.Len() > 0 {
if len(keeper.JoinBucket.RecordsAndContexts) > 0 {
cmp := compareLexically(
keeper.JoinBucket.leftFieldValues,
rightFieldValues,
@ -344,7 +343,7 @@ func (keeper *JoinBucketKeeper) prepareForFirstJoinBucket() {
if keeper.peekRecordAndContext.Record.HasSelectedKeys(keeper.leftJoinFieldNames) {
break
}
keeper.leftUnpaireds.PushBack(keeper.peekRecordAndContext)
keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext)
}
if keeper.peekRecordAndContext == nil {
@ -374,7 +373,7 @@ func (keeper *JoinBucketKeeper) prepareForNewJoinBucket(
rightFieldValues []*mlrval.Mlrval,
) {
if !keeper.JoinBucket.WasPaired {
moveRecordsAndContexts(keeper.leftUnpaireds, keeper.JoinBucket.RecordsAndContexts)
moveRecordsAndContexts(&keeper.leftUnpaireds, &keeper.JoinBucket.RecordsAndContexts)
}
keeper.JoinBucket = NewJoinBucket(nil)
@ -401,7 +400,7 @@ func (keeper *JoinBucketKeeper) prepareForNewJoinBucket(
// Keep seeking and filling the bucket until = or >; this may or may not
// end up being a match.
for {
keeper.leftUnpaireds.PushBack(keeper.peekRecordAndContext)
keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext)
keeper.peekRecordAndContext = nil
for {
@ -416,7 +415,7 @@ func (keeper *JoinBucketKeeper) prepareForNewJoinBucket(
if peekRec.HasSelectedKeys(keeper.leftJoinFieldNames) {
break
}
keeper.leftUnpaireds.PushBack(keeper.peekRecordAndContext)
keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext)
}
// Double break from double for-loop
@ -470,7 +469,7 @@ func (keeper *JoinBucketKeeper) fillNextJoinBucket() {
}
keeper.JoinBucket.leftFieldValues = mlrval.CopyMlrvalArray(peekFieldValues)
keeper.JoinBucket.RecordsAndContexts.PushBack(keeper.peekRecordAndContext)
keeper.JoinBucket.RecordsAndContexts = append(keeper.JoinBucket.RecordsAndContexts, keeper.peekRecordAndContext)
keeper.JoinBucket.WasPaired = false
keeper.peekRecordAndContext = nil
@ -497,9 +496,9 @@ func (keeper *JoinBucketKeeper) fillNextJoinBucket() {
if cmp != 0 {
break
}
keeper.JoinBucket.RecordsAndContexts.PushBack(keeper.peekRecordAndContext)
keeper.JoinBucket.RecordsAndContexts = append(keeper.JoinBucket.RecordsAndContexts, keeper.peekRecordAndContext)
} else {
keeper.leftUnpaireds.PushBack(keeper.peekRecordAndContext)
keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext)
}
keeper.peekRecordAndContext = nil
}
@ -510,13 +509,13 @@ func (keeper *JoinBucketKeeper) fillNextJoinBucket() {
func (keeper *JoinBucketKeeper) markRemainingsAsUnpaired() {
// 1. Any records already in keeper.JoinBucket.records (current bucket)
if !keeper.JoinBucket.WasPaired {
moveRecordsAndContexts(keeper.leftUnpaireds, keeper.JoinBucket.RecordsAndContexts)
moveRecordsAndContexts(&keeper.leftUnpaireds, &keeper.JoinBucket.RecordsAndContexts)
}
keeper.JoinBucket.RecordsAndContexts = nil
// 2. Peek-record, if any
if keeper.peekRecordAndContext != nil {
keeper.leftUnpaireds.PushBack(keeper.peekRecordAndContext)
keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext)
keeper.peekRecordAndContext = nil
}
@ -526,36 +525,26 @@ func (keeper *JoinBucketKeeper) markRemainingsAsUnpaired() {
if keeper.peekRecordAndContext == nil {
break
}
keeper.leftUnpaireds.PushBack(keeper.peekRecordAndContext)
keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext)
}
}
// ----------------------------------------------------------------
// TODO: comment
func (keeper *JoinBucketKeeper) OutputAndReleaseLeftUnpaireds(
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
for {
element := keeper.leftUnpaireds.Front()
if element == nil {
break
}
recordAndContext := element.Value.(*types.RecordAndContext)
outputRecordsAndContexts.PushBack(recordAndContext)
keeper.leftUnpaireds.Remove(element)
for len(keeper.leftUnpaireds) > 0 {
recordAndContext := keeper.leftUnpaireds[0]
*outputRecordsAndContexts = append(*outputRecordsAndContexts, recordAndContext)
keeper.leftUnpaireds = keeper.leftUnpaireds[1:]
}
}
func (keeper *JoinBucketKeeper) ReleaseLeftUnpaireds(
outputRecordsAndContexts *list.List, // list of *types.RecordAndContext
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
) {
for {
element := keeper.leftUnpaireds.Front()
if element == nil {
break
}
keeper.leftUnpaireds.Remove(element)
}
keeper.leftUnpaireds = keeper.leftUnpaireds[:0]
}
// ================================================================
@ -576,8 +565,8 @@ func (keeper *JoinBucketKeeper) readRecord() *types.RecordAndContext {
os.Exit(1)
case leftrecsAndContexts := <-keeper.readerChannel:
// TODO: temp
lib.InternalCodingErrorIf(leftrecsAndContexts.Len() != 1)
leftrecAndContext := leftrecsAndContexts.Front().Value.(*types.RecordAndContext)
lib.InternalCodingErrorIf(len(leftrecsAndContexts) != 1)
leftrecAndContext := leftrecsAndContexts[0]
leftrecAndContext.Record = KeepLeftFieldNames(leftrecAndContext.Record, keeper.leftKeepFieldNameSet)
if leftrecAndContext.EndOfStream { // end-of-stream marker
keeper.recordReaderDone = true
@ -594,17 +583,11 @@ func (keeper *JoinBucketKeeper) readRecord() *types.RecordAndContext {
// Pops everything off second-argument list and push to first-argument list.
func moveRecordsAndContexts(
destination *list.List,
source *list.List,
destination *[]*types.RecordAndContext,
source *[]*types.RecordAndContext,
) {
for {
element := source.Front()
if element == nil {
break
}
destination.PushBack(element.Value.(*types.RecordAndContext))
source.Remove(element)
}
*destination = append(*destination, (*source)...)
*source = (*source)[:0]
}
// ----------------------------------------------------------------

8
pkg/types/context.go
View file

@ -2,7 +2,6 @@ package types
import (
"bytes"
"container/list"
"strconv"
"github.com/johnkerl/miller/v6/pkg/mlrval"
@ -84,10 +83,9 @@ func NewEndOfStreamMarker(context *Context) *RecordAndContext {
// TODO: comment
// For the record-readers to update their initial context as each new record is read.
func NewEndOfStreamMarkerList(context *Context) *list.List {
ell := list.New()
ell.PushBack(NewEndOfStreamMarker(context))
return ell
func NewEndOfStreamMarkerList(context *Context) []*RecordAndContext {
recordsAndContexts := []*RecordAndContext{NewEndOfStreamMarker(context)}
return recordsAndContexts
}
// ----------------------------------------------------------------