mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-17 16:38:54 +00:00
a63ea57359
The uniq verb outputs only the group-by columns, and issue #1075 asks for a way to deduplicate on some fields while keeping the rest. Miller already supports this via "head -n 1 -g", but that wasn't discoverable from uniq's help text or its reference-verbs section. Add a note to "mlr uniq --help" and a short recipe (with live examples) to the uniq section of the verbs reference, and regenerate the derived man page, docs, and CLI-help golden files. Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
644 lines
20 KiB
Go
644 lines
20 KiB
Go
package transformers
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
|
|
"github.com/johnkerl/miller/v6/pkg/cli"
|
|
"github.com/johnkerl/miller/v6/pkg/lib"
|
|
"github.com/johnkerl/miller/v6/pkg/mlrval"
|
|
"github.com/johnkerl/miller/v6/pkg/types"
|
|
)
|
|
|
|
const verbNameCountDistinct = "count-distinct"
|
|
const verbNameUniq = "uniq"
|
|
const uniqDefaultOutputFieldName = "count"
|
|
|
|
var countDistinctOptions = []OptionSpec{
|
|
{Flag: "-f", Arg: "{a,b,c}", Type: "csv-list", Desc: "Field names for distinct count (synonym for -g)."},
|
|
{Flag: "-g", Arg: "{a,b,c}", Type: "csv-list", Desc: "Field names for distinct count."},
|
|
{Flag: "-x", Arg: "{a,b,c}", Type: "csv-list", Desc: "Field names to exclude for distinct count; use each record's other fields instead."},
|
|
{Flag: "-n", Type: "bool", Desc: "Show only the number of distinct values. Not compatible with -u."},
|
|
{Flag: "-o", Arg: "{name}", Type: "string", Desc: "Field name for output count. Default \"count\". Ignored with -u."},
|
|
{Flag: "-u", Type: "bool", Desc: "Do unlashed counts for multiple field names. With -f a,b and without -u, computes counts for distinct combinations of a and b field values. With -f a,b and with -u, computes counts for distinct a field values and counts for distinct b field values separately."},
|
|
}
|
|
|
|
var CountDistinctSetup = TransformerSetup{
|
|
Verb: verbNameCountDistinct,
|
|
UsageFunc: transformerCountDistinctUsage,
|
|
ParseCLIFunc: transformerCountDistinctParseCLI,
|
|
IgnoresInput: false,
|
|
Options: countDistinctOptions,
|
|
}
|
|
|
|
var uniqOptions = []OptionSpec{
|
|
{Flag: "-g", Arg: "{d,e,f}", Type: "csv-list", Desc: "Group-by field names for uniq counts."},
|
|
{Flag: "-f", Arg: "{d,e,f}", Type: "csv-list", Desc: "Synonym for -g."},
|
|
{Flag: "-x", Arg: "{a,b,c}", Type: "csv-list", Desc: "Field names to exclude for uniq; use each record's other fields instead."},
|
|
{Flag: "-c", Type: "bool", Desc: "Show repeat counts in addition to unique values."},
|
|
{Flag: "-n", Type: "bool", Desc: "Show only the number of distinct values."},
|
|
{Flag: "-o", Arg: "{name}", Type: "string", Desc: "Field name for output count. Default \"count\"."},
|
|
{Flag: "-a", Type: "bool", Desc: "Output each unique record only once. Incompatible with -g. With -c, produces unique records, with repeat counts for each. With -n, produces only one record which is the unique-record count. With neither -c nor -n, produces unique records."},
|
|
}
|
|
|
|
var UniqSetup = TransformerSetup{
|
|
Verb: verbNameUniq,
|
|
UsageFunc: transformerUniqUsage,
|
|
ParseCLIFunc: transformerUniqParseCLI,
|
|
IgnoresInput: false,
|
|
Options: uniqOptions,
|
|
}
|
|
|
|
func transformerCountDistinctUsage(
|
|
o *os.File,
|
|
) {
|
|
argv0 := "mlr"
|
|
verb := verbNameCountDistinct
|
|
fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb)
|
|
fmt.Fprintf(o, "Prints number of records having distinct values for specified field names.\n")
|
|
fmt.Fprintf(o, "Same as uniq -c.\n")
|
|
fmt.Fprintf(o, "\n")
|
|
WriteVerbOptions(o, countDistinctOptions)
|
|
}
|
|
|
|
func transformerCountDistinctParseCLI(
|
|
pargi *int,
|
|
argc int,
|
|
args []string,
|
|
_ *cli.TOptions,
|
|
doConstruct bool, // false for first pass of CLI-parse, true for second pass
|
|
) (RecordTransformer, error) {
|
|
|
|
// Skip the verb name from the current spot in the mlr command line
|
|
argi := *pargi
|
|
verb := args[argi]
|
|
argi++
|
|
|
|
// Parse local flags
|
|
var fieldNames []string = nil
|
|
invertFieldNames := false
|
|
showNumDistinctOnly := false
|
|
outputFieldName := uniqDefaultOutputFieldName
|
|
doLashed := true
|
|
|
|
var err error
|
|
for argi < argc /* variable increment: 1 or 2 depending on flag */ {
|
|
opt := args[argi]
|
|
if !strings.HasPrefix(opt, "-") {
|
|
break // No more flag options to process
|
|
}
|
|
if args[argi] == "--" {
|
|
break // All transformers must do this so main-flags can follow verb-flags
|
|
}
|
|
argi++
|
|
|
|
switch opt {
|
|
case "-h", "--help":
|
|
transformerCountDistinctUsage(os.Stdout)
|
|
return nil, cli.ErrHelpRequested
|
|
|
|
case "-g", "-f":
|
|
fieldNames, err = cli.VerbGetStringArrayArg(verb, opt, args, &argi, argc)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
case "-x":
|
|
fieldNames, err = cli.VerbGetStringArrayArg(verb, opt, args, &argi, argc)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
invertFieldNames = true
|
|
|
|
case "-n":
|
|
showNumDistinctOnly = true
|
|
|
|
case "-o":
|
|
outputFieldName, err = cli.VerbGetStringArg(verb, opt, args, &argi, argc)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
case "-u":
|
|
doLashed = false
|
|
|
|
default:
|
|
return nil, cli.VerbErrorf(verb, "option \"%s\" not recognized", opt)
|
|
}
|
|
}
|
|
|
|
if fieldNames == nil {
|
|
return nil, cli.VerbErrorf(verb, "-g or -x field names required")
|
|
}
|
|
if !doLashed && showNumDistinctOnly {
|
|
return nil, cli.VerbErrorf(verb, "-n requires -a (uniqify entire records)")
|
|
}
|
|
|
|
showCounts := true
|
|
uniqifyEntireRecords := false
|
|
|
|
*pargi = argi
|
|
if !doConstruct { // All transformers must do this for main command-line parsing
|
|
return nil, nil
|
|
}
|
|
|
|
transformer, err := NewTransformerUniq(
|
|
fieldNames,
|
|
invertFieldNames,
|
|
showCounts,
|
|
showNumDistinctOnly,
|
|
outputFieldName,
|
|
doLashed,
|
|
uniqifyEntireRecords,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return transformer, nil
|
|
}
|
|
|
|
func transformerUniqUsage(
|
|
o *os.File,
|
|
) {
|
|
argv0 := "mlr"
|
|
verb := verbNameUniq
|
|
fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb)
|
|
fmt.Fprintf(o, "Prints distinct values for specified field names. With -c, same as\n")
|
|
fmt.Fprintf(o, "count-distinct. For uniq, -f is a synonym for -g. Output fields are\n")
|
|
fmt.Fprintf(o, "written in the order in which they are named with -g or -f, not in the\n")
|
|
fmt.Fprintf(o, "order in which they appear in the input records.\n")
|
|
fmt.Fprintf(o, "To deduplicate records by one or more fields while keeping all other\n")
|
|
fmt.Fprintf(o, "fields, use head: e.g. \"%s head -n 1 -g hash\" keeps the first record\n", argv0)
|
|
fmt.Fprintf(o, "for each distinct value of the hash field, with all fields intact.\n")
|
|
fmt.Fprintf(o, "\n")
|
|
WriteVerbOptions(o, uniqOptions)
|
|
}
|
|
|
|
func transformerUniqParseCLI(
|
|
pargi *int,
|
|
argc int,
|
|
args []string,
|
|
_ *cli.TOptions,
|
|
doConstruct bool, // false for first pass of CLI-parse, true for second pass
|
|
) (RecordTransformer, error) {
|
|
|
|
// Skip the verb name from the current spot in the mlr command line
|
|
argi := *pargi
|
|
verb := args[argi]
|
|
argi++
|
|
|
|
// Parse local flags
|
|
var fieldNames []string = nil
|
|
invertFieldNames := false
|
|
showCounts := false
|
|
showNumDistinctOnly := false
|
|
outputFieldName := uniqDefaultOutputFieldName
|
|
uniqifyEntireRecords := false
|
|
|
|
var err error
|
|
for argi < argc /* variable increment: 1 or 2 depending on flag */ {
|
|
opt := args[argi]
|
|
if !strings.HasPrefix(opt, "-") {
|
|
break // No more flag options to process
|
|
}
|
|
if args[argi] == "--" {
|
|
break // All transformers must do this so main-flags can follow verb-flags
|
|
}
|
|
argi++
|
|
|
|
switch opt {
|
|
case "-h", "--help":
|
|
transformerUniqUsage(os.Stdout)
|
|
return nil, cli.ErrHelpRequested
|
|
|
|
case "-g", "-f":
|
|
fieldNames, err = cli.VerbGetStringArrayArg(verb, opt, args, &argi, argc)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
case "-x":
|
|
fieldNames, err = cli.VerbGetStringArrayArg(verb, opt, args, &argi, argc)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
invertFieldNames = true
|
|
|
|
case "-c":
|
|
showCounts = true
|
|
|
|
case "-n":
|
|
showNumDistinctOnly = true
|
|
|
|
case "-o":
|
|
outputFieldName, err = cli.VerbGetStringArg(verb, opt, args, &argi, argc)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
case "-a":
|
|
uniqifyEntireRecords = true
|
|
|
|
default:
|
|
return nil, cli.VerbErrorf(verb, "option \"%s\" not recognized", opt)
|
|
}
|
|
}
|
|
|
|
if uniqifyEntireRecords {
|
|
if fieldNames != nil {
|
|
return nil, cli.VerbErrorf(verb, "-a (uniqify entire records) is incompatible with -g/-x")
|
|
}
|
|
if showCounts && showNumDistinctOnly {
|
|
return nil, cli.VerbErrorf(verb, "-c and -n are mutually exclusive with -a")
|
|
}
|
|
} else {
|
|
if fieldNames == nil {
|
|
return nil, cli.VerbErrorf(verb, "-g or -x field names required")
|
|
}
|
|
}
|
|
|
|
doLashed := true
|
|
|
|
*pargi = argi
|
|
if !doConstruct { // All transformers must do this for main command-line parsing
|
|
return nil, nil
|
|
}
|
|
|
|
transformer, _ := NewTransformerUniq(
|
|
fieldNames,
|
|
invertFieldNames,
|
|
showCounts,
|
|
showNumDistinctOnly,
|
|
outputFieldName,
|
|
doLashed,
|
|
uniqifyEntireRecords,
|
|
)
|
|
|
|
return transformer, nil
|
|
}
|
|
|
|
type TransformerUniq struct {
|
|
fieldNames []string
|
|
fieldNamesSet map[string]bool
|
|
invertFieldNames bool
|
|
showCounts bool
|
|
outputFieldName string
|
|
|
|
// Example:
|
|
// Input is:
|
|
// a=1,b=2,c=3
|
|
// a=4,b=5,c=6
|
|
// Uniquing on fields ["a","b"]
|
|
// uniqifiedRecordCounts:
|
|
// '{"a":1,"b":2,"c":3}' => 1
|
|
// '{"a":4,"b":5,"c":6}' => 1
|
|
// uniqifiedRecords:
|
|
// '{"a":1,"b":2,"c":3}' => {"a":1,"b":2,"c":3}
|
|
// '{"a":4,"b":5,"c":6}' => {"a":4,"b":5,"c":6}
|
|
// countsByGroup:
|
|
// "1,2" -> 1
|
|
// "4,5" -> 1
|
|
// valuesByGroup:
|
|
// "1,2" -> [1,2]
|
|
// "4,5" -> [4,5]
|
|
// unlashedCounts:
|
|
// "a" => "1" => 1
|
|
// "a" => "4" => 1
|
|
// ...
|
|
// unlashedCountValues:
|
|
// "a" => "1" => 1
|
|
// "a" => "4" => 4
|
|
uniqifiedRecordCounts *lib.OrderedMap[int64] // record-as-string -> counts
|
|
uniqifiedRecords *lib.OrderedMap[*types.RecordAndContext] // record-as-string -> records
|
|
keysByGroup *lib.OrderedMap[[]string] // XXX COMMENT ME
|
|
countsByGroup *lib.OrderedMap[int64] // grouping key -> count
|
|
valuesByGroup *lib.OrderedMap[[]*mlrval.Mlrval] // grouping key -> array of values
|
|
unlashedCounts *lib.OrderedMap[*lib.OrderedMap[int64]] // field name -> string field value -> count
|
|
unlashedCountValues *lib.OrderedMap[*lib.OrderedMap[*mlrval.Mlrval]] // field name -> string field value -> typed field value
|
|
|
|
recordTransformerFunc RecordTransformerFunc
|
|
}
|
|
|
|
func NewTransformerUniq(
|
|
fieldNames []string,
|
|
invertFieldNames bool,
|
|
showCounts bool,
|
|
showNumDistinctOnly bool,
|
|
outputFieldName string,
|
|
doLashed bool,
|
|
uniqifyEntireRecords bool,
|
|
) (*TransformerUniq, error) {
|
|
|
|
tr := &TransformerUniq{
|
|
fieldNames: fieldNames,
|
|
fieldNamesSet: lib.StringListToSet(fieldNames),
|
|
invertFieldNames: invertFieldNames,
|
|
showCounts: showCounts,
|
|
outputFieldName: outputFieldName,
|
|
|
|
uniqifiedRecordCounts: lib.NewOrderedMap[int64](),
|
|
uniqifiedRecords: lib.NewOrderedMap[*types.RecordAndContext](),
|
|
keysByGroup: lib.NewOrderedMap[[]string](),
|
|
countsByGroup: lib.NewOrderedMap[int64](),
|
|
valuesByGroup: lib.NewOrderedMap[[]*mlrval.Mlrval](),
|
|
unlashedCounts: lib.NewOrderedMap[*lib.OrderedMap[int64]](),
|
|
unlashedCountValues: lib.NewOrderedMap[*lib.OrderedMap[*mlrval.Mlrval]](),
|
|
}
|
|
|
|
if uniqifyEntireRecords {
|
|
if showCounts {
|
|
tr.recordTransformerFunc = tr.transformUniqifyEntireRecordsShowCounts
|
|
} else if showNumDistinctOnly {
|
|
tr.recordTransformerFunc = tr.transformUniqifyEntireRecordsShowNumDistinctOnly
|
|
} else {
|
|
tr.recordTransformerFunc = tr.transformUniqifyEntireRecords
|
|
}
|
|
} else if !doLashed {
|
|
tr.recordTransformerFunc = tr.transformUnlashed
|
|
} else if showNumDistinctOnly {
|
|
tr.recordTransformerFunc = tr.transformNumDistinctOnly
|
|
} else if showCounts {
|
|
tr.recordTransformerFunc = tr.transformWithCounts
|
|
} else {
|
|
tr.recordTransformerFunc = tr.transformWithoutCounts
|
|
}
|
|
|
|
return tr, nil
|
|
}
|
|
|
|
func (tr *TransformerUniq) getFieldNamesForGrouping(
|
|
inrec *mlrval.Mlrmap,
|
|
) []string {
|
|
if tr.invertFieldNames {
|
|
return inrec.GetKeysExcept(tr.fieldNamesSet)
|
|
}
|
|
return tr.fieldNames
|
|
}
|
|
|
|
func (tr *TransformerUniq) Transform(
|
|
inrecAndContext *types.RecordAndContext,
|
|
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
|
|
inputDownstreamDoneChannel <-chan bool,
|
|
outputDownstreamDoneChannel chan<- bool,
|
|
) {
|
|
HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel)
|
|
tr.recordTransformerFunc(inrecAndContext, outputRecordsAndContexts, inputDownstreamDoneChannel, outputDownstreamDoneChannel)
|
|
}
|
|
|
|
// Print each unique record only once, with uniqueness counts. This means
|
|
// non-streaming, with output at end of stream.
|
|
func (tr *TransformerUniq) transformUniqifyEntireRecordsShowCounts(
|
|
inrecAndContext *types.RecordAndContext,
|
|
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
|
|
inputDownstreamDoneChannel <-chan bool,
|
|
outputDownstreamDoneChannel chan<- bool,
|
|
) {
|
|
if !inrecAndContext.EndOfStream {
|
|
inrec := inrecAndContext.Record
|
|
|
|
recordAsString := inrec.String()
|
|
icount, present := tr.uniqifiedRecordCounts.GetWithCheck(recordAsString)
|
|
if !present { // first time seen
|
|
tr.uniqifiedRecordCounts.Put(recordAsString, int64(1))
|
|
tr.uniqifiedRecords.Put(recordAsString, inrecAndContext.Copy())
|
|
} else { // have seen before
|
|
tr.uniqifiedRecordCounts.Put(recordAsString, icount+1)
|
|
}
|
|
|
|
} else { // end of record stream
|
|
|
|
for pe := tr.uniqifiedRecords.Head; pe != nil; pe = pe.Next {
|
|
outrecAndContext := pe.Value
|
|
icount := tr.uniqifiedRecordCounts.Get(pe.Key)
|
|
mcount := mlrval.FromInt(icount)
|
|
outrecAndContext.Record.PrependReference(tr.outputFieldName, mcount)
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, outrecAndContext)
|
|
}
|
|
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
|
|
}
|
|
|
|
}
|
|
|
|
// Print count of unique records. This means non-streaming, with output at end
|
|
// of stream.
|
|
func (tr *TransformerUniq) transformUniqifyEntireRecordsShowNumDistinctOnly(
|
|
inrecAndContext *types.RecordAndContext,
|
|
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
|
|
inputDownstreamDoneChannel <-chan bool,
|
|
outputDownstreamDoneChannel chan<- bool,
|
|
) {
|
|
if !inrecAndContext.EndOfStream {
|
|
inrec := inrecAndContext.Record
|
|
recordAsString := inrec.String()
|
|
if !tr.uniqifiedRecordCounts.Has(recordAsString) {
|
|
tr.uniqifiedRecordCounts.Put(recordAsString, int64(1))
|
|
}
|
|
|
|
} else { // end of record stream
|
|
outrec := mlrval.NewMlrmapAsRecord()
|
|
outrec.PutReference(
|
|
tr.outputFieldName,
|
|
mlrval.FromInt(tr.uniqifiedRecordCounts.FieldCount),
|
|
)
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
|
|
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
|
|
}
|
|
}
|
|
|
|
// Print each unique record only once (on first occurrence).
|
|
func (tr *TransformerUniq) transformUniqifyEntireRecords(
|
|
inrecAndContext *types.RecordAndContext,
|
|
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
|
|
inputDownstreamDoneChannel <-chan bool,
|
|
outputDownstreamDoneChannel chan<- bool,
|
|
) {
|
|
if !inrecAndContext.EndOfStream {
|
|
inrec := inrecAndContext.Record
|
|
|
|
recordAsString := inrec.String()
|
|
if !tr.uniqifiedRecordCounts.Has(recordAsString) {
|
|
tr.uniqifiedRecordCounts.Put(recordAsString, int64(1))
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
|
|
}
|
|
|
|
} else { // end of record stream
|
|
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
|
|
}
|
|
}
|
|
|
|
func (tr *TransformerUniq) transformUnlashed(
|
|
inrecAndContext *types.RecordAndContext,
|
|
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
|
|
inputDownstreamDoneChannel <-chan bool,
|
|
outputDownstreamDoneChannel chan<- bool,
|
|
) {
|
|
if !inrecAndContext.EndOfStream {
|
|
inrec := inrecAndContext.Record
|
|
|
|
for _, fieldName := range tr.getFieldNamesForGrouping(inrec) {
|
|
var countsForFieldName *lib.OrderedMap[int64] = nil
|
|
iCountsForFieldName, present := tr.unlashedCounts.GetWithCheck(fieldName)
|
|
if !present {
|
|
countsForFieldName = lib.NewOrderedMap[int64]()
|
|
tr.unlashedCounts.Put(fieldName, countsForFieldName)
|
|
tr.unlashedCountValues.Put(fieldName, lib.NewOrderedMap[*mlrval.Mlrval]())
|
|
} else {
|
|
countsForFieldName = iCountsForFieldName
|
|
}
|
|
|
|
fieldValue := inrec.Get(fieldName)
|
|
if fieldValue != nil {
|
|
fieldValueString := fieldValue.String()
|
|
if !countsForFieldName.Has(fieldValueString) {
|
|
countsForFieldName.Put(fieldValueString, int64(1))
|
|
tr.unlashedCountValues.Get(fieldName).Put(fieldValueString, fieldValue.Copy())
|
|
} else {
|
|
countsForFieldName.Put(fieldValueString, countsForFieldName.Get(fieldValueString)+1)
|
|
}
|
|
}
|
|
}
|
|
|
|
} else { // end of record stream
|
|
|
|
for pe := tr.unlashedCounts.Head; pe != nil; pe = pe.Next {
|
|
fieldName := pe.Key
|
|
countsForFieldName := pe.Value
|
|
for pf := countsForFieldName.Head; pf != nil; pf = pf.Next {
|
|
fieldValueString := pf.Key
|
|
outrec := mlrval.NewMlrmapAsRecord()
|
|
outrec.PutReference("field", mlrval.FromString(fieldName))
|
|
outrec.PutCopy(
|
|
"value",
|
|
tr.unlashedCountValues.Get(fieldName).Get(fieldValueString),
|
|
)
|
|
outrec.PutReference("count", mlrval.FromInt(pf.Value))
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
|
|
}
|
|
}
|
|
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
|
|
}
|
|
}
|
|
|
|
func (tr *TransformerUniq) transformNumDistinctOnly(
|
|
inrecAndContext *types.RecordAndContext,
|
|
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
|
|
inputDownstreamDoneChannel <-chan bool,
|
|
outputDownstreamDoneChannel chan<- bool,
|
|
) {
|
|
if !inrecAndContext.EndOfStream {
|
|
inrec := inrecAndContext.Record
|
|
|
|
groupingKey, ok := inrec.GetSelectedValuesJoined(tr.getFieldNamesForGrouping(inrec))
|
|
if ok {
|
|
iCount, present := tr.countsByGroup.GetWithCheck(groupingKey)
|
|
if !present {
|
|
tr.countsByGroup.Put(groupingKey, int64(1))
|
|
} else {
|
|
tr.countsByGroup.Put(groupingKey, iCount+1)
|
|
}
|
|
}
|
|
|
|
} else {
|
|
outrec := mlrval.NewMlrmapAsRecord()
|
|
outrec.PutReference(
|
|
"count",
|
|
mlrval.FromInt(tr.countsByGroup.FieldCount),
|
|
)
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
|
|
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
|
|
}
|
|
}
|
|
|
|
func (tr *TransformerUniq) transformWithCounts(
|
|
inrecAndContext *types.RecordAndContext,
|
|
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
|
|
inputDownstreamDoneChannel <-chan bool,
|
|
outputDownstreamDoneChannel chan<- bool,
|
|
) {
|
|
if !inrecAndContext.EndOfStream {
|
|
inrec := inrecAndContext.Record
|
|
|
|
fieldNamesForGrouping := tr.getFieldNamesForGrouping(inrec)
|
|
|
|
groupingKey, selectedValues, ok := inrec.GetSelectedValuesAndJoined(fieldNamesForGrouping)
|
|
if ok {
|
|
iCount, present := tr.countsByGroup.GetWithCheck(groupingKey)
|
|
if !present {
|
|
tr.countsByGroup.Put(groupingKey, int64(1))
|
|
tr.valuesByGroup.Put(groupingKey, selectedValues)
|
|
tr.keysByGroup.Put(groupingKey, fieldNamesForGrouping)
|
|
} else {
|
|
tr.countsByGroup.Put(groupingKey, iCount+1)
|
|
}
|
|
}
|
|
|
|
} else { // end of record stream
|
|
for pa := tr.countsByGroup.Head; pa != nil; pa = pa.Next {
|
|
outrec := mlrval.NewMlrmapAsRecord()
|
|
valuesForGroup := tr.valuesByGroup.Get(pa.Key)
|
|
keysForGroup := tr.keysByGroup.Get(pa.Key)
|
|
|
|
for i, fieldNameForGrouping := range keysForGroup {
|
|
outrec.PutCopy(
|
|
fieldNameForGrouping,
|
|
valuesForGroup[i],
|
|
)
|
|
}
|
|
|
|
if tr.showCounts {
|
|
outrec.PutReference(
|
|
tr.outputFieldName,
|
|
mlrval.FromInt(pa.Value),
|
|
)
|
|
}
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
|
|
}
|
|
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
|
|
}
|
|
}
|
|
|
|
func (tr *TransformerUniq) transformWithoutCounts(
|
|
inrecAndContext *types.RecordAndContext,
|
|
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
|
|
inputDownstreamDoneChannel <-chan bool,
|
|
outputDownstreamDoneChannel chan<- bool,
|
|
) {
|
|
if !inrecAndContext.EndOfStream {
|
|
inrec := inrecAndContext.Record
|
|
|
|
groupingKey, selectedValues, ok := inrec.GetSelectedValuesAndJoined(tr.getFieldNamesForGrouping(inrec))
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
iCount, present := tr.countsByGroup.GetWithCheck(groupingKey)
|
|
if !present {
|
|
tr.countsByGroup.Put(groupingKey, int64(1))
|
|
tr.valuesByGroup.Put(groupingKey, selectedValues)
|
|
outrec := mlrval.NewMlrmapAsRecord()
|
|
|
|
for i, fieldNameForGrouping := range tr.getFieldNamesForGrouping(inrec) {
|
|
outrec.PutCopy(
|
|
fieldNameForGrouping,
|
|
selectedValues[i],
|
|
)
|
|
}
|
|
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, types.NewRecordAndContext(outrec, &inrecAndContext.Context))
|
|
|
|
} else {
|
|
tr.countsByGroup.Put(groupingKey, iCount+1)
|
|
}
|
|
|
|
} else { // end of record stream
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
|
|
}
|
|
}
|