miller/pkg/transformers/flatten.go

package transformers

import (
	"fmt"
	"os"
	"strings"

	"github.com/johnkerl/miller/v6/pkg/cli"
	"github.com/johnkerl/miller/v6/pkg/lib"
	"github.com/johnkerl/miller/v6/pkg/types"
)

const verbNameFlatten = "flatten"

var flattenOptions = []OptionSpec{
	{Flag: "-f", Type: "csv-list", Desc: "Comma-separated list of field names to flatten (default all)."},
	{Flag: "-s", Arg: "{string}", Type: "string", Desc: "Separator, defaulting to mlr --flatsep value."},
}

var FlattenSetup = TransformerSetup{
	Verb:         verbNameFlatten,
	UsageFunc:    transformerFlattenUsage,
	ParseCLIFunc: transformerFlattenParseCLI,
	IgnoresInput: false,
	Options:      flattenOptions,
}

func transformerFlattenUsage(
	o *os.File,
) {
	fmt.Fprintf(o, "Usage: %s %s [options]\n", "mlr", verbNameFlatten)
	fmt.Fprint(o,
		`Flattens multi-level maps to single-level ones. Example: field with name 'a'
and value '{"b": { "c": 4 }}' becomes name 'a.b.c' and value 4.
`)
	WriteVerbOptions(o, flattenOptions)
}

func transformerFlattenParseCLI(
	pargi *int,
	argc int,
	args []string,
	options *cli.TOptions,
	doConstruct bool, // false for first pass of CLI-parse, true for second pass
) (RecordTransformer, error) {

	// Skip the verb name from the current spot in the mlr command line
	argi := *pargi
	verb := args[argi]
	argi++

	oFlatSep := "" // means take it from the record context
	var fieldNames []string = nil

	var err error
	for argi < argc /* variable increment: 1 or 2 depending on flag */ {
		opt := args[argi]
		if !strings.HasPrefix(opt, "-") {
			break // No more flag options to process
		}
		if args[argi] == "--" {
			break // All transformers must do this so main-flags can follow verb-flags
		}
		argi++

		switch opt {
		case "-h", "--help":
			transformerFlattenUsage(os.Stdout)
			return nil, cli.ErrHelpRequested

		case "-s":
			oFlatSep, err = cli.VerbGetStringArg(verb, opt, args, &argi, argc)
			if err != nil {
				return nil, err
			}

		case "-f":
			fieldNames, err = cli.VerbGetStringArrayArg(verb, opt, args, &argi, argc)
			if err != nil {
				return nil, err
			}

		default:
			return nil, cli.VerbErrorf(verb, "option \"%s\" not recognized", opt)
		}
	}

	*pargi = argi
	if !doConstruct { // All transformers must do this for main command-line parsing
		return nil, nil
	}

	transformer, err := NewTransformerFlatten(
		oFlatSep,
		options,
		fieldNames,
	)
	if err != nil {
		return nil, err
	}

	return transformer, nil
}

type TransformerFlatten struct {
	// input
	oFlatSep     string
	options      *cli.TOptions
	fieldNameSet map[string]bool

	// state
	recordTransformerFunc RecordTransformerFunc
}

func NewTransformerFlatten(
	oFlatSep string,
	options *cli.TOptions,
	fieldNames []string,
) (*TransformerFlatten, error) {
	var fieldNameSet map[string]bool = nil
	if fieldNames != nil {
		fieldNameSet = lib.StringListToSet(fieldNames)
	}

	retval := &TransformerFlatten{
		oFlatSep:     oFlatSep,
		options:      options,
		fieldNameSet: fieldNameSet,
	}

	retval.recordTransformerFunc = retval.flattenAll
	if fieldNameSet != nil {
		retval.recordTransformerFunc = retval.flattenSome
	}

	return retval, nil
}

func (tr *TransformerFlatten) Transform(
	inrecAndContext *types.RecordAndContext,
	outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
	inputDownstreamDoneChannel <-chan bool,
	outputDownstreamDoneChannel chan<- bool,
) error {
	HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel)
	return tr.recordTransformerFunc(inrecAndContext, outputRecordsAndContexts, inputDownstreamDoneChannel, outputDownstreamDoneChannel)
}

func (tr *TransformerFlatten) flattenAll(
	inrecAndContext *types.RecordAndContext,
	outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
	inputDownstreamDoneChannel <-chan bool,
	outputDownstreamDoneChannel chan<- bool,
) error {
	if !inrecAndContext.EndOfStream {
		inrec := inrecAndContext.Record
		oFlatSep := tr.oFlatSep
		if oFlatSep == "" {
			oFlatSep = tr.options.WriterOptions.FLATSEP
		}
		inrec.Flatten(oFlatSep)
		*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
	} else {
		*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
	}
	return nil
}

func (tr *TransformerFlatten) flattenSome(
	inrecAndContext *types.RecordAndContext,
	outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
	inputDownstreamDoneChannel <-chan bool,
	outputDownstreamDoneChannel chan<- bool,
) error {
	if !inrecAndContext.EndOfStream {
		inrec := inrecAndContext.Record
		oFlatSep := tr.oFlatSep
		if oFlatSep == "" {
			oFlatSep = tr.options.WriterOptions.FLATSEP
		}
		inrec.FlattenFields(tr.fieldNameSet, oFlatSep)
		*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
	} else {
		*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
	}
	return nil
}