mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-17 16:38:54 +00:00
* Tier-2 structured verb options: OptionSpec, initial migration (#2098) PR 3 of the AI-friendly roadmap (plans/plan-2098-llm.md). Infrastructure: - Add OptionSpec{Flag,Arg,Type,Desc,Repeatable,Values} to pkg/transformers/aaa_record_transformer.go alongside TransformerSetup. Type is one of: bool, string, int, float, csv-list, regex, filename, format, enum. For type=="enum", Values lists the valid choices. - Add Options []OptionSpec to TransformerSetup (nil = not yet migrated). - Emit Options in VerbInfoForJSON (omitempty so unmigrated verbs stay backward-compatible; agents check key presence for Tier-2 availability). UsageText is always present as the Tier-1 prose fallback. - Add VerbOptionsNilCheck() in aaa_verb_options_check.go: progress report of migrated vs. unmigrated verbs, analogous to FLAG_TABLE.NilCheck(). - Wire verb-options-nil-check into mlr help (internal/docgen section). Initial migration (5/70 verbs): - nothing: empty Options (no verb-specific options, explicitly migrated) - cat: -n (bool), -N (string), -g (csv-list), --filename, --filenum (bool) - head: -g (csv-list), -n (int) - tail: -g (csv-list), -n (int) - tee: -a, -p (bool) Tests: - 5 new unit tests in aaa_transformer_json_test.go covering migrated/ unmigrated paths, field population, JSON round-trip, and key-presence. - Regression test case 0003: mlr help verb-options-nil-check golden output. Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com> * Migrate all 70 verbs to structured OptionSpec; bump catalog schema to v2 Completes the Tier-2 migration started in the previous commit. Every verb in TRANSFORMER_LOOKUP_TABLE now has a non-nil Options field. - Workflow-migrated all 65 remaining verbs. Each Setup var now carries Options: []OptionSpec{...} with Flag/Arg/Type/Desc fields. Verbs with no verb-specific options (altkv, check, group-like, nothing, etc.) use an empty slice to signal "migrated but no options." - Drop `omitempty` from VerbInfoForJSON.Options: empty slices were silently dropped, making migrated-no-option verbs indistinguishable from unmigrated ones in JSON. Without omitempty: null=unmigrated, []=migrated-no-options, [...]= migrated-with-options. Bump catalogSchemaVersion 1→2 for this shape change. - Replace the two "unmigrated-verb" unit tests (which used stats1 as an example) with TestAllVerbsFullyMigrated (asserts every verb has non-nil Options) and TestAllVerbsHaveOptionsKeyInJSON (asserts every migrated verb emits the "options" key in JSON). - Regenerate test/cases/cli-help/0003/expout: now reads "Verb options migration: 70/70 migrated." Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com> * remove a transitional helper * git rms * Render verb usage Options blocks from structured OptionSpec Each verb's usage message and its Tier-2 OptionSpec list previously duplicated the option text. New WriteVerbOptions (aaa_verb_usage.go) renders the "Options:" block from the specs: aligned flag column, descriptions word-wrapped at 80, uniform trailing -h|--help line. - OptionSpec gains Aliases (JSON "aliases") so long-form spellings like join's --lk|--left-keep-field-names survive in both outputs - All 70 verbs migrated; options literals hoisted to package-level vars (usage funcs can't reference their Setup var without a Go init cycle) - Hand-written per-option details the specs had condensed away are merged into Desc, enriching the JSON catalog - Non-option prose (examples, cross-references, dynamic accumulator listings) kept verbatim - Regenerated the six usage-embedding regression expectations and the two affected doc pages Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Fix pre-existing usage-text bugs surfaced by the OptionSpec migration - gap: usage said "One of -f or -g is required" but the parser takes -n or -g - seqgen: drop description line copy-pasted from cat ("Passes input records directly to output...") which contradicted "Discards the input record stream" - utf8-to-latin1: description read inverted ("from Latin-1 to UTF-8") - sec2gmtdate: usage said "../c/mlr" instead of "mlr" - top: document the accepted-but-undocumented --max flag - stats2: add linreg-pca to the -a enum values, matching the runtime accumulator table Regression expectations and docs regenerated accordingly. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Fix check usage sentence order; stats1 usage blank line to usage stream - check: the description's second and third lines were swapped, reading "Consumes records without printing any output, / Useful for doing a well-formatted check on input data. / with the exception that warnings are printed to stderr." - stats1: a bare fmt.Println() in the usage func wrote its blank line to process stdout instead of the usage output stream Regression expectation and docs regenerated. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
181 lines
5.1 KiB
Go
181 lines
5.1 KiB
Go
package transformers
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
|
|
"github.com/johnkerl/miller/v6/pkg/cli"
|
|
"github.com/johnkerl/miller/v6/pkg/lib"
|
|
"github.com/johnkerl/miller/v6/pkg/mlrval"
|
|
"github.com/johnkerl/miller/v6/pkg/types"
|
|
)
|
|
|
|
const verbNameSec2GMT = "sec2gmt"
|
|
|
|
var sec2GMTOptions = []OptionSpec{
|
|
{Flag: "-1", Type: "bool", Desc: "Format seconds with 1 decimal place."},
|
|
{Flag: "-2", Type: "bool", Desc: "Format seconds with 2 decimal places."},
|
|
{Flag: "-3", Type: "bool", Desc: "Format seconds with 3 decimal places."},
|
|
{Flag: "-4", Type: "bool", Desc: "Format seconds with 4 decimal places."},
|
|
{Flag: "-5", Type: "bool", Desc: "Format seconds with 5 decimal places."},
|
|
{Flag: "-6", Type: "bool", Desc: "Format seconds with 6 decimal places."},
|
|
{Flag: "-7", Type: "bool", Desc: "Format seconds with 7 decimal places."},
|
|
{Flag: "-8", Type: "bool", Desc: "Format seconds with 8 decimal places."},
|
|
{Flag: "-9", Type: "bool", Desc: "Format seconds with 9 decimal places."},
|
|
{Flag: "--millis", Type: "bool", Desc: "Input numbers are treated as milliseconds since the epoch."},
|
|
{Flag: "--micros", Type: "bool", Desc: "Input numbers are treated as microseconds since the epoch."},
|
|
{Flag: "--nanos", Type: "bool", Desc: "Input numbers are treated as nanoseconds since the epoch."},
|
|
}
|
|
|
|
var Sec2GMTSetup = TransformerSetup{
|
|
Verb: verbNameSec2GMT,
|
|
UsageFunc: transformerSec2GMTUsage,
|
|
ParseCLIFunc: transformerSec2GMTParseCLI,
|
|
IgnoresInput: false,
|
|
Options: sec2GMTOptions,
|
|
}
|
|
|
|
func transformerSec2GMTUsage(
|
|
o *os.File,
|
|
) {
|
|
fmt.Fprintf(o, "Usage: %s %s [options] {comma-separated list of field names}\n", "mlr", verbNameSec2GMT)
|
|
fmt.Fprintf(o, "Replaces a numeric field representing seconds since the epoch with the\n")
|
|
fmt.Fprintf(o, "corresponding GMT timestamp; leaves non-numbers as-is. This is nothing\n")
|
|
fmt.Fprintf(o, "more than a keystroke-saver for the sec2gmt function:\n")
|
|
fmt.Fprintf(o, " %s %s time1,time2\n", "mlr", verbNameSec2GMT)
|
|
fmt.Fprintf(o, "is the same as\n")
|
|
fmt.Fprintf(o, " %s put '$time1 = sec2gmt($time1); $time2 = sec2gmt($time2)'\n", "mlr")
|
|
WriteVerbOptions(o, sec2GMTOptions)
|
|
}
|
|
|
|
func transformerSec2GMTParseCLI(
|
|
pargi *int,
|
|
argc int,
|
|
args []string,
|
|
_ *cli.TOptions,
|
|
doConstruct bool, // false for first pass of CLI-parse, true for second pass
|
|
) (RecordTransformer, error) {
|
|
|
|
// Skip the verb name from the current spot in the mlr command line
|
|
argi := *pargi
|
|
argi++
|
|
|
|
preDivide := 1.0
|
|
numDecimalPlaces := 0
|
|
|
|
for argi < argc /* variable increment: 1 or 2 depending on flag */ {
|
|
opt := args[argi]
|
|
if opt[0] != '-' {
|
|
break // No more flag options to process
|
|
}
|
|
if args[argi] == "--" {
|
|
break // All transformers must do this so main-flags can follow verb-flags
|
|
}
|
|
argi++
|
|
|
|
switch opt {
|
|
case "-h", "--help":
|
|
transformerSec2GMTUsage(os.Stdout)
|
|
return nil, cli.ErrHelpRequested
|
|
|
|
case "-1":
|
|
numDecimalPlaces = 1
|
|
case "-2":
|
|
numDecimalPlaces = 2
|
|
case "-3":
|
|
numDecimalPlaces = 3
|
|
case "-4":
|
|
numDecimalPlaces = 4
|
|
case "-5":
|
|
numDecimalPlaces = 5
|
|
case "-6":
|
|
numDecimalPlaces = 6
|
|
case "-7":
|
|
numDecimalPlaces = 7
|
|
case "-8":
|
|
numDecimalPlaces = 8
|
|
case "-9":
|
|
numDecimalPlaces = 9
|
|
|
|
case "--millis":
|
|
preDivide = 1.0e3
|
|
case "--micros":
|
|
preDivide = 1.0e6
|
|
case "--nanos":
|
|
preDivide = 1.0e9
|
|
|
|
default:
|
|
return nil, cli.VerbErrorf(verbNameSec2GMT, "option \"%s\" not recognized", opt)
|
|
}
|
|
}
|
|
|
|
if argi >= argc {
|
|
return nil, cli.VerbErrorf(verbNameSec2GMT, "field names required")
|
|
}
|
|
fieldNames := args[argi]
|
|
argi++
|
|
|
|
*pargi = argi
|
|
if !doConstruct { // All transformers must do this for main command-line parsing
|
|
return nil, nil
|
|
}
|
|
|
|
transformer, err := NewTransformerSec2GMT(
|
|
fieldNames,
|
|
preDivide,
|
|
numDecimalPlaces,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return transformer, nil
|
|
}
|
|
|
|
type TransformerSec2GMT struct {
|
|
fieldNameList []string
|
|
preDivide float64
|
|
numDecimalPlaces int
|
|
}
|
|
|
|
func NewTransformerSec2GMT(
|
|
fieldNames string,
|
|
preDivide float64,
|
|
numDecimalPlaces int,
|
|
) (*TransformerSec2GMT, error) {
|
|
tr := &TransformerSec2GMT{
|
|
fieldNameList: lib.SplitString(fieldNames, ","),
|
|
preDivide: preDivide,
|
|
numDecimalPlaces: numDecimalPlaces,
|
|
}
|
|
return tr, nil
|
|
}
|
|
|
|
func (tr *TransformerSec2GMT) Transform(
|
|
inrecAndContext *types.RecordAndContext,
|
|
outputRecordsAndContexts *[]*types.RecordAndContext, // list of *types.RecordAndContext
|
|
inputDownstreamDoneChannel <-chan bool,
|
|
outputDownstreamDoneChannel chan<- bool,
|
|
) {
|
|
HandleDefaultDownstreamDone(inputDownstreamDoneChannel, outputDownstreamDoneChannel)
|
|
if !inrecAndContext.EndOfStream {
|
|
inrec := inrecAndContext.Record
|
|
for _, fieldName := range tr.fieldNameList {
|
|
value := inrec.Get(fieldName)
|
|
if value != nil {
|
|
floatval, ok := value.GetNumericToFloatValue()
|
|
if ok {
|
|
newValue := mlrval.FromString(lib.Sec2GMT(
|
|
floatval/tr.preDivide,
|
|
tr.numDecimalPlaces,
|
|
))
|
|
inrec.PutReference(fieldName, newValue)
|
|
}
|
|
}
|
|
}
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext)
|
|
|
|
} else { // End of record stream
|
|
*outputRecordsAndContexts = append(*outputRecordsAndContexts, inrecAndContext) // end-of-stream marker
|
|
}
|
|
}
|