From 2dd94cb1817e777f4b8aeb950be81898427e8e18 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sun, 28 Jun 2026 16:55:54 -0400 Subject: [PATCH] Add machine-readable help catalog: mlr help --json (#2098) (#2099) Emit Miller's existing help catalog (verbs, functions, flags, keywords) as structured JSON so AI agents and tooling can model Miller's surface without scraping prose. The --json token may appear anywhere on a `mlr help ...` command line; plain text help is unchanged. mlr help --json # full catalog mlr help verb cat --json # one or more verbs mlr help function splitax --json # one or more functions mlr help flag --ifs --json # one or more flags mlr help keyword ENV --json # one or more keywords Functions and flags serialize fully (name/class/arity/help/examples; section/name/alt_names/arg/help). Verbs carry a summary, ignores_input, and captured raw usage_text as a Tier-1 fallback, since per-verb options are prose-only today (each verb hand-writes its UsageFunc). Structured verb options are a planned follow-on (see #2098). This is a serialization layer over the existing registries -- no refactor of the text-help path. Co-authored-by: Claude Opus 4.8 --- pkg/cli/flag_json.go | 54 +++++++ pkg/dsl/cst/builtin_function_manager_json.go | 49 +++++++ pkg/dsl/cst/keyword_usage_json.go | 74 ++++++++++ pkg/terminals/help/entry.go | 7 + pkg/terminals/help/entry_json.go | 146 +++++++++++++++++++ pkg/terminals/help/entry_json_test.go | 107 ++++++++++++++ pkg/transformers/aaa_transformer_json.go | 94 ++++++++++++ 7 files changed, 531 insertions(+) create mode 100644 pkg/cli/flag_json.go create mode 100644 pkg/dsl/cst/builtin_function_manager_json.go create mode 100644 pkg/dsl/cst/keyword_usage_json.go create mode 100644 pkg/terminals/help/entry_json.go create mode 100644 pkg/terminals/help/entry_json_test.go create mode 100644 pkg/transformers/aaa_transformer_json.go diff --git a/pkg/cli/flag_json.go b/pkg/cli/flag_json.go new file mode 100644 index 000000000..ed1ba4ca9 --- /dev/null +++ b/pkg/cli/flag_json.go @@ -0,0 +1,54 @@ +// Machine-readable (JSON) accessors over the command-line flag table, for +// `mlr help --json` and similar tooling. Unlike verb options (which are +// prose-only), the flag table is already fully structured since it drives the +// actual command-line parser -- so this is a thin serialization layer. + +package cli + +// FlagInfoForJSON is the structured view of a single command-line flag. Section +// is the human-readable flag-section name (e.g. "CSV-only flags"); Arg is the +// argument placeholder in curly braces (e.g. "{filename}") or "" for boolean +// flags; AltNames carries alternate spellings (e.g. "--csv" alongside "-c"). +type FlagInfoForJSON struct { + Section string `json:"section"` + Name string `json:"name"` + AltNames []string `json:"alt_names,omitempty"` + Arg string `json:"arg,omitempty"` + Help string `json:"help"` +} + +func makeFlagInfoForJSON(sectionName string, flag *Flag) *FlagInfoForJSON { + return &FlagInfoForJSON{ + Section: sectionName, + Name: flag.name, + AltNames: flag.altNames, + Arg: flag.arg, + Help: flag.help, + } +} + +// GetFlagInfosForJSON returns the full flag catalog, grouped by section in +// table order, flattened into a single list with each flag tagged by section. +func (ft *FlagTable) GetFlagInfosForJSON() []*FlagInfoForJSON { + infos := make([]*FlagInfoForJSON, 0) + for _, section := range ft.sections { + for i := range section.flags { + infos = append(infos, makeFlagInfoForJSON(section.name, §ion.flags[i])) + } + } + return infos +} + +// GetFlagInfoForJSON returns the structured view of a single flag by name +// (matching either its primary name or any alternate spelling), or nil if there +// is no such flag. +func (ft *FlagTable) GetFlagInfoForJSON(name string) *FlagInfoForJSON { + for _, section := range ft.sections { + for i := range section.flags { + if section.flags[i].Owns(name) { + return makeFlagInfoForJSON(section.name, §ion.flags[i]) + } + } + } + return nil +} diff --git a/pkg/dsl/cst/builtin_function_manager_json.go b/pkg/dsl/cst/builtin_function_manager_json.go new file mode 100644 index 000000000..feff968fa --- /dev/null +++ b/pkg/dsl/cst/builtin_function_manager_json.go @@ -0,0 +1,49 @@ +// Machine-readable (JSON) accessors over the built-in-function catalog. These +// mirror the human-readable listings elsewhere in this file's neighbors (e.g. +// ListBuiltinFunctionUsages) but return structured data for `mlr help --json` +// and any other tooling -- AI agents in particular -- which needs to model +// Miller's function surface without scraping prose. + +package cst + +// FunctionInfoForJSON is the structured, marshalable view of a single built-in +// function. Field contents match what the text help shows: Arity is the same +// string produced by describeNargs (e.g. "1", "2,3", "1-4", "variadic"), and +// Help is JoinHelp()'d so source-code newlines are collapsed. +type FunctionInfoForJSON struct { + Name string `json:"name"` + Class string `json:"class"` + Arity string `json:"arity"` + Help string `json:"help"` + Examples []string `json:"examples,omitempty"` +} + +func makeFunctionInfoForJSON(info *BuiltinFunctionInfo) *FunctionInfoForJSON { + return &FunctionInfoForJSON{ + Name: info.name, + Class: string(info.class), + Arity: describeNargs(info), + Help: info.JoinHelp(), + Examples: info.examples, + } +} + +// GetFunctionInfosForJSON returns the full function catalog in source-table +// (insertion) order, matching the human-readable `mlr help usage-functions`. +func (mgr *BuiltinFunctionManager) GetFunctionInfosForJSON() []*FunctionInfoForJSON { + infos := make([]*FunctionInfoForJSON, 0, len(*mgr.lookupTable)) + for i := range *mgr.lookupTable { + infos = append(infos, makeFunctionInfoForJSON(&(*mgr.lookupTable)[i])) + } + return infos +} + +// GetFunctionInfoForJSON returns the structured view of a single function, or +// nil if there is no such function. +func (mgr *BuiltinFunctionManager) GetFunctionInfoForJSON(name string) *FunctionInfoForJSON { + info := mgr.LookUp(name) + if info == nil { + return nil + } + return makeFunctionInfoForJSON(info) +} diff --git a/pkg/dsl/cst/keyword_usage_json.go b/pkg/dsl/cst/keyword_usage_json.go new file mode 100644 index 000000000..417dc7b67 --- /dev/null +++ b/pkg/dsl/cst/keyword_usage_json.go @@ -0,0 +1,74 @@ +// Machine-readable (JSON) accessors over the keyword catalog, for +// `mlr help --json` and similar tooling. The per-keyword usage functions print +// their bodies directly to stdout (they predate any structured-help need), so +// here we capture that output by temporarily redirecting os.Stdout. + +package cst + +import ( + "bytes" + "io" + "os" + "strings" +) + +// KeywordInfoForJSON is the structured view of a single DSL keyword. +type KeywordInfoForJSON struct { + Name string `json:"name"` + Help string `json:"help"` +} + +// captureStdout runs f with os.Stdout redirected to a pipe and returns whatever +// f printed. The keyword usage functions write via fmt.Println/Printf, which +// resolve os.Stdout at call time, so swapping it here captures their output. +// Help generation is single-threaded and one-shot, so the global swap is safe. +func captureStdout(f func()) string { + old := os.Stdout + r, w, err := os.Pipe() + if err != nil { + return "" + } + os.Stdout = w + + done := make(chan string) + go func() { + var buf bytes.Buffer + io.Copy(&buf, r) + done <- buf.String() + }() + + f() + + w.Close() + os.Stdout = old + s := <-done + r.Close() + return s +} + +func makeKeywordInfoForJSON(entry *tKeywordUsageEntry) *KeywordInfoForJSON { + return &KeywordInfoForJSON{ + Name: entry.name, + Help: strings.TrimRight(captureStdout(entry.usageFunc), "\n"), + } +} + +// GetKeywordInfosForJSON returns the full keyword catalog in source-table order. +func GetKeywordInfosForJSON() []*KeywordInfoForJSON { + infos := make([]*KeywordInfoForJSON, 0, len(KEYWORD_USAGE_TABLE)) + for i := range KEYWORD_USAGE_TABLE { + infos = append(infos, makeKeywordInfoForJSON(&KEYWORD_USAGE_TABLE[i])) + } + return infos +} + +// GetKeywordInfoForJSON returns the structured view of a single keyword, or nil +// if there is no such keyword. +func GetKeywordInfoForJSON(name string) *KeywordInfoForJSON { + for i := range KEYWORD_USAGE_TABLE { + if KEYWORD_USAGE_TABLE[i].name == name { + return makeKeywordInfoForJSON(&KEYWORD_USAGE_TABLE[i]) + } + } + return nil +} diff --git a/pkg/terminals/help/entry.go b/pkg/terminals/help/entry.go index 5c3ec6796..5f06881e9 100644 --- a/pkg/terminals/help/entry.go +++ b/pkg/terminals/help/entry.go @@ -232,6 +232,13 @@ func GetTerminalFlagNames() []string { func HelpMain(args []string) int { args = args[1:] + // Machine-readable help: `mlr help --json [topic [names...]]`. The --json + // token may appear anywhere; if present we emit structured JSON and the + // plain-text handlers below are bypassed. + if jsonMode, rest := extractJSONFlag(args); jsonMode { + return helpJSON(rest) + } + // "mlr help" and nothing else if len(args) == 0 { handleDefault() diff --git a/pkg/terminals/help/entry_json.go b/pkg/terminals/help/entry_json.go new file mode 100644 index 000000000..b682f994b --- /dev/null +++ b/pkg/terminals/help/entry_json.go @@ -0,0 +1,146 @@ +// Machine-readable (JSON) help, for `mlr help --json` and friends. +// +// This assembles the structured catalogs exposed by the verb, function, flag, +// and keyword registries into a single document, so AI agents and other tooling +// can model Miller's surface without scraping the human-readable prose. The +// plain (non-`--json`) help behavior is unchanged; `--json` only switches the +// rendering. + +package help + +import ( + "encoding/json" + "fmt" + + "github.com/johnkerl/miller/v6/pkg/cli" + "github.com/johnkerl/miller/v6/pkg/dsl/cst" + "github.com/johnkerl/miller/v6/pkg/transformers" + "github.com/johnkerl/miller/v6/pkg/version" +) + +// CatalogForJSON is the top-level document emitted by `mlr help --json` with no +// further topic: the entire help catalog in one machine-readable object. +type CatalogForJSON struct { + MlrVersion string `json:"mlr_version"` + Verbs []*transformers.VerbInfoForJSON `json:"verbs"` + Functions []*cst.FunctionInfoForJSON `json:"functions"` + Flags []*cli.FlagInfoForJSON `json:"flags"` + Keywords []*cst.KeywordInfoForJSON `json:"keywords"` +} + +// extractJSONFlag removes any "--json" token from args, returning whether one +// was present along with the remaining args. The flag may appear anywhere +// (e.g. `mlr help --json` or `mlr help verb cat --json`). +func extractJSONFlag(args []string) (bool, []string) { + jsonMode := false + kept := make([]string, 0, len(args)) + for _, arg := range args { + if arg == "--json" { + jsonMode = true + } else { + kept = append(kept, arg) + } + } + return jsonMode, kept +} + +// printAsJSON marshals v as indented JSON to stdout. Returns a process exit +// code. +func printAsJSON(v any) int { + bytes, err := json.MarshalIndent(v, "", " ") + if err != nil { + fmt.Printf("mlr help: could not render JSON: %v\n", err) + return 1 + } + fmt.Println(string(bytes)) + return 0 +} + +// helpJSON dispatches `mlr help --json [topic [names...]]`. With no topic it +// emits the full catalog; with a topic (verb/function/flag/keyword) it emits +// just those entries -- all of them if no names are given, or the named ones. +func helpJSON(args []string) int { + if len(args) == 0 { + return printAsJSON(buildFullCatalog()) + } + + topic := args[0] + names := args[1:] + + switch topic { + case "verb", "verbs": + return printAsJSON(collectVerbs(names)) + case "function", "functions": + return printAsJSON(collectFunctions(names)) + case "flag", "flags": + return printAsJSON(collectFlags(names)) + case "keyword", "keywords": + return printAsJSON(collectKeywords(names)) + default: + fmt.Printf("mlr help --json: unsupported topic \"%s\".\n", topic) + fmt.Printf("Supported: (no topic) for the full catalog, or one of: verb, function, flag, keyword.\n") + return 1 + } +} + +func buildFullCatalog() *CatalogForJSON { + return &CatalogForJSON{ + MlrVersion: version.STRING, + Verbs: transformers.GetVerbInfosForJSON(), + Functions: cst.BuiltinFunctionManagerInstance.GetFunctionInfosForJSON(), + Flags: cli.FLAG_TABLE.GetFlagInfosForJSON(), + Keywords: cst.GetKeywordInfosForJSON(), + } +} + +func collectVerbs(names []string) []*transformers.VerbInfoForJSON { + if len(names) == 0 { + return transformers.GetVerbInfosForJSON() + } + infos := make([]*transformers.VerbInfoForJSON, 0, len(names)) + for _, name := range names { + if info := transformers.GetVerbInfoForJSON(name); info != nil { + infos = append(infos, info) + } + } + return infos +} + +func collectFunctions(names []string) []*cst.FunctionInfoForJSON { + if len(names) == 0 { + return cst.BuiltinFunctionManagerInstance.GetFunctionInfosForJSON() + } + infos := make([]*cst.FunctionInfoForJSON, 0, len(names)) + for _, name := range names { + if info := cst.BuiltinFunctionManagerInstance.GetFunctionInfoForJSON(name); info != nil { + infos = append(infos, info) + } + } + return infos +} + +func collectFlags(names []string) []*cli.FlagInfoForJSON { + if len(names) == 0 { + return cli.FLAG_TABLE.GetFlagInfosForJSON() + } + infos := make([]*cli.FlagInfoForJSON, 0, len(names)) + for _, name := range names { + if info := cli.FLAG_TABLE.GetFlagInfoForJSON(name); info != nil { + infos = append(infos, info) + } + } + return infos +} + +func collectKeywords(names []string) []*cst.KeywordInfoForJSON { + if len(names) == 0 { + return cst.GetKeywordInfosForJSON() + } + infos := make([]*cst.KeywordInfoForJSON, 0, len(names)) + for _, name := range names { + if info := cst.GetKeywordInfoForJSON(name); info != nil { + infos = append(infos, info) + } + } + return infos +} diff --git a/pkg/terminals/help/entry_json_test.go b/pkg/terminals/help/entry_json_test.go new file mode 100644 index 000000000..e3189ff00 --- /dev/null +++ b/pkg/terminals/help/entry_json_test.go @@ -0,0 +1,107 @@ +package help + +import ( + "encoding/json" + "testing" +) + +// TestFullCatalogIsValidJSON guards the `mlr help --json` contract: the +// assembled catalog must round-trip through encoding/json and carry a non-empty +// entry in each of the four sections. +func TestFullCatalogIsValidJSON(t *testing.T) { + catalog := buildFullCatalog() + + bytes, err := json.Marshal(catalog) + if err != nil { + t.Fatalf("catalog failed to marshal: %v", err) + } + + var roundTrip CatalogForJSON + if err := json.Unmarshal(bytes, &roundTrip); err != nil { + t.Fatalf("catalog failed to unmarshal: %v", err) + } + + if catalog.MlrVersion == "" { + t.Error("catalog mlr_version is empty") + } + if len(catalog.Verbs) == 0 { + t.Error("catalog has no verbs") + } + if len(catalog.Functions) == 0 { + t.Error("catalog has no functions") + } + if len(catalog.Flags) == 0 { + t.Error("catalog has no flags") + } + if len(catalog.Keywords) == 0 { + t.Error("catalog has no keywords") + } +} + +// TestCatalogEntriesArePopulated checks that the structured fields agents rely +// on are actually filled in, not just present. +func TestCatalogEntriesArePopulated(t *testing.T) { + catalog := buildFullCatalog() + + for _, verb := range catalog.Verbs { + if verb.Name == "" { + t.Error("found a verb with an empty name") + } + if verb.UsageText == "" { + t.Errorf("verb %q has empty usage_text", verb.Name) + } + } + + for _, function := range catalog.Functions { + if function.Name == "" { + t.Error("found a function with an empty name") + } + if function.Class == "" { + t.Errorf("function %q has empty class", function.Name) + } + if function.Arity == "" { + t.Errorf("function %q has empty arity", function.Name) + } + if function.Help == "" { + t.Errorf("function %q has empty help", function.Name) + } + } + + for _, flag := range catalog.Flags { + if flag.Name == "" { + t.Error("found a flag with an empty name") + } + if flag.Section == "" { + t.Errorf("flag %q has empty section", flag.Name) + } + } + + for _, keyword := range catalog.Keywords { + if keyword.Name == "" { + t.Error("found a keyword with an empty name") + } + if keyword.Help == "" { + t.Errorf("keyword %q has empty help", keyword.Name) + } + } +} + +// TestPerTopicLookups exercises the single-entry lookups used by +// `mlr help --json`, including the not-found path. +func TestPerTopicLookups(t *testing.T) { + if got := collectVerbs([]string{"cat"}); len(got) != 1 || got[0].Name != "cat" { + t.Errorf("verb lookup for cat: got %+v", got) + } + if got := collectVerbs([]string{"no-such-verb"}); len(got) != 0 { + t.Errorf("verb lookup for bogus name should be empty, got %+v", got) + } + if got := collectFunctions([]string{"splitax"}); len(got) != 1 || got[0].Name != "splitax" { + t.Errorf("function lookup for splitax: got %+v", got) + } + if got := collectKeywords([]string{"ENV"}); len(got) != 1 || got[0].Name != "ENV" { + t.Errorf("keyword lookup for ENV: got %+v", got) + } + if got := collectFlags([]string{"--ifs"}); len(got) != 1 || got[0].Name != "--ifs" { + t.Errorf("flag lookup for --ifs: got %+v", got) + } +} diff --git a/pkg/transformers/aaa_transformer_json.go b/pkg/transformers/aaa_transformer_json.go new file mode 100644 index 000000000..0162a853d --- /dev/null +++ b/pkg/transformers/aaa_transformer_json.go @@ -0,0 +1,94 @@ +// Machine-readable (JSON) accessors over the verb (transformer) catalog, for +// `mlr help --json` and similar tooling. +// +// Tier-1 caveat: unlike functions and flags, verb options are not held in any +// structured form -- each verb hand-writes a UsageFunc that prints prose. So +// here we expose the verb name, a one-line summary, and the captured raw usage +// text. Structured per-verb options (flag/arg/type) are a planned follow-on +// (an optional Options field on TransformerSetup); when present they can be +// emitted alongside UsageText. + +package transformers + +import ( + "bytes" + "io" + "os" + "strings" +) + +// VerbInfoForJSON is the structured view of a single verb. Summary is the first +// non-"Usage:" line of the usage text; UsageText is the verb's full usage +// output verbatim (the Tier-1 fallback for not-yet-structured options). +type VerbInfoForJSON struct { + Name string `json:"name"` + Summary string `json:"summary"` + IgnoresInput bool `json:"ignores_input"` + UsageText string `json:"usage_text"` +} + +// captureUsageFunc runs a verb's UsageFunc against a pipe and returns what it +// printed. The UsageFunc signature takes an *os.File, so we hand it the +// write-end of a pipe directly -- no global-stdout swap needed. A goroutine +// drains the read-end so we never block on the OS pipe buffer. +func captureUsageFunc(usageFunc TransformerUsageFunc) string { + r, w, err := os.Pipe() + if err != nil { + return "" + } + + done := make(chan string) + go func() { + var buf bytes.Buffer + io.Copy(&buf, r) + done <- buf.String() + }() + + usageFunc(w) + w.Close() + s := <-done + r.Close() + return s +} + +// summarizeUsageText returns the first line that isn't blank or a "Usage:" +// banner -- the closest thing each verb has to a one-line description. +func summarizeUsageText(usageText string) string { + for _, line := range strings.Split(usageText, "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "Usage:") { + continue + } + return trimmed + } + return "" +} + +func makeVerbInfoForJSON(setup *TransformerSetup) *VerbInfoForJSON { + usageText := captureUsageFunc(setup.UsageFunc) + return &VerbInfoForJSON{ + Name: setup.Verb, + Summary: summarizeUsageText(usageText), + IgnoresInput: setup.IgnoresInput, + UsageText: strings.TrimRight(usageText, "\n"), + } +} + +// GetVerbInfosForJSON returns the full verb catalog in table order. +func GetVerbInfosForJSON() []*VerbInfoForJSON { + infos := make([]*VerbInfoForJSON, 0, len(TRANSFORMER_LOOKUP_TABLE)) + for i := range TRANSFORMER_LOOKUP_TABLE { + infos = append(infos, makeVerbInfoForJSON(&TRANSFORMER_LOOKUP_TABLE[i])) + } + return infos +} + +// GetVerbInfoForJSON returns the structured view of a single verb, or nil if +// there is no such verb. +func GetVerbInfoForJSON(verb string) *VerbInfoForJSON { + setup := LookUp(verb) + if setup == nil { + return nil + } + return makeVerbInfoForJSON(setup) +}