Add machine-readable help catalog: mlr help --json (#2098) (#2099)

Emit Miller's existing help catalog (verbs, functions, flags, keywords)
as structured JSON so AI agents and tooling can model Miller's surface
without scraping prose. The --json token may appear anywhere on a
`mlr help ...` command line; plain text help is unchanged.

  mlr help --json                  # full catalog
  mlr help verb cat --json         # one or more verbs
  mlr help function splitax --json # one or more functions
  mlr help flag --ifs --json       # one or more flags
  mlr help keyword ENV --json      # one or more keywords

Functions and flags serialize fully (name/class/arity/help/examples;
section/name/alt_names/arg/help). Verbs carry a summary, ignores_input,
and captured raw usage_text as a Tier-1 fallback, since per-verb options
are prose-only today (each verb hand-writes its UsageFunc). Structured
verb options are a planned follow-on (see #2098).

This is a serialization layer over the existing registries -- no
refactor of the text-help path.

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

pkg/cli/flag_json.go

@@ -0,0 +1,54 @@
+// Machine-readable (JSON) accessors over the command-line flag table, for
+// `mlr help --json` and similar tooling. Unlike verb options (which are
+// prose-only), the flag table is already fully structured since it drives the
+// actual command-line parser -- so this is a thin serialization layer.
+
+package cli
+
+// FlagInfoForJSON is the structured view of a single command-line flag. Section
+// is the human-readable flag-section name (e.g. "CSV-only flags"); Arg is the
+// argument placeholder in curly braces (e.g. "{filename}") or "" for boolean
+// flags; AltNames carries alternate spellings (e.g. "--csv" alongside "-c").
+type FlagInfoForJSON struct {
+	Section  string   `json:"section"`
+	Name     string   `json:"name"`
+	AltNames []string `json:"alt_names,omitempty"`
+	Arg      string   `json:"arg,omitempty"`
+	Help     string   `json:"help"`
+}
+
+func makeFlagInfoForJSON(sectionName string, flag *Flag) *FlagInfoForJSON {
+	return &FlagInfoForJSON{
+		Section:  sectionName,
+		Name:     flag.name,
+		AltNames: flag.altNames,
+		Arg:      flag.arg,
+		Help:     flag.help,
+	}
+}
+
+// GetFlagInfosForJSON returns the full flag catalog, grouped by section in
+// table order, flattened into a single list with each flag tagged by section.
+func (ft *FlagTable) GetFlagInfosForJSON() []*FlagInfoForJSON {
+	infos := make([]*FlagInfoForJSON, 0)
+	for _, section := range ft.sections {
+		for i := range section.flags {
+			infos = append(infos, makeFlagInfoForJSON(section.name, &section.flags[i]))
+		}
+	}
+	return infos
+}
+
+// GetFlagInfoForJSON returns the structured view of a single flag by name
+// (matching either its primary name or any alternate spelling), or nil if there
+// is no such flag.
+func (ft *FlagTable) GetFlagInfoForJSON(name string) *FlagInfoForJSON {
+	for _, section := range ft.sections {
+		for i := range section.flags {
+			if section.flags[i].Owns(name) {
+				return makeFlagInfoForJSON(section.name, &section.flags[i])
+			}
+		}
+	}
+	return nil
+}

pkg/dsl/cst/builtin_function_manager_json.go

@@ -0,0 +1,49 @@
+// Machine-readable (JSON) accessors over the built-in-function catalog. These
+// mirror the human-readable listings elsewhere in this file's neighbors (e.g.
+// ListBuiltinFunctionUsages) but return structured data for `mlr help --json`
+// and any other tooling -- AI agents in particular -- which needs to model
+// Miller's function surface without scraping prose.
+
+package cst
+
+// FunctionInfoForJSON is the structured, marshalable view of a single built-in
+// function. Field contents match what the text help shows: Arity is the same
+// string produced by describeNargs (e.g. "1", "2,3", "1-4", "variadic"), and
+// Help is JoinHelp()'d so source-code newlines are collapsed.
+type FunctionInfoForJSON struct {
+	Name     string   `json:"name"`
+	Class    string   `json:"class"`
+	Arity    string   `json:"arity"`
+	Help     string   `json:"help"`
+	Examples []string `json:"examples,omitempty"`
+}
+
+func makeFunctionInfoForJSON(info *BuiltinFunctionInfo) *FunctionInfoForJSON {
+	return &FunctionInfoForJSON{
+		Name:     info.name,
+		Class:    string(info.class),
+		Arity:    describeNargs(info),
+		Help:     info.JoinHelp(),
+		Examples: info.examples,
+	}
+}
+
+// GetFunctionInfosForJSON returns the full function catalog in source-table
+// (insertion) order, matching the human-readable `mlr help usage-functions`.
+func (mgr *BuiltinFunctionManager) GetFunctionInfosForJSON() []*FunctionInfoForJSON {
+	infos := make([]*FunctionInfoForJSON, 0, len(*mgr.lookupTable))
+	for i := range *mgr.lookupTable {
+		infos = append(infos, makeFunctionInfoForJSON(&(*mgr.lookupTable)[i]))
+	}
+	return infos
+}
+
+// GetFunctionInfoForJSON returns the structured view of a single function, or
+// nil if there is no such function.
+func (mgr *BuiltinFunctionManager) GetFunctionInfoForJSON(name string) *FunctionInfoForJSON {
+	info := mgr.LookUp(name)
+	if info == nil {
+		return nil
+	}
+	return makeFunctionInfoForJSON(info)
+}

pkg/dsl/cst/keyword_usage_json.go

@@ -0,0 +1,74 @@
+// Machine-readable (JSON) accessors over the keyword catalog, for
+// `mlr help --json` and similar tooling. The per-keyword usage functions print
+// their bodies directly to stdout (they predate any structured-help need), so
+// here we capture that output by temporarily redirecting os.Stdout.
+
+package cst
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"strings"
+)
+
+// KeywordInfoForJSON is the structured view of a single DSL keyword.
+type KeywordInfoForJSON struct {
+	Name string `json:"name"`
+	Help string `json:"help"`
+}
+
+// captureStdout runs f with os.Stdout redirected to a pipe and returns whatever
+// f printed. The keyword usage functions write via fmt.Println/Printf, which
+// resolve os.Stdout at call time, so swapping it here captures their output.
+// Help generation is single-threaded and one-shot, so the global swap is safe.
+func captureStdout(f func()) string {
+	old := os.Stdout
+	r, w, err := os.Pipe()
+	if err != nil {
+		return ""
+	}
+	os.Stdout = w
+
+	done := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, r)
+		done <- buf.String()
+	}()
+
+	f()
+
+	w.Close()
+	os.Stdout = old
+	s := <-done
+	r.Close()
+	return s
+}
+
+func makeKeywordInfoForJSON(entry *tKeywordUsageEntry) *KeywordInfoForJSON {
+	return &KeywordInfoForJSON{
+		Name: entry.name,
+		Help: strings.TrimRight(captureStdout(entry.usageFunc), "\n"),
+	}
+}
+
+// GetKeywordInfosForJSON returns the full keyword catalog in source-table order.
+func GetKeywordInfosForJSON() []*KeywordInfoForJSON {
+	infos := make([]*KeywordInfoForJSON, 0, len(KEYWORD_USAGE_TABLE))
+	for i := range KEYWORD_USAGE_TABLE {
+		infos = append(infos, makeKeywordInfoForJSON(&KEYWORD_USAGE_TABLE[i]))
+	}
+	return infos
+}
+
+// GetKeywordInfoForJSON returns the structured view of a single keyword, or nil
+// if there is no such keyword.
+func GetKeywordInfoForJSON(name string) *KeywordInfoForJSON {
+	for i := range KEYWORD_USAGE_TABLE {
+		if KEYWORD_USAGE_TABLE[i].name == name {
+			return makeKeywordInfoForJSON(&KEYWORD_USAGE_TABLE[i])
+		}
+	}
+	return nil
+}

pkg/terminals/help/entry.go

@@ -232,6 +232,13 @@ func GetTerminalFlagNames() []string {
 func HelpMain(args []string) int {
 	args = args[1:]
 
+	// Machine-readable help: `mlr help --json [topic [names...]]`. The --json
+	// token may appear anywhere; if present we emit structured JSON and the
+	// plain-text handlers below are bypassed.
+	if jsonMode, rest := extractJSONFlag(args); jsonMode {
+		return helpJSON(rest)
+	}
+
 	// "mlr help" and nothing else
 	if len(args) == 0 {
 		handleDefault()

pkg/terminals/help/entry_json.go

@@ -0,0 +1,146 @@
+// Machine-readable (JSON) help, for `mlr help --json` and friends.
+//
+// This assembles the structured catalogs exposed by the verb, function, flag,
+// and keyword registries into a single document, so AI agents and other tooling
+// can model Miller's surface without scraping the human-readable prose. The
+// plain (non-`--json`) help behavior is unchanged; `--json` only switches the
+// rendering.
+
+package help
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/johnkerl/miller/v6/pkg/cli"
+	"github.com/johnkerl/miller/v6/pkg/dsl/cst"
+	"github.com/johnkerl/miller/v6/pkg/transformers"
+	"github.com/johnkerl/miller/v6/pkg/version"
+)
+
+// CatalogForJSON is the top-level document emitted by `mlr help --json` with no
+// further topic: the entire help catalog in one machine-readable object.
+type CatalogForJSON struct {
+	MlrVersion string                          `json:"mlr_version"`
+	Verbs      []*transformers.VerbInfoForJSON `json:"verbs"`
+	Functions  []*cst.FunctionInfoForJSON      `json:"functions"`
+	Flags      []*cli.FlagInfoForJSON          `json:"flags"`
+	Keywords   []*cst.KeywordInfoForJSON       `json:"keywords"`
+}
+
+// extractJSONFlag removes any "--json" token from args, returning whether one
+// was present along with the remaining args. The flag may appear anywhere
+// (e.g. `mlr help --json` or `mlr help verb cat --json`).
+func extractJSONFlag(args []string) (bool, []string) {
+	jsonMode := false
+	kept := make([]string, 0, len(args))
+	for _, arg := range args {
+		if arg == "--json" {
+			jsonMode = true
+		} else {
+			kept = append(kept, arg)
+		}
+	}
+	return jsonMode, kept
+}
+
+// printAsJSON marshals v as indented JSON to stdout. Returns a process exit
+// code.
+func printAsJSON(v any) int {
+	bytes, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		fmt.Printf("mlr help: could not render JSON: %v\n", err)
+		return 1
+	}
+	fmt.Println(string(bytes))
+	return 0
+}
+
+// helpJSON dispatches `mlr help --json [topic [names...]]`. With no topic it
+// emits the full catalog; with a topic (verb/function/flag/keyword) it emits
+// just those entries -- all of them if no names are given, or the named ones.
+func helpJSON(args []string) int {
+	if len(args) == 0 {
+		return printAsJSON(buildFullCatalog())
+	}
+
+	topic := args[0]
+	names := args[1:]
+
+	switch topic {
+	case "verb", "verbs":
+		return printAsJSON(collectVerbs(names))
+	case "function", "functions":
+		return printAsJSON(collectFunctions(names))
+	case "flag", "flags":
+		return printAsJSON(collectFlags(names))
+	case "keyword", "keywords":
+		return printAsJSON(collectKeywords(names))
+	default:
+		fmt.Printf("mlr help --json: unsupported topic \"%s\".\n", topic)
+		fmt.Printf("Supported: (no topic) for the full catalog, or one of: verb, function, flag, keyword.\n")
+		return 1
+	}
+}
+
+func buildFullCatalog() *CatalogForJSON {
+	return &CatalogForJSON{
+		MlrVersion: version.STRING,
+		Verbs:      transformers.GetVerbInfosForJSON(),
+		Functions:  cst.BuiltinFunctionManagerInstance.GetFunctionInfosForJSON(),
+		Flags:      cli.FLAG_TABLE.GetFlagInfosForJSON(),
+		Keywords:   cst.GetKeywordInfosForJSON(),
+	}
+}
+
+func collectVerbs(names []string) []*transformers.VerbInfoForJSON {
+	if len(names) == 0 {
+		return transformers.GetVerbInfosForJSON()
+	}
+	infos := make([]*transformers.VerbInfoForJSON, 0, len(names))
+	for _, name := range names {
+		if info := transformers.GetVerbInfoForJSON(name); info != nil {
+			infos = append(infos, info)
+		}
+	}
+	return infos
+}
+
+func collectFunctions(names []string) []*cst.FunctionInfoForJSON {
+	if len(names) == 0 {
+		return cst.BuiltinFunctionManagerInstance.GetFunctionInfosForJSON()
+	}
+	infos := make([]*cst.FunctionInfoForJSON, 0, len(names))
+	for _, name := range names {
+		if info := cst.BuiltinFunctionManagerInstance.GetFunctionInfoForJSON(name); info != nil {
+			infos = append(infos, info)
+		}
+	}
+	return infos
+}
+
+func collectFlags(names []string) []*cli.FlagInfoForJSON {
+	if len(names) == 0 {
+		return cli.FLAG_TABLE.GetFlagInfosForJSON()
+	}
+	infos := make([]*cli.FlagInfoForJSON, 0, len(names))
+	for _, name := range names {
+		if info := cli.FLAG_TABLE.GetFlagInfoForJSON(name); info != nil {
+			infos = append(infos, info)
+		}
+	}
+	return infos
+}
+
+func collectKeywords(names []string) []*cst.KeywordInfoForJSON {
+	if len(names) == 0 {
+		return cst.GetKeywordInfosForJSON()
+	}
+	infos := make([]*cst.KeywordInfoForJSON, 0, len(names))
+	for _, name := range names {
+		if info := cst.GetKeywordInfoForJSON(name); info != nil {
+			infos = append(infos, info)
+		}
+	}
+	return infos
+}

pkg/terminals/help/entry_json_test.go

@@ -0,0 +1,107 @@
+package help
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+// TestFullCatalogIsValidJSON guards the `mlr help --json` contract: the
+// assembled catalog must round-trip through encoding/json and carry a non-empty
+// entry in each of the four sections.
+func TestFullCatalogIsValidJSON(t *testing.T) {
+	catalog := buildFullCatalog()
+
+	bytes, err := json.Marshal(catalog)
+	if err != nil {
+		t.Fatalf("catalog failed to marshal: %v", err)
+	}
+
+	var roundTrip CatalogForJSON
+	if err := json.Unmarshal(bytes, &roundTrip); err != nil {
+		t.Fatalf("catalog failed to unmarshal: %v", err)
+	}
+
+	if catalog.MlrVersion == "" {
+		t.Error("catalog mlr_version is empty")
+	}
+	if len(catalog.Verbs) == 0 {
+		t.Error("catalog has no verbs")
+	}
+	if len(catalog.Functions) == 0 {
+		t.Error("catalog has no functions")
+	}
+	if len(catalog.Flags) == 0 {
+		t.Error("catalog has no flags")
+	}
+	if len(catalog.Keywords) == 0 {
+		t.Error("catalog has no keywords")
+	}
+}
+
+// TestCatalogEntriesArePopulated checks that the structured fields agents rely
+// on are actually filled in, not just present.
+func TestCatalogEntriesArePopulated(t *testing.T) {
+	catalog := buildFullCatalog()
+
+	for _, verb := range catalog.Verbs {
+		if verb.Name == "" {
+			t.Error("found a verb with an empty name")
+		}
+		if verb.UsageText == "" {
+			t.Errorf("verb %q has empty usage_text", verb.Name)
+		}
+	}
+
+	for _, function := range catalog.Functions {
+		if function.Name == "" {
+			t.Error("found a function with an empty name")
+		}
+		if function.Class == "" {
+			t.Errorf("function %q has empty class", function.Name)
+		}
+		if function.Arity == "" {
+			t.Errorf("function %q has empty arity", function.Name)
+		}
+		if function.Help == "" {
+			t.Errorf("function %q has empty help", function.Name)
+		}
+	}
+
+	for _, flag := range catalog.Flags {
+		if flag.Name == "" {
+			t.Error("found a flag with an empty name")
+		}
+		if flag.Section == "" {
+			t.Errorf("flag %q has empty section", flag.Name)
+		}
+	}
+
+	for _, keyword := range catalog.Keywords {
+		if keyword.Name == "" {
+			t.Error("found a keyword with an empty name")
+		}
+		if keyword.Help == "" {
+			t.Errorf("keyword %q has empty help", keyword.Name)
+		}
+	}
+}
+
+// TestPerTopicLookups exercises the single-entry lookups used by
+// `mlr help <topic> <name> --json`, including the not-found path.
+func TestPerTopicLookups(t *testing.T) {
+	if got := collectVerbs([]string{"cat"}); len(got) != 1 || got[0].Name != "cat" {
+		t.Errorf("verb lookup for cat: got %+v", got)
+	}
+	if got := collectVerbs([]string{"no-such-verb"}); len(got) != 0 {
+		t.Errorf("verb lookup for bogus name should be empty, got %+v", got)
+	}
+	if got := collectFunctions([]string{"splitax"}); len(got) != 1 || got[0].Name != "splitax" {
+		t.Errorf("function lookup for splitax: got %+v", got)
+	}
+	if got := collectKeywords([]string{"ENV"}); len(got) != 1 || got[0].Name != "ENV" {
+		t.Errorf("keyword lookup for ENV: got %+v", got)
+	}
+	if got := collectFlags([]string{"--ifs"}); len(got) != 1 || got[0].Name != "--ifs" {
+		t.Errorf("flag lookup for --ifs: got %+v", got)
+	}
+}

pkg/transformers/aaa_transformer_json.go

@@ -0,0 +1,94 @@
+// Machine-readable (JSON) accessors over the verb (transformer) catalog, for
+// `mlr help --json` and similar tooling.
+//
+// Tier-1 caveat: unlike functions and flags, verb options are not held in any
+// structured form -- each verb hand-writes a UsageFunc that prints prose. So
+// here we expose the verb name, a one-line summary, and the captured raw usage
+// text. Structured per-verb options (flag/arg/type) are a planned follow-on
+// (an optional Options field on TransformerSetup); when present they can be
+// emitted alongside UsageText.
+
+package transformers
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"strings"
+)
+
+// VerbInfoForJSON is the structured view of a single verb. Summary is the first
+// non-"Usage:" line of the usage text; UsageText is the verb's full usage
+// output verbatim (the Tier-1 fallback for not-yet-structured options).
+type VerbInfoForJSON struct {
+	Name         string `json:"name"`
+	Summary      string `json:"summary"`
+	IgnoresInput bool   `json:"ignores_input"`
+	UsageText    string `json:"usage_text"`
+}
+
+// captureUsageFunc runs a verb's UsageFunc against a pipe and returns what it
+// printed. The UsageFunc signature takes an *os.File, so we hand it the
+// write-end of a pipe directly -- no global-stdout swap needed. A goroutine
+// drains the read-end so we never block on the OS pipe buffer.
+func captureUsageFunc(usageFunc TransformerUsageFunc) string {
+	r, w, err := os.Pipe()
+	if err != nil {
+		return ""
+	}
+
+	done := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, r)
+		done <- buf.String()
+	}()
+
+	usageFunc(w)
+	w.Close()
+	s := <-done
+	r.Close()
+	return s
+}
+
+// summarizeUsageText returns the first line that isn't blank or a "Usage:"
+// banner -- the closest thing each verb has to a one-line description.
+func summarizeUsageText(usageText string) string {
+	for _, line := range strings.Split(usageText, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" || strings.HasPrefix(trimmed, "Usage:") {
+			continue
+		}
+		return trimmed
+	}
+	return ""
+}
+
+func makeVerbInfoForJSON(setup *TransformerSetup) *VerbInfoForJSON {
+	usageText := captureUsageFunc(setup.UsageFunc)
+	return &VerbInfoForJSON{
+		Name:         setup.Verb,
+		Summary:      summarizeUsageText(usageText),
+		IgnoresInput: setup.IgnoresInput,
+		UsageText:    strings.TrimRight(usageText, "\n"),
+	}
+}
+
+// GetVerbInfosForJSON returns the full verb catalog in table order.
+func GetVerbInfosForJSON() []*VerbInfoForJSON {
+	infos := make([]*VerbInfoForJSON, 0, len(TRANSFORMER_LOOKUP_TABLE))
+	for i := range TRANSFORMER_LOOKUP_TABLE {
+		infos = append(infos, makeVerbInfoForJSON(&TRANSFORMER_LOOKUP_TABLE[i]))
+	}
+	return infos
+}
+
+// GetVerbInfoForJSON returns the structured view of a single verb, or nil if
+// there is no such verb.
+func GetVerbInfoForJSON(verb string) *VerbInfoForJSON {
+	setup := LookUp(verb)
+	if setup == nil {
+		return nil
+	}
+	return makeVerbInfoForJSON(setup)
+}