mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-18 08:55:41 +00:00
* Add JSON index and mlr which capability router (#2098) PR 2 of the AI-friendly roadmap (plans/plan-2098-llm.md): - `mlr help --as-json --index` emits a lightweight [{kind,name,summary}] index across all 651 catalog items (verbs, functions, flags, keywords), sorted by kind then name. Agents use this as a cheap first call to pick a verb before fetching its full entry. - `mlr which "<query>"` is a new terminal that tokenizes a natural-language query, scores every catalog item (name match +20/token, body match +5/token), and returns ranked JSON [{kind,name,score,summary}]. Exit code 0 means a confident match (at least one token hit the item name); exit code 2 means low confidence. Agents branch on the exit code rather than parsing prose. Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com> * Move mlr which into pkg/terminals/help, deduplicate firstLine pkg/terminals/which/ was a misplaced package: it imported the same four catalog registries as pkg/terminals/help/ and duplicated the firstLine helper. Moving the logic into help/entry_which.go fixes both issues: - WhichMain and all which helpers now live alongside the other --as-json catalog machinery in pkg/terminals/help/ - indexFirstLine (entry_json.go) and firstLine (which/entry.go) collapse into a single firstLine shared by both files - pkg/terminals/terminals.go calls help.WhichMain directly; the pkg/terminals/which/ package is deleted No behavior change. Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com> * Fix five Go-purity issues found in code review 1. Exit-code false positive: whichScore now returns (int, bool) where the bool records whether any token hit the item name. WhichMain uses results[0].nameHit for exit-code 0, so 4 body-only token hits (4×5=20) no longer incorrectly signal a confident match. 2. Flag Summary inconsistency: whichSearch was setting Summary: fl.Help directly for flags while using firstLine(...) for functions and keywords. Changed to firstLine(fl.Help) so all four kinds behave consistently. 3+5. kindOrder/whichKindRank duplication: the verb<function<flag<keyword ordering was encoded twice — as a local map[string]int in buildIndex and as a switch in whichKindRank. Replaced both with a single package-level kindRank() function. The map lookup also silently returned 0 (= verb rank) for unknown kinds; the switch correctly returns 4 (sorts last). 4. extractIndexFlag/extractAsJSONFlag duplication: both had identical loop bodies differing only in the sentinel string. Introduced a generic extractFlag(args, flag) helper; both are now one-liners. Also promoted whichStopwords to a package-level var so whichTokenize does not allocate a new map on every call. Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
109 lines
3.6 KiB
Go
109 lines
3.6 KiB
Go
package help
|
||
|
||
import (
|
||
"testing"
|
||
)
|
||
|
||
// TestWhichTokenize verifies that query tokenization drops stopwords and deduplicates.
|
||
func TestWhichTokenize(t *testing.T) {
|
||
cases := []struct {
|
||
query string
|
||
expect []string
|
||
}{
|
||
{"join two files on a key", []string{"join", "files", "key"}},
|
||
{"count distinct values", []string{"count", "distinct", "values"}},
|
||
{"group records by field", []string{"group", "records", "field"}},
|
||
{"", nil},
|
||
{"a an the", nil},
|
||
}
|
||
for _, tc := range cases {
|
||
got := whichTokenize(tc.query)
|
||
if len(got) != len(tc.expect) {
|
||
t.Errorf("whichTokenize(%q): got %v, want %v", tc.query, got, tc.expect)
|
||
continue
|
||
}
|
||
for i, tok := range got {
|
||
if tok != tc.expect[i] {
|
||
t.Errorf("whichTokenize(%q)[%d]: got %q, want %q", tc.query, i, tok, tc.expect[i])
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestWhichScoreNameMatch verifies that a name hit scores higher than a body-only
|
||
// hit and that nameHit is set correctly.
|
||
func TestWhichScoreNameMatch(t *testing.T) {
|
||
tokens := []string{"join"}
|
||
nameScore, nameHit := whichScore(tokens, "join", "combine two streams")
|
||
bodyScore, bodyHit := whichScore(tokens, "combine", "join two records")
|
||
if nameScore <= bodyScore {
|
||
t.Errorf("name match (%d) should outscore body-only match (%d)", nameScore, bodyScore)
|
||
}
|
||
if nameScore != whichNameMatchScore {
|
||
t.Errorf("single-token name match: got %d, want %d", nameScore, whichNameMatchScore)
|
||
}
|
||
if !nameHit {
|
||
t.Error("nameHit should be true when token matches name")
|
||
}
|
||
if bodyHit {
|
||
t.Error("nameHit should be false when token matches only body")
|
||
}
|
||
}
|
||
|
||
// TestWhichExitCodeRequiresNameHit verifies that body-only matches (however
|
||
// many) do not trigger the confident-match exit code. This guards against the
|
||
// failure mode where 4 body-only token hits (4×5=20) equal whichNameMatchScore
|
||
// and would incorrectly signal a confident match.
|
||
func TestWhichExitCodeRequiresNameHit(t *testing.T) {
|
||
// Tokens that appear in the body but not in the name.
|
||
tokens := []string{"statistics", "aggregate", "compute", "average"}
|
||
name := "xyz-verb"
|
||
body := "statistics aggregate compute average"
|
||
score, nameHit := whichScore(tokens, name, body)
|
||
if nameHit {
|
||
t.Error("body-only hits should not set nameHit")
|
||
}
|
||
if score < whichNameMatchScore {
|
||
t.Errorf("expected body score >= %d to demonstrate the false-positive risk, got %d", whichNameMatchScore, score)
|
||
}
|
||
}
|
||
|
||
// TestWhichSearchReturnsJoinVerb checks that "join" as a query surfaces the join verb.
|
||
func TestWhichSearchReturnsJoinVerb(t *testing.T) {
|
||
tokens := whichTokenize("join two files on a key")
|
||
results := whichSearch(tokens)
|
||
if len(results) == 0 {
|
||
t.Fatal("expected at least one result for 'join'")
|
||
}
|
||
top := results[0]
|
||
if top.Name != "join" || top.Kind != "verb" {
|
||
t.Errorf("expected top result to be verb:join, got kind=%s name=%s", top.Kind, top.Name)
|
||
}
|
||
if top.Score < whichNameMatchScore {
|
||
t.Errorf("top result score %d below confident threshold %d", top.Score, whichNameMatchScore)
|
||
}
|
||
}
|
||
|
||
// TestWhichSearchReturnsCountVerb checks that "count records" surfaces the count verb.
|
||
func TestWhichSearchReturnsCountVerb(t *testing.T) {
|
||
tokens := whichTokenize("count records")
|
||
results := whichSearch(tokens)
|
||
found := false
|
||
for _, r := range results {
|
||
if r.Kind == "verb" && r.Name == "count" {
|
||
found = true
|
||
break
|
||
}
|
||
}
|
||
if !found {
|
||
t.Error("expected 'count' verb in results for 'count records'")
|
||
}
|
||
}
|
||
|
||
// TestWhichSearchAllStopwords ensures an all-stopword query returns nothing.
|
||
func TestWhichSearchAllStopwords(t *testing.T) {
|
||
tokens := whichTokenize("a an the to")
|
||
if len(tokens) != 0 {
|
||
t.Errorf("expected empty tokens for all-stopword query, got %v", tokens)
|
||
}
|
||
}
|