Files
multica/server/internal/daemon/execenv/runtime_config_kind_test.go
Bohan Jiang abdfd3e28c refactor(skills): make the brief's skill list a names-only index (MUL-5529) (#6207)
* refactor(skills): make the brief's skill list a names-only index (MUL-5529)

Step 3 of MUL-5529. Every runtime CLI discovers the SKILL.md files the daemon
writes and builds its own listing from their frontmatter — verified against 11
locally installed CLIs plus official docs for 5 more. The brief's copy of those
descriptions was therefore the same routing signal paid for twice: measured on
a real task, `## Skills` was 13,295 chars, 40% of the entire brief, against a
16,304-char CLI listing of the same 28 skills.

Now 850 chars for that same set — roughly 3,100 tokens back per brief.

The index itself stays. It is the one skill listing Multica controls; each
CLI's own listing is theirs, and its format — or its existence — can change
with any release.

Three changes:

  - Descriptions dropped from the `## Skills` entries.

  - The per-provider branch is gone. Its fallback told providers outside a
    hardcoded list to read `.agent_context/skills/`, but the only providers
    that ever reached it were grok and traecli, whose files are written to
    `.grok/skills` and `.traecli/skills` and which discover natively. The
    pointer was wrong for everyone it addressed, so removing the branch
    deletes the bug rather than relocating it. This closes MUL-5537.

  - issue_context.md and its quick-create / autopilot variants no longer render
    `## Agent Skills`. That copy duplicated the brief once both were
    names-only, and nothing ever read it: no prompt references the path, and
    grepping the server finds only the writer. `.agent_context/skills/` had the
    same fate for hermes (issue #5242). Quick-create, previously skipped in the
    brief and served only by that unread copy, now gets the brief section like
    every other kind — one index, one place.

Not included: skills carrying `disable-model-invocation` are still written to
disk for every provider. The plan assumed that key needed provider-specific
handling for everything except claude; probing the installed CLIs shows 9 of 11
honor it, and only opencode and hermes do not. The remaining question is
narrow and a genuine product tradeoff — withholding the file honors the
author's intent but also removes explicit invocation — so it is left to a
separate decision rather than folded in here.

Co-authored-by: multica-agent <github@multica.ai>

* docs(skills): align stale comments with the names-only brief contract (MUL-5529)

Co-authored-by: multica-agent <github@multica.ai>

---------

Co-authored-by: Bohan-J <bohan@devv.ai>
Co-authored-by: multica-agent <github@multica.ai>
Co-authored-by: Steve Jobs (Multica Agent) <agent-steve-jobs@multica.ai>
2026-07-31 13:27:50 +08:00

285 lines
10 KiB
Go

package execenv
import (
"strings"
"testing"
)
// TestClassifyTask pins the precedence rule on classifyTask. All four
// kinds plus tiebreak cases for safety.
func TestClassifyTask(t *testing.T) {
t.Parallel()
cases := []struct {
name string
ctx TaskContextForEnv
want taskKind
}{
{"chat", TaskContextForEnv{ChatSessionID: "c"}, kindChat},
{"quick-create", TaskContextForEnv{QuickCreatePrompt: "p"}, kindQuickCreate},
{"autopilot", TaskContextForEnv{AutopilotRunID: "r"}, kindAutopilotRunOnly},
{"issue-comment-triggered", TaskContextForEnv{IssueID: "i", TriggerCommentID: "c"}, kindIssue},
{"issue-assignment-triggered", TaskContextForEnv{IssueID: "i"}, kindIssue},
{"issue-bare", TaskContextForEnv{}, kindIssue},
{"tiebreak-chat-vs-quick", TaskContextForEnv{ChatSessionID: "c", QuickCreatePrompt: "p"}, kindChat},
{"tiebreak-quick-vs-autopilot", TaskContextForEnv{QuickCreatePrompt: "p", AutopilotRunID: "r"}, kindQuickCreate},
{"tiebreak-autopilot-vs-comment", TaskContextForEnv{AutopilotRunID: "r", IssueID: "i", TriggerCommentID: "c"}, kindAutopilotRunOnly},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
if got := classifyTask(tc.ctx); got != tc.want {
t.Errorf("classifyTask: got %d, want %d", got, tc.want)
}
})
}
}
// TestTaskKindHasIssueContext pins the predicate that gates Project
// Context / Issue Metadata / Sub-issue Creation in the slim dispatcher.
func TestTaskKindHasIssueContext(t *testing.T) {
t.Parallel()
cases := []struct {
kind taskKind
want bool
}{
{kindIssue, true},
{kindAutopilotRunOnly, false},
{kindQuickCreate, false},
{kindChat, false},
}
for _, tc := range cases {
if got := tc.kind.hasIssueContext(); got != tc.want {
t.Errorf("kind=%d hasIssueContext: got %v, want %v", tc.kind, got, tc.want)
}
}
}
// TestBuildMetaSkillContentBriefContent pins that buildMetaSkillContent
// renders the (now sole) brief: the `issue get` one-liner is present and
// the retired legacy verbose description is not.
func TestBuildMetaSkillContentBriefContent(t *testing.T) {
t.Parallel()
out := buildMetaSkillContent("claude", TaskContextForEnv{
IssueID: "issue-1",
TriggerCommentID: "comment-1",
AgentName: "Eve",
AgentID: "eve-1",
})
if !strings.Contains(out, "- `multica issue get <id> --output json` — full issue.\n") {
t.Errorf("brief is missing the `issue get` one-liner\n---\n%s", out)
}
if strings.Contains(out, "Get full issue details.") {
t.Errorf("brief still carries the retired legacy `issue get` description\n---\n%s", out)
}
}
// TestBuildMetaSkillContentIssueBodyFormatting pins the shared issue-body
// hierarchy rule across every task kind that can author an issue.
func TestBuildMetaSkillContentIssueBodyFormatting(t *testing.T) {
t.Parallel()
fixtures := map[string]TaskContextForEnv{
"issue": {IssueID: "i-1"},
"autopilot": {AutopilotRunID: "r-1"},
"quick-create": {QuickCreatePrompt: "create an issue"},
"chat": {ChatSessionID: "c-1"},
}
for name, ctx := range fixtures {
name, ctx := name, ctx
t.Run(name, func(t *testing.T) {
t.Parallel()
out := buildMetaSkillContent("codex", ctx)
for _, want := range []string{
"## Issue Body Formatting",
"An issue title already serves as its H1.",
"do not add a Markdown H1 (`# ...`)",
"start with prose or `##` subheadings",
"Only add an H1 when the user specifically requests one",
} {
if !strings.Contains(out, want) {
t.Errorf("brief is missing issue-body formatting guidance %q\n---\n%s", want, out)
}
}
})
}
}
// TestBuildMetaSkillContentSlimKindMatrix locks in which sections the
// slim brief emits per task kind, machine-checking the matrix documented
// on `buildMetaSkillContentSlim`. Heading is matched as a discrete line
// (preceded by newline + followed by newline) so inline references like
// "see ## Comment Formatting" do not trip the absence assertions.
func TestBuildMetaSkillContentSlimKindMatrix(t *testing.T) {
baseRepo := []RepoContextForEnv{{URL: "https://example.com/x.git", Description: "x"}}
baseSkill := []SkillContextForEnv{{Name: "skill-x", Description: "x"}}
type sectionCheck struct {
heading string
mustHave map[taskKind]bool
}
allKinds := map[taskKind]bool{
kindIssue: true, kindAutopilotRunOnly: true,
kindQuickCreate: true, kindChat: true,
}
issueKinds := map[taskKind]bool{kindIssue: true}
checks := []sectionCheck{
{"# Multica Agent Runtime", allKinds},
{"## Background Task Safety", allKinds},
{"## Agent Identity", allKinds},
{"## Available Commands", allKinds},
{"## Issue Body Formatting", allKinds},
{"### Workflow", allKinds},
{"## Important: Always Use the `multica` CLI", allKinds},
{"## Output", allKinds},
{"## Comment Formatting", issueKinds},
{"## Repositories", map[taskKind]bool{
kindIssue: true, kindAutopilotRunOnly: true, kindChat: true,
}},
{"## Issue Metadata", issueKinds},
{"## Instruction Precedence", issueKinds},
{"## Sub-issue Creation", issueKinds},
// Quick-create included: it used to be skipped here and carry its own
// copy in issue_context.md, which nothing read. One index, one place.
{"## Skills", allKinds},
{"## Mentions", issueKinds},
{"## Attachments", issueKinds},
}
fixtures := map[taskKind]TaskContextForEnv{
kindChat: {ChatSessionID: "c-1", AgentName: "Eve", AgentID: "eve-1",
Repos: baseRepo, AgentSkills: baseSkill},
kindQuickCreate: {QuickCreatePrompt: "p", AgentName: "Eve", AgentID: "eve-1",
Repos: baseRepo, AgentSkills: baseSkill},
kindAutopilotRunOnly: {AutopilotRunID: "r-1", AgentName: "Eve", AgentID: "eve-1",
Repos: baseRepo, AgentSkills: baseSkill},
kindIssue: {IssueID: "i-1", AgentName: "Eve", AgentID: "eve-1",
Repos: baseRepo, AgentSkills: baseSkill},
}
for kind, ctx := range fixtures {
out := buildMetaSkillContent("claude", ctx)
for _, c := range checks {
needle := "\n" + c.heading + "\n"
firstLine := c.heading + "\n"
present := strings.HasPrefix(out, firstLine) || strings.Contains(out, needle)
want := c.mustHave[kind]
if want && !present {
t.Errorf("kind=%d: expected heading %q in slim brief", kind, c.heading)
}
if !want && present {
t.Errorf("kind=%d: heading %q should NOT be in slim brief (matrix gating regression)", kind, c.heading)
}
}
}
}
// TestSlimQuickCreateAvailableCommands locks the minimal-variant content
// for quick-create's Available Commands: `issue create` present, every
// other Core command absent (the hard guardrails forbid the call).
func TestSlimQuickCreateAvailableCommands(t *testing.T) {
out := buildMetaSkillContent("codex", TaskContextForEnv{
QuickCreatePrompt: "create an issue about flaky tests",
AgentName: "Eve", AgentID: "eve-1",
})
for _, want := range []string{
"## Available Commands",
"multica issue create --title",
"`multica --help`",
} {
if !strings.Contains(out, want) {
t.Errorf("quick_create slim Available Commands missing %q", want)
}
}
for _, banned := range []string{
"multica issue get <id>",
"multica issue comment list <issue-id>",
"multica issue update <id>",
"multica issue status <id> <status>",
"multica issue comment add <issue-id>",
"multica issue metadata list <issue-id>",
"multica issue metadata set <issue-id>",
"multica issue metadata delete <issue-id>",
"multica issue children <id>",
"multica repo checkout <url>",
"### Squad maintenance",
"multica squad member set-role",
} {
if strings.Contains(out, banned) {
t.Errorf("quick_create slim Available Commands should NOT advertise %q (hard guardrails forbid the call)", banned)
}
}
}
// TestBackgroundTaskSafetySlimHardPins asserts the slim brief carries the
// same hardened Background Task Safety pins as the legacy brief (MUL-4140).
// The verbose path is covered by
// TestInjectRuntimeConfigBackgroundTaskSafetyProviderAgnostic; this locks
// the compressed slim path so a future slim-brief trim can't quietly drop
// the no-background-and-yield / no-"standing by" guardrails that address
// the MUL-4091 mechanism.
func TestBackgroundTaskSafetySlimHardPins(t *testing.T) {
out := buildMetaSkillContent("claude", TaskContextForEnv{
IssueID: "i-1", TriggerCommentID: "tc-1",
AgentName: "Eve", AgentID: "eve-1",
})
for _, want := range []string{
"## Background Task Safety",
"Do NOT end your turn while background tasks",
"wait for a future notification/reminder",
"run the work synchronously instead",
"Never background-and-yield",
"foreground tool call that blocks",
// MUL-5274: an explicitly requested persistent local service is a
// completed handoff, not unfinished run-owned work. Pin the narrow
// exception and its readiness / cleanup / honesty requirements.
"persistent service handoff",
"running service itself is the requested deliverable",
"stdio redirected to durable logs",
"PID/profile",
"verify readiness before replying",
"survival as best-effort, not guaranteed",
"does not cover tests, builds, CI polling",
"are not agent-owned background tasks",
"GitHub Actions after a successful push",
"Do not wait for them by default",
// MUL-5223 pins: named tool-shape bans, merge requirements
// denied as acceptance criteria, replacement hand-off phrasing,
// and the scoped escape hatch that keeps an explicitly requested
// CI result both permitted and executable.
"do NOT run `gh pr checks --watch`",
"any sleep / retry loop that polls check status",
"NOT your delivery acceptance criteria",
"CI running: <PR link>",
"unless the explicit exception below applies",
"The one exception",
"ONE foreground blocking call (`gh pr checks <pr> --watch`)",
"running in the background so you can keep working",
"standing by",
} {
if !strings.Contains(out, want) {
t.Errorf("slim Background Task Safety missing hardened pin %q\n---\n%s", want, out)
}
}
// `gh run watch` may only appear as a banned command, never as the
// section's example of how to wait properly.
if strings.Contains(out, "e.g. `gh run watch`") {
t.Errorf("slim Background Task Safety should not suggest waiting for external GitHub CI\n---\n%s", out)
}
// MUL-5274 review: with the persistent-service exception in the list, a
// "The rules above ..." scoping sentence would sweep in work that is
// precisely no longer run-owned after handoff.
if strings.Contains(out, "The rules above") {
t.Errorf("slim Background Task Safety must not reintroduce the ambiguous \"The rules above\" scoping sentence\n---\n%s", out)
}
}