From 8724ba1bb8747305dc95699cebc8a862731ef23f Mon Sep 17 00:00:00 2001 From: Jiayuan Zhang Date: Mon, 13 Jul 2026 14:55:10 +0800 Subject: [PATCH] fix(agents): constrain builder models to runtime --- .../components/agent-creation-studio.test.ts | 100 +++++++++++++++++- .../components/agent-creation-studio.tsx | 80 +++++++++++++- server/internal/handler/agent_builder.go | 2 + server/internal/handler/agent_builder_test.go | 12 +++ 4 files changed, 188 insertions(+), 6 deletions(-) diff --git a/packages/views/agents/components/agent-creation-studio.test.ts b/packages/views/agents/components/agent-creation-studio.test.ts index 256356f975..34d86fee9f 100644 --- a/packages/views/agents/components/agent-creation-studio.test.ts +++ b/packages/views/agents/components/agent-creation-studio.test.ts @@ -50,9 +50,26 @@ Return findings."}`; }); it("round-trips only the user's natural-language request for chat display", () => { - const content = encodeBuilderInput("Create a release manager", draft(), [], []); + const content = encodeBuilderInput( + "Create a release manager", + draft(), + [], + [], + { id: "runtime-1", name: "Codex", provider: "codex" }, + [{ id: "gpt-5.5", label: "GPT-5.5", provider: "openai" }], + ); expect(decodeBuilderInput(content)).toBe("Create a release manager"); + expect(JSON.parse(content.slice(content.indexOf("\n") + 1))).toMatchObject({ + selected_runtime: { + id: "runtime-1", + name: "Codex", + provider: "codex", + }, + available_runtime_models: [ + { id: "gpt-5.5", label: "GPT-5.5", provider: "openai" }, + ], + }); expect(decodeBuilderInput("ordinary chat message")).toBe( "ordinary chat message", ); @@ -70,6 +87,7 @@ Return findings."}`; }, new Set(["skill-1", "skill-2"]), new Set(["member-1"]), + new Set(["model-1"]), ); expect(result.name).toBe("Release manager"); @@ -79,6 +97,86 @@ Return findings."}`; expect([...result.memberIds]).toEqual(["member-1"]); }); + it("accepts catalog models and rejects invented or cross-runtime models", () => { + const validModelIds = new Set(["gpt-5.5", "gpt-5.3-codex"]); + + expect( + mergeBuilderDraft( + draft(), + { model: "gpt-5.5" }, + new Set(), + new Set(), + validModelIds, + ).model, + ).toBe("gpt-5.5"); + expect( + mergeBuilderDraft( + draft(), + { model: "claude-3-5-sonnet" }, + new Set(), + new Set(), + validModelIds, + ).model, + ).toBe("model-1"); + expect( + mergeBuilderDraft( + draft(), + { model: "invented-model" }, + new Set(), + new Set(), + validModelIds, + ).model, + ).toBe("model-1"); + expect( + mergeBuilderDraft( + draft(), + { model: "" }, + new Set(), + new Set(), + validModelIds, + ).model, + ).toBe(""); + }); + + it("preserves a user-selected custom model when the catalog is unavailable", () => { + expect( + mergeBuilderDraft( + draft(), + { model: "invented-model" }, + new Set(), + new Set(), + null, + ).model, + ).toBe("model-1"); + expect( + mergeBuilderDraft( + draft(), + { model: "model-1" }, + new Set(), + new Set(), + null, + ).model, + ).toBe("model-1"); + expect( + mergeBuilderDraft( + draft(), + { model: "invented-model" }, + new Set(), + new Set(), + new Set(), + ).model, + ).toBe("model-1"); + expect( + mergeBuilderDraft( + draft(), + { model: "" }, + new Set(), + new Set(), + new Set(), + ).model, + ).toBe("model-1"); + }); + it("preserves scoped member and team grants when duplicating an agent", () => { const access = deriveDuplicateAccess({ permission_mode: "public_to", diff --git a/packages/views/agents/components/agent-creation-studio.tsx b/packages/views/agents/components/agent-creation-studio.tsx index 796822610d..db6a296908 100644 --- a/packages/views/agents/components/agent-creation-studio.tsx +++ b/packages/views/agents/components/agent-creation-studio.tsx @@ -26,7 +26,10 @@ import { } from "@multica/core/chat/queries"; import { useWorkspaceId } from "@multica/core/hooks"; import { useWorkspacePaths } from "@multica/core/paths"; -import { runtimeListOptions } from "@multica/core/runtimes"; +import { + runtimeListOptions, + runtimeModelsOptions, +} from "@multica/core/runtimes"; import type { Agent, AgentInvocationTargetInput, @@ -35,6 +38,7 @@ import type { CreateAgentRequest, MemberWithUser, RuntimeDevice, + RuntimeModel, } from "@multica/core/types"; import { agentListOptions, @@ -223,6 +227,34 @@ export function AgentCreationStudio() { ), [currentUser?.id, runtimes], ); + const selectedRuntime = + runtimes.find((runtime) => runtime.id === draft.runtimeId) ?? null; + const builderModelsQuery = useQuery( + runtimeModelsOptions( + mode === "ai" && selectedRuntime?.status === "online" + ? selectedRuntime.id + : null, + ), + ); + // `null` means discovery is not available yet (or failed), while `[]` is + // an authoritative catalog with no selectable models. In both cases the + // builder may preserve the user's current value but cannot invent one. + const builderModelCatalog = useMemo( + () => + builderModelsQuery.isSuccess + ? builderModelsQuery.data.supported + ? builderModelsQuery.data.models + : [] + : null, + [builderModelsQuery.data, builderModelsQuery.isSuccess], + ); + const validBuilderModelIds = useMemo( + () => + builderModelCatalog === null + ? null + : new Set(builderModelCatalog.map((model) => model.id)), + [builderModelCatalog], + ); useEffect(() => { if (draft.runtimeId || usableRuntimes.length === 0) return; @@ -284,12 +316,21 @@ export function AgentCreationStudio() { const payload = parseBuilderDraft(latestBuilderDraftMessageContent); if (!payload) return; appliedAssistantMessageRef.current = latestBuilderDraftMessageId; - setDraft((current) => mergeBuilderDraft(current, payload, skillIdSet, memberIdSet)); + setDraft((current) => + mergeBuilderDraft( + current, + payload, + skillIdSet, + memberIdSet, + validBuilderModelIds, + ), + ); }, [ latestBuilderDraftMessageContent, latestBuilderDraftMessageId, memberIdSet, skillIdSet, + validBuilderModelIds, ]); const filteredTemplates = useMemo(() => { @@ -303,8 +344,6 @@ export function AgentCreationStudio() { ); }, [templateSearch, templates]); - const selectedRuntime = - runtimes.find((runtime) => runtime.id === draft.runtimeId) ?? null; const accessInvalid = draft.permissionScope === "members" && draft.memberIds.size === 0 && @@ -425,6 +464,8 @@ export function AgentCreationStudio() { draft, workspaceSkills, members, + selectedRuntime, + builderModelCatalog, ); const result = await api.sendChatMessage( builderSessionId, @@ -1429,6 +1470,8 @@ export function encodeBuilderInput( draft: AgentDraft, skills: Array<{ id: string; name: string; description: string }>, members: Array<{ user_id: string; name: string }>, + runtime: Pick | null, + models: RuntimeModel[] | null, ): string { return ( BUILDER_INPUT_PREFIX + @@ -1444,6 +1487,21 @@ export function encodeBuilderInput( permission_scope: draft.permissionScope, member_ids: [...draft.memberIds], }, + selected_runtime: runtime + ? { + id: runtime.id, + name: runtime.name, + provider: runtime.provider, + } + : null, + available_runtime_models: + models === null + ? null + : models.map((model) => ({ + id: model.id, + label: model.label, + provider: model.provider, + })), available_workspace_skills: skills.map((skill) => ({ id: skill.id, name: skill.name, @@ -1479,6 +1537,7 @@ export function mergeBuilderDraft( payload: BuilderDraftPayload, validSkillIds: Set, validMemberIds: Set, + validModelIds: ReadonlySet | null, ): AgentDraft { const scope = payload.permission_scope === "workspace" || @@ -1498,6 +1557,17 @@ export function mergeBuilderDraft( typeof id === "string" && validMemberIds.has(id), ) : [...current.memberIds]; + // The current value may be a deliberate custom entry from ModelDropdown, + // so preserving it is always safe. Only catalog IDs may be introduced by + // the builder; failed discovery therefore cannot turn into fail-open input. + const model = + typeof payload.model === "string" && + (payload.model === current.model || + (validModelIds !== null && + validModelIds.size > 0 && + (payload.model === "" || validModelIds.has(payload.model)))) + ? payload.model + : current.model; return { ...current, @@ -1510,7 +1580,7 @@ export function mergeBuilderDraft( typeof payload.instructions === "string" ? payload.instructions : current.instructions, - model: typeof payload.model === "string" ? payload.model : current.model, + model, skillIds: new Set(skillIds), permissionScope: scope, memberIds: new Set(scope === "members" ? memberIds : []), diff --git a/server/internal/handler/agent_builder.go b/server/internal/handler/agent_builder.go index e66f6c537e..84ac9033fb 100644 --- a/server/internal/handler/agent_builder.go +++ b/server/internal/handler/agent_builder.go @@ -26,6 +26,8 @@ Rules: - name is concise and suitable for a workspace list. - description is one sentence, at most 200 characters. - instructions are a complete Markdown system prompt describing role, workflow, output, and constraints. +- model must be empty, preserve current_draft.model, or exactly match an id explicitly listed in AVAILABLE RUNTIME MODELS. Never use a model label as the id. +- When AVAILABLE RUNTIME MODELS is null or empty, preserve current_draft.model and never invent a model id. - skill_ids may only contain IDs explicitly listed in AVAILABLE WORKSPACE SKILLS. - permission_scope must be private, workspace, or members. Default to private unless the user explicitly requests sharing. - member_ids may only contain IDs explicitly listed in AVAILABLE WORKSPACE MEMBERS, and only when permission_scope is members. diff --git a/server/internal/handler/agent_builder_test.go b/server/internal/handler/agent_builder_test.go index a59ed4eade..2f1c566c1b 100644 --- a/server/internal/handler/agent_builder_test.go +++ b/server/internal/handler/agent_builder_test.go @@ -9,6 +9,18 @@ import ( "testing" ) +func TestAgentBuilderInstructionsConstrainModelsToRuntimeCatalog(t *testing.T) { + for _, requirement := range []string{ + "AVAILABLE RUNTIME MODELS", + "Never use a model label as the id", + "never invent a model id", + } { + if !strings.Contains(agentBuilderInstructions, requirement) { + t.Fatalf("agent builder instructions missing model constraint %q", requirement) + } + } +} + func TestCreateAgentBuilderSessionCreatesIsolatedHiddenBuilder(t *testing.T) { if testHandler == nil { t.Skip("database not available")