mirror of
https://github.com/multica-ai/multica.git
synced 2026-08-05 17:40:11 +02:00
* feat(runtime): unbind agents on runtime delete instead of destroying them Deleting a runtime archived its agents and then hard-deleted the rows, so the agents and every conversation with them disappeared — while the confirmation dialog said "archive", which a user reasonably reads as recoverable. Retiring a laptop is an ordinary action; losing the agents configured on it is not an ordinary consequence. An agent is now a persistent business object and a runtime is replaceable execution capacity: deleting a runtime unbinds its agents. `runtime_id IS NULL` means unbound — orthogonal to archived — and the agent keeps its instructions, skills, chats, labels, channel installations, autopilots and task history. service.AgentReadiness already refused an agent with no runtime, so the scheduling safety gate needed no change. Two columns become nullable, not one. Without `agent_task_queue.runtime_id`, deleting the runtime still cascades the task history away (and task_message / task_usage / task_token with it), so the agents would survive with no record of anything they did — the same class of loss. A NOT VALID CHECK keeps NULL confined to history: an active task must always have a runtime, so claim / dispatch / delivery-CAS paths can never observe one without. It is written against completed_at rather than a status list so a future non-terminal status fails closed instead of slipping through. Two prerequisites this depends on: - 'deferred' (migration 128) was missing from CancelAgentTasksByRuntimeOrAgent. It went unnoticed because the delete used to cascade those rows away; with the new CHECK it would abort the delete and make the runtime undeletable. - The channel-installation / label / chat-pin / invocation-target / draft-restore cleanups were scoped to "archived agents on this runtime". Archived user agents now survive, so that scope is narrowed to kind='system' — otherwise the fix would produce a subtler loss: agent alive, configuration wiped. Also removes the squad guard that refused (409) when an active squad's leader was an archived agent on the runtime, plus the archived-squad delete that existed only to get past squad.leader_id's RESTRICT FK. The leader is no longer deleted, so nothing needs to be given up to retire a machine. Autopilots are no longer paused either: their assignee survives, and a rebind restores them without the owner having to remember to re-enable. Reason codes: an unbound agent reports agent_runtime_required, not runtime_offline. The copy for runtime_offline tells users to reconnect a machine; an unbound agent has no machine to reconnect, and the fix is to bind a runtime. Chat's bare 409 string gains the same code so the composer can offer that action. API: agents gain runtime_bound. runtime_id stays a string (empty when unbound) so installed clients keep parsing and no gated two-release rollout is needed. The confirmed-delete endpoint is /unbind-agents-and-delete; /archive-agents-and-delete still routes to it, and the compared expected_active_agent_ids set is unchanged — widening it would 409 every older client forever. Co-authored-by: multica-agent <github@multica.ai> * fix: make runtime unbinding recoverable Co-authored-by: multica-agent <github@multica.ai> * fix: address runtime unbind review nits Co-authored-by: multica-agent <github@multica.ai> * fix: resolve runtime unbind review blockers Co-authored-by: multica-agent <github@multica.ai> * fix(migrations): renumber runtime unbind after main merge Co-authored-by: multica-agent <github@multica.ai> * test(daemon): avoid late-request lease flake Co-authored-by: multica-agent <github@multica.ai> * test(autopilots): bind validation fixture runtime Co-authored-by: multica-agent <github@multica.ai> --------- Co-authored-by: Eve <eve@multica-ai.local> Co-authored-by: multica-agent <github@multica.ai>
340 lines
13 KiB
Go
340 lines
13 KiB
Go
package handler
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
// Resource-label junction tables (agent_to_label / skill_to_label) deliberately
|
|
// carry no foreign keys, so every bulk hard-delete entry point that removes the
|
|
// owning agents/skills must clear their label links in the same transaction.
|
|
// These tests pin that cleanup on the four batch paths that never pass through a
|
|
// per-entity delete: runtime delete (strict + cascade), runtime-profile delete,
|
|
// and workspace delete. Without the sweep, a labelled agent/skill leaves a
|
|
// permanent, invisible orphan row once resource labels are enabled.
|
|
|
|
// insertLabelRow creates a real issue_label so the seeded junction row is valid
|
|
// regardless of whether a given database still carries the pre-release label_id
|
|
// foreign key. Registers cleanup.
|
|
func insertLabelRow(t *testing.T, ctx context.Context, workspaceID, resourceType string) string {
|
|
t.Helper()
|
|
var labelID string
|
|
if err := testPool.QueryRow(ctx, `
|
|
INSERT INTO issue_label (workspace_id, resource_type, name, color)
|
|
VALUES ($1, $2, $3, '#3b82f6')
|
|
RETURNING id
|
|
`, workspaceID, resourceType, resourceType+"-"+uuid.NewString()[:8]).Scan(&labelID); err != nil {
|
|
t.Fatalf("insert issue_label: %v", err)
|
|
}
|
|
t.Cleanup(func() {
|
|
_, _ = testPool.Exec(context.Background(), `DELETE FROM issue_label WHERE id = $1`, labelID)
|
|
})
|
|
return labelID
|
|
}
|
|
|
|
func seedAgentLabel(t *testing.T, ctx context.Context, workspaceID, agentID string) {
|
|
t.Helper()
|
|
labelID := insertLabelRow(t, ctx, workspaceID, "agent")
|
|
if _, err := testPool.Exec(ctx,
|
|
`INSERT INTO agent_to_label (agent_id, label_id) VALUES ($1, $2)`,
|
|
agentID, labelID); err != nil {
|
|
t.Fatalf("seed agent_to_label: %v", err)
|
|
}
|
|
}
|
|
|
|
func seedSkillLabel(t *testing.T, ctx context.Context, workspaceID, skillID string) {
|
|
t.Helper()
|
|
labelID := insertLabelRow(t, ctx, workspaceID, "skill")
|
|
if _, err := testPool.Exec(ctx,
|
|
`INSERT INTO skill_to_label (skill_id, label_id) VALUES ($1, $2)`,
|
|
skillID, labelID); err != nil {
|
|
t.Fatalf("seed skill_to_label: %v", err)
|
|
}
|
|
}
|
|
|
|
func countAgentLabelAssignments(t *testing.T, ctx context.Context, agentID string) int {
|
|
t.Helper()
|
|
var n int
|
|
if err := testPool.QueryRow(ctx,
|
|
`SELECT count(*) FROM agent_to_label WHERE agent_id = $1`, agentID).Scan(&n); err != nil {
|
|
t.Fatalf("count agent_to_label: %v", err)
|
|
}
|
|
return n
|
|
}
|
|
|
|
func countSkillLabelAssignments(t *testing.T, ctx context.Context, skillID string) int {
|
|
t.Helper()
|
|
var n int
|
|
if err := testPool.QueryRow(ctx,
|
|
`SELECT count(*) FROM skill_to_label WHERE skill_id = $1`, skillID).Scan(&n); err != nil {
|
|
t.Fatalf("count skill_to_label: %v", err)
|
|
}
|
|
return n
|
|
}
|
|
|
|
// TestDeleteAgentRuntime_KeepsUnboundAgentLabelAssignments: since MUL-5559 the
|
|
// strict runtime delete unbinds the archived agent instead of hard-deleting it,
|
|
// so its label links must SURVIVE. Clearing them by runtime — which is what the
|
|
// old sweep did — would strip labels off an agent that is still there.
|
|
func TestDeleteAgentRuntime_KeepsUnboundAgentLabelAssignments(t *testing.T) {
|
|
if testHandler == nil {
|
|
t.Skip("database not available")
|
|
}
|
|
ctx := context.Background()
|
|
|
|
runtimeID := seedIsolatedRuntime(t, "Label Cleanup Runtime")
|
|
agentID := seedAgentOnRuntime(t, runtimeID, "Label Cleanup Archived Agent", true)
|
|
seedAgentLabel(t, ctx, testWorkspaceID, agentID)
|
|
|
|
w := httptest.NewRecorder()
|
|
req := newRequest("DELETE", "/api/runtimes/"+runtimeID, nil)
|
|
req = withURLParam(req, "runtimeId", runtimeID)
|
|
testHandler.DeleteAgentRuntime(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("DeleteAgentRuntime: expected 200, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
|
|
if !agentExists(t, agentID) {
|
|
t.Fatalf("archived agent must survive its runtime as an unbound agent")
|
|
}
|
|
if n := countAgentLabelAssignments(t, ctx, agentID); n != 1 {
|
|
t.Fatalf("agent_to_label rows for a surviving agent: got %d, want 1", n)
|
|
}
|
|
}
|
|
|
|
// TestUnbindAgentsAndDeleteRuntime_KeepsAgentLabelAssignments: the confirmed
|
|
// endpoint unbinds the active agent, so its labels stay attached too.
|
|
func TestUnbindAgentsAndDeleteRuntime_KeepsAgentLabelAssignments(t *testing.T) {
|
|
if testHandler == nil {
|
|
t.Skip("database not available")
|
|
}
|
|
ctx := context.Background()
|
|
|
|
runtimeID := createCascadeFixtureRuntime(t, ctx, "Label Cascade Runtime")
|
|
agentID := createCascadeFixtureAgent(t, ctx, runtimeID, "Label Cascade Agent")
|
|
seedAgentLabel(t, ctx, testWorkspaceID, agentID)
|
|
|
|
w := httptest.NewRecorder()
|
|
req := newRequest("POST", "/api/runtimes/"+runtimeID+"/unbind-agents-and-delete",
|
|
map[string]any{"expected_active_agent_ids": []string{agentID}})
|
|
req = withURLParam(req, "runtimeId", runtimeID)
|
|
testHandler.UnbindAgentsAndDeleteRuntime(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("UnbindAgentsAndDeleteRuntime: expected 200, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
|
|
if n := countAgentLabelAssignments(t, ctx, agentID); n != 1 {
|
|
t.Fatalf("agent_to_label rows for a surviving agent: got %d, want 1", n)
|
|
}
|
|
}
|
|
|
|
// TestDeleteRuntimeProfile_KeepsAgentLabelAssignments: the profile teardown runs
|
|
// the same unbind, so the archived agent and its label links survive there too.
|
|
func TestDeleteRuntimeProfile_KeepsAgentLabelAssignments(t *testing.T) {
|
|
if testHandler == nil {
|
|
t.Skip("database not available")
|
|
}
|
|
ctx := context.Background()
|
|
|
|
profileID := insertRuntimeProfileFixture(t, ctx, "Label Cleanup Profile", "codex", "company-codex-label")
|
|
runtimeID := insertProfileRuntimeFixture(t, ctx, profileID, "Label Cleanup Profile Runtime", "codex")
|
|
agentID := createCascadeFixtureAgent(t, ctx, runtimeID, "Label Cleanup Profile Agent")
|
|
if _, err := testPool.Exec(ctx, `UPDATE agent SET archived_at = now() WHERE id = $1`, agentID); err != nil {
|
|
t.Fatalf("archive agent: %v", err)
|
|
}
|
|
seedAgentLabel(t, ctx, testWorkspaceID, agentID)
|
|
|
|
w := httptest.NewRecorder()
|
|
req := newRequest("DELETE", "/api/workspaces/"+testWorkspaceID+"/runtime-profiles/"+profileID, nil)
|
|
req = withURLParams(req, "id", testWorkspaceID, "profileId", profileID)
|
|
testHandler.DeleteRuntimeProfile(w, req)
|
|
if w.Code != http.StatusNoContent {
|
|
t.Fatalf("DeleteRuntimeProfile: expected 204, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
|
|
if !agentExists(t, agentID) {
|
|
t.Fatalf("archived agent must survive its runtime profile as an unbound agent")
|
|
}
|
|
if n := countAgentLabelAssignments(t, ctx, agentID); n != 1 {
|
|
t.Fatalf("agent_to_label rows for a surviving agent: got %d, want 1", n)
|
|
}
|
|
}
|
|
|
|
// TestDeleteAgentRuntime_CleansSystemAgentLabelAssignments: system agents are
|
|
// still hard-deleted with their runtime (they are invisible infrastructure with
|
|
// no rebind affordance), so their label links must still be swept — otherwise
|
|
// they become the invisible orphan rows the sweep exists to prevent.
|
|
func TestDeleteAgentRuntime_CleansSystemAgentLabelAssignments(t *testing.T) {
|
|
if testHandler == nil {
|
|
t.Skip("database not available")
|
|
}
|
|
ctx := context.Background()
|
|
|
|
runtimeID := seedIsolatedRuntime(t, "Label Cleanup System Runtime")
|
|
agentID := seedAgentOnRuntime(t, runtimeID, "Label Cleanup System Agent", false)
|
|
if _, err := testPool.Exec(ctx,
|
|
`UPDATE agent SET kind = 'system', system_key = 'label_cleanup_probe' WHERE id = $1`,
|
|
agentID); err != nil {
|
|
t.Fatalf("make agent a system agent: %v", err)
|
|
}
|
|
seedAgentLabel(t, ctx, testWorkspaceID, agentID)
|
|
|
|
w := httptest.NewRecorder()
|
|
req := newRequest("DELETE", "/api/runtimes/"+runtimeID, nil)
|
|
req = withURLParam(req, "runtimeId", runtimeID)
|
|
testHandler.DeleteAgentRuntime(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("DeleteAgentRuntime: expected 200, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
|
|
if agentExists(t, agentID) {
|
|
t.Fatalf("system agent should still be hard-deleted with its runtime")
|
|
}
|
|
if n := countAgentLabelAssignments(t, ctx, agentID); n != 0 {
|
|
t.Fatalf("agent_to_label rows survived system-agent delete: %d", n)
|
|
}
|
|
}
|
|
|
|
func seedWorkspaceResourceLabelFixture(t *testing.T, ctx context.Context, slug string) (string, string, string) {
|
|
t.Helper()
|
|
_, _ = testPool.Exec(ctx, `DELETE FROM workspace WHERE slug = $1`, slug)
|
|
|
|
var wsID string
|
|
if err := testPool.QueryRow(ctx, `
|
|
INSERT INTO workspace (name, slug, description)
|
|
VALUES ($1, $2, $3)
|
|
RETURNING id
|
|
`, "Handler Test Delete Labels", slug, "resource-label atomic cleanup test").Scan(&wsID); err != nil {
|
|
t.Fatalf("create workspace: %v", err)
|
|
}
|
|
t.Cleanup(func() {
|
|
_, _ = testPool.Exec(context.Background(), `DELETE FROM workspace WHERE id = $1`, wsID)
|
|
})
|
|
if _, err := testPool.Exec(ctx,
|
|
`INSERT INTO member (workspace_id, user_id, role) VALUES ($1, $2, 'owner')`,
|
|
wsID, testUserID); err != nil {
|
|
t.Fatalf("create owner member: %v", err)
|
|
}
|
|
|
|
// agent.runtime_id is NOT NULL, so the labelled agent needs a runtime in the
|
|
// same workspace. Both cascade away with the workspace.
|
|
var runtimeID string
|
|
if err := testPool.QueryRow(ctx, `
|
|
INSERT INTO agent_runtime (
|
|
workspace_id, daemon_id, name, runtime_mode, provider, status,
|
|
device_info, metadata, owner_id, last_seen_at
|
|
)
|
|
VALUES ($1, NULL, 'ws-label-runtime', 'cloud', 'ws-label-test', 'online', 'dev', '{}'::jsonb, $2, now())
|
|
RETURNING id
|
|
`, wsID, testUserID).Scan(&runtimeID); err != nil {
|
|
t.Fatalf("insert runtime: %v", err)
|
|
}
|
|
var agentID string
|
|
if err := testPool.QueryRow(ctx, `
|
|
INSERT INTO agent (
|
|
workspace_id, name, description, runtime_mode, runtime_config,
|
|
runtime_id, visibility, max_concurrent_tasks, owner_id
|
|
)
|
|
VALUES ($1, 'ws-label-agent', '', 'cloud', '{}'::jsonb, $2, 'workspace', 1, $3)
|
|
RETURNING id
|
|
`, wsID, runtimeID, testUserID).Scan(&agentID); err != nil {
|
|
t.Fatalf("insert agent: %v", err)
|
|
}
|
|
var skillID string
|
|
if err := testPool.QueryRow(ctx, `
|
|
INSERT INTO skill (workspace_id, name, description, content, config, created_by)
|
|
VALUES ($1, 'ws-label-skill', 'fixture', '# x', '{}'::jsonb, $2)
|
|
RETURNING id
|
|
`, wsID, testUserID).Scan(&skillID); err != nil {
|
|
t.Fatalf("insert skill: %v", err)
|
|
}
|
|
seedAgentLabel(t, ctx, wsID, agentID)
|
|
seedSkillLabel(t, ctx, wsID, skillID)
|
|
t.Cleanup(func() {
|
|
_, _ = testPool.Exec(context.Background(), `DELETE FROM agent_to_label WHERE agent_id = $1`, agentID)
|
|
_, _ = testPool.Exec(context.Background(), `DELETE FROM skill_to_label WHERE skill_id = $1`, skillID)
|
|
_, _ = testPool.Exec(context.Background(), `DELETE FROM workspace WHERE id = $1`, wsID)
|
|
})
|
|
|
|
return wsID, agentID, skillID
|
|
}
|
|
|
|
// TestDeleteWorkspace_CleansResourceLabelAssignments: workspace delete cascades
|
|
// away the agents and skills, but the junction tables have no workspace_id and
|
|
// no foreign key, so both must be swept before the cascade or they orphan.
|
|
func TestDeleteWorkspace_CleansResourceLabelAssignments(t *testing.T) {
|
|
if testHandler == nil {
|
|
t.Skip("database not available")
|
|
}
|
|
ctx := context.Background()
|
|
wsID, agentID, skillID := seedWorkspaceResourceLabelFixture(t, ctx, "handler-tests-delete-labels")
|
|
|
|
w := httptest.NewRecorder()
|
|
req := newRequest("DELETE", "/api/workspaces/"+wsID, nil)
|
|
req = withURLParam(req, "id", wsID)
|
|
testHandler.DeleteWorkspace(w, req)
|
|
if w.Code != http.StatusNoContent {
|
|
t.Fatalf("DeleteWorkspace: expected 204, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
|
|
if n := countAgentLabelAssignments(t, ctx, agentID); n != 0 {
|
|
t.Fatalf("agent_to_label rows survived workspace delete: %d", n)
|
|
}
|
|
if n := countSkillLabelAssignments(t, ctx, skillID); n != 0 {
|
|
t.Fatalf("skill_to_label rows survived workspace delete: %d", n)
|
|
}
|
|
}
|
|
|
|
// TestDeleteWorkspace_RollsBackResourceLabelCleanup verifies the cleanup and
|
|
// final workspace delete share one database statement. A restrictive test-only
|
|
// foreign key makes the final delete fail; both junction rows must remain.
|
|
func TestDeleteWorkspace_RollsBackResourceLabelCleanup(t *testing.T) {
|
|
if testHandler == nil {
|
|
t.Skip("database not available")
|
|
}
|
|
ctx := context.Background()
|
|
wsID, agentID, skillID := seedWorkspaceResourceLabelFixture(t, ctx, "handler-tests-delete-labels-rollback")
|
|
|
|
const guardTable = "workspace_delete_resource_label_rollback_guard"
|
|
_, _ = testPool.Exec(ctx, `DROP TABLE IF EXISTS `+guardTable)
|
|
if _, err := testPool.Exec(ctx, `
|
|
CREATE TABLE `+guardTable+` (
|
|
workspace_id UUID NOT NULL REFERENCES workspace(id)
|
|
)
|
|
`); err != nil {
|
|
t.Fatalf("create workspace delete guard: %v", err)
|
|
}
|
|
if _, err := testPool.Exec(ctx, `INSERT INTO `+guardTable+` (workspace_id) VALUES ($1)`, wsID); err != nil {
|
|
t.Fatalf("insert workspace delete guard: %v", err)
|
|
}
|
|
t.Cleanup(func() {
|
|
_, _ = testPool.Exec(context.Background(), `DROP TABLE IF EXISTS `+guardTable)
|
|
})
|
|
|
|
w := httptest.NewRecorder()
|
|
req := newRequest("DELETE", "/api/workspaces/"+wsID, nil)
|
|
req = withURLParam(req, "id", wsID)
|
|
testHandler.DeleteWorkspace(w, req)
|
|
if w.Code != http.StatusInternalServerError {
|
|
t.Fatalf("DeleteWorkspace: expected 500, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
|
|
var workspaceExists bool
|
|
if err := testPool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM workspace WHERE id = $1)`, wsID).Scan(&workspaceExists); err != nil {
|
|
t.Fatalf("check workspace after failed delete: %v", err)
|
|
}
|
|
if !workspaceExists {
|
|
t.Fatal("workspace was removed despite the injected delete failure")
|
|
}
|
|
if n := countAgentLabelAssignments(t, ctx, agentID); n != 1 {
|
|
t.Fatalf("agent_to_label rows after failed workspace delete = %d, want 1", n)
|
|
}
|
|
if n := countSkillLabelAssignments(t, ctx, skillID); n != 1 {
|
|
t.Fatalf("skill_to_label rows after failed workspace delete = %d, want 1", n)
|
|
}
|
|
}
|