Files
multica/server/internal/handler/resource_label_cascade_test.go
Multica Eve b06af2ae17 feat(runtime): unbind agents on runtime delete instead of destroying them (#6220)
* feat(runtime): unbind agents on runtime delete instead of destroying them

Deleting a runtime archived its agents and then hard-deleted the rows, so the
agents and every conversation with them disappeared — while the confirmation
dialog said "archive", which a user reasonably reads as recoverable. Retiring a
laptop is an ordinary action; losing the agents configured on it is not an
ordinary consequence.

An agent is now a persistent business object and a runtime is replaceable
execution capacity: deleting a runtime unbinds its agents. `runtime_id IS NULL`
means unbound — orthogonal to archived — and the agent keeps its instructions,
skills, chats, labels, channel installations, autopilots and task history.
service.AgentReadiness already refused an agent with no runtime, so the
scheduling safety gate needed no change.

Two columns become nullable, not one. Without `agent_task_queue.runtime_id`,
deleting the runtime still cascades the task history away (and task_message /
task_usage / task_token with it), so the agents would survive with no record of
anything they did — the same class of loss. A NOT VALID CHECK keeps NULL confined
to history: an active task must always have a runtime, so claim / dispatch /
delivery-CAS paths can never observe one without. It is written against
completed_at rather than a status list so a future non-terminal status fails
closed instead of slipping through.

Two prerequisites this depends on:

- 'deferred' (migration 128) was missing from CancelAgentTasksByRuntimeOrAgent.
  It went unnoticed because the delete used to cascade those rows away; with the
  new CHECK it would abort the delete and make the runtime undeletable.
- The channel-installation / label / chat-pin / invocation-target / draft-restore
  cleanups were scoped to "archived agents on this runtime". Archived user agents
  now survive, so that scope is narrowed to kind='system' — otherwise the fix
  would produce a subtler loss: agent alive, configuration wiped.

Also removes the squad guard that refused (409) when an active squad's leader was
an archived agent on the runtime, plus the archived-squad delete that existed
only to get past squad.leader_id's RESTRICT FK. The leader is no longer deleted,
so nothing needs to be given up to retire a machine. Autopilots are no longer
paused either: their assignee survives, and a rebind restores them without the
owner having to remember to re-enable.

Reason codes: an unbound agent reports agent_runtime_required, not
runtime_offline. The copy for runtime_offline tells users to reconnect a machine;
an unbound agent has no machine to reconnect, and the fix is to bind a runtime.
Chat's bare 409 string gains the same code so the composer can offer that action.

API: agents gain runtime_bound. runtime_id stays a string (empty when unbound) so
installed clients keep parsing and no gated two-release rollout is needed. The
confirmed-delete endpoint is /unbind-agents-and-delete; /archive-agents-and-delete
still routes to it, and the compared expected_active_agent_ids set is unchanged —
widening it would 409 every older client forever.

Co-authored-by: multica-agent <github@multica.ai>

* fix: make runtime unbinding recoverable

Co-authored-by: multica-agent <github@multica.ai>

* fix: address runtime unbind review nits

Co-authored-by: multica-agent <github@multica.ai>

* fix: resolve runtime unbind review blockers

Co-authored-by: multica-agent <github@multica.ai>

* fix(migrations): renumber runtime unbind after main merge

Co-authored-by: multica-agent <github@multica.ai>

* test(daemon): avoid late-request lease flake

Co-authored-by: multica-agent <github@multica.ai>

* test(autopilots): bind validation fixture runtime

Co-authored-by: multica-agent <github@multica.ai>

---------

Co-authored-by: Eve <eve@multica-ai.local>
Co-authored-by: multica-agent <github@multica.ai>
2026-08-03 12:39:27 +08:00

340 lines
13 KiB
Go

package handler
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"github.com/google/uuid"
)
// Resource-label junction tables (agent_to_label / skill_to_label) deliberately
// carry no foreign keys, so every bulk hard-delete entry point that removes the
// owning agents/skills must clear their label links in the same transaction.
// These tests pin that cleanup on the four batch paths that never pass through a
// per-entity delete: runtime delete (strict + cascade), runtime-profile delete,
// and workspace delete. Without the sweep, a labelled agent/skill leaves a
// permanent, invisible orphan row once resource labels are enabled.
// insertLabelRow creates a real issue_label so the seeded junction row is valid
// regardless of whether a given database still carries the pre-release label_id
// foreign key. Registers cleanup.
func insertLabelRow(t *testing.T, ctx context.Context, workspaceID, resourceType string) string {
t.Helper()
var labelID string
if err := testPool.QueryRow(ctx, `
INSERT INTO issue_label (workspace_id, resource_type, name, color)
VALUES ($1, $2, $3, '#3b82f6')
RETURNING id
`, workspaceID, resourceType, resourceType+"-"+uuid.NewString()[:8]).Scan(&labelID); err != nil {
t.Fatalf("insert issue_label: %v", err)
}
t.Cleanup(func() {
_, _ = testPool.Exec(context.Background(), `DELETE FROM issue_label WHERE id = $1`, labelID)
})
return labelID
}
func seedAgentLabel(t *testing.T, ctx context.Context, workspaceID, agentID string) {
t.Helper()
labelID := insertLabelRow(t, ctx, workspaceID, "agent")
if _, err := testPool.Exec(ctx,
`INSERT INTO agent_to_label (agent_id, label_id) VALUES ($1, $2)`,
agentID, labelID); err != nil {
t.Fatalf("seed agent_to_label: %v", err)
}
}
func seedSkillLabel(t *testing.T, ctx context.Context, workspaceID, skillID string) {
t.Helper()
labelID := insertLabelRow(t, ctx, workspaceID, "skill")
if _, err := testPool.Exec(ctx,
`INSERT INTO skill_to_label (skill_id, label_id) VALUES ($1, $2)`,
skillID, labelID); err != nil {
t.Fatalf("seed skill_to_label: %v", err)
}
}
func countAgentLabelAssignments(t *testing.T, ctx context.Context, agentID string) int {
t.Helper()
var n int
if err := testPool.QueryRow(ctx,
`SELECT count(*) FROM agent_to_label WHERE agent_id = $1`, agentID).Scan(&n); err != nil {
t.Fatalf("count agent_to_label: %v", err)
}
return n
}
func countSkillLabelAssignments(t *testing.T, ctx context.Context, skillID string) int {
t.Helper()
var n int
if err := testPool.QueryRow(ctx,
`SELECT count(*) FROM skill_to_label WHERE skill_id = $1`, skillID).Scan(&n); err != nil {
t.Fatalf("count skill_to_label: %v", err)
}
return n
}
// TestDeleteAgentRuntime_KeepsUnboundAgentLabelAssignments: since MUL-5559 the
// strict runtime delete unbinds the archived agent instead of hard-deleting it,
// so its label links must SURVIVE. Clearing them by runtime — which is what the
// old sweep did — would strip labels off an agent that is still there.
func TestDeleteAgentRuntime_KeepsUnboundAgentLabelAssignments(t *testing.T) {
if testHandler == nil {
t.Skip("database not available")
}
ctx := context.Background()
runtimeID := seedIsolatedRuntime(t, "Label Cleanup Runtime")
agentID := seedAgentOnRuntime(t, runtimeID, "Label Cleanup Archived Agent", true)
seedAgentLabel(t, ctx, testWorkspaceID, agentID)
w := httptest.NewRecorder()
req := newRequest("DELETE", "/api/runtimes/"+runtimeID, nil)
req = withURLParam(req, "runtimeId", runtimeID)
testHandler.DeleteAgentRuntime(w, req)
if w.Code != http.StatusOK {
t.Fatalf("DeleteAgentRuntime: expected 200, got %d: %s", w.Code, w.Body.String())
}
if !agentExists(t, agentID) {
t.Fatalf("archived agent must survive its runtime as an unbound agent")
}
if n := countAgentLabelAssignments(t, ctx, agentID); n != 1 {
t.Fatalf("agent_to_label rows for a surviving agent: got %d, want 1", n)
}
}
// TestUnbindAgentsAndDeleteRuntime_KeepsAgentLabelAssignments: the confirmed
// endpoint unbinds the active agent, so its labels stay attached too.
func TestUnbindAgentsAndDeleteRuntime_KeepsAgentLabelAssignments(t *testing.T) {
if testHandler == nil {
t.Skip("database not available")
}
ctx := context.Background()
runtimeID := createCascadeFixtureRuntime(t, ctx, "Label Cascade Runtime")
agentID := createCascadeFixtureAgent(t, ctx, runtimeID, "Label Cascade Agent")
seedAgentLabel(t, ctx, testWorkspaceID, agentID)
w := httptest.NewRecorder()
req := newRequest("POST", "/api/runtimes/"+runtimeID+"/unbind-agents-and-delete",
map[string]any{"expected_active_agent_ids": []string{agentID}})
req = withURLParam(req, "runtimeId", runtimeID)
testHandler.UnbindAgentsAndDeleteRuntime(w, req)
if w.Code != http.StatusOK {
t.Fatalf("UnbindAgentsAndDeleteRuntime: expected 200, got %d: %s", w.Code, w.Body.String())
}
if n := countAgentLabelAssignments(t, ctx, agentID); n != 1 {
t.Fatalf("agent_to_label rows for a surviving agent: got %d, want 1", n)
}
}
// TestDeleteRuntimeProfile_KeepsAgentLabelAssignments: the profile teardown runs
// the same unbind, so the archived agent and its label links survive there too.
func TestDeleteRuntimeProfile_KeepsAgentLabelAssignments(t *testing.T) {
if testHandler == nil {
t.Skip("database not available")
}
ctx := context.Background()
profileID := insertRuntimeProfileFixture(t, ctx, "Label Cleanup Profile", "codex", "company-codex-label")
runtimeID := insertProfileRuntimeFixture(t, ctx, profileID, "Label Cleanup Profile Runtime", "codex")
agentID := createCascadeFixtureAgent(t, ctx, runtimeID, "Label Cleanup Profile Agent")
if _, err := testPool.Exec(ctx, `UPDATE agent SET archived_at = now() WHERE id = $1`, agentID); err != nil {
t.Fatalf("archive agent: %v", err)
}
seedAgentLabel(t, ctx, testWorkspaceID, agentID)
w := httptest.NewRecorder()
req := newRequest("DELETE", "/api/workspaces/"+testWorkspaceID+"/runtime-profiles/"+profileID, nil)
req = withURLParams(req, "id", testWorkspaceID, "profileId", profileID)
testHandler.DeleteRuntimeProfile(w, req)
if w.Code != http.StatusNoContent {
t.Fatalf("DeleteRuntimeProfile: expected 204, got %d: %s", w.Code, w.Body.String())
}
if !agentExists(t, agentID) {
t.Fatalf("archived agent must survive its runtime profile as an unbound agent")
}
if n := countAgentLabelAssignments(t, ctx, agentID); n != 1 {
t.Fatalf("agent_to_label rows for a surviving agent: got %d, want 1", n)
}
}
// TestDeleteAgentRuntime_CleansSystemAgentLabelAssignments: system agents are
// still hard-deleted with their runtime (they are invisible infrastructure with
// no rebind affordance), so their label links must still be swept — otherwise
// they become the invisible orphan rows the sweep exists to prevent.
func TestDeleteAgentRuntime_CleansSystemAgentLabelAssignments(t *testing.T) {
if testHandler == nil {
t.Skip("database not available")
}
ctx := context.Background()
runtimeID := seedIsolatedRuntime(t, "Label Cleanup System Runtime")
agentID := seedAgentOnRuntime(t, runtimeID, "Label Cleanup System Agent", false)
if _, err := testPool.Exec(ctx,
`UPDATE agent SET kind = 'system', system_key = 'label_cleanup_probe' WHERE id = $1`,
agentID); err != nil {
t.Fatalf("make agent a system agent: %v", err)
}
seedAgentLabel(t, ctx, testWorkspaceID, agentID)
w := httptest.NewRecorder()
req := newRequest("DELETE", "/api/runtimes/"+runtimeID, nil)
req = withURLParam(req, "runtimeId", runtimeID)
testHandler.DeleteAgentRuntime(w, req)
if w.Code != http.StatusOK {
t.Fatalf("DeleteAgentRuntime: expected 200, got %d: %s", w.Code, w.Body.String())
}
if agentExists(t, agentID) {
t.Fatalf("system agent should still be hard-deleted with its runtime")
}
if n := countAgentLabelAssignments(t, ctx, agentID); n != 0 {
t.Fatalf("agent_to_label rows survived system-agent delete: %d", n)
}
}
func seedWorkspaceResourceLabelFixture(t *testing.T, ctx context.Context, slug string) (string, string, string) {
t.Helper()
_, _ = testPool.Exec(ctx, `DELETE FROM workspace WHERE slug = $1`, slug)
var wsID string
if err := testPool.QueryRow(ctx, `
INSERT INTO workspace (name, slug, description)
VALUES ($1, $2, $3)
RETURNING id
`, "Handler Test Delete Labels", slug, "resource-label atomic cleanup test").Scan(&wsID); err != nil {
t.Fatalf("create workspace: %v", err)
}
t.Cleanup(func() {
_, _ = testPool.Exec(context.Background(), `DELETE FROM workspace WHERE id = $1`, wsID)
})
if _, err := testPool.Exec(ctx,
`INSERT INTO member (workspace_id, user_id, role) VALUES ($1, $2, 'owner')`,
wsID, testUserID); err != nil {
t.Fatalf("create owner member: %v", err)
}
// agent.runtime_id is NOT NULL, so the labelled agent needs a runtime in the
// same workspace. Both cascade away with the workspace.
var runtimeID string
if err := testPool.QueryRow(ctx, `
INSERT INTO agent_runtime (
workspace_id, daemon_id, name, runtime_mode, provider, status,
device_info, metadata, owner_id, last_seen_at
)
VALUES ($1, NULL, 'ws-label-runtime', 'cloud', 'ws-label-test', 'online', 'dev', '{}'::jsonb, $2, now())
RETURNING id
`, wsID, testUserID).Scan(&runtimeID); err != nil {
t.Fatalf("insert runtime: %v", err)
}
var agentID string
if err := testPool.QueryRow(ctx, `
INSERT INTO agent (
workspace_id, name, description, runtime_mode, runtime_config,
runtime_id, visibility, max_concurrent_tasks, owner_id
)
VALUES ($1, 'ws-label-agent', '', 'cloud', '{}'::jsonb, $2, 'workspace', 1, $3)
RETURNING id
`, wsID, runtimeID, testUserID).Scan(&agentID); err != nil {
t.Fatalf("insert agent: %v", err)
}
var skillID string
if err := testPool.QueryRow(ctx, `
INSERT INTO skill (workspace_id, name, description, content, config, created_by)
VALUES ($1, 'ws-label-skill', 'fixture', '# x', '{}'::jsonb, $2)
RETURNING id
`, wsID, testUserID).Scan(&skillID); err != nil {
t.Fatalf("insert skill: %v", err)
}
seedAgentLabel(t, ctx, wsID, agentID)
seedSkillLabel(t, ctx, wsID, skillID)
t.Cleanup(func() {
_, _ = testPool.Exec(context.Background(), `DELETE FROM agent_to_label WHERE agent_id = $1`, agentID)
_, _ = testPool.Exec(context.Background(), `DELETE FROM skill_to_label WHERE skill_id = $1`, skillID)
_, _ = testPool.Exec(context.Background(), `DELETE FROM workspace WHERE id = $1`, wsID)
})
return wsID, agentID, skillID
}
// TestDeleteWorkspace_CleansResourceLabelAssignments: workspace delete cascades
// away the agents and skills, but the junction tables have no workspace_id and
// no foreign key, so both must be swept before the cascade or they orphan.
func TestDeleteWorkspace_CleansResourceLabelAssignments(t *testing.T) {
if testHandler == nil {
t.Skip("database not available")
}
ctx := context.Background()
wsID, agentID, skillID := seedWorkspaceResourceLabelFixture(t, ctx, "handler-tests-delete-labels")
w := httptest.NewRecorder()
req := newRequest("DELETE", "/api/workspaces/"+wsID, nil)
req = withURLParam(req, "id", wsID)
testHandler.DeleteWorkspace(w, req)
if w.Code != http.StatusNoContent {
t.Fatalf("DeleteWorkspace: expected 204, got %d: %s", w.Code, w.Body.String())
}
if n := countAgentLabelAssignments(t, ctx, agentID); n != 0 {
t.Fatalf("agent_to_label rows survived workspace delete: %d", n)
}
if n := countSkillLabelAssignments(t, ctx, skillID); n != 0 {
t.Fatalf("skill_to_label rows survived workspace delete: %d", n)
}
}
// TestDeleteWorkspace_RollsBackResourceLabelCleanup verifies the cleanup and
// final workspace delete share one database statement. A restrictive test-only
// foreign key makes the final delete fail; both junction rows must remain.
func TestDeleteWorkspace_RollsBackResourceLabelCleanup(t *testing.T) {
if testHandler == nil {
t.Skip("database not available")
}
ctx := context.Background()
wsID, agentID, skillID := seedWorkspaceResourceLabelFixture(t, ctx, "handler-tests-delete-labels-rollback")
const guardTable = "workspace_delete_resource_label_rollback_guard"
_, _ = testPool.Exec(ctx, `DROP TABLE IF EXISTS `+guardTable)
if _, err := testPool.Exec(ctx, `
CREATE TABLE `+guardTable+` (
workspace_id UUID NOT NULL REFERENCES workspace(id)
)
`); err != nil {
t.Fatalf("create workspace delete guard: %v", err)
}
if _, err := testPool.Exec(ctx, `INSERT INTO `+guardTable+` (workspace_id) VALUES ($1)`, wsID); err != nil {
t.Fatalf("insert workspace delete guard: %v", err)
}
t.Cleanup(func() {
_, _ = testPool.Exec(context.Background(), `DROP TABLE IF EXISTS `+guardTable)
})
w := httptest.NewRecorder()
req := newRequest("DELETE", "/api/workspaces/"+wsID, nil)
req = withURLParam(req, "id", wsID)
testHandler.DeleteWorkspace(w, req)
if w.Code != http.StatusInternalServerError {
t.Fatalf("DeleteWorkspace: expected 500, got %d: %s", w.Code, w.Body.String())
}
var workspaceExists bool
if err := testPool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM workspace WHERE id = $1)`, wsID).Scan(&workspaceExists); err != nil {
t.Fatalf("check workspace after failed delete: %v", err)
}
if !workspaceExists {
t.Fatal("workspace was removed despite the injected delete failure")
}
if n := countAgentLabelAssignments(t, ctx, agentID); n != 1 {
t.Fatalf("agent_to_label rows after failed workspace delete = %d, want 1", n)
}
if n := countSkillLabelAssignments(t, ctx, skillID); n != 1 {
t.Fatalf("skill_to_label rows after failed workspace delete = %d, want 1", n)
}
}