Files
multica/server/pkg/db/queries/agent.sql
LinYushen 0b1333fb00 feat(server): orphan-task recovery + auto-retry + manual rerun (MUL-1128) (#1476)
* feat(server): orphan-task recovery + auto-retry + manual rerun (MUL-1128)

When the daemon process crashed mid-task the issue was stuck at
in_progress for up to 2.5h: the in-flight task timeout was the only
mechanism that ever moved the row, and the runtime heartbeat sweeper
only fires after the runtime stays offline for 45s — a quick restart
beats both windows.

This change implements the A+B plan from the issue thread:

A. lifecycle hygiene
- migration 055 adds attempt / max_attempts / parent_task_id /
  failure_reason / last_heartbeat_at to agent_task_queue
- new daemon-auth endpoint POST /runtimes/{id}/recover-orphans:
  daemon calls it on every register so the server fails any
  dispatched/running tasks the previous process left behind
- new daemon-auth endpoint POST /tasks/{id}/session: persists the
  agent's session_id + work_dir mid-flight so a crash doesn't
  lose the resume pointer (claude+codex emit MessageStatus with
  SessionID; daemon forwards on the first one it sees)
- FailAgentTask / FailStaleTasks / FailTasksForOfflineRuntimes
  now set failure_reason ('agent_error' / 'timeout' /
  'runtime_offline')

B. auto-retry with resume context
- TaskService.MaybeRetryFailedTask spawns a fresh queued attempt
  carrying parent's session_id/work_dir when the failure reason
  is infrastructure-shaped (timeout, runtime_offline,
  runtime_recovery) and attempt < max_attempts; skips autopilot
- wired into the runtime sweeper paths and TaskService.FailTask
  so the user transparently sees a new in_progress run instead of
  a stuck row
- new user-auth POST /api/issues/{id}/rerun + multica issue rerun
  CLI for the manual escape hatch

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix(server): address PR review for orphan-task recovery (MUL-1128)

Three review-must-fix items on top of the A+B implementation:

1. recover-orphans now funnels through TaskService.HandleFailedTasks,
   the same shared post-failure pipeline used by the runtime sweeper.
   This guarantees task:failed events are emitted, agent status is
   reconciled, and issues stuck in_progress with no remaining active
   task are reset to todo even when no auto-retry is created
   (max_attempts exhausted, autopilot, non-retryable reason).

2. RerunIssue now uses CancelAgentTasksByIssueAndAgent, scoped to the
   issue's current assignee. The previous implementation called
   CancelAgentTasksByIssue, which would collateral-cancel parallel
   @-mention agents on the same issue.

3. GetLastTaskSession now considers both completed and failed tasks
   (mirroring GetLastChatTaskSession), ordering by the most recent
   timestamp. With UpdateAgentTaskSession pinning session_id/work_dir
   mid-flight, an auto-retry or manual rerun of a daemon-crash failure
   now actually resumes the prior conversation context instead of
   starting fresh — matching the stated B-branch behaviour.

go build / go vet pass; the existing service and agent test suites pass.
runtime_sweeper / handler integration tests require a local DB with the
055 migration (and the pre-existing 050 first_executed_at column).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-04-22 13:08:37 +08:00

272 lines
11 KiB
SQL

-- name: ListAgents :many
SELECT * FROM agent
WHERE workspace_id = $1 AND archived_at IS NULL
ORDER BY created_at ASC;
-- name: ListAllAgents :many
SELECT * FROM agent
WHERE workspace_id = $1
ORDER BY created_at ASC;
-- name: GetAgent :one
SELECT * FROM agent
WHERE id = $1;
-- name: GetAgentInWorkspace :one
SELECT * FROM agent
WHERE id = $1 AND workspace_id = $2;
-- name: CreateAgent :one
INSERT INTO agent (
workspace_id, name, description, avatar_url, runtime_mode,
runtime_config, runtime_id, visibility, max_concurrent_tasks, owner_id,
instructions, custom_env, custom_args, mcp_config, model
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
RETURNING *;
-- name: UpdateAgent :one
UPDATE agent SET
name = COALESCE(sqlc.narg('name'), name),
description = COALESCE(sqlc.narg('description'), description),
avatar_url = COALESCE(sqlc.narg('avatar_url'), avatar_url),
runtime_config = COALESCE(sqlc.narg('runtime_config'), runtime_config),
runtime_mode = COALESCE(sqlc.narg('runtime_mode'), runtime_mode),
runtime_id = COALESCE(sqlc.narg('runtime_id'), runtime_id),
visibility = COALESCE(sqlc.narg('visibility'), visibility),
status = COALESCE(sqlc.narg('status'), status),
max_concurrent_tasks = COALESCE(sqlc.narg('max_concurrent_tasks'), max_concurrent_tasks),
instructions = COALESCE(sqlc.narg('instructions'), instructions),
custom_env = COALESCE(sqlc.narg('custom_env'), custom_env),
custom_args = COALESCE(sqlc.narg('custom_args'), custom_args),
mcp_config = COALESCE(sqlc.narg('mcp_config'), mcp_config),
model = COALESCE(sqlc.narg('model'), model),
updated_at = now()
WHERE id = $1
RETURNING *;
-- name: ClearAgentMcpConfig :one
UPDATE agent SET mcp_config = NULL, updated_at = now()
WHERE id = $1
RETURNING *;
-- name: ArchiveAgent :one
UPDATE agent SET archived_at = now(), archived_by = $2, updated_at = now()
WHERE id = $1
RETURNING *;
-- name: RestoreAgent :one
UPDATE agent SET archived_at = NULL, archived_by = NULL, updated_at = now()
WHERE id = $1
RETURNING *;
-- name: ListAgentTasks :many
SELECT * FROM agent_task_queue
WHERE agent_id = $1
ORDER BY created_at DESC;
-- name: CreateAgentTask :one
INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, priority, trigger_comment_id)
VALUES ($1, $2, $3, 'queued', $4, sqlc.narg(trigger_comment_id))
RETURNING *;
-- name: CreateRetryTask :one
-- Clones a parent task into a fresh queued attempt. Carries forward the
-- agent's resume context (session_id/work_dir) so the child can continue
-- the conversation when the backend supports it. attempt is incremented;
-- max_attempts and trigger_comment_id are inherited.
INSERT INTO agent_task_queue (
agent_id, runtime_id, issue_id, chat_session_id, autopilot_run_id,
status, priority, trigger_comment_id, context,
session_id, work_dir,
attempt, max_attempts, parent_task_id
)
SELECT
p.agent_id, p.runtime_id, p.issue_id, p.chat_session_id, p.autopilot_run_id,
'queued', p.priority, p.trigger_comment_id, p.context,
p.session_id, p.work_dir,
p.attempt + 1, p.max_attempts, p.id
FROM agent_task_queue p
WHERE p.id = $1
RETURNING *;
-- name: CancelAgentTasksByIssue :exec
UPDATE agent_task_queue
SET status = 'cancelled'
WHERE issue_id = $1 AND status IN ('queued', 'dispatched', 'running');
-- name: CancelAgentTasksByIssueAndAgent :many
-- Cancels active tasks for a single (issue, agent) pair without touching
-- tasks belonging to other agents on the same issue. Used by the manual
-- rerun flow so re-running the assignee doesn't collateral-cancel a
-- still-running @-mention agent on the same issue.
UPDATE agent_task_queue
SET status = 'cancelled', completed_at = now()
WHERE issue_id = $1 AND agent_id = $2 AND status IN ('queued', 'dispatched', 'running')
RETURNING *;
-- name: CancelAgentTasksByAgent :exec
UPDATE agent_task_queue
SET status = 'cancelled'
WHERE agent_id = $1 AND status IN ('queued', 'dispatched', 'running');
-- name: GetAgentTask :one
SELECT * FROM agent_task_queue
WHERE id = $1;
-- name: ClaimAgentTask :one
-- Claims the next queued task for an agent, enforcing per-(issue, agent) serialization:
-- a task is only claimable when no other task for the same issue AND same agent is
-- already dispatched or running. This allows different agents to work on the same
-- issue in parallel while preventing a single agent from running duplicate tasks.
-- Chat tasks (issue_id IS NULL) use chat_session_id for serialization instead.
UPDATE agent_task_queue
SET status = 'dispatched', dispatched_at = now()
WHERE id = (
SELECT atq.id FROM agent_task_queue atq
WHERE atq.agent_id = $1 AND atq.status = 'queued'
AND NOT EXISTS (
SELECT 1 FROM agent_task_queue active
WHERE active.agent_id = atq.agent_id
AND active.status IN ('dispatched', 'running')
AND (
(atq.issue_id IS NOT NULL AND active.issue_id = atq.issue_id)
OR (atq.chat_session_id IS NOT NULL AND active.chat_session_id = atq.chat_session_id)
)
)
ORDER BY atq.priority DESC, atq.created_at ASC
LIMIT 1
FOR UPDATE SKIP LOCKED
)
RETURNING *;
-- name: StartAgentTask :one
UPDATE agent_task_queue
SET status = 'running', started_at = now()
WHERE id = $1 AND status = 'dispatched'
RETURNING *;
-- name: CompleteAgentTask :one
UPDATE agent_task_queue
SET status = 'completed', completed_at = now(), result = $2, session_id = $3, work_dir = $4
WHERE id = $1 AND status = 'running'
RETURNING *;
-- name: GetLastTaskSession :one
-- Returns the session_id and work_dir from the most recent task for a given
-- (agent_id, issue_id) pair, used for session resumption. We accept both
-- 'completed' and 'failed' tasks: a failed task may have established a real
-- agent session before crashing (orphaned by a daemon restart, runtime offline,
-- or sweeper timeout), and the daemon pins the resume pointer mid-flight via
-- UpdateAgentTaskSession. Without this, an auto-retry / manual rerun of a
-- mid-run failure would silently start a fresh conversation and lose the
-- in-flight context — exactly what MUL-1128's B branch is meant to fix.
SELECT session_id, work_dir FROM agent_task_queue
WHERE agent_id = $1 AND issue_id = $2
AND status IN ('completed', 'failed')
AND session_id IS NOT NULL
ORDER BY COALESCE(completed_at, started_at, dispatched_at, created_at) DESC
LIMIT 1;
-- name: FailAgentTask :one
-- Marks a task as failed. session_id and work_dir are merged via COALESCE so
-- if the agent already established a real session before failing (e.g. it
-- crashed mid-conversation, was cancelled, or hit a tool error) the resume
-- pointer is preserved on the task row. The next chat task can then fall
-- back to GetLastChatTaskSession and continue the conversation instead of
-- silently starting over.
--
-- failure_reason is a coarse classifier consumed by the auto-retry path;
-- 'agent_error' is the safe default when the daemon doesn't supply one.
UPDATE agent_task_queue
SET status = 'failed',
completed_at = now(),
error = $2,
failure_reason = COALESCE(sqlc.narg('failure_reason'), 'agent_error'),
session_id = COALESCE(sqlc.narg('session_id'), session_id),
work_dir = COALESCE(sqlc.narg('work_dir'), work_dir)
WHERE id = $1 AND status IN ('dispatched', 'running')
RETURNING *;
-- name: UpdateAgentTaskSession :exec
-- Pins the resume pointer mid-flight so a daemon crash leaves a usable
-- session_id/work_dir on the task row. No-op if the task is no longer
-- in dispatched/running.
UPDATE agent_task_queue
SET session_id = COALESCE(sqlc.narg('session_id'), session_id),
work_dir = COALESCE(sqlc.narg('work_dir'), work_dir),
last_heartbeat_at = now()
WHERE id = $1 AND status IN ('dispatched', 'running');
-- name: RecoverOrphanedTasksForRuntime :many
-- Called by the daemon at startup. Atomically fails any dispatched/running
-- task that the prior incarnation of this runtime owned but did not
-- finalize. Returns the failed rows so callers can hand them to the
-- auto-retry path.
UPDATE agent_task_queue
SET status = 'failed',
completed_at = now(),
error = 'daemon restarted while task was in flight',
failure_reason = 'runtime_recovery'
WHERE runtime_id = $1 AND status IN ('dispatched', 'running')
RETURNING *;
-- name: FailStaleTasks :many
-- Fails tasks stuck in dispatched/running beyond the given thresholds.
-- Handles cases where the daemon is alive but the task is orphaned
-- (e.g. agent process hung, daemon failed to report completion).
UPDATE agent_task_queue
SET status = 'failed', completed_at = now(), error = 'task timed out',
failure_reason = 'timeout'
WHERE (status = 'dispatched' AND dispatched_at < now() - make_interval(secs => @dispatch_timeout_secs::double precision))
OR (status = 'running' AND started_at < now() - make_interval(secs => @running_timeout_secs::double precision))
RETURNING *;
-- name: CancelAgentTask :one
UPDATE agent_task_queue
SET status = 'cancelled', completed_at = now()
WHERE id = $1 AND status IN ('queued', 'dispatched', 'running')
RETURNING *;
-- name: CountRunningTasks :one
SELECT count(*) FROM agent_task_queue
WHERE agent_id = $1 AND status IN ('dispatched', 'running');
-- name: HasActiveTaskForIssue :one
-- Returns true if there is any queued, dispatched, or running task for the issue.
SELECT count(*) > 0 AS has_active FROM agent_task_queue
WHERE issue_id = $1 AND status IN ('queued', 'dispatched', 'running');
-- name: HasPendingTaskForIssue :one
-- Returns true if there is a queued or dispatched (but not yet running) task for the issue.
-- Used by the coalescing queue: allow enqueue when a task is running (so
-- the agent picks up new comments on the next cycle) but skip if a pending
-- task already exists (natural dedup).
SELECT count(*) > 0 AS has_pending FROM agent_task_queue
WHERE issue_id = $1 AND status IN ('queued', 'dispatched');
-- name: HasPendingTaskForIssueAndAgent :one
-- Returns true if a specific agent already has a queued or dispatched task
-- for the given issue. Used by @mention trigger dedup.
SELECT count(*) > 0 AS has_pending FROM agent_task_queue
WHERE issue_id = $1 AND agent_id = $2 AND status IN ('queued', 'dispatched');
-- name: ListPendingTasksByRuntime :many
SELECT * FROM agent_task_queue
WHERE runtime_id = $1 AND status IN ('queued', 'dispatched')
ORDER BY priority DESC, created_at ASC;
-- name: ListActiveTasksByIssue :many
SELECT * FROM agent_task_queue
WHERE issue_id = $1 AND status IN ('dispatched', 'running')
ORDER BY created_at DESC;
-- name: ListTasksByIssue :many
SELECT * FROM agent_task_queue
WHERE issue_id = $1
ORDER BY created_at DESC;
-- name: UpdateAgentStatus :one
UPDATE agent SET status = $2, updated_at = now()
WHERE id = $1
RETURNING *;