Files
multica/server/internal/handler/task_lifecycle.go
LinYushen 0b1333fb00 feat(server): orphan-task recovery + auto-retry + manual rerun (MUL-1128) (#1476)
* feat(server): orphan-task recovery + auto-retry + manual rerun (MUL-1128)

When the daemon process crashed mid-task the issue was stuck at
in_progress for up to 2.5h: the in-flight task timeout was the only
mechanism that ever moved the row, and the runtime heartbeat sweeper
only fires after the runtime stays offline for 45s — a quick restart
beats both windows.

This change implements the A+B plan from the issue thread:

A. lifecycle hygiene
- migration 055 adds attempt / max_attempts / parent_task_id /
  failure_reason / last_heartbeat_at to agent_task_queue
- new daemon-auth endpoint POST /runtimes/{id}/recover-orphans:
  daemon calls it on every register so the server fails any
  dispatched/running tasks the previous process left behind
- new daemon-auth endpoint POST /tasks/{id}/session: persists the
  agent's session_id + work_dir mid-flight so a crash doesn't
  lose the resume pointer (claude+codex emit MessageStatus with
  SessionID; daemon forwards on the first one it sees)
- FailAgentTask / FailStaleTasks / FailTasksForOfflineRuntimes
  now set failure_reason ('agent_error' / 'timeout' /
  'runtime_offline')

B. auto-retry with resume context
- TaskService.MaybeRetryFailedTask spawns a fresh queued attempt
  carrying parent's session_id/work_dir when the failure reason
  is infrastructure-shaped (timeout, runtime_offline,
  runtime_recovery) and attempt < max_attempts; skips autopilot
- wired into the runtime sweeper paths and TaskService.FailTask
  so the user transparently sees a new in_progress run instead of
  a stuck row
- new user-auth POST /api/issues/{id}/rerun + multica issue rerun
  CLI for the manual escape hatch

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix(server): address PR review for orphan-task recovery (MUL-1128)

Three review-must-fix items on top of the A+B implementation:

1. recover-orphans now funnels through TaskService.HandleFailedTasks,
   the same shared post-failure pipeline used by the runtime sweeper.
   This guarantees task:failed events are emitted, agent status is
   reconciled, and issues stuck in_progress with no remaining active
   task are reset to todo even when no auto-retry is created
   (max_attempts exhausted, autopilot, non-retryable reason).

2. RerunIssue now uses CancelAgentTasksByIssueAndAgent, scoped to the
   issue's current assignee. The previous implementation called
   CancelAgentTasksByIssue, which would collateral-cancel parallel
   @-mention agents on the same issue.

3. GetLastTaskSession now considers both completed and failed tasks
   (mirroring GetLastChatTaskSession), ordering by the most recent
   timestamp. With UpdateAgentTaskSession pinning session_id/work_dir
   mid-flight, an auto-retry or manual rerun of a daemon-crash failure
   now actually resumes the prior conversation context instead of
   starting fresh — matching the stated B-branch behaviour.

go build / go vet pass; the existing service and agent test suites pass.
runtime_sweeper / handler integration tests require a local DB with the
055 migration (and the pre-existing 050 first_executed_at column).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-04-22 13:08:37 +08:00

116 lines
4.1 KiB
Go

package handler
import (
"encoding/json"
"log/slog"
"net/http"
"github.com/go-chi/chi/v5"
"github.com/jackc/pgx/v5/pgtype"
db "github.com/multica-ai/multica/server/pkg/db/generated"
)
// RecoverOrphanedTasks is called by the daemon at startup for each runtime
// it owns. It atomically fails any dispatched/running tasks the server still
// believes belong to that runtime — those are the tasks the previous daemon
// process was running when it died — and triggers MaybeRetryFailedTask for
// each so the user sees a fresh attempt instead of a permanently stuck row.
//
// This is the targeted fix for "issue stuck at in_progress when daemon
// restarts mid-task": the runtime heartbeat sweeper takes up to 75s + the
// in-process task timeout (2.5h) to notice such tasks; the daemon itself
// knows the moment it comes back up, so we let it report orphan recovery.
func (h *Handler) RecoverOrphanedTasks(w http.ResponseWriter, r *http.Request) {
runtimeID := chi.URLParam(r, "runtimeId")
if _, ok := h.requireDaemonRuntimeAccess(w, r, runtimeID); !ok {
return
}
rows, err := h.Queries.RecoverOrphanedTasksForRuntime(r.Context(), parseUUID(runtimeID))
if err != nil {
slog.Warn("recover-orphans failed", "runtime_id", runtimeID, "error", err)
writeError(w, http.StatusInternalServerError, "recover orphans failed")
return
}
// Funnel through the shared post-failure pipeline so we get the same
// task:failed events, agent reconcile, issue rollback, and auto-retry
// behaviour as the runtime sweeper. This was previously a fast-path
// that bypassed those side effects, leaving the UI stale when no retry
// was created (max_attempts exhausted, autopilot, non-retryable reason).
retried := h.TaskService.HandleFailedTasks(r.Context(), rows)
if len(rows) > 0 {
slog.Info("recover-orphans completed",
"runtime_id", runtimeID,
"orphaned", len(rows),
"retried", retried,
)
}
writeJSON(w, http.StatusOK, map[string]any{
"orphaned": len(rows),
"retried": retried,
})
}
// PinTaskSession lets the daemon persist the agent's session_id and
// work_dir as soon as they're known — typically right after the agent
// emits its first system message — so a crash mid-run doesn't lose the
// resume pointer needed to continue the conversation on the next attempt.
type PinTaskSessionRequest struct {
SessionID string `json:"session_id,omitempty"`
WorkDir string `json:"work_dir,omitempty"`
}
func (h *Handler) PinTaskSession(w http.ResponseWriter, r *http.Request) {
taskID := chi.URLParam(r, "taskId")
if _, ok := h.requireDaemonTaskAccess(w, r, taskID); !ok {
return
}
var req PinTaskSessionRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
if req.SessionID == "" && req.WorkDir == "" {
writeError(w, http.StatusBadRequest, "session_id or work_dir required")
return
}
params := db.UpdateAgentTaskSessionParams{ID: parseUUID(taskID)}
if req.SessionID != "" {
params.SessionID = pgtype.Text{String: req.SessionID, Valid: true}
}
if req.WorkDir != "" {
params.WorkDir = pgtype.Text{String: req.WorkDir, Valid: true}
}
if err := h.Queries.UpdateAgentTaskSession(r.Context(), params); err != nil {
slog.Warn("pin-session failed", "task_id", taskID, "error", err)
writeError(w, http.StatusInternalServerError, "pin session failed")
return
}
w.WriteHeader(http.StatusNoContent)
}
// RerunIssue manually re-enqueues the issue's current agent assignment as a
// fresh task. Useful when an issue is stuck or the user wants to retry a
// failed run. The new task carries the most recent session_id/work_dir so
// the agent can resume where it left off when the backend supports it.
func (h *Handler) RerunIssue(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
issue, ok := h.loadIssueForUser(w, r, id)
if !ok {
return
}
task, err := h.TaskService.RerunIssue(r.Context(), issue.ID, pgtype.UUID{})
if err != nil {
slog.Warn("issue rerun failed", "issue_id", id, "error", err)
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusAccepted, taskToResponse(*task))
}