mirror of
https://github.com/multica-ai/multica.git
synced 2026-07-05 21:39:54 +02:00
* feat(server): orphan-task recovery + auto-retry + manual rerun (MUL-1128)
When the daemon process crashed mid-task the issue was stuck at
in_progress for up to 2.5h: the in-flight task timeout was the only
mechanism that ever moved the row, and the runtime heartbeat sweeper
only fires after the runtime stays offline for 45s — a quick restart
beats both windows.
This change implements the A+B plan from the issue thread:
A. lifecycle hygiene
- migration 055 adds attempt / max_attempts / parent_task_id /
failure_reason / last_heartbeat_at to agent_task_queue
- new daemon-auth endpoint POST /runtimes/{id}/recover-orphans:
daemon calls it on every register so the server fails any
dispatched/running tasks the previous process left behind
- new daemon-auth endpoint POST /tasks/{id}/session: persists the
agent's session_id + work_dir mid-flight so a crash doesn't
lose the resume pointer (claude+codex emit MessageStatus with
SessionID; daemon forwards on the first one it sees)
- FailAgentTask / FailStaleTasks / FailTasksForOfflineRuntimes
now set failure_reason ('agent_error' / 'timeout' /
'runtime_offline')
B. auto-retry with resume context
- TaskService.MaybeRetryFailedTask spawns a fresh queued attempt
carrying parent's session_id/work_dir when the failure reason
is infrastructure-shaped (timeout, runtime_offline,
runtime_recovery) and attempt < max_attempts; skips autopilot
- wired into the runtime sweeper paths and TaskService.FailTask
so the user transparently sees a new in_progress run instead of
a stuck row
- new user-auth POST /api/issues/{id}/rerun + multica issue rerun
CLI for the manual escape hatch
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
* fix(server): address PR review for orphan-task recovery (MUL-1128)
Three review-must-fix items on top of the A+B implementation:
1. recover-orphans now funnels through TaskService.HandleFailedTasks,
the same shared post-failure pipeline used by the runtime sweeper.
This guarantees task:failed events are emitted, agent status is
reconciled, and issues stuck in_progress with no remaining active
task are reset to todo even when no auto-retry is created
(max_attempts exhausted, autopilot, non-retryable reason).
2. RerunIssue now uses CancelAgentTasksByIssueAndAgent, scoped to the
issue's current assignee. The previous implementation called
CancelAgentTasksByIssue, which would collateral-cancel parallel
@-mention agents on the same issue.
3. GetLastTaskSession now considers both completed and failed tasks
(mirroring GetLastChatTaskSession), ordering by the most recent
timestamp. With UpdateAgentTaskSession pinning session_id/work_dir
mid-flight, an auto-retry or manual rerun of a daemon-crash failure
now actually resumes the prior conversation context instead of
starting fresh — matching the stated B-branch behaviour.
go build / go vet pass; the existing service and agent test suites pass.
runtime_sweeper / handler integration tests require a local DB with the
055 migration (and the pre-existing 050 first_executed_at column).
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---------
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
116 lines
4.1 KiB
Go
116 lines
4.1 KiB
Go
package handler
|
|
|
|
import (
|
|
"encoding/json"
|
|
"log/slog"
|
|
"net/http"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
db "github.com/multica-ai/multica/server/pkg/db/generated"
|
|
)
|
|
|
|
// RecoverOrphanedTasks is called by the daemon at startup for each runtime
|
|
// it owns. It atomically fails any dispatched/running tasks the server still
|
|
// believes belong to that runtime — those are the tasks the previous daemon
|
|
// process was running when it died — and triggers MaybeRetryFailedTask for
|
|
// each so the user sees a fresh attempt instead of a permanently stuck row.
|
|
//
|
|
// This is the targeted fix for "issue stuck at in_progress when daemon
|
|
// restarts mid-task": the runtime heartbeat sweeper takes up to 75s + the
|
|
// in-process task timeout (2.5h) to notice such tasks; the daemon itself
|
|
// knows the moment it comes back up, so we let it report orphan recovery.
|
|
func (h *Handler) RecoverOrphanedTasks(w http.ResponseWriter, r *http.Request) {
|
|
runtimeID := chi.URLParam(r, "runtimeId")
|
|
if _, ok := h.requireDaemonRuntimeAccess(w, r, runtimeID); !ok {
|
|
return
|
|
}
|
|
|
|
rows, err := h.Queries.RecoverOrphanedTasksForRuntime(r.Context(), parseUUID(runtimeID))
|
|
if err != nil {
|
|
slog.Warn("recover-orphans failed", "runtime_id", runtimeID, "error", err)
|
|
writeError(w, http.StatusInternalServerError, "recover orphans failed")
|
|
return
|
|
}
|
|
|
|
// Funnel through the shared post-failure pipeline so we get the same
|
|
// task:failed events, agent reconcile, issue rollback, and auto-retry
|
|
// behaviour as the runtime sweeper. This was previously a fast-path
|
|
// that bypassed those side effects, leaving the UI stale when no retry
|
|
// was created (max_attempts exhausted, autopilot, non-retryable reason).
|
|
retried := h.TaskService.HandleFailedTasks(r.Context(), rows)
|
|
|
|
if len(rows) > 0 {
|
|
slog.Info("recover-orphans completed",
|
|
"runtime_id", runtimeID,
|
|
"orphaned", len(rows),
|
|
"retried", retried,
|
|
)
|
|
}
|
|
|
|
writeJSON(w, http.StatusOK, map[string]any{
|
|
"orphaned": len(rows),
|
|
"retried": retried,
|
|
})
|
|
}
|
|
|
|
// PinTaskSession lets the daemon persist the agent's session_id and
|
|
// work_dir as soon as they're known — typically right after the agent
|
|
// emits its first system message — so a crash mid-run doesn't lose the
|
|
// resume pointer needed to continue the conversation on the next attempt.
|
|
type PinTaskSessionRequest struct {
|
|
SessionID string `json:"session_id,omitempty"`
|
|
WorkDir string `json:"work_dir,omitempty"`
|
|
}
|
|
|
|
func (h *Handler) PinTaskSession(w http.ResponseWriter, r *http.Request) {
|
|
taskID := chi.URLParam(r, "taskId")
|
|
if _, ok := h.requireDaemonTaskAccess(w, r, taskID); !ok {
|
|
return
|
|
}
|
|
|
|
var req PinTaskSessionRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid request body")
|
|
return
|
|
}
|
|
if req.SessionID == "" && req.WorkDir == "" {
|
|
writeError(w, http.StatusBadRequest, "session_id or work_dir required")
|
|
return
|
|
}
|
|
|
|
params := db.UpdateAgentTaskSessionParams{ID: parseUUID(taskID)}
|
|
if req.SessionID != "" {
|
|
params.SessionID = pgtype.Text{String: req.SessionID, Valid: true}
|
|
}
|
|
if req.WorkDir != "" {
|
|
params.WorkDir = pgtype.Text{String: req.WorkDir, Valid: true}
|
|
}
|
|
if err := h.Queries.UpdateAgentTaskSession(r.Context(), params); err != nil {
|
|
slog.Warn("pin-session failed", "task_id", taskID, "error", err)
|
|
writeError(w, http.StatusInternalServerError, "pin session failed")
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}
|
|
|
|
// RerunIssue manually re-enqueues the issue's current agent assignment as a
|
|
// fresh task. Useful when an issue is stuck or the user wants to retry a
|
|
// failed run. The new task carries the most recent session_id/work_dir so
|
|
// the agent can resume where it left off when the backend supports it.
|
|
func (h *Handler) RerunIssue(w http.ResponseWriter, r *http.Request) {
|
|
id := chi.URLParam(r, "id")
|
|
issue, ok := h.loadIssueForUser(w, r, id)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
task, err := h.TaskService.RerunIssue(r.Context(), issue.ID, pgtype.UUID{})
|
|
if err != nil {
|
|
slog.Warn("issue rerun failed", "issue_id", id, "error", err)
|
|
writeError(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusAccepted, taskToResponse(*task))
|
|
}
|