Files
multica/server/internal/service/task_complete_race_test.go
LinYushen 0b1333fb00 feat(server): orphan-task recovery + auto-retry + manual rerun (MUL-1128) (#1476)
* feat(server): orphan-task recovery + auto-retry + manual rerun (MUL-1128)

When the daemon process crashed mid-task the issue was stuck at
in_progress for up to 2.5h: the in-flight task timeout was the only
mechanism that ever moved the row, and the runtime heartbeat sweeper
only fires after the runtime stays offline for 45s — a quick restart
beats both windows.

This change implements the A+B plan from the issue thread:

A. lifecycle hygiene
- migration 055 adds attempt / max_attempts / parent_task_id /
  failure_reason / last_heartbeat_at to agent_task_queue
- new daemon-auth endpoint POST /runtimes/{id}/recover-orphans:
  daemon calls it on every register so the server fails any
  dispatched/running tasks the previous process left behind
- new daemon-auth endpoint POST /tasks/{id}/session: persists the
  agent's session_id + work_dir mid-flight so a crash doesn't
  lose the resume pointer (claude+codex emit MessageStatus with
  SessionID; daemon forwards on the first one it sees)
- FailAgentTask / FailStaleTasks / FailTasksForOfflineRuntimes
  now set failure_reason ('agent_error' / 'timeout' /
  'runtime_offline')

B. auto-retry with resume context
- TaskService.MaybeRetryFailedTask spawns a fresh queued attempt
  carrying parent's session_id/work_dir when the failure reason
  is infrastructure-shaped (timeout, runtime_offline,
  runtime_recovery) and attempt < max_attempts; skips autopilot
- wired into the runtime sweeper paths and TaskService.FailTask
  so the user transparently sees a new in_progress run instead of
  a stuck row
- new user-auth POST /api/issues/{id}/rerun + multica issue rerun
  CLI for the manual escape hatch

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix(server): address PR review for orphan-task recovery (MUL-1128)

Three review-must-fix items on top of the A+B implementation:

1. recover-orphans now funnels through TaskService.HandleFailedTasks,
   the same shared post-failure pipeline used by the runtime sweeper.
   This guarantees task:failed events are emitted, agent status is
   reconciled, and issues stuck in_progress with no remaining active
   task are reset to todo even when no auto-retry is created
   (max_attempts exhausted, autopilot, non-retryable reason).

2. RerunIssue now uses CancelAgentTasksByIssueAndAgent, scoped to the
   issue's current assignee. The previous implementation called
   CancelAgentTasksByIssue, which would collateral-cancel parallel
   @-mention agents on the same issue.

3. GetLastTaskSession now considers both completed and failed tasks
   (mirroring GetLastChatTaskSession), ordering by the most recent
   timestamp. With UpdateAgentTaskSession pinning session_id/work_dir
   mid-flight, an auto-retry or manual rerun of a daemon-crash failure
   now actually resumes the prior conversation context instead of
   starting fresh — matching the stated B-branch behaviour.

go build / go vet pass; the existing service and agent test suites pass.
runtime_sweeper / handler integration tests require a local DB with the
055 migration (and the pre-existing 050 first_executed_at column).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-04-22 13:08:37 +08:00

169 lines
3.9 KiB
Go

package service
import (
"context"
"strings"
"testing"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgtype"
"github.com/multica-ai/multica/server/internal/events"
db "github.com/multica-ai/multica/server/pkg/db/generated"
)
// mockRow implements pgx.Row, returning either a scanned task or pgx.ErrNoRows.
type mockRow struct {
task *db.AgentTaskQueue
err error
}
func (r *mockRow) Scan(dest ...any) error {
if r.err != nil {
return r.err
}
t := r.task
ptrs := []any{
&t.ID, &t.AgentID, &t.IssueID, &t.Status, &t.Priority,
&t.DispatchedAt, &t.StartedAt, &t.CompletedAt, &t.Result,
&t.Error, &t.CreatedAt, &t.Context, &t.RuntimeID,
&t.SessionID, &t.WorkDir, &t.TriggerCommentID,
&t.ChatSessionID, &t.AutopilotRunID,
}
for i, p := range ptrs {
if i >= len(dest) {
break
}
// Copy value from source to dest by assigning through the pointer.
switch d := dest[i].(type) {
case *pgtype.UUID:
*d = *(p.(*pgtype.UUID))
case *string:
*d = *(p.(*string))
case *int32:
*d = *(p.(*int32))
case *pgtype.Timestamptz:
*d = *(p.(*pgtype.Timestamptz))
case *[]byte:
*d = *(p.(*[]byte))
case *pgtype.Text:
*d = *(p.(*pgtype.Text))
}
}
return nil
}
// mockDBTX routes QueryRow calls: complete/fail queries return ErrNoRows,
// getAgentTask returns the stored task.
type mockDBTX struct {
task db.AgentTaskQueue
}
func (m *mockDBTX) Exec(_ context.Context, _ string, _ ...interface{}) (pgconn.CommandTag, error) {
return pgconn.NewCommandTag(""), nil
}
func (m *mockDBTX) Query(_ context.Context, _ string, _ ...interface{}) (pgx.Rows, error) {
return nil, pgx.ErrNoRows
}
func (m *mockDBTX) QueryRow(_ context.Context, sql string, _ ...interface{}) pgx.Row {
// CompleteAgentTask and FailAgentTask SQL contain "SET status ="
if strings.Contains(sql, "SET status =") {
return &mockRow{err: pgx.ErrNoRows}
}
// GetAgentTask — return the existing task
return &mockRow{task: &m.task}
}
func testUUID(b byte) pgtype.UUID {
var u pgtype.UUID
u.Valid = true
u.Bytes[0] = b
return u
}
func TestCompleteTask_AlreadyFinalized(t *testing.T) {
taskID := testUUID(1)
agentID := testUUID(2)
tests := []struct {
name string
status string
}{
{"already completed", "completed"},
{"already cancelled", "cancelled"},
{"already failed", "failed"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mock := &mockDBTX{task: db.AgentTaskQueue{
ID: taskID,
AgentID: agentID,
Status: tt.status,
}}
svc := &TaskService{
Queries: db.New(mock),
Bus: events.New(),
}
got, err := svc.CompleteTask(context.Background(), taskID, nil, "", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if got == nil {
t.Fatal("expected task, got nil")
}
if got.Status != tt.status {
t.Errorf("expected status %q, got %q", tt.status, got.Status)
}
if got.ID != taskID {
t.Error("returned task ID doesn't match")
}
})
}
}
func TestFailTask_AlreadyFinalized(t *testing.T) {
taskID := testUUID(1)
agentID := testUUID(2)
tests := []struct {
name string
status string
}{
{"already completed", "completed"},
{"already cancelled", "cancelled"},
{"already failed", "failed"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mock := &mockDBTX{task: db.AgentTaskQueue{
ID: taskID,
AgentID: agentID,
Status: tt.status,
}}
svc := &TaskService{
Queries: db.New(mock),
Bus: events.New(),
}
got, err := svc.FailTask(context.Background(), taskID, "agent crashed", "", "", "")
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if got == nil {
t.Fatal("expected task, got nil")
}
if got.Status != tt.status {
t.Errorf("expected status %q, got %q", tt.status, got.Status)
}
if got.ID != taskID {
t.Error("returned task ID doesn't match")
}
})
}
}