package service import ( "context" "encoding/json" "errors" "fmt" "log/slog" "strconv" "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/multica-ai/multica/server/internal/events" "github.com/multica-ai/multica/server/internal/mention" "github.com/multica-ai/multica/server/internal/realtime" "github.com/multica-ai/multica/server/internal/util" db "github.com/multica-ai/multica/server/pkg/db/generated" "github.com/multica-ai/multica/server/pkg/protocol" "github.com/multica-ai/multica/server/pkg/redact" ) type TaskService struct { Queries *db.Queries TxStarter TxStarter Hub *realtime.Hub Bus *events.Bus } func NewTaskService(q *db.Queries, tx TxStarter, hub *realtime.Hub, bus *events.Bus) *TaskService { return &TaskService{Queries: q, TxStarter: tx, Hub: hub, Bus: bus} } // EnqueueTaskForIssue creates a queued task for an agent-assigned issue. // No context snapshot is stored — the agent fetches all data it needs at // runtime via the multica CLI. func (s *TaskService) EnqueueTaskForIssue(ctx context.Context, issue db.Issue, triggerCommentID ...pgtype.UUID) (db.AgentTaskQueue, error) { if !issue.AssigneeID.Valid { slog.Error("task enqueue failed", "issue_id", util.UUIDToString(issue.ID), "error", "issue has no assignee") return db.AgentTaskQueue{}, fmt.Errorf("issue has no assignee") } agent, err := s.Queries.GetAgent(ctx, issue.AssigneeID) if err != nil { slog.Error("task enqueue failed", "issue_id", util.UUIDToString(issue.ID), "error", err) return db.AgentTaskQueue{}, fmt.Errorf("load agent: %w", err) } if agent.ArchivedAt.Valid { slog.Debug("task enqueue skipped: agent is archived", "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(agent.ID)) return db.AgentTaskQueue{}, fmt.Errorf("agent is archived") } if !agent.RuntimeID.Valid { slog.Error("task enqueue failed", "issue_id", util.UUIDToString(issue.ID), "error", "agent has no runtime") return db.AgentTaskQueue{}, fmt.Errorf("agent has no runtime") } var commentID pgtype.UUID if len(triggerCommentID) > 0 { commentID = triggerCommentID[0] } task, err := s.Queries.CreateAgentTask(ctx, db.CreateAgentTaskParams{ AgentID: issue.AssigneeID, RuntimeID: agent.RuntimeID, IssueID: issue.ID, Priority: priorityToInt(issue.Priority), TriggerCommentID: commentID, }) if err != nil { slog.Error("task enqueue failed", "issue_id", util.UUIDToString(issue.ID), "error", err) return db.AgentTaskQueue{}, fmt.Errorf("create task: %w", err) } slog.Info("task enqueued", "task_id", util.UUIDToString(task.ID), "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(issue.AssigneeID)) return task, nil } // EnqueueTaskForMention creates a queued task for a mentioned agent on an issue. // Unlike EnqueueTaskForIssue, this takes an explicit agent ID rather than // deriving it from the issue assignee. func (s *TaskService) EnqueueTaskForMention(ctx context.Context, issue db.Issue, agentID pgtype.UUID, triggerCommentID pgtype.UUID) (db.AgentTaskQueue, error) { agent, err := s.Queries.GetAgent(ctx, agentID) if err != nil { slog.Error("mention task enqueue failed: agent not found", "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(agentID), "error", err) return db.AgentTaskQueue{}, fmt.Errorf("load agent: %w", err) } if agent.ArchivedAt.Valid { slog.Debug("mention task enqueue skipped: agent is archived", "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(agentID)) return db.AgentTaskQueue{}, fmt.Errorf("agent is archived") } if !agent.RuntimeID.Valid { slog.Error("mention task enqueue failed: agent has no runtime", "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(agentID)) return db.AgentTaskQueue{}, fmt.Errorf("agent has no runtime") } task, err := s.Queries.CreateAgentTask(ctx, db.CreateAgentTaskParams{ AgentID: agentID, RuntimeID: agent.RuntimeID, IssueID: issue.ID, Priority: priorityToInt(issue.Priority), TriggerCommentID: triggerCommentID, }) if err != nil { slog.Error("mention task enqueue failed", "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(agentID), "error", err) return db.AgentTaskQueue{}, fmt.Errorf("create task: %w", err) } slog.Info("mention task enqueued", "task_id", util.UUIDToString(task.ID), "issue_id", util.UUIDToString(issue.ID), "agent_id", util.UUIDToString(agentID)) return task, nil } // QuickCreateContext is the JSON payload stored on a quick-create task's // context column. The daemon detects this variant via Type == "quick_create" // and switches to the quick-create prompt template; the completion path // uses RequesterID + WorkspaceID to write the inbox notification. type QuickCreateContext struct { Type string `json:"type"` Prompt string `json:"prompt"` RequesterID string `json:"requester_id"` WorkspaceID string `json:"workspace_id"` } // QuickCreateContextType marks a task as a quick-create job. const QuickCreateContextType = "quick_create" // EnqueueQuickCreateTask creates a queued task that has no issue / chat / // autopilot link — the user's natural-language prompt is stored in the // task's context JSONB and the agent is expected to translate it into a // `multica issue create` call. Pre-validates that the agent is reachable // (not archived, has a runtime) so the API can reject up-front rather than // queue a task no one will ever claim. func (s *TaskService) EnqueueQuickCreateTask(ctx context.Context, workspaceID, requesterID pgtype.UUID, agentID pgtype.UUID, prompt string) (db.AgentTaskQueue, error) { agent, err := s.Queries.GetAgent(ctx, agentID) if err != nil { return db.AgentTaskQueue{}, fmt.Errorf("load agent: %w", err) } if agent.ArchivedAt.Valid { return db.AgentTaskQueue{}, fmt.Errorf("agent is archived") } if !agent.RuntimeID.Valid { return db.AgentTaskQueue{}, fmt.Errorf("agent has no runtime") } payload := QuickCreateContext{ Type: QuickCreateContextType, Prompt: prompt, RequesterID: util.UUIDToString(requesterID), WorkspaceID: util.UUIDToString(workspaceID), } contextJSON, err := json.Marshal(payload) if err != nil { return db.AgentTaskQueue{}, fmt.Errorf("marshal quick-create context: %w", err) } task, err := s.Queries.CreateQuickCreateTask(ctx, db.CreateQuickCreateTaskParams{ AgentID: agentID, RuntimeID: agent.RuntimeID, Priority: priorityToInt("high"), Context: contextJSON, }) if err != nil { return db.AgentTaskQueue{}, fmt.Errorf("create quick-create task: %w", err) } slog.Info("quick-create task enqueued", "task_id", util.UUIDToString(task.ID), "agent_id", util.UUIDToString(agentID), "requester_id", util.UUIDToString(requesterID), "workspace_id", util.UUIDToString(workspaceID), ) return task, nil } // EnqueueChatTask creates a queued task for a chat session. // Unlike issue tasks, chat tasks have no issue_id. func (s *TaskService) EnqueueChatTask(ctx context.Context, chatSession db.ChatSession) (db.AgentTaskQueue, error) { agent, err := s.Queries.GetAgent(ctx, chatSession.AgentID) if err != nil { slog.Error("chat task enqueue failed", "chat_session_id", util.UUIDToString(chatSession.ID), "error", err) return db.AgentTaskQueue{}, fmt.Errorf("load agent: %w", err) } if agent.ArchivedAt.Valid { return db.AgentTaskQueue{}, fmt.Errorf("agent is archived") } if !agent.RuntimeID.Valid { return db.AgentTaskQueue{}, fmt.Errorf("agent has no runtime") } task, err := s.Queries.CreateChatTask(ctx, db.CreateChatTaskParams{ AgentID: chatSession.AgentID, RuntimeID: agent.RuntimeID, Priority: 2, // medium priority for chat ChatSessionID: chatSession.ID, }) if err != nil { slog.Error("chat task enqueue failed", "chat_session_id", util.UUIDToString(chatSession.ID), "error", err) return db.AgentTaskQueue{}, fmt.Errorf("create chat task: %w", err) } slog.Info("chat task enqueued", "task_id", util.UUIDToString(task.ID), "chat_session_id", util.UUIDToString(chatSession.ID), "agent_id", util.UUIDToString(chatSession.AgentID)) return task, nil } // CancelTasksForIssue cancels every active task on the issue, reconciles each // affected agent's status, and broadcasts task:cancelled events so frontends // clear their live cards. // // Before #1587 this path was "cancel rows and return" — issue-status flips // (e.g. user marks the issue `done` or `cancelled` while a task is still // running) left the agent stuck at status="working" indefinitely, requiring a // manual `multica agent update --status idle` to unwedge. Matches the // pattern already used by CancelTask and RerunIssue. func (s *TaskService) CancelTasksForIssue(ctx context.Context, issueID pgtype.UUID) error { cancelled, err := s.Queries.CancelAgentTasksByIssue(ctx, issueID) if err != nil { return err } for _, t := range cancelled { s.ReconcileAgentStatus(ctx, t.AgentID) s.broadcastTaskEvent(ctx, protocol.EventTaskCancelled, t) } return nil } // CancelTasksByTriggerComment cancels active tasks whose trigger is the given // comment. Called from DeleteComment so an agent does not run with the // now-deleted content already embedded in its prompt. Must be invoked BEFORE // the comment row is deleted because the FK ON DELETE SET NULL would // otherwise nullify trigger_comment_id and we'd lose the ability to find // the affected tasks. func (s *TaskService) CancelTasksByTriggerComment(ctx context.Context, commentID pgtype.UUID) error { cancelled, err := s.Queries.CancelAgentTasksByTriggerComment(ctx, commentID) if err != nil { return err } for _, t := range cancelled { s.ReconcileAgentStatus(ctx, t.AgentID) s.broadcastTaskEvent(ctx, protocol.EventTaskCancelled, t) } return nil } // CancelTask cancels a single task by ID. It broadcasts a task:cancelled event // so frontends can update immediately. func (s *TaskService) CancelTask(ctx context.Context, taskID pgtype.UUID) (*db.AgentTaskQueue, error) { task, err := s.Queries.CancelAgentTask(ctx, taskID) if errors.Is(err, pgx.ErrNoRows) { existing, err := s.Queries.GetAgentTask(ctx, taskID) if err != nil { return nil, fmt.Errorf("cancel task: %w", err) } return &existing, nil } if err != nil { return nil, fmt.Errorf("cancel task: %w", err) } slog.Info("task cancelled", "task_id", util.UUIDToString(task.ID), "issue_id", util.UUIDToString(task.IssueID)) // Reconcile agent status s.ReconcileAgentStatus(ctx, task.AgentID) // Broadcast cancellation as a task:failed event so frontends clear the live card s.broadcastTaskEvent(ctx, protocol.EventTaskCancelled, task) return &task, nil } // ClaimTask atomically claims the next queued task for an agent, // respecting max_concurrent_tasks. func (s *TaskService) ClaimTask(ctx context.Context, agentID pgtype.UUID) (*db.AgentTaskQueue, error) { start := time.Now() var ( outcome = "unknown" getAgentMs, countRunningMs, claimAgentMs, updateStatusMs, dispatchMs int64 ) defer func() { s.maybeLogClaimSlow(agentID, outcome, start, getAgentMs, countRunningMs, claimAgentMs, updateStatusMs, dispatchMs) }() t0 := start agent, err := s.Queries.GetAgent(ctx, agentID) getAgentMs = time.Since(t0).Milliseconds() if err != nil { outcome = "error_get_agent" return nil, fmt.Errorf("agent not found: %w", err) } t0 = time.Now() running, err := s.Queries.CountRunningTasks(ctx, agentID) countRunningMs = time.Since(t0).Milliseconds() if err != nil { outcome = "error_count_running" return nil, fmt.Errorf("count running tasks: %w", err) } if running >= int64(agent.MaxConcurrentTasks) { slog.Debug("task claim: no capacity", "agent_id", util.UUIDToString(agentID), "running", running, "max", agent.MaxConcurrentTasks) outcome = "no_capacity" return nil, nil // No capacity } t0 = time.Now() task, err := s.Queries.ClaimAgentTask(ctx, agentID) claimAgentMs = time.Since(t0).Milliseconds() if err != nil { if errors.Is(err, pgx.ErrNoRows) { slog.Debug("task claim: no tasks available", "agent_id", util.UUIDToString(agentID)) outcome = "no_tasks" return nil, nil // No tasks available } outcome = "error_claim" return nil, fmt.Errorf("claim task: %w", err) } slog.Info("task claimed", "task_id", util.UUIDToString(task.ID), "agent_id", util.UUIDToString(agentID)) // Refresh agent status from active tasks. This avoids a stale unconditional // working write racing after a just-cancelled claim. t0 = time.Now() s.ReconcileAgentStatus(ctx, agentID) updateStatusMs = time.Since(t0).Milliseconds() // Broadcast task:dispatch. ResolveTaskWorkspaceID inside this path can // re-query issue/chat_session/autopilot_run, so it can also be a real // contributor to claim latency. t0 = time.Now() s.broadcastTaskDispatch(ctx, task) dispatchMs = time.Since(t0).Milliseconds() outcome = "claimed" return &task, nil } // ClaimTaskForRuntime claims the next runnable task for a runtime while // still respecting each agent's max_concurrent_tasks limit. func (s *TaskService) ClaimTaskForRuntime(ctx context.Context, runtimeID pgtype.UUID) (*db.AgentTaskQueue, error) { start := time.Now() var ( outcome = "no_task" listMs, loopMs int64 listCount, tried int claimedFlag bool ) defer func() { totalMs := time.Since(start).Milliseconds() if totalMs < 300 { return } slog.Info("claim_for_runtime slow", "runtime_id", util.UUIDToString(runtimeID), "outcome", outcome, "total_ms", totalMs, "list_pending_ms", listMs, "list_pending_count", listCount, "agents_tried", tried, "claim_loop_ms", loopMs, "claimed", claimedFlag, ) }() t0 := start tasks, err := s.Queries.ListPendingTasksByRuntime(ctx, runtimeID) listMs = time.Since(t0).Milliseconds() listCount = len(tasks) if err != nil { outcome = "error_list" return nil, fmt.Errorf("list pending tasks: %w", err) } loopStart := time.Now() triedAgents := map[string]struct{}{} var claimed *db.AgentTaskQueue for _, candidate := range tasks { agentKey := util.UUIDToString(candidate.AgentID) if _, seen := triedAgents[agentKey]; seen { continue } triedAgents[agentKey] = struct{}{} tried++ task, err := s.ClaimTask(ctx, candidate.AgentID) if err != nil { loopMs = time.Since(loopStart).Milliseconds() outcome = "error_claim" return nil, err } if task != nil && task.RuntimeID == runtimeID { claimed = task break } } loopMs = time.Since(loopStart).Milliseconds() if claimed != nil { claimedFlag = true outcome = "claimed" } return claimed, nil } // maybeLogClaimSlow emits one structured log per ClaimTask call when its total // latency exceeds 300ms, so the prod tail can be diagnosed without flooding // logs at normal poll rates. Called via defer so it captures the full path // including post-claim updateAgentStatus / broadcastTaskDispatch (both of // which can hit the DB) and any error exit. func (s *TaskService) maybeLogClaimSlow(agentID pgtype.UUID, outcome string, start time.Time, getAgentMs, countRunningMs, claimAgentMs, updateStatusMs, dispatchMs int64) { totalMs := time.Since(start).Milliseconds() if totalMs < 300 { return } slog.Info("claim_task slow", "agent_id", util.UUIDToString(agentID), "outcome", outcome, "total_ms", totalMs, "get_agent_ms", getAgentMs, "count_running_ms", countRunningMs, "claim_agent_ms", claimAgentMs, "update_status_ms", updateStatusMs, "dispatch_ms", dispatchMs, ) } // StartTask transitions a dispatched task to running. // Issue status is NOT changed here — the agent manages it via the CLI. func (s *TaskService) StartTask(ctx context.Context, taskID pgtype.UUID) (*db.AgentTaskQueue, error) { task, err := s.Queries.StartAgentTask(ctx, taskID) if err != nil { return nil, fmt.Errorf("start task: %w", err) } slog.Info("task started", "task_id", util.UUIDToString(task.ID), "issue_id", util.UUIDToString(task.IssueID)) return &task, nil } // CompleteTask marks a task as completed. // Issue status is NOT changed here — the agent manages it via the CLI. // // For chat tasks, CompleteAgentTask and the chat_session resume-pointer // update run in a single transaction. This closes a race where the next // queued chat message could be claimed in the window between the task // flipping to 'completed' and chat_session.session_id being refreshed, // causing the new task to resume against a stale (or NULL) session. func (s *TaskService) CompleteTask(ctx context.Context, taskID pgtype.UUID, result []byte, sessionID, workDir string) (*db.AgentTaskQueue, error) { var task db.AgentTaskQueue if err := s.runInTx(ctx, func(qtx *db.Queries) error { t, err := qtx.CompleteAgentTask(ctx, db.CompleteAgentTaskParams{ ID: taskID, Result: result, SessionID: pgtype.Text{String: sessionID, Valid: sessionID != ""}, WorkDir: pgtype.Text{String: workDir, Valid: workDir != ""}, }) if err != nil { return err } task = t if t.ChatSessionID.Valid { // COALESCE in SQL guarantees empty inputs don't wipe the // existing resume pointer; we still surface DB errors. if err := qtx.UpdateChatSessionSession(ctx, db.UpdateChatSessionSessionParams{ ID: t.ChatSessionID, SessionID: pgtype.Text{String: sessionID, Valid: sessionID != ""}, WorkDir: pgtype.Text{String: workDir, Valid: workDir != ""}, }); err != nil { return fmt.Errorf("update chat session resume pointer: %w", err) } } return nil }); err != nil { // When parallel agents race, a task may already be completed, // cancelled, or failed by the time this call runs. The UPDATE // … WHERE status = 'running' returns no rows in that case. // Treat it as an idempotent success — same pattern as CancelTask. if existing, lookupErr := s.Queries.GetAgentTask(ctx, taskID); lookupErr == nil { if errors.Is(err, pgx.ErrNoRows) { slog.Info("complete task: already finalized", "task_id", util.UUIDToString(taskID), "current_status", existing.Status, "agent_id", util.UUIDToString(existing.AgentID), ) return &existing, nil } slog.Warn("complete task failed", "task_id", util.UUIDToString(taskID), "current_status", existing.Status, "issue_id", util.UUIDToString(existing.IssueID), "chat_session_id", util.UUIDToString(existing.ChatSessionID), "agent_id", util.UUIDToString(existing.AgentID), "error", err, ) } else { slog.Warn("complete task failed: task not found", "task_id", util.UUIDToString(taskID), "lookup_error", lookupErr, ) } return nil, fmt.Errorf("complete task: %w", err) } slog.Info("task completed", "task_id", util.UUIDToString(task.ID), "issue_id", util.UUIDToString(task.IssueID)) // Invariant: every completed issue task must have at least one agent // comment on the issue, so the user always sees something when a run // ends. If the agent posted a comment during execution (result, progress // ping, or CLI reply), HasAgentCommentedSince returns true and we skip. // Otherwise, synthesize one from the final output. For comment-triggered // tasks, TriggerCommentID threads the fallback under the original comment; // for assignment-triggered tasks it is NULL and the fallback is top-level. // Chat tasks have no IssueID and are handled separately below. if task.IssueID.Valid { agentCommented, _ := s.Queries.HasAgentCommentedSince(ctx, db.HasAgentCommentedSinceParams{ IssueID: task.IssueID, AuthorID: task.AgentID, Since: task.StartedAt, }) if !agentCommented { var payload protocol.TaskCompletedPayload if err := json.Unmarshal(result, &payload); err == nil { if payload.Output != "" { s.createAgentComment(ctx, task.IssueID, task.AgentID, redact.Text(payload.Output), "comment", task.TriggerCommentID) } } } } // Quick-create tasks: locate the issue the agent just created and push // an inbox confirmation to the requester. The agent has no issue / chat // link, so the regular completion paths above don't apply. We find the // new issue by querying for the most recent issue this agent created in // the requester's workspace since the task started — more robust than // parsing the agent's stdout for an identifier. if qc, ok := s.parseQuickCreateContext(task); ok { s.notifyQuickCreateCompleted(ctx, task, qc) } // For chat tasks, save assistant reply and broadcast chat:done. The // resume pointer was already persisted inside the transaction above. if task.ChatSessionID.Valid { var payload protocol.TaskCompletedPayload if err := json.Unmarshal(result, &payload); err == nil && payload.Output != "" { if _, err := s.Queries.CreateChatMessage(ctx, db.CreateChatMessageParams{ ChatSessionID: task.ChatSessionID, Role: "assistant", Content: redact.Text(payload.Output), TaskID: task.ID, }); err != nil { slog.Error("failed to save assistant chat message", "task_id", util.UUIDToString(task.ID), "error", err) } else { // Event-driven unread: stamp unread_since on the first unread // assistant message. No-op if the session already has unread. // If the user is actively viewing the session, the frontend's // auto-mark-read effect will clear this within a tick. if err := s.Queries.SetUnreadSinceIfNull(ctx, task.ChatSessionID); err != nil { slog.Warn("failed to set unread_since", "chat_session_id", util.UUIDToString(task.ChatSessionID), "error", err) } } } s.broadcastChatDone(ctx, task) } // Reconcile agent status s.ReconcileAgentStatus(ctx, task.AgentID) // Broadcast s.broadcastTaskEvent(ctx, protocol.EventTaskCompleted, task) return &task, nil } // FailTask marks a task as failed. // Issue status is NOT changed here — the agent manages it via the CLI. // // sessionID/workDir are optional: when the agent established a real session // before failing (e.g. crashed mid-conversation, was cancelled, or hit a // tool error), the daemon should pass them so we can preserve the resume // pointer on both the task row and the chat_session — otherwise the next // chat turn would silently start a brand-new session and lose memory. // // failureReason is a coarse classifier consumed by the auto-retry path. // Pass "" when unknown (treated as 'agent_error'). func (s *TaskService) FailTask(ctx context.Context, taskID pgtype.UUID, errMsg, sessionID, workDir, failureReason string) (*db.AgentTaskQueue, error) { var task db.AgentTaskQueue if err := s.runInTx(ctx, func(qtx *db.Queries) error { t, err := qtx.FailAgentTask(ctx, db.FailAgentTaskParams{ ID: taskID, Error: pgtype.Text{String: errMsg, Valid: true}, FailureReason: pgtype.Text{String: failureReason, Valid: failureReason != ""}, SessionID: pgtype.Text{String: sessionID, Valid: sessionID != ""}, WorkDir: pgtype.Text{String: workDir, Valid: workDir != ""}, }) if err != nil { return err } task = t if t.ChatSessionID.Valid { if err := qtx.UpdateChatSessionSession(ctx, db.UpdateChatSessionSessionParams{ ID: t.ChatSessionID, SessionID: pgtype.Text{String: sessionID, Valid: sessionID != ""}, WorkDir: pgtype.Text{String: workDir, Valid: workDir != ""}, }); err != nil { return fmt.Errorf("update chat session resume pointer: %w", err) } } return nil }); err != nil { if existing, lookupErr := s.Queries.GetAgentTask(ctx, taskID); lookupErr == nil { if errors.Is(err, pgx.ErrNoRows) { slog.Info("fail task: already finalized", "task_id", util.UUIDToString(taskID), "current_status", existing.Status, "agent_id", util.UUIDToString(existing.AgentID), ) return &existing, nil } slog.Warn("fail task failed", "task_id", util.UUIDToString(taskID), "current_status", existing.Status, "issue_id", util.UUIDToString(existing.IssueID), "chat_session_id", util.UUIDToString(existing.ChatSessionID), "agent_id", util.UUIDToString(existing.AgentID), "error", err, ) } else { slog.Warn("fail task failed: task not found", "task_id", util.UUIDToString(taskID), "lookup_error", lookupErr, ) } return nil, fmt.Errorf("fail task: %w", err) } slog.Warn("task failed", "task_id", util.UUIDToString(task.ID), "issue_id", util.UUIDToString(task.IssueID), "error", errMsg, "failure_reason", failureReason) // Auto-retry eligible failures (orphan, timeout, runtime_offline, // runtime_recovery). The helper itself enforces attempt < max_attempts // and only triggers for issue/chat tasks. retried, _ := s.MaybeRetryFailedTask(ctx, task) // Skip the per-failure system comment when we'll immediately retry — // the new task will surface its own status to the user, and we don't // want to spam the issue with "task timed out" messages on every // daemon hiccup. if errMsg != "" && task.IssueID.Valid && retried == nil { s.createAgentComment(ctx, task.IssueID, task.AgentID, redact.Text(errMsg), "system", task.TriggerCommentID) } // Quick-create tasks: push a failure inbox notification to the // requester so they can either retry or fall back to the advanced form // without losing their original prompt. Skipped when an auto-retry is // pending — the new attempt will write its own outcome. if retried == nil { if qc, ok := s.parseQuickCreateContext(task); ok { s.notifyQuickCreateFailed(ctx, task, qc, errMsg) } } // Reconcile agent status s.ReconcileAgentStatus(ctx, task.AgentID) // Broadcast s.broadcastTaskEvent(ctx, protocol.EventTaskFailed, task) return &task, nil } // retryableReasons enumerates failure reasons that the auto-retry path is // allowed to act on. Agent-side errors (compile failures, model rejections, // etc.) are intentionally excluded — those are real problems that the user // should see, not infrastructure flakiness. var retryableReasons = map[string]bool{ "runtime_offline": true, "runtime_recovery": true, "timeout": true, } // MaybeRetryFailedTask spawns a fresh queued attempt for a recently-failed // task when the failure was infrastructure-shaped (daemon crash, runtime // went offline, dispatch/run timeout) and the task hasn't exhausted its // max_attempts budget. The child task inherits agent/runtime/issue/chat // links and the parent's session_id/work_dir so the agent can resume the // conversation when the backend supports it. Returns the new task, or nil // when no retry was created. // // Autopilot tasks are NOT auto-retried here; the autopilot scheduler owns // its own re-run cadence and we don't want to double-fire it. func (s *TaskService) MaybeRetryFailedTask(ctx context.Context, parent db.AgentTaskQueue) (*db.AgentTaskQueue, error) { if parent.Status != "failed" { return nil, nil } reason := "" if parent.FailureReason.Valid { reason = parent.FailureReason.String } if !retryableReasons[reason] { return nil, nil } if parent.Attempt >= parent.MaxAttempts { slog.Info("task auto-retry skipped: budget exhausted", "task_id", util.UUIDToString(parent.ID), "attempt", parent.Attempt, "max_attempts", parent.MaxAttempts, ) return nil, nil } if parent.AutopilotRunID.Valid { // Autopilot has its own retry semantics; do not double-trigger. return nil, nil } if !parent.IssueID.Valid && !parent.ChatSessionID.Valid { return nil, nil } child, err := s.Queries.CreateRetryTask(ctx, parent.ID) if err != nil { slog.Warn("task auto-retry failed", "parent_task_id", util.UUIDToString(parent.ID), "reason", reason, "error", err, ) return nil, err } slog.Info("task auto-retry enqueued", "parent_task_id", util.UUIDToString(parent.ID), "child_task_id", util.UUIDToString(child.ID), "reason", reason, "attempt", child.Attempt, "max_attempts", child.MaxAttempts, ) s.broadcastTaskEvent(ctx, protocol.EventTaskDispatch, child) return &child, nil } // RerunIssue creates a fresh queued task for the agent currently assigned // to the issue. Used by the manual rerun endpoint. Carries the most recent // session_id/work_dir on the issue (across any status) so the new run // resumes from where the prior one left off when the backend supports it. // // Only tasks belonging to the issue's current assignee are cancelled. // Tasks owned by other agents on the same issue (e.g. a parallel // @-mention agent) are left alone — rerun must not collateral-cancel // them. func (s *TaskService) RerunIssue(ctx context.Context, issueID pgtype.UUID, triggerCommentID pgtype.UUID) (*db.AgentTaskQueue, error) { issue, err := s.Queries.GetIssue(ctx, issueID) if err != nil { return nil, fmt.Errorf("load issue: %w", err) } if !issue.AssigneeID.Valid || issue.AssigneeType.String != "agent" { return nil, fmt.Errorf("issue is not assigned to an agent") } // Cancel only the assignee's active/queued tasks on this issue. This // covers both the unique-index conflict (queued/dispatched) and a // stuck running task without touching other agents on the issue. cancelled, err := s.Queries.CancelAgentTasksByIssueAndAgent(ctx, db.CancelAgentTasksByIssueAndAgentParams{ IssueID: issueID, AgentID: issue.AssigneeID, }) if err != nil { slog.Warn("rerun: cancel prior tasks failed", "issue_id", util.UUIDToString(issueID), "agent_id", util.UUIDToString(issue.AssigneeID), "error", err, ) } for _, t := range cancelled { s.ReconcileAgentStatus(ctx, t.AgentID) s.broadcastTaskEvent(ctx, protocol.EventTaskCancelled, t) } task, err := s.EnqueueTaskForIssue(ctx, issue, triggerCommentID) if err != nil { return nil, err } slog.Info("issue rerun enqueued", "task_id", util.UUIDToString(task.ID), "issue_id", util.UUIDToString(issueID), "agent_id", util.UUIDToString(issue.AssigneeID), "cancelled_prior", len(cancelled), ) return &task, nil } // HandleFailedTasks runs the post-failure side effects for a batch of // freshly-failed tasks: optional auto-retry, task:failed event broadcast, // agent status reconciliation, and (when an issue has no remaining active // task and isn't being retried) resetting the issue back to todo so the // daemon can pick it up again. // // All callers that surface a task as failed — sweepers, FailTask, // recover-orphans — funnel through here so the same UI-consistency // guarantees apply on every code path. func (s *TaskService) HandleFailedTasks(ctx context.Context, tasks []db.AgentTaskQueue) int { if len(tasks) == 0 { return 0 } affectedAgents := make(map[string]pgtype.UUID) processedIssues := make(map[string]bool) retriedIssues := make(map[string]bool) retried := 0 for _, t := range tasks { // Auto-retry first so the issue stays in_progress rather than // flapping todo → in_progress within a tick. if child, _ := s.MaybeRetryFailedTask(ctx, t); child != nil { retried++ if t.IssueID.Valid { retriedIssues[util.UUIDToString(t.IssueID)] = true } } failureReason := "agent_error" if t.FailureReason.Valid && t.FailureReason.String != "" { failureReason = t.FailureReason.String } workspaceID := "" if t.IssueID.Valid { if issue, err := s.Queries.GetIssue(ctx, t.IssueID); err == nil { workspaceID = util.UUIDToString(issue.WorkspaceID) // Reset stuck in_progress issues only when no other active // task exists for the issue and no retry was just enqueued. issueKey := util.UUIDToString(t.IssueID) if issue.Status == "in_progress" && !processedIssues[issueKey] && !retriedIssues[issueKey] { processedIssues[issueKey] = true hasActive, checkErr := s.Queries.HasActiveTaskForIssue(ctx, t.IssueID) if checkErr != nil { slog.Warn("handle failed tasks: active check failed", "issue_id", issueKey, "error", checkErr, ) } else if !hasActive { if _, updateErr := s.Queries.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{ ID: t.IssueID, Status: "todo", }); updateErr != nil { slog.Warn("handle failed tasks: reset stuck issue failed", "issue_id", issueKey, "error", updateErr, ) } } } } } if workspaceID == "" { workspaceID = s.ResolveTaskWorkspaceID(ctx, t) } if workspaceID != "" { s.Bus.Publish(events.Event{ Type: protocol.EventTaskFailed, WorkspaceID: workspaceID, ActorType: "system", Payload: map[string]any{ "task_id": util.UUIDToString(t.ID), "agent_id": util.UUIDToString(t.AgentID), "issue_id": util.UUIDToString(t.IssueID), "status": "failed", "failure_reason": failureReason, }, }) } affectedAgents[util.UUIDToString(t.AgentID)] = t.AgentID } for _, agentID := range affectedAgents { s.ReconcileAgentStatus(ctx, agentID) } return retried } // runInTx executes fn inside a single DB transaction. If TxStarter is nil // (e.g. some tests construct TaskService directly), fn runs against the // regular Queries handle without transactional guarantees. func (s *TaskService) runInTx(ctx context.Context, fn func(*db.Queries) error) error { if s.TxStarter == nil { return fn(s.Queries) } tx, err := s.TxStarter.Begin(ctx) if err != nil { return fmt.Errorf("begin tx: %w", err) } defer tx.Rollback(ctx) if err := fn(s.Queries.WithTx(tx)); err != nil { return err } return tx.Commit(ctx) } // ReportProgress broadcasts a progress update via the event bus. func (s *TaskService) ReportProgress(ctx context.Context, taskID string, workspaceID string, summary string, step, total int) { s.Bus.Publish(events.Event{ Type: protocol.EventTaskProgress, WorkspaceID: workspaceID, ActorType: "system", ActorID: "", TaskID: taskID, Payload: protocol.TaskProgressPayload{ TaskID: taskID, Summary: summary, Step: step, Total: total, }, }) } // ReconcileAgentStatus refreshes agent status from the current active task set. func (s *TaskService) ReconcileAgentStatus(ctx context.Context, agentID pgtype.UUID) { agent, err := s.Queries.RefreshAgentStatusFromTasks(ctx, agentID) if err != nil { return } slog.Debug("agent status reconciled", "agent_id", util.UUIDToString(agentID), "status", agent.Status) s.publishAgentStatus(agent) } func (s *TaskService) updateAgentStatus(ctx context.Context, agentID pgtype.UUID, status string) { agent, err := s.Queries.UpdateAgentStatus(ctx, db.UpdateAgentStatusParams{ ID: agentID, Status: status, }) if err != nil { return } s.publishAgentStatus(agent) } func (s *TaskService) publishAgentStatus(agent db.Agent) { s.Bus.Publish(events.Event{ Type: protocol.EventAgentStatus, WorkspaceID: util.UUIDToString(agent.WorkspaceID), ActorType: "system", ActorID: "", Payload: map[string]any{"agent": agentToMap(agent)}, }) } // LoadAgentSkills loads an agent's skills with their files for task execution. func (s *TaskService) LoadAgentSkills(ctx context.Context, agentID pgtype.UUID) []AgentSkillData { skills, err := s.Queries.ListAgentSkills(ctx, agentID) if err != nil || len(skills) == 0 { return nil } result := make([]AgentSkillData, 0, len(skills)) for _, sk := range skills { data := AgentSkillData{Name: sk.Name, Content: sk.Content} files, _ := s.Queries.ListSkillFiles(ctx, sk.ID) for _, f := range files { data.Files = append(data.Files, AgentSkillFileData{Path: f.Path, Content: f.Content}) } result = append(result, data) } return result } // AgentSkillData represents a skill for task execution responses. type AgentSkillData struct { Name string `json:"name"` Content string `json:"content"` Files []AgentSkillFileData `json:"files,omitempty"` } // AgentSkillFileData represents a supporting file within a skill. type AgentSkillFileData struct { Path string `json:"path"` Content string `json:"content"` } func priorityToInt(p string) int32 { switch p { case "urgent": return 4 case "high": return 3 case "medium": return 2 case "low": return 1 default: return 0 } } func (s *TaskService) broadcastTaskDispatch(ctx context.Context, task db.AgentTaskQueue) { var payload map[string]any if task.Context != nil { json.Unmarshal(task.Context, &payload) } if payload == nil { payload = map[string]any{} } payload["task_id"] = util.UUIDToString(task.ID) payload["runtime_id"] = util.UUIDToString(task.RuntimeID) payload["issue_id"] = util.UUIDToString(task.IssueID) payload["agent_id"] = util.UUIDToString(task.AgentID) workspaceID := s.ResolveTaskWorkspaceID(ctx, task) if workspaceID == "" { return } s.Bus.Publish(events.Event{ Type: protocol.EventTaskDispatch, WorkspaceID: workspaceID, ActorType: "system", ActorID: "", Payload: payload, }) } func (s *TaskService) broadcastTaskEvent(ctx context.Context, eventType string, task db.AgentTaskQueue) { workspaceID := s.ResolveTaskWorkspaceID(ctx, task) if workspaceID == "" { return } payload := map[string]any{ "task_id": util.UUIDToString(task.ID), "agent_id": util.UUIDToString(task.AgentID), "issue_id": util.UUIDToString(task.IssueID), "status": task.Status, } if task.ChatSessionID.Valid { payload["chat_session_id"] = util.UUIDToString(task.ChatSessionID) } s.Bus.Publish(events.Event{ Type: eventType, WorkspaceID: workspaceID, ActorType: "system", ActorID: "", Payload: payload, }) } // ResolveTaskWorkspaceID determines the workspace ID for a task. // For issue tasks, it comes from the issue. For chat tasks, from the chat session. // For autopilot tasks, from the autopilot via its run. // Returns "" when none of the links resolve — callers treat that as "not found". func (s *TaskService) ResolveTaskWorkspaceID(ctx context.Context, task db.AgentTaskQueue) string { if task.IssueID.Valid { if issue, err := s.Queries.GetIssue(ctx, task.IssueID); err == nil { return util.UUIDToString(issue.WorkspaceID) } } if task.ChatSessionID.Valid { if cs, err := s.Queries.GetChatSession(ctx, task.ChatSessionID); err == nil { return util.UUIDToString(cs.WorkspaceID) } } if task.AutopilotRunID.Valid { if run, err := s.Queries.GetAutopilotRun(ctx, task.AutopilotRunID); err == nil { if ap, err := s.Queries.GetAutopilot(ctx, run.AutopilotID); err == nil { return util.UUIDToString(ap.WorkspaceID) } } } return "" } func (s *TaskService) broadcastChatDone(ctx context.Context, task db.AgentTaskQueue) { workspaceID := s.ResolveTaskWorkspaceID(ctx, task) if workspaceID == "" { return } s.Bus.Publish(events.Event{ Type: protocol.EventChatDone, WorkspaceID: workspaceID, ActorType: "system", ActorID: "", ChatSessionID: util.UUIDToString(task.ChatSessionID), Payload: protocol.ChatDonePayload{ ChatSessionID: util.UUIDToString(task.ChatSessionID), TaskID: util.UUIDToString(task.ID), }, }) } func (s *TaskService) broadcastIssueUpdated(issue db.Issue) { prefix := s.getIssuePrefix(issue.WorkspaceID) s.Bus.Publish(events.Event{ Type: protocol.EventIssueUpdated, WorkspaceID: util.UUIDToString(issue.WorkspaceID), ActorType: "system", ActorID: "", Payload: map[string]any{"issue": issueToMap(issue, prefix)}, }) } func (s *TaskService) getIssuePrefix(workspaceID pgtype.UUID) string { ws, err := s.Queries.GetWorkspace(context.Background(), workspaceID) if err != nil { return "" } return ws.IssuePrefix } func (s *TaskService) createAgentComment(ctx context.Context, issueID, agentID pgtype.UUID, content, commentType string, parentID pgtype.UUID) { if content == "" { return } // Look up issue to get workspace ID for mention expansion and broadcasting. issue, err := s.Queries.GetIssue(ctx, issueID) if err != nil { return } // Resolve thread root: if parentID points to a reply (has its own parent), // use that parent instead so the comment lands in the top-level thread. if parentID.Valid { if parent, err := s.Queries.GetComment(ctx, parentID); err == nil && parent.ParentID.Valid { parentID = parent.ParentID } } // Expand bare issue identifiers (e.g. MUL-117) into mention links. content = mention.ExpandIssueIdentifiers(ctx, s.Queries, issue.WorkspaceID, content) comment, err := s.Queries.CreateComment(ctx, db.CreateCommentParams{ IssueID: issueID, WorkspaceID: issue.WorkspaceID, AuthorType: "agent", AuthorID: agentID, Content: content, Type: commentType, ParentID: parentID, }) if err != nil { return } s.Bus.Publish(events.Event{ Type: protocol.EventCommentCreated, WorkspaceID: util.UUIDToString(issue.WorkspaceID), ActorType: "agent", ActorID: util.UUIDToString(agentID), Payload: map[string]any{ "comment": map[string]any{ "id": util.UUIDToString(comment.ID), "issue_id": util.UUIDToString(comment.IssueID), "author_type": comment.AuthorType, "author_id": util.UUIDToString(comment.AuthorID), "content": comment.Content, "type": comment.Type, "parent_id": util.UUIDToPtr(comment.ParentID), "created_at": comment.CreatedAt.Time.Format("2006-01-02T15:04:05Z"), }, "issue_title": issue.Title, "issue_status": issue.Status, }, }) } func issueToMap(issue db.Issue, issuePrefix string) map[string]any { return map[string]any{ "id": util.UUIDToString(issue.ID), "workspace_id": util.UUIDToString(issue.WorkspaceID), "number": issue.Number, "identifier": issuePrefix + "-" + strconv.Itoa(int(issue.Number)), "title": issue.Title, "description": util.TextToPtr(issue.Description), "status": issue.Status, "priority": issue.Priority, "assignee_type": util.TextToPtr(issue.AssigneeType), "assignee_id": util.UUIDToPtr(issue.AssigneeID), "creator_type": issue.CreatorType, "creator_id": util.UUIDToString(issue.CreatorID), "parent_issue_id": util.UUIDToPtr(issue.ParentIssueID), "position": issue.Position, "due_date": util.TimestampToPtr(issue.DueDate), "created_at": util.TimestampToString(issue.CreatedAt), "updated_at": util.TimestampToString(issue.UpdatedAt), } } // parseQuickCreateContext returns the quick-create payload if the task's // context JSONB contains type == "quick_create"; otherwise the bool is // false so callers can short-circuit. Tasks linked to an issue / chat / // autopilot are never quick-create even if they happen to carry a // context blob, so those are filtered up front. func (s *TaskService) parseQuickCreateContext(task db.AgentTaskQueue) (QuickCreateContext, bool) { if task.IssueID.Valid || task.ChatSessionID.Valid || task.AutopilotRunID.Valid { return QuickCreateContext{}, false } if len(task.Context) == 0 { return QuickCreateContext{}, false } var qc QuickCreateContext if err := json.Unmarshal(task.Context, &qc); err != nil { return QuickCreateContext{}, false } if qc.Type != QuickCreateContextType { return QuickCreateContext{}, false } return qc, true } // notifyQuickCreateCompleted writes a success inbox notification to the // requester pointing at the issue the agent just created. The issue is // stamped with origin_type=quick_create + origin_id= by the // daemon-injected MULTICA_QUICK_CREATE_TASK_ID env var, so this lookup is // deterministic — robust against the same agent creating other issues in // parallel (e.g. assignment task running while max_concurrent_tasks > 1 // permits another quick-create alongside it). func (s *TaskService) notifyQuickCreateCompleted(ctx context.Context, task db.AgentTaskQueue, qc QuickCreateContext) { requesterID, err := util.ParseUUID(qc.RequesterID) if err != nil { slog.Warn("quick-create completion: invalid requester id", "task_id", util.UUIDToString(task.ID), "error", err) return } workspaceID, err := util.ParseUUID(qc.WorkspaceID) if err != nil { slog.Warn("quick-create completion: invalid workspace id", "task_id", util.UUIDToString(task.ID), "error", err) return } issue, err := s.Queries.GetIssueByOrigin(ctx, db.GetIssueByOriginParams{ WorkspaceID: workspaceID, OriginType: pgtype.Text{String: "quick_create", Valid: true}, OriginID: task.ID, }) if err != nil { // No issue created — agent ran to completion but the CLI call must // have failed. Surface as a failure inbox so the user sees something. slog.Warn("quick-create completion: no issue found, writing failure inbox", "task_id", util.UUIDToString(task.ID), "agent_id", util.UUIDToString(task.AgentID), "workspace_id", qc.WorkspaceID, ) s.notifyQuickCreateFailed(ctx, task, qc, "agent finished without creating an issue") return } prefix := s.getIssuePrefix(workspaceID) identifier := fmt.Sprintf("%s-%d", prefix, issue.Number) details, _ := json.Marshal(map[string]any{ "task_id": util.UUIDToString(task.ID), "agent_id": util.UUIDToString(task.AgentID), "issue_id": util.UUIDToString(issue.ID), "identifier": identifier, }) item, err := s.Queries.CreateInboxItem(ctx, db.CreateInboxItemParams{ WorkspaceID: workspaceID, RecipientType: "member", RecipientID: requesterID, Type: "quick_create_done", Severity: "info", IssueID: issue.ID, Title: fmt.Sprintf("Created %s: %s", identifier, issue.Title), Body: pgtype.Text{}, ActorType: pgtype.Text{String: "agent", Valid: true}, ActorID: task.AgentID, Details: details, }) if err != nil { slog.Error("quick-create completion: inbox write failed", "task_id", util.UUIDToString(task.ID), "error", err) return } s.publishQuickCreateInbox(item, qc.WorkspaceID, util.UUIDToString(task.AgentID), issue.Status) } // notifyQuickCreateFailed writes a failure inbox notification carrying the // original prompt + agent ID so the frontend can render an "Edit as // advanced form" entry that pre-fills the legacy create-issue modal // without asking the user to retype. func (s *TaskService) notifyQuickCreateFailed(ctx context.Context, task db.AgentTaskQueue, qc QuickCreateContext, errMsg string) { requesterID, err := util.ParseUUID(qc.RequesterID) if err != nil { return } workspaceID, err := util.ParseUUID(qc.WorkspaceID) if err != nil { return } if errMsg == "" { errMsg = "Quick create did not finish successfully" } details, _ := json.Marshal(map[string]any{ "task_id": util.UUIDToString(task.ID), "agent_id": util.UUIDToString(task.AgentID), "original_prompt": qc.Prompt, "error": redact.Text(errMsg), }) item, err := s.Queries.CreateInboxItem(ctx, db.CreateInboxItemParams{ WorkspaceID: workspaceID, RecipientType: "member", RecipientID: requesterID, Type: "quick_create_failed", Severity: "action_required", IssueID: pgtype.UUID{}, Title: "Quick create failed", Body: pgtype.Text{String: redact.Text(errMsg), Valid: true}, ActorType: pgtype.Text{String: "agent", Valid: true}, ActorID: task.AgentID, Details: details, }) if err != nil { slog.Error("quick-create failure: inbox write failed", "task_id", util.UUIDToString(task.ID), "error", err) return } s.publishQuickCreateInbox(item, qc.WorkspaceID, util.UUIDToString(task.AgentID), "") } // publishQuickCreateInbox emits the WS event so the requester's inbox list // updates immediately. Mirrors the payload shape used by the other inbox // listeners (notification_listeners.go). func (s *TaskService) publishQuickCreateInbox(item db.InboxItem, workspaceID, agentID, issueStatus string) { resp := map[string]any{ "id": util.UUIDToString(item.ID), "workspace_id": util.UUIDToString(item.WorkspaceID), "recipient_type": item.RecipientType, "recipient_id": util.UUIDToString(item.RecipientID), "type": item.Type, "severity": item.Severity, "issue_id": util.UUIDToPtr(item.IssueID), "title": item.Title, "body": util.TextToPtr(item.Body), "read": item.Read, "archived": item.Archived, "created_at": util.TimestampToString(item.CreatedAt), "actor_type": util.TextToPtr(item.ActorType), "actor_id": util.UUIDToPtr(item.ActorID), "details": json.RawMessage(item.Details), "issue_status": issueStatus, } s.Bus.Publish(events.Event{ Type: protocol.EventInboxNew, WorkspaceID: workspaceID, ActorType: "agent", ActorID: agentID, Payload: map[string]any{"item": resp}, }) } // agentToMap builds a simple map for broadcasting agent status updates. func agentToMap(a db.Agent) map[string]any { var rc any if a.RuntimeConfig != nil { json.Unmarshal(a.RuntimeConfig, &rc) } return map[string]any{ "id": util.UUIDToString(a.ID), "workspace_id": util.UUIDToString(a.WorkspaceID), "runtime_id": util.UUIDToString(a.RuntimeID), "name": a.Name, "description": a.Description, "avatar_url": util.TextToPtr(a.AvatarUrl), "runtime_mode": a.RuntimeMode, "runtime_config": rc, "visibility": a.Visibility, "status": a.Status, "max_concurrent_tasks": a.MaxConcurrentTasks, "owner_id": util.UUIDToPtr(a.OwnerID), "skills": []any{}, "created_at": util.TimestampToString(a.CreatedAt), "updated_at": util.TimestampToString(a.UpdatedAt), "archived_at": util.TimestampToPtr(a.ArchivedAt), "archived_by": util.UUIDToPtr(a.ArchivedBy), } }