mirror of
https://github.com/multica-ai/multica.git
synced 2026-07-28 05:46:58 +02:00
* feat(daemon): Redis empty-claim fast path for /tasks/claim polling Daemons poll /tasks/claim every 30s per runtime; the steady-state warm-empty case currently runs ListPendingTasksByRuntime against Postgres on every poll. This collapses that path: - New ListQueuedClaimCandidatesByRuntime query restricts to status = 'queued' (the old query also returned 'dispatched' rows that can never be reclaimed) and is backed by a partial index keyed on (runtime_id, priority DESC, created_at ASC). - New EmptyClaimCache caches the negative verdict in Redis with a 30s TTL. ClaimTaskForRuntime checks the cache before SELECT and populates it on confirmed-empty results. - notifyTaskAvailable now invalidates the runtime's empty key before kicking the daemon WS, so newly enqueued tasks become claimable immediately rather than waiting out the TTL. - AutopilotService.dispatchRunOnly now goes through TaskService.NotifyTaskEnqueued so run_only tasks get the same invalidate-then-wakeup contract as every other enqueue path. Co-authored-by: multica-agent <github@multica.ai> * fix(daemon): close MarkEmpty/Bump race in empty-claim fast path GPT-Boy's review on PR #1860 caught a real concurrency bug. Under the prior implementation it was possible for a slow claim to write an empty verdict AFTER a concurrent enqueue had already invalidated it: T1 claim: SELECT -> empty T2 enqueue: INSERT row, DEL empty key (no-op, key not set yet), wakeup T1 claim: SET empty (writes a stale "empty" verdict) T3 wakeup: IsEmpty -> hit -> returns null The just-queued task would then sit idle until the empty key's TTL expired (up to 30s). Replace the DEL-based invalidation with a per-runtime version counter: - CurrentVersion(rt) is a Redis INCR counter at mul:claim:runtime:version:<rt> with a 24h sliding TTL. - Claim samples version BEFORE the SELECT and passes it to MarkEmpty, which stores the verdict's value as the observed-version string. - IsEmpty MGETs both keys and trusts the verdict only when the empty-key value equals the current version. - Enqueue Bumps the version (INCR + EXPIRE) before the wakeup, causing any verdict written under a prior version to be rejected on the next read. Also bound every Redis call from this cache with a 250ms timeout — notifyTaskAvailable uses a background context so a wedged Redis must not block enqueue. Tests against a real Redis (REDIS_TEST_URL) cover: - MarkEmpty + IsEmpty under matching version returns hit - Bump invalidates a prior empty verdict (race-fix pin) - A MarkEmpty written under a stale pre-Bump version is rejected - TTL clamping, per-runtime isolation, nil-cache safety - notifyTaskAvailable Bumps before the wakeup fires Co-authored-by: multica-agent <github@multica.ai> * chore(daemon): renumber claim-candidate index migration to 067 Slot 064 was taken on main by 064_notification_preference. The migration runner tracks per-version in schema_migrations and would silently skip the second 064_*, leaving the index uncreated. Rename to 067 (next free slot). Co-authored-by: multica-agent <github@multica.ai> --------- Co-authored-by: multica-agent <github@multica.ai>
361 lines
11 KiB
Go
361 lines
11 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
"github.com/multica-ai/multica/server/internal/events"
|
|
"github.com/multica-ai/multica/server/internal/util"
|
|
db "github.com/multica-ai/multica/server/pkg/db/generated"
|
|
"github.com/multica-ai/multica/server/pkg/protocol"
|
|
)
|
|
|
|
// TxStarter abstracts transaction creation (satisfied by pgxpool.Pool).
|
|
type TxStarter interface {
|
|
Begin(ctx context.Context) (pgx.Tx, error)
|
|
}
|
|
|
|
type AutopilotService struct {
|
|
Queries *db.Queries
|
|
TxStarter TxStarter
|
|
Bus *events.Bus
|
|
TaskSvc *TaskService
|
|
}
|
|
|
|
func NewAutopilotService(q *db.Queries, tx TxStarter, bus *events.Bus, taskSvc *TaskService) *AutopilotService {
|
|
return &AutopilotService{Queries: q, TxStarter: tx, Bus: bus, TaskSvc: taskSvc}
|
|
}
|
|
|
|
// DispatchAutopilot is the core execution entry point.
|
|
// It creates a run and either creates an issue or enqueues a direct agent task
|
|
// depending on execution_mode.
|
|
func (s *AutopilotService) DispatchAutopilot(
|
|
ctx context.Context,
|
|
autopilot db.Autopilot,
|
|
triggerID pgtype.UUID,
|
|
source string,
|
|
payload []byte,
|
|
) (*db.AutopilotRun, error) {
|
|
// Determine initial status based on execution mode.
|
|
initialStatus := "issue_created"
|
|
if autopilot.ExecutionMode == "run_only" {
|
|
initialStatus = "running"
|
|
}
|
|
|
|
run, err := s.Queries.CreateAutopilotRun(ctx, db.CreateAutopilotRunParams{
|
|
AutopilotID: autopilot.ID,
|
|
TriggerID: triggerID,
|
|
Source: source,
|
|
Status: initialStatus,
|
|
TriggerPayload: payload,
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("create run: %w", err)
|
|
}
|
|
|
|
switch autopilot.ExecutionMode {
|
|
case "create_issue":
|
|
if err := s.dispatchCreateIssue(ctx, autopilot, &run); err != nil {
|
|
s.failRun(ctx, run.ID, err.Error())
|
|
return &run, fmt.Errorf("dispatch create_issue: %w", err)
|
|
}
|
|
case "run_only":
|
|
if err := s.dispatchRunOnly(ctx, autopilot, &run); err != nil {
|
|
s.failRun(ctx, run.ID, err.Error())
|
|
return &run, fmt.Errorf("dispatch run_only: %w", err)
|
|
}
|
|
default:
|
|
s.failRun(ctx, run.ID, "unknown execution_mode: "+autopilot.ExecutionMode)
|
|
return &run, fmt.Errorf("unknown execution_mode: %s", autopilot.ExecutionMode)
|
|
}
|
|
|
|
// Update last_run_at on the autopilot.
|
|
s.Queries.UpdateAutopilotLastRunAt(ctx, autopilot.ID)
|
|
|
|
// Publish run start event.
|
|
s.Bus.Publish(events.Event{
|
|
Type: protocol.EventAutopilotRunStart,
|
|
WorkspaceID: util.UUIDToString(autopilot.WorkspaceID),
|
|
ActorType: "system",
|
|
Payload: map[string]any{
|
|
"run_id": util.UUIDToString(run.ID),
|
|
"autopilot_id": util.UUIDToString(autopilot.ID),
|
|
"source": source,
|
|
"status": run.Status,
|
|
},
|
|
})
|
|
|
|
return &run, nil
|
|
}
|
|
|
|
// dispatchCreateIssue creates an issue and enqueues a task for the agent.
|
|
func (s *AutopilotService) dispatchCreateIssue(ctx context.Context, ap db.Autopilot, run *db.AutopilotRun) error {
|
|
tx, err := s.TxStarter.Begin(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("begin tx: %w", err)
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
qtx := s.Queries.WithTx(tx)
|
|
|
|
// Get next issue number.
|
|
issueNumber, err := qtx.IncrementIssueCounter(ctx, ap.WorkspaceID)
|
|
if err != nil {
|
|
return fmt.Errorf("increment issue counter: %w", err)
|
|
}
|
|
|
|
title := s.interpolateTemplate(ap)
|
|
description := s.buildIssueDescription(ap)
|
|
|
|
issue, err := qtx.CreateIssueWithOrigin(ctx, db.CreateIssueWithOriginParams{
|
|
WorkspaceID: ap.WorkspaceID,
|
|
Title: title,
|
|
Description: description,
|
|
Status: "todo",
|
|
Priority: "none",
|
|
AssigneeType: pgtype.Text{String: "agent", Valid: true},
|
|
AssigneeID: ap.AssigneeID,
|
|
CreatorType: ap.CreatedByType,
|
|
CreatorID: ap.CreatedByID,
|
|
ParentIssueID: pgtype.UUID{},
|
|
Position: 0,
|
|
DueDate: pgtype.Timestamptz{},
|
|
Number: issueNumber,
|
|
ProjectID: pgtype.UUID{},
|
|
OriginType: pgtype.Text{String: "autopilot", Valid: true},
|
|
OriginID: ap.ID,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("create issue: %w", err)
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return fmt.Errorf("commit tx: %w", err)
|
|
}
|
|
|
|
// Update run with the linked issue.
|
|
updatedRun, err := s.Queries.UpdateAutopilotRunIssueCreated(ctx, db.UpdateAutopilotRunIssueCreatedParams{
|
|
ID: run.ID,
|
|
IssueID: issue.ID,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("link run to issue: %w", err)
|
|
}
|
|
*run = updatedRun
|
|
|
|
// Publish issue:created so the existing event chain fires
|
|
// (subscriber listeners, activity listeners, notification listeners).
|
|
prefix := s.getIssuePrefix(ap.WorkspaceID)
|
|
s.Bus.Publish(events.Event{
|
|
Type: protocol.EventIssueCreated,
|
|
WorkspaceID: util.UUIDToString(ap.WorkspaceID),
|
|
ActorType: ap.CreatedByType,
|
|
ActorID: util.UUIDToString(ap.CreatedByID),
|
|
Payload: map[string]any{
|
|
"issue": issueToMap(issue, prefix),
|
|
},
|
|
})
|
|
|
|
// Enqueue agent task via the existing flow.
|
|
if _, err := s.TaskSvc.EnqueueTaskForIssue(ctx, issue); err != nil {
|
|
return fmt.Errorf("enqueue task for issue: %w", err)
|
|
}
|
|
|
|
slog.Info("autopilot dispatched (create_issue)",
|
|
"autopilot_id", util.UUIDToString(ap.ID),
|
|
"issue_id", util.UUIDToString(issue.ID),
|
|
"run_id", util.UUIDToString(run.ID),
|
|
)
|
|
return nil
|
|
}
|
|
|
|
// dispatchRunOnly enqueues a direct agent task without creating an issue.
|
|
func (s *AutopilotService) dispatchRunOnly(ctx context.Context, ap db.Autopilot, run *db.AutopilotRun) error {
|
|
agent, err := s.Queries.GetAgent(ctx, ap.AssigneeID)
|
|
if err != nil {
|
|
return fmt.Errorf("load agent: %w", err)
|
|
}
|
|
if agent.ArchivedAt.Valid {
|
|
return fmt.Errorf("agent is archived")
|
|
}
|
|
if !agent.RuntimeID.Valid {
|
|
return fmt.Errorf("agent has no runtime")
|
|
}
|
|
|
|
task, err := s.Queries.CreateAutopilotTask(ctx, db.CreateAutopilotTaskParams{
|
|
AgentID: ap.AssigneeID,
|
|
RuntimeID: agent.RuntimeID,
|
|
Priority: 0,
|
|
AutopilotRunID: run.ID,
|
|
// Snapshot the autopilot title so task rows self-describe later
|
|
// without joining back to autopilot. Truncated for the same
|
|
// transmission-cost reason as comment-driven summaries.
|
|
TriggerSummary: pgtype.Text{
|
|
String: truncateForSummary(ap.Title, triggerSummaryMaxLen),
|
|
Valid: ap.Title != "",
|
|
},
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("create autopilot task: %w", err)
|
|
}
|
|
|
|
// Update run with task reference.
|
|
updatedRun, err := s.Queries.UpdateAutopilotRunRunning(ctx, db.UpdateAutopilotRunRunningParams{
|
|
ID: run.ID,
|
|
TaskID: task.ID,
|
|
})
|
|
if err != nil {
|
|
slog.Warn("failed to update run with task_id", "run_id", util.UUIDToString(run.ID), "error", err)
|
|
} else {
|
|
*run = updatedRun
|
|
}
|
|
|
|
// Drop the empty-claim cache and wake the daemon. dispatchRunOnly
|
|
// inserts the task row directly via Queries.CreateAutopilotTask
|
|
// (bypassing TaskService.Enqueue*), so without this the runtime
|
|
// would not get a wakeup and any cached "empty" verdict would
|
|
// stall the task until the TTL expired.
|
|
s.TaskSvc.NotifyTaskEnqueued(task)
|
|
|
|
slog.Info("autopilot dispatched (run_only)",
|
|
"autopilot_id", util.UUIDToString(ap.ID),
|
|
"task_id", util.UUIDToString(task.ID),
|
|
"run_id", util.UUIDToString(run.ID),
|
|
)
|
|
return nil
|
|
}
|
|
|
|
// SyncRunFromIssue updates the autopilot run when its linked issue reaches a terminal status.
|
|
func (s *AutopilotService) SyncRunFromIssue(ctx context.Context, issue db.Issue) {
|
|
if !issue.OriginType.Valid || issue.OriginType.String != "autopilot" {
|
|
return
|
|
}
|
|
|
|
run, err := s.Queries.GetAutopilotRunByIssue(ctx, issue.ID)
|
|
if err != nil {
|
|
return // no active run linked to this issue
|
|
}
|
|
|
|
wsID := util.UUIDToString(issue.WorkspaceID)
|
|
|
|
switch issue.Status {
|
|
case "done", "in_review":
|
|
if _, err := s.Queries.UpdateAutopilotRunCompleted(ctx, db.UpdateAutopilotRunCompletedParams{
|
|
ID: run.ID,
|
|
}); err != nil {
|
|
slog.Warn("failed to complete autopilot run", "run_id", util.UUIDToString(run.ID), "error", err)
|
|
return
|
|
}
|
|
s.publishRunDone(wsID, run, "completed")
|
|
case "cancelled", "blocked":
|
|
reason := "issue " + issue.Status
|
|
if _, err := s.Queries.UpdateAutopilotRunFailed(ctx, db.UpdateAutopilotRunFailedParams{
|
|
ID: run.ID,
|
|
FailureReason: pgtype.Text{String: reason, Valid: true},
|
|
}); err != nil {
|
|
slog.Warn("failed to fail autopilot run", "run_id", util.UUIDToString(run.ID), "error", err)
|
|
return
|
|
}
|
|
s.publishRunDone(wsID, run, "failed")
|
|
}
|
|
}
|
|
|
|
// SyncRunFromTask updates the autopilot run when a run_only task completes or fails.
|
|
func (s *AutopilotService) SyncRunFromTask(ctx context.Context, task db.AgentTaskQueue) {
|
|
if !task.AutopilotRunID.Valid {
|
|
return
|
|
}
|
|
|
|
run, err := s.Queries.GetAutopilotRun(ctx, task.AutopilotRunID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
autopilot, err := s.Queries.GetAutopilot(ctx, run.AutopilotID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
wsID := util.UUIDToString(autopilot.WorkspaceID)
|
|
|
|
switch task.Status {
|
|
case "completed":
|
|
if _, err := s.Queries.UpdateAutopilotRunCompleted(ctx, db.UpdateAutopilotRunCompletedParams{
|
|
ID: run.ID,
|
|
Result: task.Result,
|
|
}); err != nil {
|
|
slog.Warn("failed to complete autopilot run from task", "run_id", util.UUIDToString(run.ID), "error", err)
|
|
return
|
|
}
|
|
s.publishRunDone(wsID, run, "completed")
|
|
case "failed", "cancelled":
|
|
reason := "task " + task.Status
|
|
if task.Error.Valid {
|
|
reason = task.Error.String
|
|
}
|
|
if _, err := s.Queries.UpdateAutopilotRunFailed(ctx, db.UpdateAutopilotRunFailedParams{
|
|
ID: run.ID,
|
|
FailureReason: pgtype.Text{String: reason, Valid: true},
|
|
}); err != nil {
|
|
slog.Warn("failed to fail autopilot run from task", "run_id", util.UUIDToString(run.ID), "error", err)
|
|
return
|
|
}
|
|
s.publishRunDone(wsID, run, "failed")
|
|
}
|
|
}
|
|
|
|
|
|
func (s *AutopilotService) failRun(ctx context.Context, runID pgtype.UUID, reason string) {
|
|
if _, err := s.Queries.UpdateAutopilotRunFailed(ctx, db.UpdateAutopilotRunFailedParams{
|
|
ID: runID,
|
|
FailureReason: pgtype.Text{String: reason, Valid: true},
|
|
}); err != nil {
|
|
slog.Warn("failed to mark autopilot run as failed", "run_id", util.UUIDToString(runID), "error", err)
|
|
}
|
|
}
|
|
|
|
func (s *AutopilotService) publishRunDone(workspaceID string, run db.AutopilotRun, status string) {
|
|
s.Bus.Publish(events.Event{
|
|
Type: protocol.EventAutopilotRunDone,
|
|
WorkspaceID: workspaceID,
|
|
ActorType: "system",
|
|
Payload: map[string]any{
|
|
"run_id": util.UUIDToString(run.ID),
|
|
"autopilot_id": util.UUIDToString(run.AutopilotID),
|
|
"status": status,
|
|
},
|
|
})
|
|
}
|
|
|
|
// buildIssueDescription appends an autopilot system instruction to the
|
|
// user-provided description, asking the agent to rename the issue after
|
|
// it understands the actual work.
|
|
func (s *AutopilotService) buildIssueDescription(ap db.Autopilot) pgtype.Text {
|
|
now := time.Now().UTC().Format("2006-01-02 15:04 UTC")
|
|
note := fmt.Sprintf("\n\n---\n*Autopilot run triggered at %s. After starting work, rename this issue to accurately reflect what you are doing.*", now)
|
|
base := ap.Description.String
|
|
return pgtype.Text{String: base + note, Valid: true}
|
|
}
|
|
|
|
// interpolateTemplate replaces {{date}} in the issue title template.
|
|
func (s *AutopilotService) interpolateTemplate(ap db.Autopilot) string {
|
|
tmpl := ap.Title
|
|
if ap.IssueTitleTemplate.Valid && ap.IssueTitleTemplate.String != "" {
|
|
tmpl = ap.IssueTitleTemplate.String
|
|
}
|
|
now := time.Now().UTC().Format("2006-01-02")
|
|
return strings.ReplaceAll(tmpl, "{{date}}", now)
|
|
}
|
|
|
|
func (s *AutopilotService) getIssuePrefix(workspaceID pgtype.UUID) string {
|
|
ws, err := s.Queries.GetWorkspace(context.Background(), workspaceID)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return ws.IssuePrefix
|
|
}
|