mirror of
https://github.com/multica-ai/multica.git
synced 2026-07-13 13:18:56 +02:00
Post-deploy of the new scheduled-dispatch scheduler (PR #4444), an autopilot configured for "weekdays 17:10 Asia/Shanghai" fired at ~12:30 Beijing the day after deploy — ~4h 38m before the next scheduled time the UI showed. Traced to a cold-start regression in the planner hook: Old behaviour ------------- On the first tick after migration the hook found no `sys_cron_executions` row for the trigger (`latestPlan(...).Found == false`) and anchored on the trigger's `created_at`, then applied the 24h replay cap: after := cfg.CreatedAt if oldest := now.Add(-replayWindow); after.Before(oldest) { after = oldest // now - 24h } For a trigger created days/weeks earlier and last fired by the legacy goroutine at Mon 17:10 Beijing (= Mon 09:10 UTC), this set `after = Tue 04:13 UTC - 24h ≈ Mon 04:13 UTC`. The half-open enumeration `(Mon 04:13 UTC, Tue 04:13 UTC]` STILL contained Mon 09:10 UTC — the occurrence the legacy code had already handled — so the new scheduler dispatched it again the moment it took over. The result: a SCHEDULED-source autopilot_run with planned_at = Mon 17:10 Beijing but a wall-clock dispatch at Tue ~12:30 Beijing. Timezone math was correct; the bug was purely the cold-start anchor not respecting prior-fire history. Fix Co-authored-by: multica-agent <github@multica.ai> --- The `autopilot_trigger.last_fired_at` column is maintained by both the legacy goroutine and the new scheduler (via TouchAutopilotTriggerFiredAt), so it is the authoritative "most-recent successful fire" cursor across the migration boundary. The planner hook now anchors cold-start enumeration on it: case latest.Found: after = latest.PlanTime case lastFiredAt != zero: after = lastFiredAt default: after = cfg.CreatedAt For the regressed case, `after = Mon 17:10 Beijing`, the next enumeration window is `(Mon 17:10, Tue 12:30]`, and Tue 17:10 is in the future — the hook returns nothing and the trigger waits quietly for Tue 17:10 as the UI promised. For brand-new triggers (last_fired_at NULL), the original `created_at` path still applies. For long-dormant triggers the `replayWindow` cap remains. Changes ------- * `ListSchedulableAutopilotTriggers` SQL now returns `last_fired_at`. * `autopilotTriggerConfig.LastFiredAt` is populated by the scope provider on every tick. * `autopilotPlansForScope` cold-start branch uses the new anchor. Tests ----- * TestAutopilotScheduleJobColdStartHonorsLastFiredAt — seeds the exact dev-environment shape (created 3 days ago, last_fired_at 5 hours ago, no sys_cron_executions row), runs a tick, asserts zero exec rows AND zero autopilot_run rows. Without the fix this test produces one of each at a historical plan_time. * TestAutopilotScheduleJobColdStartBrandNewTriggerStillFires — asserts a brand-new trigger (last_fired_at NULL) still fires its first due occurrence on cold start. All existing `TestAutopilotScheduleJob*` tests still pass. Refs MUL-3551 Co-authored-by: Eve <eve@multica-ai.local>
415 lines
15 KiB
SQL
415 lines
15 KiB
SQL
-- =====================
|
|
-- Autopilot CRUD
|
|
-- =====================
|
|
|
|
-- name: ListAutopilots :many
|
|
-- List rows carry three derived columns the list UI needs (trigger badges,
|
|
-- next run, last-run outcome) so the page never has to N+1 into the detail
|
|
-- endpoint. trigger_kinds/next_run_at only consider ENABLED triggers — the
|
|
-- columns answer "how does this fire today", not "what is configured".
|
|
-- last_run_status is COALESCEd to '' (never ran) because sqlc cannot infer
|
|
-- nullability through a scalar subquery; the handler maps '' back to omitted.
|
|
SELECT
|
|
sqlc.embed(a),
|
|
(
|
|
SELECT array_agg(DISTINCT t.kind ORDER BY t.kind)
|
|
FROM autopilot_trigger t
|
|
WHERE t.autopilot_id = a.id AND t.enabled
|
|
)::text[] AS trigger_kinds,
|
|
(
|
|
SELECT min(t.next_run_at)
|
|
FROM autopilot_trigger t
|
|
WHERE t.autopilot_id = a.id AND t.enabled AND t.kind = 'schedule'
|
|
)::timestamptz AS next_run_at,
|
|
COALESCE((
|
|
SELECT r.status
|
|
FROM autopilot_run r
|
|
WHERE r.autopilot_id = a.id
|
|
ORDER BY r.triggered_at DESC
|
|
LIMIT 1
|
|
), '')::text AS last_run_status
|
|
FROM autopilot a
|
|
WHERE a.workspace_id = $1
|
|
AND (sqlc.narg('status')::text IS NULL OR a.status = sqlc.narg('status'))
|
|
ORDER BY a.created_at DESC;
|
|
|
|
-- name: GetAutopilot :one
|
|
SELECT * FROM autopilot
|
|
WHERE id = $1;
|
|
|
|
-- name: GetAutopilotInWorkspace :one
|
|
SELECT * FROM autopilot
|
|
WHERE id = $1 AND workspace_id = $2;
|
|
|
|
-- name: CreateAutopilot :one
|
|
INSERT INTO autopilot (
|
|
workspace_id, title, description, assignee_type, assignee_id,
|
|
status, execution_mode, issue_title_template, project_id,
|
|
created_by_type, created_by_id
|
|
) VALUES (
|
|
$1, $2, sqlc.narg('description'), $3, $4,
|
|
$5, $6, sqlc.narg('issue_title_template'), sqlc.narg('project_id'),
|
|
$7, $8
|
|
) RETURNING *;
|
|
|
|
-- name: UpdateAutopilot :one
|
|
UPDATE autopilot SET
|
|
title = COALESCE(sqlc.narg('title'), title),
|
|
description = COALESCE(sqlc.narg('description'), description),
|
|
assignee_type = COALESCE(sqlc.narg('assignee_type'), assignee_type),
|
|
assignee_id = COALESCE(sqlc.narg('assignee_id')::uuid, assignee_id),
|
|
status = COALESCE(sqlc.narg('status'), status),
|
|
execution_mode = COALESCE(sqlc.narg('execution_mode'), execution_mode),
|
|
issue_title_template = sqlc.narg('issue_title_template'),
|
|
project_id = sqlc.narg('project_id'),
|
|
updated_at = now()
|
|
WHERE id = $1
|
|
RETURNING *;
|
|
|
|
-- name: DeleteAutopilot :exec
|
|
DELETE FROM autopilot WHERE id = $1;
|
|
|
|
-- name: UpdateAutopilotLastRunAt :exec
|
|
UPDATE autopilot SET last_run_at = now(), updated_at = now()
|
|
WHERE id = $1;
|
|
|
|
-- =====================
|
|
-- Autopilot Trigger CRUD
|
|
-- =====================
|
|
|
|
-- name: ListAutopilotTriggers :many
|
|
SELECT * FROM autopilot_trigger
|
|
WHERE autopilot_id = $1
|
|
ORDER BY created_at ASC;
|
|
|
|
-- name: GetAutopilotTrigger :one
|
|
SELECT * FROM autopilot_trigger
|
|
WHERE id = $1;
|
|
|
|
-- name: CreateAutopilotTrigger :one
|
|
INSERT INTO autopilot_trigger (
|
|
autopilot_id, kind, enabled, cron_expression, timezone,
|
|
next_run_at, webhook_token, label, provider, event_filters
|
|
) VALUES (
|
|
$1, $2, $3, sqlc.narg('cron_expression'), sqlc.narg('timezone'),
|
|
sqlc.narg('next_run_at'), sqlc.narg('webhook_token'), sqlc.narg('label'),
|
|
COALESCE(sqlc.narg('provider')::text, 'generic'),
|
|
sqlc.narg('event_filters')
|
|
) RETURNING *;
|
|
|
|
-- name: UpdateAutopilotTrigger :one
|
|
UPDATE autopilot_trigger SET
|
|
enabled = COALESCE(sqlc.narg('enabled')::boolean, enabled),
|
|
cron_expression = COALESCE(sqlc.narg('cron_expression'), cron_expression),
|
|
timezone = COALESCE(sqlc.narg('timezone'), timezone),
|
|
next_run_at = sqlc.narg('next_run_at'),
|
|
label = COALESCE(sqlc.narg('label'), label),
|
|
event_filters = COALESCE(sqlc.narg('event_filters'), event_filters),
|
|
updated_at = now()
|
|
WHERE id = $1
|
|
RETURNING *;
|
|
|
|
-- name: DeleteAutopilotTrigger :exec
|
|
DELETE FROM autopilot_trigger WHERE id = $1;
|
|
|
|
-- name: AdvanceTriggerNextRun :exec
|
|
UPDATE autopilot_trigger
|
|
SET next_run_at = sqlc.narg('next_run_at'),
|
|
last_fired_at = now(),
|
|
updated_at = now()
|
|
WHERE id = $1;
|
|
|
|
-- name: GetWebhookTriggerByToken :one
|
|
-- Look up a webhook trigger by its public bearer token. Joined to autopilot
|
|
-- so the webhook handler can derive the workspace from the trigger's parent
|
|
-- without trusting any request header. The handler still re-loads the
|
|
-- Autopilot via GetAutopilot and cross-checks WorkspaceID matches the row's
|
|
-- autopilot_workspace_id.
|
|
SELECT t.*, a.workspace_id AS autopilot_workspace_id
|
|
FROM autopilot_trigger t
|
|
JOIN autopilot a ON a.id = t.autopilot_id
|
|
WHERE t.kind = 'webhook'
|
|
AND t.webhook_token = $1;
|
|
|
|
-- name: TouchAutopilotTriggerFiredAt :exec
|
|
-- Bumps last_fired_at after a webhook fires, regardless of whether the
|
|
-- dispatch succeeded, was admission-skipped, or even if Autopilot status
|
|
-- transitioned to paused/disabled at exactly the wrong moment. Disabled /
|
|
-- paused early-return paths in the handler never call this.
|
|
UPDATE autopilot_trigger
|
|
SET last_fired_at = now(),
|
|
updated_at = now()
|
|
WHERE id = $1;
|
|
|
|
-- name: RotateAutopilotTriggerWebhookToken :one
|
|
-- Rotates the bearer token for a webhook trigger. Restricted to kind='webhook'
|
|
-- so an accidental call against a schedule/api trigger is a no-op (returns no
|
|
-- rows) rather than corrupting unrelated state.
|
|
UPDATE autopilot_trigger
|
|
SET webhook_token = $2,
|
|
updated_at = now()
|
|
WHERE id = $1
|
|
AND kind = 'webhook'
|
|
RETURNING *;
|
|
|
|
-- name: SetAutopilotTriggerWebhookToken :one
|
|
-- Sets the webhook token at creation time. CreateAutopilotTrigger inserts the
|
|
-- row first (using its full 8-arg signature), then this query attaches the
|
|
-- token. Splitting the create + token-set keeps the existing CreateAutopilotTrigger
|
|
-- query usable by the schedule path without forcing every caller to think
|
|
-- about webhook_token.
|
|
UPDATE autopilot_trigger
|
|
SET webhook_token = $2,
|
|
updated_at = now()
|
|
WHERE id = $1
|
|
RETURNING *;
|
|
|
|
-- name: SetAutopilotTriggerSigningSecret :one
|
|
-- Writes the signing secret for a webhook trigger. Kept as a dedicated query
|
|
-- (not a field on UpdateAutopilotTrigger) so the request body for the
|
|
-- write-only endpoint only ever carries the secret value, with no risk of an
|
|
-- accidental log line leaking it alongside other fields. Restricted to
|
|
-- webhook triggers to avoid corrupting unrelated state.
|
|
UPDATE autopilot_trigger
|
|
SET signing_secret = sqlc.narg('signing_secret'),
|
|
updated_at = now()
|
|
WHERE id = $1
|
|
AND kind = 'webhook'
|
|
RETURNING *;
|
|
|
|
-- =====================
|
|
-- Autopilot Run Management
|
|
-- =====================
|
|
|
|
-- name: CreateAutopilotRun :one
|
|
-- squad_id is an attribution hook: set to the assignee squad when the
|
|
-- parent autopilot has assignee_type='squad', NULL otherwise. The executing
|
|
-- agent_id on agent_task_queue still records who actually ran the work
|
|
-- (the squad leader); squad_id lets reports group by squad without a join.
|
|
--
|
|
-- planned_at carries the canonical UTC fire time for scheduled triggers
|
|
-- (source='schedule'); it stays NULL for manual / webhook / api sources
|
|
-- which have no canonical occurrence. Combined with the partial unique
|
|
-- index uq_autopilot_run_trigger_planned, this gives dispatch-layer
|
|
-- idempotency: a stale-steal retry at the same plan_time cannot create
|
|
-- a second run for the same (trigger_id, planned_at) pair (MUL-3551).
|
|
INSERT INTO autopilot_run (
|
|
autopilot_id, trigger_id, source, status, trigger_payload, squad_id, planned_at
|
|
) VALUES (
|
|
$1, sqlc.narg('trigger_id'), $2, $3, sqlc.narg('trigger_payload'),
|
|
sqlc.narg('squad_id'), sqlc.narg('planned_at')
|
|
) RETURNING *;
|
|
|
|
-- name: GetAutopilotRunByTriggerAndPlanned :one
|
|
-- Idempotent lookup used by DispatchAutopilotForPlan to detect a
|
|
-- crash-during-dispatch retry: if a row already exists for this
|
|
-- (trigger_id, planned_at), the caller reuses it instead of creating a
|
|
-- duplicate. The partial unique index covers the same key, so a race
|
|
-- between "look up then insert" still resolves to a single row — this
|
|
-- query is just the fast path that lets us skip the INSERT when we
|
|
-- can see the prior row clearly. Returns no rows for the (much more
|
|
-- common) first-time dispatch.
|
|
SELECT * FROM autopilot_run
|
|
WHERE trigger_id = $1
|
|
AND planned_at = $2
|
|
LIMIT 1;
|
|
|
|
-- name: RecoverPartialAutopilotRun :exec
|
|
-- Recovers a partial-state autopilot_run from a crashed first attempt
|
|
-- (the runner wrote the run row but died before creating the downstream
|
|
-- issue/task) so that a subsequent DispatchAutopilotForPlan call can
|
|
-- create a fresh run at the same (trigger_id, planned_at).
|
|
--
|
|
-- Setting planned_at = NULL clears the partial-unique slot held by
|
|
-- uq_autopilot_run_trigger_planned, letting the new INSERT proceed.
|
|
-- The row stays in autopilot_run as a FAILED record (with a recovery
|
|
-- reason) so ops still see the abandoned attempt in the run history —
|
|
-- it is not silently deleted.
|
|
UPDATE autopilot_run
|
|
SET status = 'failed',
|
|
completed_at = now(),
|
|
failure_reason = 'recovered partial dispatch (crashed before downstream creation)',
|
|
planned_at = NULL
|
|
WHERE id = $1;
|
|
|
|
-- name: GetAutopilotRun :one
|
|
SELECT * FROM autopilot_run
|
|
WHERE id = $1;
|
|
|
|
-- name: ListAutopilotRuns :many
|
|
SELECT * FROM autopilot_run
|
|
WHERE autopilot_id = $1
|
|
ORDER BY created_at DESC
|
|
LIMIT $2 OFFSET $3;
|
|
|
|
-- name: UpdateAutopilotRunIssueCreated :one
|
|
UPDATE autopilot_run
|
|
SET status = 'issue_created', issue_id = $2
|
|
WHERE id = $1
|
|
RETURNING *;
|
|
|
|
-- name: UpdateAutopilotRunRunning :one
|
|
UPDATE autopilot_run
|
|
SET status = 'running', task_id = $2
|
|
WHERE id = $1
|
|
RETURNING *;
|
|
|
|
-- name: UpdateAutopilotRunCompleted :one
|
|
UPDATE autopilot_run
|
|
SET status = 'completed', completed_at = now(), result = sqlc.narg('result')
|
|
WHERE id = $1
|
|
RETURNING *;
|
|
|
|
-- name: UpdateAutopilotRunFailed :one
|
|
UPDATE autopilot_run
|
|
SET status = 'failed', completed_at = now(), failure_reason = $2
|
|
WHERE id = $1
|
|
RETURNING *;
|
|
|
|
-- name: UpdateAutopilotRunSkipped :one
|
|
-- Marks an autopilot_run as skipped without enqueueing any task. Used by the
|
|
-- pre-flight admission check when the assignee agent's runtime is offline:
|
|
-- creating an issue / task in that state would just pile a doomed job onto
|
|
-- agent_task_queue (the canonical "持续给离线 local agent 入队" symptom from
|
|
-- MUL-1899). Recording the skip + reason gives the UI / failure monitor / ops
|
|
-- a paper trail without polluting the failure ratio.
|
|
UPDATE autopilot_run
|
|
SET status = 'skipped', completed_at = now(), failure_reason = $2
|
|
WHERE id = $1
|
|
RETURNING *;
|
|
|
|
-- name: UpdateAutopilotRunSkippedWithResult :one
|
|
UPDATE autopilot_run
|
|
SET status = 'skipped',
|
|
completed_at = now(),
|
|
failure_reason = $2,
|
|
result = sqlc.narg('result')
|
|
WHERE id = $1
|
|
RETURNING *;
|
|
|
|
-- =====================
|
|
-- Scheduler Queries
|
|
-- =====================
|
|
|
|
-- name: ListSchedulableAutopilotTriggers :many
|
|
-- Lists every schedule trigger the autopilot_schedule_dispatch JobSpec
|
|
-- should consider this tick. Returns just the columns the scheduler's
|
|
-- scope provider + PlansForScope hook need; the full trigger row is
|
|
-- re-loaded by the handler so a trigger update between scope-list and
|
|
-- handler-run sees the latest enabled / cron values.
|
|
--
|
|
-- last_fired_at is read so the planner hook can anchor cold-start
|
|
-- enumeration on the most recent successful fire (set by either the
|
|
-- legacy goroutine before the new scheduler took over, or the new
|
|
-- scheduler's own TouchAutopilotTriggerFiredAt call). Without it,
|
|
-- a trigger that was created days ago and fired by the legacy code
|
|
-- looks like a brand-new trigger to the new scheduler on first tick
|
|
-- and the half-open `(created_at, now]` enumeration replays the most
|
|
-- recent already-fired occurrence — exactly the post-deploy
|
|
-- spurious-fire reported on MUL-3551 dev.
|
|
--
|
|
-- Filters out webhook / api triggers, disabled triggers, paused/archived
|
|
-- autopilots, and any trigger missing its cron expression. ORDER BY id
|
|
-- keeps the per-tick scope list stable across replicas.
|
|
SELECT t.id, t.autopilot_id, t.cron_expression, t.timezone, t.created_at, t.last_fired_at
|
|
FROM autopilot_trigger t
|
|
JOIN autopilot a ON a.id = t.autopilot_id
|
|
WHERE t.kind = 'schedule'
|
|
AND t.enabled = TRUE
|
|
AND a.status = 'active'
|
|
AND t.cron_expression IS NOT NULL
|
|
AND t.cron_expression <> ''
|
|
ORDER BY t.id;
|
|
|
|
-- =====================
|
|
-- Task Queue (run_only mode)
|
|
-- =====================
|
|
|
|
-- name: CreateAutopilotTask :one
|
|
INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, priority, autopilot_run_id, trigger_summary)
|
|
VALUES ($1, $2, NULL, 'queued', $3, $4, sqlc.narg(trigger_summary))
|
|
RETURNING *;
|
|
|
|
-- =====================
|
|
-- Run lookup by linked entities
|
|
-- =====================
|
|
|
|
-- name: GetAutopilotRunByIssue :one
|
|
SELECT * FROM autopilot_run
|
|
WHERE issue_id = $1 AND status IN ('issue_created', 'running')
|
|
LIMIT 1;
|
|
|
|
-- name: FailAutopilotRunsByIssue :exec
|
|
-- Fails active autopilot runs linked to a given issue.
|
|
-- Must be called BEFORE issue deletion (ON DELETE SET NULL clears issue_id).
|
|
UPDATE autopilot_run
|
|
SET status = 'failed', completed_at = now(), failure_reason = 'linked issue was deleted'
|
|
WHERE issue_id = $1
|
|
AND status IN ('issue_created', 'running');
|
|
|
|
-- =====================
|
|
-- Failure-rate auto-pause
|
|
-- =====================
|
|
|
|
-- name: SelectAutopilotsExceedingFailureThreshold :many
|
|
-- Find active autopilots whose recent run failure rate exceeds the threshold.
|
|
-- Counts only "real" terminal runs (completed | failed). 'skipped' is
|
|
-- excluded from BOTH numerator and denominator: an admission-skipped run
|
|
-- (e.g. assignee runtime offline at dispatch time, MUL-1899) is neither a
|
|
-- success nor a failure, so it must not dilute the failure ratio (which
|
|
-- would let a 100%-failing autopilot mask itself behind a wall of skips)
|
|
-- nor inflate it. issue_created/running are still excluded so in-flight
|
|
-- work isn't penalised.
|
|
-- Used by the failure monitor to auto-pause sustained-failure autopilots
|
|
-- (the canonical example from MUL-1336 was an autopilot scheduled every 5 min
|
|
-- that 100% failed for days, burning ~1.5k useless tasks per week).
|
|
WITH stats AS (
|
|
SELECT autopilot_id,
|
|
count(*) FILTER (WHERE status IN ('completed', 'failed')) AS total,
|
|
count(*) FILTER (WHERE status = 'failed') AS failed
|
|
FROM autopilot_run
|
|
WHERE created_at >= sqlc.arg('since')::timestamptz
|
|
GROUP BY autopilot_id
|
|
)
|
|
SELECT a.id, a.workspace_id, a.title, a.assignee_id,
|
|
a.created_by_type, a.created_by_id,
|
|
s.total::bigint AS total_runs,
|
|
s.failed::bigint AS failed_runs
|
|
FROM autopilot a
|
|
JOIN stats s ON s.autopilot_id = a.id
|
|
WHERE a.status = 'active'
|
|
AND s.total >= sqlc.arg('min_runs')::bigint
|
|
AND s.failed::float8 / NULLIF(s.total, 0)::float8 >= sqlc.arg('fail_ratio_threshold')::float8
|
|
ORDER BY s.failed DESC, a.id ASC;
|
|
|
|
-- name: SystemPauseAutopilot :one
|
|
-- Atomically pauses an autopilot only if it is currently active. Returns no
|
|
-- rows when the autopilot was already paused/archived (or another worker
|
|
-- raced first), letting the caller treat that as a benign no-op rather than
|
|
-- an error.
|
|
UPDATE autopilot
|
|
SET status = 'paused', updated_at = now()
|
|
WHERE id = $1 AND status = 'active'
|
|
RETURNING *;
|
|
|
|
-- =====================
|
|
-- Autopilot Subscribers
|
|
-- =====================
|
|
|
|
-- name: ListAutopilotSubscribers :many
|
|
-- ORDER BY created_at keeps chip rendering stable across refreshes.
|
|
SELECT * FROM autopilot_subscriber
|
|
WHERE autopilot_id = $1
|
|
ORDER BY created_at ASC, user_id ASC;
|
|
|
|
-- name: AddAutopilotSubscriber :exec
|
|
INSERT INTO autopilot_subscriber (autopilot_id, user_type, user_id)
|
|
VALUES ($1, $2, $3)
|
|
ON CONFLICT (autopilot_id, user_type, user_id) DO NOTHING;
|
|
|
|
-- name: DeleteAutopilotSubscribersForAutopilot :exec
|
|
-- Paired with a re-insert loop to implement full-replace PATCH semantics.
|
|
DELETE FROM autopilot_subscriber
|
|
WHERE autopilot_id = $1;
|
|
|