Files
multica/packages/core/dashboard/failure-class.ts
Bohan Jiang c271f80999 MUL-5370 fix: label stalled skill-bundle downloads, align failure-reason copy with the backend taxonomy (#6001)
* fix(daemon): label stalled skill-bundle downloads and make them retryable

A skill bundle that could not be downloaded during task preparation surfaced
as the bare string "resolve skill bundles: context deadline exceeded".
taskfailure.Classify has no rule for a Go context deadline, so it landed in
agent_error.unknown — a bucket that is NOT on the server's retry allowlist.
A transient stall therefore became a terminal chat failure carrying a label
nobody could act on, and the failure was invisible on the Usage page's Errors
breakdown. (MUL-5370)

- Add the platform-side reason skill_bundle_unavailable and put it on
  retryableReasons. Retrying is cheap and safe: the agent process never
  started, and bundles that did arrive are already cached on disk, so
  successive attempts converge.
- Carry a sentinel error from the resolve loop so the reason is derived
  structurally rather than by matching the wrapped transport error's text,
  and name the skill, its declared size and the elapsed wait in the wrap —
  enough to tell "this bundle is too big for the link" from "the link is
  dead" without reading daemon logs.
- Normalise the wire shape an OLD daemon produces (a non-empty catchall plus
  the previous "resolve skill bundles:" wrapper) on the server side. Installed
  daemons upgrade on their own cadence, and FailTask only classifies when the
  caller supplied nothing, so without this the fix would reach only hosts that
  happened to update — while the un-upgraded hosts most likely to be hitting
  the bug kept failing terminally.
- Teach Classify about "deadline exceeded" and net/http's "Client.Timeout
  exceeded while awaiting" so any other Go-side deadline that reaches it as
  text stops falling into the unknown bucket too.
- Backfill historical rows in both agent_task_queue and chat_message. Scoped
  to agent_error.unknown alone — the old wrapper string postdates the
  in-flight classifier by three weeks, so no row carrying it can hold the
  legacy coarse value — which keeps the down migration an exact inverse.

Co-authored-by: multica-agent <github@multica.ai>

* fix(chat): give chat its own failure copy for the refined reasons

#5991 rebuilt the operator-facing failure labels around an open wire string
with a raw-value fallback, but the chat bubble kept its own exact-key lookup
against the six coarse values from migration 055. So all 14 agent_error.*
values still missed and rendered the generic "Something went wrong and the
agent couldn't finish replying" — the classification the backend had already
computed was discarded at the last step, and that is the message the MUL-5370
reporter saw.

- Add resolveFailureReasonKey in packages/core: exact match, else degrade an
  `agent_error.*` value to its family, else undefined. A reason newer than the
  shipped client now lands on the family line instead of the fallback.
- Rekey the chat copy map by wire value and route it through the helper.
  Chat deliberately degrades to friendly copy rather than adopting the
  operator surfaces' raw-value fallback: it is read by the person who just
  sent a message, and the raw error is one click away under the collapsible.
- Add refined chat copy (en / zh-Hans / ja / ko) only where it can say
  something the family line can't — a different next step: network, auth,
  quota, rate limit, context overflow, missing/outdated CLI, skill download.
- Give skill_bundle_unavailable a label on the web and mobile surfaces and a
  class on the Usage page's Errors breakdown (runtime — the operator response
  is "check the daemon's link to Multica", the provider is not involved).
- Mobile's two label maps were still coarse-only for the same reason; rekey
  them by wire value and fill in the refined taxonomy.

Co-authored-by: multica-agent <github@multica.ai>

---------

Co-authored-by: Bohan-J <bohan@devv.ai>
Co-authored-by: multica-agent <github@multica.ai>
2026-07-28 13:31:29 +08:00

98 lines
4.0 KiB
TypeScript

// Display grouping for `agent_task_queue.failure_reason`.
//
// The backend taxonomy (server/pkg/taskfailure) has 22 reasons, which is far
// too many series for a stacked chart or a scannable breakdown list. These
// seven classes are the granularity an operator actually acts on: an auth
// spike means "go re-auth", a rate-limit spike means "back off or raise the
// cap", a runtime spike means "a daemon is down". The raw reason stays on the
// wire and is still shown in the breakdown, so nothing is lost.
//
// Ordering below is the render order everywhere (stack segments, legend,
// breakdown rows): most-actionable first, catchall last, so the chart's
// colour ramp reads consistently regardless of which classes a workspace
// actually hits.
export const FAILURE_CLASSES = [
"auth",
"rate_limit",
"timeout",
"provider",
"runtime",
"agent",
"other",
] as const;
export type FailureClass = (typeof FAILURE_CLASSES)[number];
// Reason → class. Keys are the wire values written by the backend: the 22
// canonical `taskfailure.Reason` strings, the `"unclassified"` sentinel the
// failure rollups substitute for a failed row with an empty column, and the
// pre-MUL-1949 coarse values that still sit in historical rows.
//
// Anything absent from this map falls through to "other" — including a new
// reason from a backend newer than this client, which is the case that makes
// a total-coverage exhaustive `Record<Reason, …>` the wrong shape here.
const REASON_CLASS: Record<string, FailureClass> = {
// Credentials / access.
"agent_error.provider_auth_or_access": "auth",
"agent_error.missing_config": "auth",
// Capacity the account ran out of — rate limits and billing quota share a
// class because the operator response is the same: wait, or raise a cap.
"agent_error.provider_capacity_or_rate_limit": "rate_limit",
"agent_error.provider_quota_limit": "rate_limit",
// Ran too long. Platform-side sweeper timeout and the agent's own hard
// timeout land together — from the dashboard both read as "this run hung".
timeout: "timeout",
"agent_error.agent_timeout": "timeout",
codex_semantic_inactivity: "timeout",
// The upstream model API misbehaved or was asked for something it rejected.
"agent_error.provider_server_error": "provider",
"agent_error.provider_network": "provider",
"agent_error.model_not_found_or_unavailable": "provider",
api_invalid_request: "provider",
// Multica-side execution substrate: daemon offline / restarted, task never
// got picked up, runner binary missing or too old.
runtime_offline: "runtime",
runtime_recovery: "runtime",
queued_expired: "runtime",
"agent_error.runtime_missing_executable": "runtime",
"agent_error.runtime_version_unsupported": "runtime",
// The daemon could not fetch the agent's skills from the control plane, so
// the run never started. Grouped with runtime rather than provider: the
// operator response is "check the daemon's link to Multica", the same as a
// daemon that went offline — the model provider is not involved.
skill_bundle_unavailable: "runtime",
// The agent process itself produced the failure.
"agent_error.process_failure": "agent",
"agent_error.empty_or_unparseable_output": "agent",
"agent_error.context_overflow": "agent",
iteration_limit: "agent",
agent_blocked: "agent",
// Catchall + legacy coarse values.
"agent_error.unknown": "other",
agent_error: "other",
manual: "other",
unclassified: "other",
};
/**
* Fold a raw `failure_reason` into its display class.
*
* Unknown reasons — including ones a newer backend introduced — resolve to
* "other" rather than being dropped, so the class totals always reconcile
* with the raw failure count.
*
* Callers must not pass the empty string: in the dashboard failure rollups
* that value is the *succeeded* bucket, not a failure. It resolves to "other"
* here so a caller that leaks one in inflates a visible bucket instead of
* silently corrupting the error rate.
*/
export function failureClassOf(reason: string): FailureClass {
return REASON_CLASS[reason] ?? "other";
}