mirror of
https://github.com/multica-ai/multica.git
synced 2026-08-03 19:20:07 +02:00
* fix(daemon): label stalled skill-bundle downloads and make them retryable A skill bundle that could not be downloaded during task preparation surfaced as the bare string "resolve skill bundles: context deadline exceeded". taskfailure.Classify has no rule for a Go context deadline, so it landed in agent_error.unknown — a bucket that is NOT on the server's retry allowlist. A transient stall therefore became a terminal chat failure carrying a label nobody could act on, and the failure was invisible on the Usage page's Errors breakdown. (MUL-5370) - Add the platform-side reason skill_bundle_unavailable and put it on retryableReasons. Retrying is cheap and safe: the agent process never started, and bundles that did arrive are already cached on disk, so successive attempts converge. - Carry a sentinel error from the resolve loop so the reason is derived structurally rather than by matching the wrapped transport error's text, and name the skill, its declared size and the elapsed wait in the wrap — enough to tell "this bundle is too big for the link" from "the link is dead" without reading daemon logs. - Normalise the wire shape an OLD daemon produces (a non-empty catchall plus the previous "resolve skill bundles:" wrapper) on the server side. Installed daemons upgrade on their own cadence, and FailTask only classifies when the caller supplied nothing, so without this the fix would reach only hosts that happened to update — while the un-upgraded hosts most likely to be hitting the bug kept failing terminally. - Teach Classify about "deadline exceeded" and net/http's "Client.Timeout exceeded while awaiting" so any other Go-side deadline that reaches it as text stops falling into the unknown bucket too. - Backfill historical rows in both agent_task_queue and chat_message. Scoped to agent_error.unknown alone — the old wrapper string postdates the in-flight classifier by three weeks, so no row carrying it can hold the legacy coarse value — which keeps the down migration an exact inverse. Co-authored-by: multica-agent <github@multica.ai> * fix(chat): give chat its own failure copy for the refined reasons #5991 rebuilt the operator-facing failure labels around an open wire string with a raw-value fallback, but the chat bubble kept its own exact-key lookup against the six coarse values from migration 055. So all 14 agent_error.* values still missed and rendered the generic "Something went wrong and the agent couldn't finish replying" — the classification the backend had already computed was discarded at the last step, and that is the message the MUL-5370 reporter saw. - Add resolveFailureReasonKey in packages/core: exact match, else degrade an `agent_error.*` value to its family, else undefined. A reason newer than the shipped client now lands on the family line instead of the fallback. - Rekey the chat copy map by wire value and route it through the helper. Chat deliberately degrades to friendly copy rather than adopting the operator surfaces' raw-value fallback: it is read by the person who just sent a message, and the raw error is one click away under the collapsible. - Add refined chat copy (en / zh-Hans / ja / ko) only where it can say something the family line can't — a different next step: network, auth, quota, rate limit, context overflow, missing/outdated CLI, skill download. - Give skill_bundle_unavailable a label on the web and mobile surfaces and a class on the Usage page's Errors breakdown (runtime — the operator response is "check the daemon's link to Multica", the provider is not involved). - Mobile's two label maps were still coarse-only for the same reason; rekey them by wire value and fill in the refined taxonomy. Co-authored-by: multica-agent <github@multica.ai> --------- Co-authored-by: Bohan-J <bohan@devv.ai> Co-authored-by: multica-agent <github@multica.ai>
187 lines
6.0 KiB
Go
187 lines
6.0 KiB
Go
package taskfailure
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// TestReasonStringWireValues pins the on-the-wire string for every
|
|
// canonical reason. These strings are persisted into
|
|
// agent_task_queue.failure_reason and surfaced as Prometheus labels —
|
|
// renaming any of them is a breaking change. If this test fails because
|
|
// you intended to rename a value, also update the SQL classifier in
|
|
// MUL-1949 and ship a backfill migration before changing the constant.
|
|
func TestReasonStringWireValues(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
cases := []struct {
|
|
reason Reason
|
|
want string
|
|
}{
|
|
// Platform-side.
|
|
{ReasonQueuedExpired, "queued_expired"},
|
|
{ReasonRuntimeOffline, "runtime_offline"},
|
|
{ReasonRuntimeRecovery, "runtime_recovery"},
|
|
{ReasonTimeout, "timeout"},
|
|
{ReasonIterationLimit, "iteration_limit"},
|
|
{ReasonAgentBlocked, "agent_blocked"},
|
|
{ReasonAPIInvalidRequest, "api_invalid_request"},
|
|
{ReasonSkillBundleUnavailable, "skill_bundle_unavailable"},
|
|
// Agent-side.
|
|
{ReasonAgentProviderAuthOrAccess, "agent_error.provider_auth_or_access"},
|
|
{ReasonAgentProviderQuotaLimit, "agent_error.provider_quota_limit"},
|
|
{ReasonAgentProviderCapacityOrRateLimit, "agent_error.provider_capacity_or_rate_limit"},
|
|
{ReasonAgentProviderServerError, "agent_error.provider_server_error"},
|
|
{ReasonAgentProviderNetwork, "agent_error.provider_network"},
|
|
{ReasonAgentProcessFailure, "agent_error.process_failure"},
|
|
{ReasonAgentEmptyOrUnparseableOutput, "agent_error.empty_or_unparseable_output"},
|
|
{ReasonAgentTimeout, "agent_error.agent_timeout"},
|
|
{ReasonAgentContextOverflow, "agent_error.context_overflow"},
|
|
{ReasonAgentMissingConfig, "agent_error.missing_config"},
|
|
{ReasonAgentModelNotFoundOrUnavailable, "agent_error.model_not_found_or_unavailable"},
|
|
{ReasonAgentRuntimeVersionUnsupported, "agent_error.runtime_version_unsupported"},
|
|
{ReasonAgentRuntimeMissingExecutable, "agent_error.runtime_missing_executable"},
|
|
{ReasonAgentUnknown, "agent_error.unknown"},
|
|
}
|
|
|
|
if got, want := len(cases), 22; got != want {
|
|
t.Fatalf("constant count = %d, want %d (canonical taxonomy size)", got, want)
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.want, func(t *testing.T) {
|
|
if got := c.reason.String(); got != c.want {
|
|
t.Errorf("Reason(%q).String() = %q, want %q", c.reason, got, c.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestIsAgentError pins the platform-side vs agent-side split so a
|
|
// future Prometheus collector / retry policy can rely on the prefix
|
|
// rather than maintaining a parallel allow-list.
|
|
func TestIsAgentError(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
platformSide := []Reason{
|
|
ReasonQueuedExpired,
|
|
ReasonRuntimeOffline,
|
|
ReasonRuntimeRecovery,
|
|
ReasonTimeout,
|
|
ReasonIterationLimit,
|
|
ReasonAgentBlocked,
|
|
ReasonAPIInvalidRequest,
|
|
ReasonSkillBundleUnavailable,
|
|
}
|
|
for _, r := range platformSide {
|
|
if r.IsAgentError() {
|
|
t.Errorf("%q.IsAgentError() = true, want false (platform-side)", r)
|
|
}
|
|
}
|
|
|
|
agentSide := []Reason{
|
|
ReasonAgentProviderAuthOrAccess,
|
|
ReasonAgentProviderQuotaLimit,
|
|
ReasonAgentProviderCapacityOrRateLimit,
|
|
ReasonAgentProviderServerError,
|
|
ReasonAgentProviderNetwork,
|
|
ReasonAgentProcessFailure,
|
|
ReasonAgentEmptyOrUnparseableOutput,
|
|
ReasonAgentTimeout,
|
|
ReasonAgentContextOverflow,
|
|
ReasonAgentMissingConfig,
|
|
ReasonAgentModelNotFoundOrUnavailable,
|
|
ReasonAgentRuntimeVersionUnsupported,
|
|
ReasonAgentRuntimeMissingExecutable,
|
|
ReasonAgentUnknown,
|
|
}
|
|
for _, r := range agentSide {
|
|
if !r.IsAgentError() {
|
|
t.Errorf("%q.IsAgentError() = false, want true (agent-side)", r)
|
|
}
|
|
if !strings.HasPrefix(r.String(), "agent_error.") {
|
|
t.Errorf("%q missing required agent_error. prefix", r)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestAllReasonsContents verifies that AllReasons() returns the
|
|
// complete canonical taxonomy with no duplicates and no surprise
|
|
// values. Prometheus pre-warming relies on this fixture being stable.
|
|
func TestAllReasonsContents(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got := AllReasons()
|
|
if len(got) != 22 {
|
|
t.Fatalf("AllReasons() returned %d entries, want 22", len(got))
|
|
}
|
|
|
|
seen := make(map[Reason]bool, len(got))
|
|
var platformCount, agentCount int
|
|
for _, r := range got {
|
|
if seen[r] {
|
|
t.Errorf("AllReasons() returned duplicate %q", r)
|
|
}
|
|
seen[r] = true
|
|
if r.IsAgentError() {
|
|
agentCount++
|
|
} else {
|
|
platformCount++
|
|
}
|
|
}
|
|
|
|
if platformCount != 8 {
|
|
t.Errorf("AllReasons(): platform-side count = %d, want 8", platformCount)
|
|
}
|
|
if agentCount != 14 {
|
|
t.Errorf("AllReasons(): agent-side count = %d, want 14", agentCount)
|
|
}
|
|
|
|
// Sanity-check that every constant declared at package level
|
|
// shows up in AllReasons. This catches a future drift where
|
|
// someone adds a constant but forgets to register it in the
|
|
// allReasons slice.
|
|
required := []Reason{
|
|
ReasonQueuedExpired, ReasonRuntimeOffline, ReasonRuntimeRecovery,
|
|
ReasonTimeout, ReasonIterationLimit, ReasonAgentBlocked,
|
|
ReasonAPIInvalidRequest, ReasonSkillBundleUnavailable,
|
|
ReasonAgentProviderAuthOrAccess, ReasonAgentProviderQuotaLimit,
|
|
ReasonAgentProviderCapacityOrRateLimit, ReasonAgentProviderServerError,
|
|
ReasonAgentProviderNetwork, ReasonAgentProcessFailure,
|
|
ReasonAgentEmptyOrUnparseableOutput, ReasonAgentTimeout,
|
|
ReasonAgentContextOverflow, ReasonAgentMissingConfig,
|
|
ReasonAgentModelNotFoundOrUnavailable,
|
|
ReasonAgentRuntimeVersionUnsupported,
|
|
ReasonAgentRuntimeMissingExecutable,
|
|
ReasonAgentUnknown,
|
|
}
|
|
for _, r := range required {
|
|
if !seen[r] {
|
|
t.Errorf("AllReasons() missing canonical reason %q", r)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestAllReasonsIsDefensiveCopy guards the contract that mutating the
|
|
// returned slice cannot corrupt the package-level fixture. Without
|
|
// this, two callers (e.g. two Prometheus collectors at startup) could
|
|
// race on a shared slice.
|
|
func TestAllReasonsIsDefensiveCopy(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
first := AllReasons()
|
|
if len(first) == 0 {
|
|
t.Fatal("AllReasons() returned empty slice")
|
|
}
|
|
original := first[0]
|
|
first[0] = "tampered"
|
|
|
|
second := AllReasons()
|
|
if second[0] == "tampered" {
|
|
t.Fatalf("AllReasons() leaked package state: second call returned tampered value %q", second[0])
|
|
}
|
|
if second[0] != original {
|
|
t.Fatalf("AllReasons()[0] = %q, want %q", second[0], original)
|
|
}
|
|
}
|