Files
multica/packages/core/api/ws-client.ts
MeloMei 67cd1e645a fix(core): add exponential backoff with jitter to WSClient reconnect (#5036)
* fix(core): add exponential backoff with jitter to WSClient reconnect

The WebSocket client used a flat 3-second reconnect delay with no
backoff, jitter, or attempt limit. When the server restarts, every
connected client (web + desktop) reconnects at exactly T+3s, creating
a thundering-herd connection spike.

Replace the fixed delay with exponential backoff:
- Base delay 1 s, doubling each attempt (1 → 2 → 4 → 8 → …)
- Cap at 30 s to keep recovery time reasonable
- ±20 % jitter to decorrelate clients that disconnect simultaneously
- Give up after 20 consecutive failures (log error, allow manual retry)
- Reset the counter on successful authentication

Add 7 unit tests covering the backoff curve, cap, jitter range,
counter reset, max-attempt cutoff, and disconnect cancellation.

Closes #5035

* fix(core): clamp jittered delay to max and make jitter test deterministic

Address Copilot review feedback:

1. Clamp the final delay to RECONNECT_MAX_DELAY_MS after jitter is
   applied. Previously, when base was already at the 30s cap, +20%
   jitter could push the delay to 36s, violating the configured max.

2. Replace the nondeterministic jitter test (which relied on real
   Math.random() producing ≥2 distinct values in 20 samples) with a
   deterministic stub that alternates between 0 and 1, asserting
   exact min/max delays (800ms and 1200ms).

* fix(core): remove reconnect attempt limit, retry indefinitely with capped backoff

Address maintainer feedback (NevilleQingNY):

The web/desktop UI does not currently expose a visible disconnected
state or manual retry action, so the 20-attempt give-up limit would
leave an open tab silently stale after a long outage. Remove the
limit and let the client retry indefinitely with the 30s capped
jittered delay.

- Drop RECONNECT_MAX_ATTEMPTS constant and the give-up early-return
- Update JSDoc to document the indefinite-retry contract
- Replace "stops after max attempts" test with "keeps retrying
  indefinitely with capped delay" that verifies 25+ attempts still
  schedule reconnects at 30s
2026-07-09 08:55:19 +08:00

246 lines
8.2 KiB
TypeScript

import type { WSMessage, WSEventType } from "../types/events";
import { type Logger, noopLogger } from "../logger";
type EventHandler = (payload: unknown, actorId?: string, actorType?: string) => void;
// Cap how much of an unparseable frame we put into the log. A malformed or
// rogue server can stream arbitrarily large garbage, and the warn handler may
// be a console / IPC bridge whose buffers we don't want to blow.
const UNPARSEABLE_LOG_MAX_CHARS = 200;
// Reconnect backoff parameters. A flat delay causes a thundering herd when many
// clients reconnect after a server restart; exponential backoff with jitter
// spreads the reconnection attempts over time. The client retries indefinitely
// (capped at RECONNECT_MAX_DELAY_MS) because the web/desktop UI does not yet
// expose a visible disconnected state or manual retry action.
const RECONNECT_BASE_DELAY_MS = 1_000;
const RECONNECT_MAX_DELAY_MS = 30_000;
function summarizeUnparseable(data: unknown): string {
const text = typeof data === "string" ? data : String(data);
if (text.length <= UNPARSEABLE_LOG_MAX_CHARS) return text;
return `${text.slice(0, UNPARSEABLE_LOG_MAX_CHARS)}… (truncated, ${text.length} chars total)`;
}
/** Identifies the WS client to the server. Sent as `client_platform`,
* `client_version`, and `client_os` query parameters on the upgrade URL —
* browsers cannot set custom headers on WebSocket handshakes, so query
* params are the only portable channel. */
export interface WSClientIdentity {
platform?: string;
version?: string;
os?: string;
}
export class WSClient {
private ws: WebSocket | null = null;
private baseUrl: string;
private token: string | null = null;
private workspaceSlug: string | null = null;
private cookieAuth = false;
private identity: WSClientIdentity | undefined;
private handlers = new Map<WSEventType, Set<EventHandler>>();
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
private reconnectAttempt = 0;
private hasConnectedBefore = false;
// One-shot per connection. A non-conforming frame can repeat hundreds of
// times per session, so we log the first drop and suppress the rest. Reset
// on each connect() so a fresh connection logs once again.
private badFrameLogged = false;
private onReconnectCallbacks = new Set<() => void>();
private anyHandlers = new Set<(msg: WSMessage) => void>();
private logger: Logger;
constructor(
url: string,
options?: {
logger?: Logger;
cookieAuth?: boolean;
identity?: WSClientIdentity;
},
) {
this.baseUrl = url;
this.logger = options?.logger ?? noopLogger;
this.cookieAuth = options?.cookieAuth ?? false;
this.identity = options?.identity;
}
setAuth(token: string | null, workspaceSlug: string) {
this.token = token;
this.workspaceSlug = workspaceSlug;
}
connect() {
this.badFrameLogged = false;
const url = new URL(this.baseUrl);
// Token is never sent as a URL query parameter — it would be logged by
// proxies, CDNs, and browser history. In cookie mode the HttpOnly cookie
// is sent automatically with the upgrade request. In token mode the token
// is delivered as the first WebSocket message after the connection opens.
if (this.workspaceSlug)
url.searchParams.set("workspace_slug", this.workspaceSlug);
if (this.identity?.platform)
url.searchParams.set("client_platform", this.identity.platform);
if (this.identity?.version)
url.searchParams.set("client_version", this.identity.version);
if (this.identity?.os)
url.searchParams.set("client_os", this.identity.os);
this.ws = new WebSocket(url.toString());
this.ws.onopen = () => {
if (!this.cookieAuth && this.token) {
this.ws!.send(
JSON.stringify({ type: "auth", payload: { token: this.token } }),
);
return;
}
this.onAuthenticated();
};
this.ws.onmessage = (event) => {
let msg: WSMessage;
try {
msg = JSON.parse(event.data as string) as WSMessage;
} catch {
this.logger.warn(
"ws: received unparseable message",
summarizeUnparseable(event.data),
);
return;
}
// Trust boundary: a frame must be an object carrying a string `type`.
// The server protocol guarantees this for every frame, but a
// non-conforming frame — an out-of-protocol frame injected by a proxy /
// browser extension, or a bare JSON primitive — must degrade to a no-op
// here. Without this guard every downstream consumer (the onAny
// dispatcher and every ws.on subscriber) runs against a bad shape;
// `msg.type.split(...)` in the realtime sync threw an uncaught TypeError
// out of onmessage and surfaced as a flood of global `$exception` events
// (MUL-3418). Validate once at the boundary, trust the shape downstream.
if (!msg || typeof (msg as { type?: unknown }).type !== "string") {
if (!this.badFrameLogged) {
this.badFrameLogged = true;
this.logger.warn(
"ws: dropping frame without a string type",
summarizeUnparseable(event.data),
);
}
return;
}
if ((msg as any).type === "auth_ack") {
this.onAuthenticated();
return;
}
this.logger.debug("received", msg.type);
const eventHandlers = this.handlers.get(msg.type);
if (eventHandlers) {
for (const handler of eventHandlers) {
handler(msg.payload, msg.actor_id, msg.actor_type);
}
}
for (const handler of this.anyHandlers) {
handler(msg);
}
};
this.ws.onclose = () => {
this.scheduleReconnect();
};
this.ws.onerror = () => {
// Suppress — onclose handles reconnect; errors during StrictMode
// double-fire are expected in dev and harmless.
};
}
/**
* Schedule a reconnection attempt with exponential backoff and jitter.
* Retries indefinitely with a capped delay because the web/desktop UI
* does not yet expose a visible disconnected state or manual retry action.
*/
private scheduleReconnect() {
const base = Math.min(
RECONNECT_BASE_DELAY_MS * 2 ** this.reconnectAttempt,
RECONNECT_MAX_DELAY_MS,
);
// ±20 % jitter so clients that disconnected at the same time don't
// reconnect in lockstep.
const jitter = base * 0.2 * (Math.random() * 2 - 1);
const delay = Math.round(
Math.min(base + jitter, RECONNECT_MAX_DELAY_MS),
);
this.reconnectAttempt++;
this.logger.warn(
`ws: disconnected, reconnecting in ${delay}ms (attempt ${this.reconnectAttempt})`,
);
this.reconnectTimer = setTimeout(() => this.connect(), delay);
}
private onAuthenticated() {
this.logger.info("connected");
this.reconnectAttempt = 0;
if (this.hasConnectedBefore) {
for (const cb of this.onReconnectCallbacks) {
try {
cb();
} catch {
// ignore reconnect callback errors
}
}
}
this.hasConnectedBefore = true;
}
disconnect() {
if (this.reconnectTimer) {
clearTimeout(this.reconnectTimer);
this.reconnectTimer = null;
}
if (this.ws) {
// Remove handlers before close to prevent onclose from scheduling a reconnect
this.ws.onclose = null;
this.ws.onerror = null;
this.ws.close();
this.ws = null;
}
this.hasConnectedBefore = false;
this.reconnectAttempt = 0;
this.handlers.clear();
this.anyHandlers.clear();
this.onReconnectCallbacks.clear();
}
on(event: WSEventType, handler: EventHandler) {
if (!this.handlers.has(event)) {
this.handlers.set(event, new Set());
}
this.handlers.get(event)!.add(handler);
return () => {
this.handlers.get(event)?.delete(handler);
};
}
onAny(handler: (msg: WSMessage) => void) {
this.anyHandlers.add(handler);
return () => {
this.anyHandlers.delete(handler);
};
}
onReconnect(callback: () => void) {
this.onReconnectCallbacks.add(callback);
return () => {
this.onReconnectCallbacks.delete(callback);
};
}
send(message: WSMessage) {
if (this.ws?.readyState === WebSocket.OPEN) {
this.ws.send(JSON.stringify(message));
}
}
}