mirror of
https://github.com/multica-ai/multica.git
synced 2026-07-13 05:16:29 +02:00
* fix(core): add exponential backoff with jitter to WSClient reconnect The WebSocket client used a flat 3-second reconnect delay with no backoff, jitter, or attempt limit. When the server restarts, every connected client (web + desktop) reconnects at exactly T+3s, creating a thundering-herd connection spike. Replace the fixed delay with exponential backoff: - Base delay 1 s, doubling each attempt (1 → 2 → 4 → 8 → …) - Cap at 30 s to keep recovery time reasonable - ±20 % jitter to decorrelate clients that disconnect simultaneously - Give up after 20 consecutive failures (log error, allow manual retry) - Reset the counter on successful authentication Add 7 unit tests covering the backoff curve, cap, jitter range, counter reset, max-attempt cutoff, and disconnect cancellation. Closes #5035 * fix(core): clamp jittered delay to max and make jitter test deterministic Address Copilot review feedback: 1. Clamp the final delay to RECONNECT_MAX_DELAY_MS after jitter is applied. Previously, when base was already at the 30s cap, +20% jitter could push the delay to 36s, violating the configured max. 2. Replace the nondeterministic jitter test (which relied on real Math.random() producing ≥2 distinct values in 20 samples) with a deterministic stub that alternates between 0 and 1, asserting exact min/max delays (800ms and 1200ms). * fix(core): remove reconnect attempt limit, retry indefinitely with capped backoff Address maintainer feedback (NevilleQingNY): The web/desktop UI does not currently expose a visible disconnected state or manual retry action, so the 20-attempt give-up limit would leave an open tab silently stale after a long outage. Remove the limit and let the client retry indefinitely with the 30s capped jittered delay. - Drop RECONNECT_MAX_ATTEMPTS constant and the give-up early-return - Update JSDoc to document the indefinite-retry contract - Replace "stops after max attempts" test with "keeps retrying indefinitely with capped delay" that verifies 25+ attempts still schedule reconnects at 30s
246 lines
8.2 KiB
TypeScript
246 lines
8.2 KiB
TypeScript
import type { WSMessage, WSEventType } from "../types/events";
|
|
import { type Logger, noopLogger } from "../logger";
|
|
|
|
type EventHandler = (payload: unknown, actorId?: string, actorType?: string) => void;
|
|
|
|
// Cap how much of an unparseable frame we put into the log. A malformed or
|
|
// rogue server can stream arbitrarily large garbage, and the warn handler may
|
|
// be a console / IPC bridge whose buffers we don't want to blow.
|
|
const UNPARSEABLE_LOG_MAX_CHARS = 200;
|
|
|
|
// Reconnect backoff parameters. A flat delay causes a thundering herd when many
|
|
// clients reconnect after a server restart; exponential backoff with jitter
|
|
// spreads the reconnection attempts over time. The client retries indefinitely
|
|
// (capped at RECONNECT_MAX_DELAY_MS) because the web/desktop UI does not yet
|
|
// expose a visible disconnected state or manual retry action.
|
|
const RECONNECT_BASE_DELAY_MS = 1_000;
|
|
const RECONNECT_MAX_DELAY_MS = 30_000;
|
|
|
|
function summarizeUnparseable(data: unknown): string {
|
|
const text = typeof data === "string" ? data : String(data);
|
|
if (text.length <= UNPARSEABLE_LOG_MAX_CHARS) return text;
|
|
return `${text.slice(0, UNPARSEABLE_LOG_MAX_CHARS)}… (truncated, ${text.length} chars total)`;
|
|
}
|
|
|
|
/** Identifies the WS client to the server. Sent as `client_platform`,
|
|
* `client_version`, and `client_os` query parameters on the upgrade URL —
|
|
* browsers cannot set custom headers on WebSocket handshakes, so query
|
|
* params are the only portable channel. */
|
|
export interface WSClientIdentity {
|
|
platform?: string;
|
|
version?: string;
|
|
os?: string;
|
|
}
|
|
|
|
export class WSClient {
|
|
private ws: WebSocket | null = null;
|
|
private baseUrl: string;
|
|
private token: string | null = null;
|
|
private workspaceSlug: string | null = null;
|
|
private cookieAuth = false;
|
|
private identity: WSClientIdentity | undefined;
|
|
private handlers = new Map<WSEventType, Set<EventHandler>>();
|
|
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
|
private reconnectAttempt = 0;
|
|
private hasConnectedBefore = false;
|
|
// One-shot per connection. A non-conforming frame can repeat hundreds of
|
|
// times per session, so we log the first drop and suppress the rest. Reset
|
|
// on each connect() so a fresh connection logs once again.
|
|
private badFrameLogged = false;
|
|
private onReconnectCallbacks = new Set<() => void>();
|
|
private anyHandlers = new Set<(msg: WSMessage) => void>();
|
|
private logger: Logger;
|
|
|
|
constructor(
|
|
url: string,
|
|
options?: {
|
|
logger?: Logger;
|
|
cookieAuth?: boolean;
|
|
identity?: WSClientIdentity;
|
|
},
|
|
) {
|
|
this.baseUrl = url;
|
|
this.logger = options?.logger ?? noopLogger;
|
|
this.cookieAuth = options?.cookieAuth ?? false;
|
|
this.identity = options?.identity;
|
|
}
|
|
|
|
setAuth(token: string | null, workspaceSlug: string) {
|
|
this.token = token;
|
|
this.workspaceSlug = workspaceSlug;
|
|
}
|
|
|
|
connect() {
|
|
this.badFrameLogged = false;
|
|
const url = new URL(this.baseUrl);
|
|
// Token is never sent as a URL query parameter — it would be logged by
|
|
// proxies, CDNs, and browser history. In cookie mode the HttpOnly cookie
|
|
// is sent automatically with the upgrade request. In token mode the token
|
|
// is delivered as the first WebSocket message after the connection opens.
|
|
if (this.workspaceSlug)
|
|
url.searchParams.set("workspace_slug", this.workspaceSlug);
|
|
if (this.identity?.platform)
|
|
url.searchParams.set("client_platform", this.identity.platform);
|
|
if (this.identity?.version)
|
|
url.searchParams.set("client_version", this.identity.version);
|
|
if (this.identity?.os)
|
|
url.searchParams.set("client_os", this.identity.os);
|
|
|
|
this.ws = new WebSocket(url.toString());
|
|
|
|
this.ws.onopen = () => {
|
|
if (!this.cookieAuth && this.token) {
|
|
this.ws!.send(
|
|
JSON.stringify({ type: "auth", payload: { token: this.token } }),
|
|
);
|
|
return;
|
|
}
|
|
|
|
this.onAuthenticated();
|
|
};
|
|
|
|
this.ws.onmessage = (event) => {
|
|
let msg: WSMessage;
|
|
try {
|
|
msg = JSON.parse(event.data as string) as WSMessage;
|
|
} catch {
|
|
this.logger.warn(
|
|
"ws: received unparseable message",
|
|
summarizeUnparseable(event.data),
|
|
);
|
|
return;
|
|
}
|
|
// Trust boundary: a frame must be an object carrying a string `type`.
|
|
// The server protocol guarantees this for every frame, but a
|
|
// non-conforming frame — an out-of-protocol frame injected by a proxy /
|
|
// browser extension, or a bare JSON primitive — must degrade to a no-op
|
|
// here. Without this guard every downstream consumer (the onAny
|
|
// dispatcher and every ws.on subscriber) runs against a bad shape;
|
|
// `msg.type.split(...)` in the realtime sync threw an uncaught TypeError
|
|
// out of onmessage and surfaced as a flood of global `$exception` events
|
|
// (MUL-3418). Validate once at the boundary, trust the shape downstream.
|
|
if (!msg || typeof (msg as { type?: unknown }).type !== "string") {
|
|
if (!this.badFrameLogged) {
|
|
this.badFrameLogged = true;
|
|
this.logger.warn(
|
|
"ws: dropping frame without a string type",
|
|
summarizeUnparseable(event.data),
|
|
);
|
|
}
|
|
return;
|
|
}
|
|
if ((msg as any).type === "auth_ack") {
|
|
this.onAuthenticated();
|
|
return;
|
|
}
|
|
this.logger.debug("received", msg.type);
|
|
const eventHandlers = this.handlers.get(msg.type);
|
|
if (eventHandlers) {
|
|
for (const handler of eventHandlers) {
|
|
handler(msg.payload, msg.actor_id, msg.actor_type);
|
|
}
|
|
}
|
|
for (const handler of this.anyHandlers) {
|
|
handler(msg);
|
|
}
|
|
};
|
|
|
|
this.ws.onclose = () => {
|
|
this.scheduleReconnect();
|
|
};
|
|
|
|
this.ws.onerror = () => {
|
|
// Suppress — onclose handles reconnect; errors during StrictMode
|
|
// double-fire are expected in dev and harmless.
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Schedule a reconnection attempt with exponential backoff and jitter.
|
|
* Retries indefinitely with a capped delay because the web/desktop UI
|
|
* does not yet expose a visible disconnected state or manual retry action.
|
|
*/
|
|
private scheduleReconnect() {
|
|
const base = Math.min(
|
|
RECONNECT_BASE_DELAY_MS * 2 ** this.reconnectAttempt,
|
|
RECONNECT_MAX_DELAY_MS,
|
|
);
|
|
// ±20 % jitter so clients that disconnected at the same time don't
|
|
// reconnect in lockstep.
|
|
const jitter = base * 0.2 * (Math.random() * 2 - 1);
|
|
const delay = Math.round(
|
|
Math.min(base + jitter, RECONNECT_MAX_DELAY_MS),
|
|
);
|
|
|
|
this.reconnectAttempt++;
|
|
this.logger.warn(
|
|
`ws: disconnected, reconnecting in ${delay}ms (attempt ${this.reconnectAttempt})`,
|
|
);
|
|
this.reconnectTimer = setTimeout(() => this.connect(), delay);
|
|
}
|
|
|
|
private onAuthenticated() {
|
|
this.logger.info("connected");
|
|
this.reconnectAttempt = 0;
|
|
if (this.hasConnectedBefore) {
|
|
for (const cb of this.onReconnectCallbacks) {
|
|
try {
|
|
cb();
|
|
} catch {
|
|
// ignore reconnect callback errors
|
|
}
|
|
}
|
|
}
|
|
this.hasConnectedBefore = true;
|
|
}
|
|
|
|
disconnect() {
|
|
if (this.reconnectTimer) {
|
|
clearTimeout(this.reconnectTimer);
|
|
this.reconnectTimer = null;
|
|
}
|
|
if (this.ws) {
|
|
// Remove handlers before close to prevent onclose from scheduling a reconnect
|
|
this.ws.onclose = null;
|
|
this.ws.onerror = null;
|
|
this.ws.close();
|
|
this.ws = null;
|
|
}
|
|
this.hasConnectedBefore = false;
|
|
this.reconnectAttempt = 0;
|
|
this.handlers.clear();
|
|
this.anyHandlers.clear();
|
|
this.onReconnectCallbacks.clear();
|
|
}
|
|
|
|
on(event: WSEventType, handler: EventHandler) {
|
|
if (!this.handlers.has(event)) {
|
|
this.handlers.set(event, new Set());
|
|
}
|
|
this.handlers.get(event)!.add(handler);
|
|
return () => {
|
|
this.handlers.get(event)?.delete(handler);
|
|
};
|
|
}
|
|
|
|
onAny(handler: (msg: WSMessage) => void) {
|
|
this.anyHandlers.add(handler);
|
|
return () => {
|
|
this.anyHandlers.delete(handler);
|
|
};
|
|
}
|
|
|
|
onReconnect(callback: () => void) {
|
|
this.onReconnectCallbacks.add(callback);
|
|
return () => {
|
|
this.onReconnectCallbacks.delete(callback);
|
|
};
|
|
}
|
|
|
|
send(message: WSMessage) {
|
|
if (this.ws?.readyState === WebSocket.OPEN) {
|
|
this.ws.send(JSON.stringify(message));
|
|
}
|
|
}
|
|
}
|