Files
multica/packages/core/api/ws-client.test.ts
MeloMei 67cd1e645a fix(core): add exponential backoff with jitter to WSClient reconnect (#5036)
* fix(core): add exponential backoff with jitter to WSClient reconnect

The WebSocket client used a flat 3-second reconnect delay with no
backoff, jitter, or attempt limit. When the server restarts, every
connected client (web + desktop) reconnects at exactly T+3s, creating
a thundering-herd connection spike.

Replace the fixed delay with exponential backoff:
- Base delay 1 s, doubling each attempt (1 → 2 → 4 → 8 → …)
- Cap at 30 s to keep recovery time reasonable
- ±20 % jitter to decorrelate clients that disconnect simultaneously
- Give up after 20 consecutive failures (log error, allow manual retry)
- Reset the counter on successful authentication

Add 7 unit tests covering the backoff curve, cap, jitter range,
counter reset, max-attempt cutoff, and disconnect cancellation.

Closes #5035

* fix(core): clamp jittered delay to max and make jitter test deterministic

Address Copilot review feedback:

1. Clamp the final delay to RECONNECT_MAX_DELAY_MS after jitter is
   applied. Previously, when base was already at the 30s cap, +20%
   jitter could push the delay to 36s, violating the configured max.

2. Replace the nondeterministic jitter test (which relied on real
   Math.random() producing ≥2 distinct values in 20 samples) with a
   deterministic stub that alternates between 0 and 1, asserting
   exact min/max delays (800ms and 1200ms).

* fix(core): remove reconnect attempt limit, retry indefinitely with capped backoff

Address maintainer feedback (NevilleQingNY):

The web/desktop UI does not currently expose a visible disconnected
state or manual retry action, so the 20-attempt give-up limit would
leave an open tab silently stale after a long outage. Remove the
limit and let the client retry indefinitely with the 30s capped
jittered delay.

- Drop RECONNECT_MAX_ATTEMPTS constant and the give-up early-return
- Update JSDoc to document the indefinite-retry contract
- Replace "stops after max attempts" test with "keeps retrying
  indefinitely with capped delay" that verifies 25+ attempts still
  schedule reconnects at 30s
2026-07-09 08:55:19 +08:00

463 lines
14 KiB
TypeScript

import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { WSClient } from "./ws-client";
import type { WSMessage } from "../types/events";
// Capture URL passed to WebSocket so we can assert the connect-time
// query string. We don't simulate the full WS lifecycle here — only the
// upgrade URL construction, which is what carries client identity.
class FakeWebSocket {
static lastUrl: string | null = null;
static lastInstance: FakeWebSocket | null = null;
// Fields read by WSClient.connect()/disconnect(), all no-op here.
onopen: (() => void) | null = null;
onmessage: ((ev: { data: string }) => void) | null = null;
onclose: (() => void) | null = null;
onerror: (() => void) | null = null;
readyState = 0;
constructor(url: string) {
FakeWebSocket.lastUrl = url;
FakeWebSocket.lastInstance = this;
}
close() {}
send() {}
}
describe("WSClient", () => {
beforeEach(() => {
FakeWebSocket.lastUrl = null;
FakeWebSocket.lastInstance = null;
vi.stubGlobal("WebSocket", FakeWebSocket as unknown as typeof WebSocket);
});
afterEach(() => {
vi.unstubAllGlobals();
});
it("includes client identity in the upgrade URL when configured", () => {
const ws = new WSClient("ws://example.test/ws", {
identity: { platform: "desktop", version: "1.2.3", os: "macos" },
});
ws.setAuth("tok", "acme");
ws.connect();
const url = new URL(FakeWebSocket.lastUrl!);
expect(url.searchParams.get("workspace_slug")).toBe("acme");
expect(url.searchParams.get("client_platform")).toBe("desktop");
expect(url.searchParams.get("client_version")).toBe("1.2.3");
expect(url.searchParams.get("client_os")).toBe("macos");
// Token must never appear in the URL — it is delivered as the first
// WS message in token mode.
expect(url.searchParams.has("token")).toBe(false);
});
it("omits client_* params when identity is not configured", () => {
const ws = new WSClient("ws://example.test/ws");
ws.setAuth("tok", "acme");
ws.connect();
const url = new URL(FakeWebSocket.lastUrl!);
expect(url.searchParams.has("client_platform")).toBe(false);
expect(url.searchParams.has("client_version")).toBe(false);
expect(url.searchParams.has("client_os")).toBe(false);
});
it("only includes the identity fields that are set", () => {
const ws = new WSClient("ws://example.test/ws", {
identity: { platform: "cli" },
});
ws.setAuth("tok", "acme");
ws.connect();
const url = new URL(FakeWebSocket.lastUrl!);
expect(url.searchParams.get("client_platform")).toBe("cli");
expect(url.searchParams.has("client_version")).toBe(false);
expect(url.searchParams.has("client_os")).toBe(false);
});
it("truncates the logged payload when an unparseable frame is large", () => {
const logger = {
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
};
const ws = new WSClient("ws://example.test/ws", { logger });
ws.connect();
const huge = "x".repeat(5000);
FakeWebSocket.lastInstance!.onmessage?.({ data: huge });
expect(logger.warn).toHaveBeenCalledTimes(1);
const [, summary] = logger.warn.mock.calls[0] as [string, string];
expect(summary.length).toBeLessThan(huge.length);
expect(summary).toContain("truncated");
expect(summary).toContain("5000");
expect(summary.startsWith("x".repeat(200))).toBe(true);
});
it("logs and skips malformed frames without breaking later messages", () => {
const logger = {
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
};
const ws = new WSClient("ws://example.test/ws", { logger });
const handler = vi.fn();
ws.on("issue:updated", handler);
ws.connect();
expect(() => {
FakeWebSocket.lastInstance!.onmessage?.({ data: `{"type":"issue` });
}).not.toThrow();
FakeWebSocket.lastInstance!.onmessage?.({
data: JSON.stringify({
type: "issue:updated",
payload: { id: "issue-1" },
}),
});
expect(logger.warn).toHaveBeenCalledWith(
"ws: received unparseable message",
`{"type":"issue`,
);
expect(handler).toHaveBeenCalledWith(
{ id: "issue-1" },
undefined,
undefined,
);
});
it("drops frames without a string type without throwing, and keeps dispatching", () => {
// Regression for MUL-3418: a frame whose parsed JSON lacks a string `type`
// (an out-of-protocol frame, or a bare JSON primitive) used to throw an
// uncaught TypeError out of onmessage via `msg.type.split(...)` in a
// downstream onAny handler, flooding `$exception` telemetry.
const logger = {
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
};
const ws = new WSClient("ws://example.test/ws", { logger });
// A downstream consumer that assumes a string type, exactly like the
// realtime sync's onAny dispatcher.
const anyHandler = vi.fn((msg: WSMessage) => msg.type.split(":")[0]);
ws.onAny(anyHandler);
const issueHandler = vi.fn();
ws.on("issue:updated", issueHandler);
ws.connect();
const badFrames = [
JSON.stringify({ payload: {} }), // object, no type
"42", // bare number
"true", // bare bool
"[]", // array
];
for (const data of badFrames) {
expect(() => {
FakeWebSocket.lastInstance!.onmessage?.({ data });
}).not.toThrow();
}
// Bad frames never reached any handler.
expect(anyHandler).not.toHaveBeenCalled();
expect(issueHandler).not.toHaveBeenCalled();
// A valid frame after the bad ones still dispatches normally.
FakeWebSocket.lastInstance!.onmessage?.({
data: JSON.stringify({ type: "issue:updated", payload: { id: "i-1" } }),
});
expect(issueHandler).toHaveBeenCalledWith({ id: "i-1" }, undefined, undefined);
expect(anyHandler).toHaveBeenCalledTimes(1);
// The drop is logged at most once per connection despite four bad frames.
expect(logger.warn).toHaveBeenCalledTimes(1);
expect(logger.warn.mock.calls[0]?.[0]).toBe(
"ws: dropping frame without a string type",
);
});
it("passes actor_id and actor_type to event handlers", () => {
const ws = new WSClient("ws://example.test/ws");
ws.setAuth("tok", "acme");
ws.connect();
const handler = vi.fn();
ws.on("issue:created", handler);
const fakeWs = (ws as any).ws as FakeWebSocket;
fakeWs.onmessage?.({
data: JSON.stringify({
type: "issue:created",
payload: { id: "issue-1" },
actor_id: "user-123",
actor_type: "user",
}),
});
expect(handler).toHaveBeenCalledWith(
{ id: "issue-1" },
"user-123",
"user",
);
});
// ── Reconnect backoff tests ────────────────────────────────────────
describe("reconnect backoff", () => {
let setTimeoutSpy: ReturnType<typeof vi.spyOn>;
beforeEach(() => {
vi.useFakeTimers();
setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
});
afterEach(() => {
setTimeoutSpy.mockRestore();
vi.useRealTimers();
});
/** Return the delay (ms) passed to the most recent setTimeout call. */
function lastTimerDelay(): number {
const calls = setTimeoutSpy.mock.calls;
return calls[calls.length - 1]?.[1] as number;
}
function simulateDisconnect() {
FakeWebSocket.lastInstance!.onclose?.();
}
/** Simulate the server acknowledging the auth token. In token mode the
* client sends `{type:"auth"}` on open, and the server replies with
* `{type:"auth_ack"}`. Only then does `onAuthenticated()` fire and
* reset the reconnect counter. */
function simulateAuthAck() {
FakeWebSocket.lastInstance!.onmessage?.({
data: JSON.stringify({ type: "auth_ack" }),
});
}
it("uses ~1000ms base delay for the first reconnect attempt", () => {
// Pin Math.random so jitter is deterministic: random()=0.5 → jitter=0.
vi.stubGlobal(
"Math",
new Proxy(Math, {
get(target, prop) {
if (prop === "random") return () => 0.5;
return (target as any)[prop];
},
}),
);
const ws = new WSClient("ws://example.test/ws");
ws.connect();
simulateDisconnect();
expect(lastTimerDelay()).toBe(1000);
});
it("doubles the base delay on consecutive failures (exponential)", () => {
vi.stubGlobal(
"Math",
new Proxy(Math, {
get(target, prop) {
if (prop === "random") return () => 0.5;
return (target as any)[prop];
},
}),
);
const ws = new WSClient("ws://example.test/ws");
ws.connect();
// Attempt 1: base = 1000 * 2^0 = 1000
simulateDisconnect();
expect(lastTimerDelay()).toBe(1000);
// Fire the timer → connect() → new FakeWebSocket → simulate disconnect
vi.advanceTimersByTime(1000);
// Attempt 2: base = 1000 * 2^1 = 2000
simulateDisconnect();
expect(lastTimerDelay()).toBe(2000);
vi.advanceTimersByTime(2000);
// Attempt 3: base = 1000 * 2^2 = 4000
simulateDisconnect();
expect(lastTimerDelay()).toBe(4000);
vi.advanceTimersByTime(4000);
// Attempt 4: base = 1000 * 2^3 = 8000
simulateDisconnect();
expect(lastTimerDelay()).toBe(8000);
});
it("caps the delay at 30s even after many failures", () => {
vi.stubGlobal(
"Math",
new Proxy(Math, {
get(target, prop) {
if (prop === "random") return () => 0.5;
return (target as any)[prop];
},
}),
);
const ws = new WSClient("ws://example.test/ws");
ws.connect();
// Drive through enough failures to exceed the cap:
// 2^5 = 32000 > 30000, so attempt 6 should be capped.
const delays = [1000, 2000, 4000, 8000, 16000];
for (const d of delays) {
simulateDisconnect();
expect(lastTimerDelay()).toBe(d);
vi.advanceTimersByTime(d);
}
// Attempt 6: 1000 * 2^5 = 32000 → capped to 30000
simulateDisconnect();
expect(lastTimerDelay()).toBe(30000);
});
it("applies jitter so delays vary with Math.random", () => {
// Stub Math.random to alternate between 0 (jitter = -20%) and
// 1 (jitter = +20%), producing deterministic min/max delays.
let callCount = 0;
vi.stubGlobal(
"Math",
new Proxy(Math, {
get(target, prop) {
if (prop === "random") return () => (callCount++ % 2 === 0 ? 0 : 1);
return (target as any)[prop];
},
}),
);
const ws = new WSClient("ws://example.test/ws");
// First disconnect: random()=0 → jitter = 1000 * 0.2 * (0*2-1) = -200 → 800ms
ws.connect();
simulateDisconnect();
const delay1 = lastTimerDelay();
// Second disconnect: random()=1 → jitter = 1000 * 0.2 * (1*2-1) = +200 → 1200ms
vi.clearAllTimers();
ws.disconnect();
ws.connect();
simulateDisconnect();
const delay2 = lastTimerDelay();
// The two delays must differ and fall within [800, 1200].
expect(delay1).toBe(800);
expect(delay2).toBe(1200);
});
it("resets the attempt counter on successful authentication", () => {
vi.stubGlobal(
"Math",
new Proxy(Math, {
get(target, prop) {
if (prop === "random") return () => 0.5;
return (target as any)[prop];
},
}),
);
const ws = new WSClient("ws://example.test/ws");
ws.setAuth("tok", "acme");
ws.connect();
// Two failures: delays 1000, 2000
simulateDisconnect();
expect(lastTimerDelay()).toBe(1000);
vi.advanceTimersByTime(1000);
simulateDisconnect();
expect(lastTimerDelay()).toBe(2000);
vi.advanceTimersByTime(2000);
// Successful connection resets the counter.
simulateAuthAck();
// Next failure should be back to the base delay.
simulateDisconnect();
expect(lastTimerDelay()).toBe(1000);
});
it("keeps retrying indefinitely with capped delay", () => {
vi.stubGlobal(
"Math",
new Proxy(Math, {
get(target, prop) {
if (prop === "random") return () => 0.5;
return (target as any)[prop];
},
}),
);
const logger = {
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
};
const ws = new WSClient("ws://example.test/ws", { logger });
ws.connect();
// Drive past the old 20-attempt limit — all should schedule a reconnect.
for (let i = 0; i < 25; i++) {
simulateDisconnect();
const delay = lastTimerDelay();
// Once past attempt 5 (base > 30000), delay should be capped at 30s.
if (i >= 5) {
expect(delay).toBe(30000);
}
vi.advanceTimersByTime(delay);
}
// The 26th disconnect should STILL schedule a reconnect (no give-up).
const timerCountBefore = setTimeoutSpy.mock.calls.length;
simulateDisconnect();
expect(setTimeoutSpy.mock.calls.length).toBe(timerCountBefore + 1);
expect(lastTimerDelay()).toBe(30000);
// No "giving up" error should have been logged.
expect(logger.error).not.toHaveBeenCalled();
});
it("disconnect() cancels a pending reconnect and resets the counter", () => {
vi.stubGlobal(
"Math",
new Proxy(Math, {
get(target, prop) {
if (prop === "random") return () => 0.5;
return (target as any)[prop];
},
}),
);
const ws = new WSClient("ws://example.test/ws");
ws.connect();
// Two failures to bump the counter.
simulateDisconnect();
vi.advanceTimersByTime(1000);
simulateDisconnect();
// A reconnect timer is now pending.
// Explicit disconnect should cancel the pending timer.
ws.disconnect();
vi.advanceTimersByTime(10_000);
// No new WebSocket should have been created after the disconnect.
// The last URL should still be from the second connect() call.
// A fresh connect() after disconnect should start from attempt 0.
ws.connect();
simulateDisconnect();
expect(lastTimerDelay()).toBe(1000);
});
});
});