mirror of
https://github.com/multica-ai/multica.git
synced 2026-07-27 21:33:41 +02:00
* feat(realtime): phase 0 — extract Broadcaster interface + add metrics Phase 0 of the WebSocket horizontal-scaling plan tracked in MUL-1138. This change is intentionally behavior-preserving: it sets up the seams needed for later phases (subscribe/unsubscribe protocol, scope-level fanout, Redis Streams relay) without altering any wire protocol or producer call sites. What changed - New realtime.Broadcaster interface covering the three fanout methods producers already use on *Hub (BroadcastToWorkspace, SendToUser, Broadcast). *Hub continues to satisfy it; a future Redis-backed implementation can be dropped in without touching listeners. - registerListeners now depends on realtime.Broadcaster instead of *realtime.Hub, isolating the bus → realtime fanout layer behind an interface. - New realtime.Metrics singleton with atomic counters: connects, disconnects, active connections, slow-client evictions, total messages sent/dropped, and per-event-type send counters. Wired into Hub register/unregister/broadcast paths and into every listener. - New GET /health/realtime endpoint returning a JSON snapshot of the metrics so we can observe baseline fanout pressure before phase 1. Why phase 0 first GPT-Boy's only-Redis plan and CC-Girl's review both call out the same prerequisite: get a Broadcaster seam and visibility in place before introducing scope-level subscriptions or a Redis relay. Doing this as a standalone step keeps each later PR focused and trivially revertable. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * feat(realtime): only-Redis fanout — scopes, subscribe protocol, Redis Streams relay (MUL-1138) Implements the final-version plan agreed in MUL-1138 on top of phase 0: * Hub: 4 scope types (workspace/user/task/chat), per-client subscription set, subscribe/unsubscribe WS frames, ScopeAuthorizer hook for task/chat scope auth, first/last-subscriber callbacks for the relay, workspace+user auto-subscribe on connect. * RedisRelay: Broadcaster impl that XADDs every event into ws:scope:{type}:{id}:stream and XREADGROUPs only the scopes for which this node has live subscribers. Per-node consumer group, heartbeat, stale-consumer sweeper, MAXLEN cap, lag/disconnect metrics. * Listeners: route task:* events to ScopeTask, chat:* events to ScopeChat; workspace remains the default for everything else. * events.Event: optional TaskID / ChatSessionID hints so the listener layer can pick the right scope without re-parsing payloads. * Handler: publishTask / publishChat helpers; chat + task message publishers updated to use them. * main.go: when REDIS_URL is set, wrap the hub with NewRedisRelay and pass the relay (instead of the hub) to registerListeners. A db-backed ScopeAuthorizer enforces that task/chat subscribes belong to the caller's workspace. * Metrics: per-scope subscribe/deny counters, redis connect state, node id, lag/dropped counters surfaced via /health/realtime. Behavior in single-node mode (REDIS_URL unset) is unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix(realtime): address PR #1429 review must-fix items (MUL-1138) - listeners: keep task/chat events on workspace fanout until the WS client supports scope-subscribe + reconnect-replay. Routing them through BroadcastToScope today (without any client subscriber) would silently drop every chat / task message and break the live timeline, chat unread badges, and pending-task UI. The server-side scope infra (Hub subscribe/unsubscribe, ScopeAuthorizer, Redis Streams relay) stays in place so flipping the switch in the client follow-up PR is a one-line change. - scope_authorizer: ScopeChat now enforces CreatorID == userID, mirroring the HTTP layer (handler/chat.go: GetChatSession / SendChatMessage / MarkChatSessionRead). Without this, any workspace member who learned a session_id could subscribe to chat:message / chat:done / chat:session_read for a peer's private chat. The same creator-only check is applied to ScopeTask when the task is a chat task (task.ChatSessionID set). Issue tasks remain workspace-scoped. - Refactor scope authorizer to depend on a narrow scopeAuthQuerier interface so its decisions can be unit-tested without a live DB. - Add tests: * listeners_scope_test.go pins the workspace-fanout fallback for task:message / task:progress / chat:message / chat:done / chat:session_read. * scope_authorizer_test.go covers chat creator-only access, chat-task creator-only access, and issue-task workspace-only access (creator allowed, peer denied, cross-workspace denied, missing session denied, empty userID denied). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: CC-Girl <cc-girl@multica.ai>
314 lines
7.9 KiB
Go
314 lines
7.9 KiB
Go
package realtime
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/golang-jwt/jwt/v5"
|
|
"github.com/gorilla/websocket"
|
|
"github.com/multica-ai/multica/server/internal/auth"
|
|
)
|
|
|
|
const testWorkspaceID = "test-workspace"
|
|
const testUserID = "test-user"
|
|
|
|
// mockMembershipChecker always returns true.
|
|
type mockMembershipChecker struct{}
|
|
|
|
func (m *mockMembershipChecker) IsMember(_ context.Context, _, _ string) bool {
|
|
return true
|
|
}
|
|
|
|
func makeTestToken(t *testing.T) string {
|
|
t.Helper()
|
|
token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
|
|
"sub": testUserID,
|
|
})
|
|
signed, err := token.SignedString(auth.JWTSecret())
|
|
if err != nil {
|
|
t.Fatalf("failed to sign test JWT: %v", err)
|
|
}
|
|
return signed
|
|
}
|
|
|
|
func newTestHub(t *testing.T) (*Hub, *httptest.Server) {
|
|
t.Helper()
|
|
hub := NewHub()
|
|
go hub.Run()
|
|
|
|
mc := &mockMembershipChecker{}
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) {
|
|
HandleWebSocket(hub, mc, nil, nil, w, r)
|
|
})
|
|
server := httptest.NewServer(mux)
|
|
return hub, server
|
|
}
|
|
|
|
func connectWS(t *testing.T, server *httptest.Server) *websocket.Conn {
|
|
t.Helper()
|
|
token := makeTestToken(t)
|
|
wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + "/ws?workspace_id=" + testWorkspaceID
|
|
conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
|
|
if err != nil {
|
|
t.Fatalf("failed to connect WebSocket: %v", err)
|
|
}
|
|
authMsg, _ := json.Marshal(map[string]any{
|
|
"type": "auth",
|
|
"payload": map[string]string{"token": token},
|
|
})
|
|
if err := conn.WriteMessage(websocket.TextMessage, authMsg); err != nil {
|
|
t.Fatalf("failed to send auth message: %v", err)
|
|
}
|
|
// Read auth_ack before returning the connection.
|
|
conn.SetReadDeadline(time.Now().Add(2 * time.Second))
|
|
_, ack, err := conn.ReadMessage()
|
|
if err != nil {
|
|
t.Fatalf("failed to read auth_ack: %v", err)
|
|
}
|
|
if !strings.Contains(string(ack), "auth_ack") {
|
|
t.Fatalf("expected auth_ack, got %s", ack)
|
|
}
|
|
conn.SetReadDeadline(time.Time{})
|
|
return conn
|
|
}
|
|
|
|
// totalClients counts all currently registered clients.
|
|
func totalClients(hub *Hub) int {
|
|
hub.mu.RLock()
|
|
defer hub.mu.RUnlock()
|
|
return len(hub.clients)
|
|
}
|
|
|
|
func TestHub_ClientRegistration(t *testing.T) {
|
|
hub, server := newTestHub(t)
|
|
defer server.Close()
|
|
|
|
conn := connectWS(t, server)
|
|
defer conn.Close()
|
|
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
count := totalClients(hub)
|
|
if count != 1 {
|
|
t.Fatalf("expected 1 client, got %d", count)
|
|
}
|
|
}
|
|
|
|
func TestHub_Broadcast(t *testing.T) {
|
|
hub, server := newTestHub(t)
|
|
defer server.Close()
|
|
|
|
conn1 := connectWS(t, server)
|
|
defer conn1.Close()
|
|
conn2 := connectWS(t, server)
|
|
defer conn2.Close()
|
|
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
msg := []byte(`{"type":"issue:created","data":"test"}`)
|
|
hub.Broadcast(msg)
|
|
|
|
conn1.SetReadDeadline(time.Now().Add(2 * time.Second))
|
|
_, received1, err := conn1.ReadMessage()
|
|
if err != nil {
|
|
t.Fatalf("client 1 read error: %v", err)
|
|
}
|
|
if string(received1) != string(msg) {
|
|
t.Fatalf("client 1: expected %s, got %s", msg, received1)
|
|
}
|
|
|
|
conn2.SetReadDeadline(time.Now().Add(2 * time.Second))
|
|
_, received2, err := conn2.ReadMessage()
|
|
if err != nil {
|
|
t.Fatalf("client 2 read error: %v", err)
|
|
}
|
|
if string(received2) != string(msg) {
|
|
t.Fatalf("client 2: expected %s, got %s", msg, received2)
|
|
}
|
|
}
|
|
|
|
func TestHub_ClientDisconnect(t *testing.T) {
|
|
hub, server := newTestHub(t)
|
|
defer server.Close()
|
|
|
|
conn := connectWS(t, server)
|
|
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
countBefore := totalClients(hub)
|
|
if countBefore != 1 {
|
|
t.Fatalf("expected 1 client before disconnect, got %d", countBefore)
|
|
}
|
|
|
|
conn.Close()
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
countAfter := totalClients(hub)
|
|
if countAfter != 0 {
|
|
t.Fatalf("expected 0 clients after disconnect, got %d", countAfter)
|
|
}
|
|
}
|
|
|
|
func TestHub_BroadcastToMultipleClients(t *testing.T) {
|
|
hub, server := newTestHub(t)
|
|
defer server.Close()
|
|
|
|
const numClients = 5
|
|
conns := make([]*websocket.Conn, numClients)
|
|
for i := 0; i < numClients; i++ {
|
|
conns[i] = connectWS(t, server)
|
|
defer conns[i].Close()
|
|
}
|
|
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
count := totalClients(hub)
|
|
if count != numClients {
|
|
t.Fatalf("expected %d clients, got %d", numClients, count)
|
|
}
|
|
|
|
msg := []byte(`{"type":"test","count":5}`)
|
|
hub.Broadcast(msg)
|
|
|
|
for i, conn := range conns {
|
|
conn.SetReadDeadline(time.Now().Add(2 * time.Second))
|
|
_, received, err := conn.ReadMessage()
|
|
if err != nil {
|
|
t.Fatalf("client %d read error: %v", i, err)
|
|
}
|
|
if string(received) != string(msg) {
|
|
t.Fatalf("client %d: expected %s, got %s", i, msg, received)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestHub_MultipleBroadcasts(t *testing.T) {
|
|
hub, server := newTestHub(t)
|
|
defer server.Close()
|
|
|
|
conn := connectWS(t, server)
|
|
defer conn.Close()
|
|
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
messages := []string{
|
|
`{"type":"issue:created"}`,
|
|
`{"type":"issue:updated"}`,
|
|
`{"type":"issue:deleted"}`,
|
|
}
|
|
|
|
for _, msg := range messages {
|
|
hub.Broadcast([]byte(msg))
|
|
}
|
|
|
|
for i, expected := range messages {
|
|
conn.SetReadDeadline(time.Now().Add(2 * time.Second))
|
|
_, received, err := conn.ReadMessage()
|
|
if err != nil {
|
|
t.Fatalf("message %d read error: %v", i, err)
|
|
}
|
|
if string(received) != expected {
|
|
t.Fatalf("message %d: expected %s, got %s", i, expected, received)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestHandleWebSocket_ClientIdentityFromQuery verifies that client_platform,
|
|
// client_version, and client_os query params on the WS upgrade URL are read
|
|
// by the handler and surfaced to the access log. Browsers cannot set custom
|
|
// headers on WS upgrades, so this query-param channel is the only way to
|
|
// preserve the same observability dimensions HTTP clients get via X-Client-*.
|
|
func TestHandleWebSocket_ClientIdentityFromQuery(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
var mu sync.Mutex
|
|
handler := slog.NewJSONHandler(&lockedWriter{w: &buf, mu: &mu}, &slog.HandlerOptions{Level: slog.LevelDebug})
|
|
prevDefault := slog.Default()
|
|
slog.SetDefault(slog.New(handler))
|
|
t.Cleanup(func() { slog.SetDefault(prevDefault) })
|
|
|
|
_, server := newTestHub(t)
|
|
defer server.Close()
|
|
|
|
token := makeTestToken(t)
|
|
wsURL := "ws" + strings.TrimPrefix(server.URL, "http") +
|
|
"/ws?workspace_id=" + testWorkspaceID +
|
|
"&client_platform=desktop&client_version=1.2.3&client_os=macos"
|
|
conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
|
|
if err != nil {
|
|
t.Fatalf("dial: %v", err)
|
|
}
|
|
defer conn.Close()
|
|
|
|
authMsg, _ := json.Marshal(map[string]any{
|
|
"type": "auth",
|
|
"payload": map[string]string{"token": token},
|
|
})
|
|
if err := conn.WriteMessage(websocket.TextMessage, authMsg); err != nil {
|
|
t.Fatalf("write auth: %v", err)
|
|
}
|
|
conn.SetReadDeadline(time.Now().Add(2 * time.Second))
|
|
if _, _, err := conn.ReadMessage(); err != nil {
|
|
t.Fatalf("read auth_ack: %v", err)
|
|
}
|
|
|
|
// Wait briefly for the "websocket connected" log line to be flushed.
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
var found map[string]any
|
|
for time.Now().Before(deadline) {
|
|
mu.Lock()
|
|
raw := buf.String()
|
|
mu.Unlock()
|
|
for _, line := range strings.Split(raw, "\n") {
|
|
if line == "" {
|
|
continue
|
|
}
|
|
var entry map[string]any
|
|
if err := json.Unmarshal([]byte(line), &entry); err != nil {
|
|
continue
|
|
}
|
|
if msg, _ := entry["msg"].(string); msg == "websocket connected" {
|
|
found = entry
|
|
break
|
|
}
|
|
}
|
|
if found != nil {
|
|
break
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
|
|
if found == nil {
|
|
t.Fatalf("did not observe \"websocket connected\" log entry; buffered logs:\n%s", buf.String())
|
|
}
|
|
if got, _ := found["client_platform"].(string); got != "desktop" {
|
|
t.Errorf("client_platform = %q, want %q", got, "desktop")
|
|
}
|
|
if got, _ := found["client_version"].(string); got != "1.2.3" {
|
|
t.Errorf("client_version = %q, want %q", got, "1.2.3")
|
|
}
|
|
if got, _ := found["client_os"].(string); got != "macos" {
|
|
t.Errorf("client_os = %q, want %q", got, "macos")
|
|
}
|
|
}
|
|
|
|
// lockedWriter is a thread-safe writer used to capture concurrent slog output.
|
|
type lockedWriter struct {
|
|
w *bytes.Buffer
|
|
mu *sync.Mutex
|
|
}
|
|
|
|
func (l *lockedWriter) Write(p []byte) (int, error) {
|
|
l.mu.Lock()
|
|
defer l.mu.Unlock()
|
|
return l.w.Write(p)
|
|
}
|