package daemon import ( "context" "encoding/json" "errors" "log/slog" "net" "net/http" "net/http/httptest" "strings" "sync/atomic" "testing" "time" "github.com/gorilla/websocket" "github.com/multica-ai/multica/server/pkg/protocol" ) func TestTaskWakeupURL(t *testing.T) { tests := []struct { name string baseURL string runtimeIDs []string want string }{ { name: "http base", baseURL: "http://localhost:8080", runtimeIDs: []string{"runtime-b", "runtime-a"}, want: "ws://localhost:8080/api/daemon/ws?runtime_ids=runtime-a%2Cruntime-b", }, { name: "https base", baseURL: "https://api.example.com", runtimeIDs: []string{"runtime-1"}, want: "wss://api.example.com/api/daemon/ws?runtime_ids=runtime-1", }, { name: "base path", baseURL: "https://api.example.com/multica", runtimeIDs: []string{"runtime-1"}, want: "wss://api.example.com/multica/api/daemon/ws?runtime_ids=runtime-1", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := taskWakeupURL(tt.baseURL, tt.runtimeIDs) if err != nil { t.Fatalf("taskWakeupURL: %v", err) } if got != tt.want { t.Fatalf("taskWakeupURL() = %q, want %q", got, tt.want) } }) } } // TestWSHeartbeatFreshnessSuppressesHTTP pins the WS-vs-HTTP coordination: // once a runtime acked over WS within the freshness window the HTTP // heartbeat loop must skip it to avoid duplicate DB writes. func TestWSHeartbeatFreshnessSuppressesHTTP(t *testing.T) { d := New(Config{HeartbeatInterval: 15 * time.Second}, slog.Default()) if d.wsHeartbeatRecentlyAcked("runtime-1") { t.Fatalf("expected unrecorded runtime to be stale") } d.recordWSHeartbeatAck("runtime-1") if !d.wsHeartbeatRecentlyAcked("runtime-1") { t.Fatalf("expected just-acked runtime to be fresh") } // Force the entry past the freshness window. d.wsHBMu.Lock() d.wsHBLastAck["runtime-1"] = time.Now().Add(-d.wsHeartbeatFreshness() - time.Second) d.wsHBMu.Unlock() if d.wsHeartbeatRecentlyAcked("runtime-1") { t.Fatalf("expected aged runtime to be stale (HTTP heartbeat must resume)") } d.recordWSHeartbeatAck("runtime-2") d.clearWSHeartbeatAcks() if d.wsHeartbeatRecentlyAcked("runtime-2") { t.Fatalf("expected clearWSHeartbeatAcks to drop all entries") } } func TestReadTaskWakeupMessagesTimesOutWithoutPeerTraffic(t *testing.T) { overrideTaskWakeupTimings(t, 60*time.Millisecond, 20*time.Millisecond, taskWakeupBackoffResetAfter) upgrader := websocket.Upgrader{} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := upgrader.Upgrade(w, r, nil) if err != nil { return } defer conn.Close() time.Sleep(300 * time.Millisecond) })) defer srv.Close() conn, _, err := websocket.DefaultDialer.Dial(taskWakeupTestWSURL(srv.URL), nil) if err != nil { t.Fatalf("dial websocket: %v", err) } defer conn.Close() d := New(Config{}, slog.Default()) errCh := make(chan error, 1) go func() { errCh <- d.readTaskWakeupMessages(conn, make(chan taskWakeup, 1)) }() select { case err := <-errCh: var netErr net.Error if !errors.As(err, &netErr) || !netErr.Timeout() { t.Fatalf("readTaskWakeupMessages error = %v, want timeout", err) } case <-time.After(time.Second): t.Fatal("readTaskWakeupMessages did not time out") } } func TestReadTaskWakeupMessagesExtendsDeadlineOnServerPing(t *testing.T) { overrideTaskWakeupTimings(t, 120*time.Millisecond, 50*time.Millisecond, taskWakeupBackoffResetAfter) clientReceived := make(chan struct{}) taskFrame := mustProtocolFrame(t, protocol.Message{ Type: protocol.EventDaemonTaskAvailable, Payload: marshalRaw(protocol.TaskAvailablePayload{ RuntimeID: "runtime-1", TaskID: "task-1", }), }) upgrader := websocket.Upgrader{} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := upgrader.Upgrade(w, r, nil) if err != nil { return } defer conn.Close() for i := 0; i < 3; i++ { time.Sleep(50 * time.Millisecond) conn.SetWriteDeadline(time.Now().Add(50 * time.Millisecond)) if err := conn.WriteMessage(websocket.PingMessage, []byte("keepalive")); err != nil { return } } if !writeWSMessage(t, conn, websocket.TextMessage, taskFrame) { return } waitForClientWakeup(t, clientReceived) })) defer srv.Close() conn, _, err := websocket.DefaultDialer.Dial(taskWakeupTestWSURL(srv.URL), nil) if err != nil { t.Fatalf("dial websocket: %v", err) } defer conn.Close() d := New(Config{}, slog.Default()) taskWakeups := make(chan taskWakeup, 1) errCh := make(chan error, 1) go func() { errCh <- d.readTaskWakeupMessages(conn, taskWakeups) }() select { case wakeup := <-taskWakeups: if wakeup.runtimeID != "runtime-1" { t.Fatalf("wakeup runtimeID = %q, want runtime-1", wakeup.runtimeID) } close(clientReceived) case err := <-errCh: t.Fatalf("readTaskWakeupMessages returned before task frame: %v", err) case <-time.After(time.Second): t.Fatal("timed out waiting for task wakeup") } } func TestReadTaskWakeupMessagesExtendsDeadlineOnApplicationMessage(t *testing.T) { overrideTaskWakeupTimings(t, 120*time.Millisecond, 50*time.Millisecond, taskWakeupBackoffResetAfter) clientReceived := make(chan struct{}) ackFrame := mustProtocolFrame(t, protocol.Message{ Type: protocol.EventDaemonHeartbeatAck, Payload: marshalRaw(HeartbeatResponse{ RuntimeID: "runtime-1", }), }) taskFrame := mustProtocolFrame(t, protocol.Message{ Type: protocol.EventDaemonTaskAvailable, Payload: marshalRaw(protocol.TaskAvailablePayload{ RuntimeID: "runtime-1", TaskID: "task-1", }), }) upgrader := websocket.Upgrader{} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := upgrader.Upgrade(w, r, nil) if err != nil { return } defer conn.Close() for i := 0; i < 3; i++ { time.Sleep(50 * time.Millisecond) if !writeWSMessage(t, conn, websocket.TextMessage, ackFrame) { return } } time.Sleep(50 * time.Millisecond) if !writeWSMessage(t, conn, websocket.TextMessage, taskFrame) { return } waitForClientWakeup(t, clientReceived) })) defer srv.Close() conn, _, err := websocket.DefaultDialer.Dial(taskWakeupTestWSURL(srv.URL), nil) if err != nil { t.Fatalf("dial websocket: %v", err) } defer conn.Close() d := New(Config{}, slog.Default()) taskWakeups := make(chan taskWakeup, 1) errCh := make(chan error, 1) go func() { errCh <- d.readTaskWakeupMessages(conn, taskWakeups) }() select { case wakeup := <-taskWakeups: if wakeup.runtimeID != "runtime-1" { t.Fatalf("wakeup runtimeID = %q, want runtime-1", wakeup.runtimeID) } close(clientReceived) case err := <-errCh: t.Fatalf("readTaskWakeupMessages returned before task frame: %v", err) case <-time.After(time.Second): t.Fatal("timed out waiting for task wakeup") } } func TestReadTaskWakeupMessagesExtendsDeadlineOnPong(t *testing.T) { overrideTaskWakeupTimings(t, 120*time.Millisecond, 50*time.Millisecond, taskWakeupBackoffResetAfter) clientReceived := make(chan struct{}) taskFrame := mustProtocolFrame(t, protocol.Message{ Type: protocol.EventDaemonTaskAvailable, Payload: marshalRaw(protocol.TaskAvailablePayload{ RuntimeID: "runtime-1", TaskID: "task-1", }), }) upgrader := websocket.Upgrader{} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := upgrader.Upgrade(w, r, nil) if err != nil { return } defer conn.Close() for i := 0; i < 3; i++ { time.Sleep(50 * time.Millisecond) if !writeWSMessage(t, conn, websocket.PongMessage, []byte("keepalive")) { return } } time.Sleep(50 * time.Millisecond) if !writeWSMessage(t, conn, websocket.TextMessage, taskFrame) { return } waitForClientWakeup(t, clientReceived) })) defer srv.Close() conn, _, err := websocket.DefaultDialer.Dial(taskWakeupTestWSURL(srv.URL), nil) if err != nil { t.Fatalf("dial websocket: %v", err) } defer conn.Close() d := New(Config{}, slog.Default()) taskWakeups := make(chan taskWakeup, 1) errCh := make(chan error, 1) go func() { errCh <- d.readTaskWakeupMessages(conn, taskWakeups) }() select { case wakeup := <-taskWakeups: if wakeup.runtimeID != "runtime-1" { t.Fatalf("wakeup runtimeID = %q, want runtime-1", wakeup.runtimeID) } close(clientReceived) case err := <-errCh: t.Fatalf("readTaskWakeupMessages returned before task frame: %v", err) case <-time.After(time.Second): t.Fatal("timed out waiting for task wakeup") } } func TestShouldResetTaskWakeupBackoffRequiresStableConnection(t *testing.T) { old := taskWakeupBackoffResetAfter taskWakeupBackoffResetAfter = 10 * time.Second t.Cleanup(func() { taskWakeupBackoffResetAfter = old }) if shouldResetTaskWakeupBackoff(0) { t.Fatal("zero connection uptime reset backoff") } if shouldResetTaskWakeupBackoff(9 * time.Second) { t.Fatal("short connection uptime reset backoff") } if !shouldResetTaskWakeupBackoff(10 * time.Second) { t.Fatal("stable connection uptime did not reset backoff") } } func TestRuntimeHeartbeatClosesIdleConnectionsAfterRepeatedTransientFailures(t *testing.T) { transport := &closeCountingTransport{} client := NewClient("http://daemon.test") client.client = &http.Client{ Timeout: time.Second, Transport: transport, } d := New(Config{HeartbeatInterval: 10 * time.Millisecond}, slog.Default()) d.client = client ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { defer close(done) d.runRuntimeHeartbeat(ctx, "runtime-1") }() deadline := time.After(time.Second) ticker := time.NewTicker(5 * time.Millisecond) defer ticker.Stop() for transport.closeCount.Load() == 0 { select { case <-ticker.C: case <-deadline: cancel() t.Fatal("CloseIdleConnections was not called") } } cancel() select { case <-done: case <-time.After(time.Second): t.Fatal("runRuntimeHeartbeat did not stop after context cancellation") } if got := transport.roundTrips.Load(); got < 2 { t.Fatalf("RoundTrip count = %d, want at least 2", got) } } type closeCountingTransport struct { roundTrips atomic.Int32 closeCount atomic.Int32 } func (t *closeCountingTransport) RoundTrip(*http.Request) (*http.Response, error) { t.roundTrips.Add(1) return nil, errors.New("dial failed") } func (t *closeCountingTransport) CloseIdleConnections() { t.closeCount.Add(1) } func overrideTaskWakeupTimings(t *testing.T, pongWait, writeWait, backoffResetAfter time.Duration) { t.Helper() oldPongWait := taskWakeupPongWait oldWriteWait := taskWakeupWriteWait oldBackoffResetAfter := taskWakeupBackoffResetAfter taskWakeupPongWait = pongWait taskWakeupWriteWait = writeWait taskWakeupBackoffResetAfter = backoffResetAfter t.Cleanup(func() { taskWakeupPongWait = oldPongWait taskWakeupWriteWait = oldWriteWait taskWakeupBackoffResetAfter = oldBackoffResetAfter }) } func taskWakeupTestWSURL(httpURL string) string { return strings.Replace(httpURL, "http", "ws", 1) } func mustProtocolFrame(t *testing.T, msg protocol.Message) []byte { t.Helper() frame, err := json.Marshal(msg) if err != nil { t.Fatalf("marshal websocket frame: %v", err) } return frame } func writeWSMessage(t *testing.T, conn *websocket.Conn, messageType int, frame []byte) bool { t.Helper() conn.SetWriteDeadline(time.Now().Add(50 * time.Millisecond)) if err := conn.WriteMessage(messageType, frame); err != nil { t.Errorf("write websocket frame: %v", err) return false } return true } func waitForClientWakeup(t *testing.T, clientReceived <-chan struct{}) { t.Helper() select { case <-clientReceived: case <-time.After(time.Second): t.Errorf("server timed out waiting for client wakeup") } }