feat(server): add readiness health endpoints (#1605)

* feat(server): add readiness health endpoints

* fix(server): cache readiness checks

* fix(server): raise readiness cache ttl

---------

Co-authored-by: Eve <eve@multica.ai>

SELF_HOSTING_ADVANCED.md

@@ -290,14 +290,23 @@ HTTP requests (issues, comments, uploads) work on LAN out of the box — Next.js
 
 ## Health Check
 
-The backend exposes a health check endpoint:
+The backend exposes public health endpoints:
 
-```
+```text
 GET /health
 → {"status":"ok"}
+
+GET /readyz
+→ {"status":"ok","checks":{"db":"ok","migrations":"ok"}}
+
+GET /healthz
+→ same response as /readyz
 ```
 
-Use this for load balancer health checks or monitoring.
+Use `/health` for basic liveness / reachability checks. Use `/readyz` for
+dependency-aware readiness probes and external monitoring that should fail when
+the database is unavailable or migrations are not fully applied. `/healthz` is
+kept as an alias for operator familiarity.
 
 ## Upgrading
 

SELF_HOSTING_AI.md

@@ -73,4 +73,4 @@ If the default ports (8080/3000) are in use:
 - **Backend not ready:** `docker compose -f docker-compose.selfhost.yml logs backend`
 - **Frontend not ready:** `docker compose -f docker-compose.selfhost.yml logs frontend`
 - **Daemon issues:** `multica daemon logs`
-- **Health check:** `curl http://localhost:8080/health`
+- **Health checks:** `curl http://localhost:8080/health` for liveness, `curl http://localhost:8080/readyz` for dependency-aware readiness

apps/docs/content/docs/getting-started/self-hosting.zh.mdx

@@ -408,14 +408,23 @@ NEXT_PUBLIC_WS_URL=wss://api.example.com/ws
 
 ## Health Check
 
-The backend exposes a health check endpoint:
+The backend exposes public health endpoints:
 
-```
+```text
 GET /health
 → {"status":"ok"}
+
+GET /readyz
+→ {"status":"ok","checks":{"db":"ok","migrations":"ok"}}
+
+GET /healthz
+→ same response as /readyz
 ```
 
-Use this for load balancer health checks or monitoring.
+Use `/health` for basic liveness / reachability checks. Use `/readyz` for
+dependency-aware readiness probes and external monitoring that should fail when
+the database is unavailable or migrations are not fully applied. `/healthz` is
+kept as an alias for operator familiarity.
 
 ## Upgrading
 

apps/docs/content/docs/self-host-quickstart.mdx

@@ -31,6 +31,9 @@ make selfhost
 3. Bring up every service using `docker-compose.selfhost.yml`
 4. Wait until the backend's `/health` endpoint is ready
 
+For ongoing production probes after startup, use `/readyz` when you want the
+check to fail on database or migration problems.
+
 The backend container **runs database migrations automatically** on startup (`docker/entrypoint.sh` runs `./migrate up` before the server starts) — you'll see the migration output in the backend logs. Version upgrades are handled the same way.
 
 <Callout type="info">

apps/docs/content/docs/self-host-quickstart.zh.mdx

@@ -31,6 +31,8 @@ make selfhost
 3. 用 `docker-compose.selfhost.yml` 启动全部服务
 4. 等后端 `/health` 端点准备就绪
 
+如果是启动完成后的生产探针，想让数据库或 migration 异常也体现为失败，请改用 `/readyz`。
+
 后端容器启动时会**自动跑数据库 migration**（`docker/entrypoint.sh` 在启动 server 前执行 `./migrate up`）——你会在 backend 日志里看到 migration 输出。升级版本时同样自动处理。
 
 <Callout type="info">

apps/docs/content/docs/troubleshooting.mdx

@@ -25,6 +25,7 @@ Look up issues by symptom. Each entry gives you **symptom / likely causes / how
 multica daemon logs --lines 100    # look for daemon-side errors
 echo $MULTICA_SERVER_URL          # confirm the address is set
 curl -i http://<server-host>:8080/health   # hit the server directly
+curl -i http://<server-host>:8080/readyz  # include DB + migration readiness
 cat ~/.multica/config.json        # verify api_token exists
 multica workspace list            # confirm you're a member of the target workspace
 ```

apps/docs/content/docs/troubleshooting.zh.mdx

@@ -25,6 +25,7 @@ import { Callout } from "fumadocs-ui/components/callout";
 multica daemon logs --lines 100    # 看 daemon 侧错误
 echo $MULTICA_SERVER_URL          # 确认地址配对
 curl -i http://<server-host>:8080/health   # 直接戳 server
+curl -i http://<server-host>:8080/readyz  # 连同 DB + migration readiness 一起检查
 cat ~/.multica/config.json        # 看 api_token 是否存在
 multica workspace list            # 确认你是目标工作区成员
 ```

packages/core/paths/reserved-slugs.ts

@@ -85,10 +85,13 @@ export const RESERVED_SLUGS = new Set([
   "tokens",
   "cli",
 
-  // Backend ops / observability. `/health` and `/ws` exist on the backend
+  // Backend ops / observability. `/health`, `/readyz`, `/healthz`, and `/ws`
+  // exist on the backend
   // host; reserving them on the workspace slug space prevents naming
   // confusion if/when these paths are ever proxied through the web origin.
   "health",
+  "readyz",
+  "healthz",
   "ws",
   "metrics",
   "ping",

server/cmd/migrate/main.go

@@ -5,12 +5,10 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
-	"path/filepath"
-	"sort"
-	"strings"
 
 	"github.com/jackc/pgx/v5/pgxpool"
 	"github.com/multica-ai/multica/server/internal/logger"
+	"github.com/multica-ai/multica/server/internal/migrations"
 )
 
 func main() {
@@ -57,28 +55,14 @@ func main() {
 		os.Exit(1)
 	}
 
-	// Find migration files
-	migrationsDir := "migrations"
-	if _, err := os.Stat(migrationsDir); os.IsNotExist(err) {
-		// Try from server/ directory
-		migrationsDir = "server/migrations"
-	}
-
-	suffix := "." + direction + ".sql"
-	files, err := filepath.Glob(filepath.Join(migrationsDir, "*"+suffix))
+	files, err := migrations.Files(direction)
 	if err != nil {
 		slog.Error("failed to find migration files", "error", err)
 		os.Exit(1)
 	}
 
-	if direction == "up" {
-		sort.Strings(files)
-	} else {
-		sort.Sort(sort.Reverse(sort.StringSlice(files)))
-	}
-
 	for _, file := range files {
-		version := extractVersion(file)
+		version := migrations.ExtractVersion(file)
 
 		if direction == "up" {
 			// Check if already applied
@@ -133,11 +117,3 @@ func main() {
 
 	fmt.Println("Done.")
 }
-
-func extractVersion(filename string) string {
-	base := filepath.Base(filename)
-	// Remove .up.sql or .down.sql
-	base = strings.TrimSuffix(base, ".up.sql")
-	base = strings.TrimSuffix(base, ".down.sql")
-	return base
-}

server/cmd/server/health.go

@@ -0,0 +1,161 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+
+	"github.com/multica-ai/multica/server/internal/migrations"
+)
+
+const readinessQuery = `SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version = $1)`
+
+const readinessCacheTTL = 3 * time.Second
+
+type readinessDB interface {
+	Ping(ctx context.Context) error
+	QueryRow(ctx context.Context, sql string, args ...any) pgx.Row
+}
+
+type serverHealth struct {
+	db              readinessDB
+	latestMigration string
+	initErr         error
+	cacheTTL        time.Duration
+	refreshMu       sync.Mutex
+	cache           atomic.Pointer[cachedReadiness]
+}
+
+type cachedReadiness struct {
+	response   readinessResponse
+	statusCode int
+	expiresAt  time.Time
+}
+
+type liveResponse struct {
+	Status string `json:"status"`
+}
+
+type readinessResponse struct {
+	Status string          `json:"status"`
+	Checks readinessChecks `json:"checks"`
+}
+
+type readinessChecks struct {
+	DB         string `json:"db"`
+	Migrations string `json:"migrations"`
+}
+
+func newServerHealth(pool *pgxpool.Pool) *serverHealth {
+	latestMigration, err := migrations.LatestVersion()
+	return &serverHealth{
+		db:              pool,
+		latestMigration: latestMigration,
+		initErr:         err,
+		cacheTTL:        readinessCacheTTL,
+	}
+}
+
+func (h *serverHealth) liveHandler(w http.ResponseWriter, _ *http.Request) {
+	writeJSON(w, http.StatusOK, liveResponse{Status: "ok"})
+}
+
+func (h *serverHealth) readyHandler(w http.ResponseWriter, r *http.Request) {
+	resp, status := h.readiness(r.Context())
+	writeJSON(w, status, resp)
+}
+
+func (h *serverHealth) readiness(parent context.Context) (readinessResponse, int) {
+	if h.cacheTTL <= 0 {
+		return h.computeReadiness(parent)
+	}
+
+	now := time.Now()
+	if cached := h.loadCachedReadiness(now); cached != nil {
+		return cached.response, cached.statusCode
+	}
+
+	h.refreshMu.Lock()
+	defer h.refreshMu.Unlock()
+
+	now = time.Now()
+	if cached := h.loadCachedReadiness(now); cached != nil {
+		return cached.response, cached.statusCode
+	}
+
+	resp, status := h.computeReadiness(parent)
+	h.cache.Store(&cachedReadiness{
+		response:   resp,
+		statusCode: status,
+		expiresAt:  now.Add(h.cacheTTL),
+	})
+	return resp, status
+}
+
+func (h *serverHealth) loadCachedReadiness(now time.Time) *cachedReadiness {
+	cached := h.cache.Load()
+	if cached == nil || !now.Before(cached.expiresAt) {
+		return nil
+	}
+	return cached
+}
+
+func (h *serverHealth) computeReadiness(parent context.Context) (readinessResponse, int) {
+	resp := readinessResponse{
+		Status: "ok",
+		Checks: readinessChecks{
+			DB:         "ok",
+			Migrations: "ok",
+		},
+	}
+
+	if h.db == nil {
+		resp.Status = "not_ready"
+		resp.Checks.DB = "error"
+		resp.Checks.Migrations = "unknown"
+		return resp, http.StatusServiceUnavailable
+	}
+
+	ctx, cancel := context.WithTimeout(parent, 2*time.Second)
+	defer cancel()
+
+	if err := h.db.Ping(ctx); err != nil {
+		resp.Status = "not_ready"
+		resp.Checks.DB = "error"
+		resp.Checks.Migrations = "unknown"
+		return resp, http.StatusServiceUnavailable
+	}
+
+	if h.initErr != nil || h.latestMigration == "" {
+		resp.Status = "not_ready"
+		resp.Checks.Migrations = "error"
+		return resp, http.StatusServiceUnavailable
+	}
+
+	var applied bool
+	if err := h.db.QueryRow(ctx, readinessQuery, h.latestMigration).Scan(&applied); err != nil {
+		resp.Status = "not_ready"
+		resp.Checks.Migrations = "error"
+		return resp, http.StatusServiceUnavailable
+	}
+
+	if !applied {
+		resp.Status = "not_ready"
+		resp.Checks.Migrations = "out_of_date"
+		return resp, http.StatusServiceUnavailable
+	}
+
+	return resp, http.StatusOK
+}
+
+func writeJSON(w http.ResponseWriter, status int, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(v)
+}

server/cmd/server/health_test.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+)
+
+type stubReadinessDB struct {
+	pingErr    error
+	queryErr   error
+	applied    bool
+	pingCalls  atomic.Int32
+	queryCalls atomic.Int32
+}
+
+func (s *stubReadinessDB) Ping(context.Context) error {
+	s.pingCalls.Add(1)
+	return s.pingErr
+}
+
+func (s *stubReadinessDB) QueryRow(context.Context, string, ...any) pgx.Row {
+	s.queryCalls.Add(1)
+	return stubRow{applied: s.applied, err: s.queryErr}
+}
+
+type stubRow struct {
+	applied bool
+	err     error
+}
+
+func (r stubRow) Scan(dest ...any) error {
+	if r.err != nil {
+		return r.err
+	}
+	*(dest[0].(*bool)) = r.applied
+	return nil
+}
+
+func TestServerHealthReadyHandlerDBPingFailure(t *testing.T) {
+	db := &stubReadinessDB{pingErr: errors.New("db unavailable")}
+	h := &serverHealth{
+		db:              db,
+		latestMigration: "056_example",
+	}
+
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodGet, "/readyz", nil)
+	h.readyHandler(rec, req)
+
+	if rec.Code != http.StatusServiceUnavailable {
+		t.Fatalf("expected 503, got %d", rec.Code)
+	}
+
+	var resp readinessResponse
+	if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
+		t.Fatalf("decode response: %v", err)
+	}
+
+	if resp.Status != "not_ready" {
+		t.Fatalf("status = %q, want %q", resp.Status, "not_ready")
+	}
+	if resp.Checks.DB != "error" {
+		t.Fatalf("db check = %q, want %q", resp.Checks.DB, "error")
+	}
+	if resp.Checks.Migrations != "unknown" {
+		t.Fatalf("migrations check = %q, want %q", resp.Checks.Migrations, "unknown")
+	}
+}
+
+func TestServerHealthReadyHandlerMigrationOutOfDate(t *testing.T) {
+	db := &stubReadinessDB{applied: false}
+	h := &serverHealth{
+		db:              db,
+		latestMigration: "056_example",
+	}
+
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodGet, "/readyz", nil)
+	h.readyHandler(rec, req)
+
+	if rec.Code != http.StatusServiceUnavailable {
+		t.Fatalf("expected 503, got %d", rec.Code)
+	}
+
+	var resp readinessResponse
+	if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
+		t.Fatalf("decode response: %v", err)
+	}
+
+	if resp.Status != "not_ready" {
+		t.Fatalf("status = %q, want %q", resp.Status, "not_ready")
+	}
+	if resp.Checks.DB != "ok" {
+		t.Fatalf("db check = %q, want %q", resp.Checks.DB, "ok")
+	}
+	if resp.Checks.Migrations != "out_of_date" {
+		t.Fatalf("migrations check = %q, want %q", resp.Checks.Migrations, "out_of_date")
+	}
+}
+
+func TestServerHealthReadinessCachesResult(t *testing.T) {
+	db := &stubReadinessDB{applied: true}
+	h := &serverHealth{
+		db:              db,
+		latestMigration: "056_example",
+		cacheTTL:        time.Minute,
+	}
+
+	resp1, status1 := h.readiness(context.Background())
+	resp2, status2 := h.readiness(context.Background())
+
+	if status1 != http.StatusOK || status2 != http.StatusOK {
+		t.Fatalf("expected cached readiness status 200, got %d and %d", status1, status2)
+	}
+	if resp1.Status != "ok" || resp2.Status != "ok" {
+		t.Fatalf("expected cached readiness status ok, got %q and %q", resp1.Status, resp2.Status)
+	}
+	if got := db.pingCalls.Load(); got != 1 {
+		t.Fatalf("Ping calls = %d, want 1", got)
+	}
+	if got := db.queryCalls.Load(); got != 1 {
+		t.Fatalf("QueryRow calls = %d, want 1", got)
+	}
+}

server/cmd/server/integration_test.go

@@ -222,6 +222,39 @@ func TestHealth(t *testing.T) {
 	}
 }
 
+func TestReadinessEndpoints(t *testing.T) {
+	for _, path := range []string{"/readyz", "/healthz"} {
+		t.Run(path, func(t *testing.T) {
+			resp, err := http.Get(testServer.URL + path)
+			if err != nil {
+				t.Fatalf("readiness check failed: %v", err)
+			}
+			defer resp.Body.Close()
+
+			if resp.StatusCode != http.StatusOK {
+				t.Fatalf("expected 200, got %d", resp.StatusCode)
+			}
+
+			var result struct {
+				Status string            `json:"status"`
+				Checks map[string]string `json:"checks"`
+			}
+			if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+				t.Fatalf("decode response: %v", err)
+			}
+			if result.Status != "ok" {
+				t.Fatalf("expected status ok, got %s", result.Status)
+			}
+			if result.Checks["db"] != "ok" {
+				t.Fatalf("expected db check ok, got %s", result.Checks["db"])
+			}
+			if result.Checks["migrations"] != "ok" {
+				t.Fatalf("expected migrations check ok, got %s", result.Checks["migrations"])
+			}
+		})
+	}
+}
+
 func TestConfigRouteIsPublic(t *testing.T) {
 	resp, err := http.Get(testServer.URL + "/api/config")
 	if err != nil {

server/cmd/server/router.go

@@ -88,6 +88,7 @@ func NewRouter(pool *pgxpool.Pool, hub *realtime.Hub, bus *events.Bus, analytics
 		h.LocalSkillListStore = handler.NewRedisLocalSkillListStore(rdb)
 		h.LocalSkillImportStore = handler.NewRedisLocalSkillImportStore(rdb)
 	}
+	health := newServerHealth(pool)
 
 	r := chi.NewRouter()
 
@@ -110,11 +111,10 @@ func NewRouter(pool *pgxpool.Pool, hub *realtime.Hub, bus *events.Bus, analytics
 		MaxAge:           300,
 	}))
 
-	// Health check
-	r.Get("/health", func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "application/json")
-		w.Write([]byte(`{"status":"ok"}`))
-	})
+	// Health / readiness checks
+	r.Get("/health", health.liveHandler)
+	r.Get("/readyz", health.readyHandler)
+	r.Get("/healthz", health.readyHandler)
 
 	// Realtime subsystem metrics — connection counts, slow-client evictions,
 	// and per-event-type send QPS counters. Exposed as JSON so it can be

server/internal/handler/workspace_reserved_slugs.go

@@ -84,10 +84,13 @@ var reservedSlugs = map[string]bool{
 	"tokens":   true,
 	"cli":      true,
 
-	// Backend ops / observability. `/health` and `/ws` exist on the backend
+	// Backend ops / observability. `/health`, `/readyz`, `/healthz`, and `/ws`
+	// exist on the backend
 	// host; reserving them on the workspace slug space prevents naming
 	// confusion if/when these paths are ever proxied through the web origin.
 	"health":  true,
+	"readyz":  true,
+	"healthz": true,
 	"ws":      true,
 	"metrics": true,
 	"ping":    true,

server/internal/middleware/request_logger.go

@@ -12,7 +12,7 @@ import (
 // It replaces Chi's built-in chimw.Logger with colored, structured output.
 func RequestLogger(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// Skip noisy endpoints.
+		// Skip the hot liveness endpoint to keep logs readable.
 		if r.URL.Path == "/health" {
 			next.ServeHTTP(w, r)
 			return

server/internal/migrations/migrations.go

@@ -0,0 +1,90 @@
+package migrations
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+const maxSearchDepth = 4
+
+var candidateLeaves = []string{
+	"migrations",
+	filepath.Join("server", "migrations"),
+}
+
+// ResolveDir returns the first migrations directory that exists from the
+// current working directory.
+func ResolveDir() (string, error) {
+	seen := make(map[string]bool)
+	for _, root := range searchRoots() {
+		base := root
+		for range maxSearchDepth + 1 {
+			for _, leaf := range candidateLeaves {
+				dir := filepath.Clean(filepath.Join(base, leaf))
+				if seen[dir] {
+					continue
+				}
+				seen[dir] = true
+				info, err := os.Stat(dir)
+				if err == nil && info.IsDir() {
+					return dir, nil
+				}
+			}
+			base = filepath.Join(base, "..")
+		}
+	}
+	return "", fmt.Errorf("migrations directory not found")
+}
+
+func searchRoots() []string {
+	roots := []string{"."}
+	if exe, err := os.Executable(); err == nil {
+		roots = append(roots, filepath.Dir(exe))
+	}
+	return roots
+}
+
+// Files returns sorted migration files for the given direction ("up" or
+// "down").
+func Files(direction string) ([]string, error) {
+	dir, err := ResolveDir()
+	if err != nil {
+		return nil, err
+	}
+
+	suffix := "." + direction + ".sql"
+	files, err := filepath.Glob(filepath.Join(dir, "*"+suffix))
+	if err != nil {
+		return nil, err
+	}
+
+	if direction == "down" {
+		sort.Sort(sort.Reverse(sort.StringSlice(files)))
+	} else {
+		sort.Strings(files)
+	}
+	return files, nil
+}
+
+// LatestVersion returns the latest "up" migration version found on disk.
+func LatestVersion() (string, error) {
+	files, err := Files("up")
+	if err != nil {
+		return "", err
+	}
+	if len(files) == 0 {
+		return "", fmt.Errorf("no up migrations found")
+	}
+	return ExtractVersion(files[len(files)-1]), nil
+}
+
+// ExtractVersion strips the .up.sql / .down.sql suffix from a migration file.
+func ExtractVersion(filename string) string {
+	base := filepath.Base(filename)
+	base = strings.TrimSuffix(base, ".up.sql")
+	base = strings.TrimSuffix(base, ".down.sql")
+	return base
+}