diff --git a/SELF_HOSTING_ADVANCED.md b/SELF_HOSTING_ADVANCED.md index aa840257e..7d86a5624 100644 --- a/SELF_HOSTING_ADVANCED.md +++ b/SELF_HOSTING_ADVANCED.md @@ -290,14 +290,23 @@ HTTP requests (issues, comments, uploads) work on LAN out of the box — Next.js ## Health Check -The backend exposes a health check endpoint: +The backend exposes public health endpoints: -``` +```text GET /health → {"status":"ok"} + +GET /readyz +→ {"status":"ok","checks":{"db":"ok","migrations":"ok"}} + +GET /healthz +→ same response as /readyz ``` -Use this for load balancer health checks or monitoring. +Use `/health` for basic liveness / reachability checks. Use `/readyz` for +dependency-aware readiness probes and external monitoring that should fail when +the database is unavailable or migrations are not fully applied. `/healthz` is +kept as an alias for operator familiarity. ## Upgrading diff --git a/SELF_HOSTING_AI.md b/SELF_HOSTING_AI.md index dba17ddc7..2c534c415 100644 --- a/SELF_HOSTING_AI.md +++ b/SELF_HOSTING_AI.md @@ -73,4 +73,4 @@ If the default ports (8080/3000) are in use: - **Backend not ready:** `docker compose -f docker-compose.selfhost.yml logs backend` - **Frontend not ready:** `docker compose -f docker-compose.selfhost.yml logs frontend` - **Daemon issues:** `multica daemon logs` -- **Health check:** `curl http://localhost:8080/health` +- **Health checks:** `curl http://localhost:8080/health` for liveness, `curl http://localhost:8080/readyz` for dependency-aware readiness diff --git a/apps/docs/content/docs/getting-started/self-hosting.zh.mdx b/apps/docs/content/docs/getting-started/self-hosting.zh.mdx index c6de2db3d..96700ba00 100644 --- a/apps/docs/content/docs/getting-started/self-hosting.zh.mdx +++ b/apps/docs/content/docs/getting-started/self-hosting.zh.mdx @@ -408,14 +408,23 @@ NEXT_PUBLIC_WS_URL=wss://api.example.com/ws ## Health Check -The backend exposes a health check endpoint: +The backend exposes public health endpoints: -``` +```text GET /health → {"status":"ok"} + +GET /readyz +→ {"status":"ok","checks":{"db":"ok","migrations":"ok"}} + +GET /healthz +→ same response as /readyz ``` -Use this for load balancer health checks or monitoring. +Use `/health` for basic liveness / reachability checks. Use `/readyz` for +dependency-aware readiness probes and external monitoring that should fail when +the database is unavailable or migrations are not fully applied. `/healthz` is +kept as an alias for operator familiarity. ## Upgrading diff --git a/apps/docs/content/docs/self-host-quickstart.mdx b/apps/docs/content/docs/self-host-quickstart.mdx index a33dde432..e5be0cf43 100644 --- a/apps/docs/content/docs/self-host-quickstart.mdx +++ b/apps/docs/content/docs/self-host-quickstart.mdx @@ -31,6 +31,9 @@ make selfhost 3. Bring up every service using `docker-compose.selfhost.yml` 4. Wait until the backend's `/health` endpoint is ready +For ongoing production probes after startup, use `/readyz` when you want the +check to fail on database or migration problems. + The backend container **runs database migrations automatically** on startup (`docker/entrypoint.sh` runs `./migrate up` before the server starts) — you'll see the migration output in the backend logs. Version upgrades are handled the same way. diff --git a/apps/docs/content/docs/self-host-quickstart.zh.mdx b/apps/docs/content/docs/self-host-quickstart.zh.mdx index c9644249f..b333fd8eb 100644 --- a/apps/docs/content/docs/self-host-quickstart.zh.mdx +++ b/apps/docs/content/docs/self-host-quickstart.zh.mdx @@ -31,6 +31,8 @@ make selfhost 3. 用 `docker-compose.selfhost.yml` 启动全部服务 4. 等后端 `/health` 端点准备就绪 +如果是启动完成后的生产探针，想让数据库或 migration 异常也体现为失败，请改用 `/readyz`。 + 后端容器启动时会**自动跑数据库 migration**（`docker/entrypoint.sh` 在启动 server 前执行 `./migrate up`）——你会在 backend 日志里看到 migration 输出。升级版本时同样自动处理。 diff --git a/apps/docs/content/docs/troubleshooting.mdx b/apps/docs/content/docs/troubleshooting.mdx index 7172057e0..2b0a0a9be 100644 --- a/apps/docs/content/docs/troubleshooting.mdx +++ b/apps/docs/content/docs/troubleshooting.mdx @@ -25,6 +25,7 @@ Look up issues by symptom. Each entry gives you **symptom / likely causes / how multica daemon logs --lines 100 # look for daemon-side errors echo $MULTICA_SERVER_URL # confirm the address is set curl -i http://:8080/health # hit the server directly +curl -i http://:8080/readyz # include DB + migration readiness cat ~/.multica/config.json # verify api_token exists multica workspace list # confirm you're a member of the target workspace ``` diff --git a/apps/docs/content/docs/troubleshooting.zh.mdx b/apps/docs/content/docs/troubleshooting.zh.mdx index 6fb0d3d88..5b3390e34 100644 --- a/apps/docs/content/docs/troubleshooting.zh.mdx +++ b/apps/docs/content/docs/troubleshooting.zh.mdx @@ -25,6 +25,7 @@ import { Callout } from "fumadocs-ui/components/callout"; multica daemon logs --lines 100 # 看 daemon 侧错误 echo $MULTICA_SERVER_URL # 确认地址配对 curl -i http://:8080/health # 直接戳 server +curl -i http://:8080/readyz # 连同 DB + migration readiness 一起检查 cat ~/.multica/config.json # 看 api_token 是否存在 multica workspace list # 确认你是目标工作区成员 ``` diff --git a/packages/core/paths/reserved-slugs.ts b/packages/core/paths/reserved-slugs.ts index a75de1936..caa2fe148 100644 --- a/packages/core/paths/reserved-slugs.ts +++ b/packages/core/paths/reserved-slugs.ts @@ -85,10 +85,13 @@ export const RESERVED_SLUGS = new Set([ "tokens", "cli", - // Backend ops / observability. `/health` and `/ws` exist on the backend + // Backend ops / observability. `/health`, `/readyz`, `/healthz`, and `/ws` + // exist on the backend // host; reserving them on the workspace slug space prevents naming // confusion if/when these paths are ever proxied through the web origin. "health", + "readyz", + "healthz", "ws", "metrics", "ping", diff --git a/server/cmd/migrate/main.go b/server/cmd/migrate/main.go index 8703542df..1b7c0fccc 100644 --- a/server/cmd/migrate/main.go +++ b/server/cmd/migrate/main.go @@ -5,12 +5,10 @@ import ( "fmt" "log/slog" "os" - "path/filepath" - "sort" - "strings" "github.com/jackc/pgx/v5/pgxpool" "github.com/multica-ai/multica/server/internal/logger" + "github.com/multica-ai/multica/server/internal/migrations" ) func main() { @@ -57,28 +55,14 @@ func main() { os.Exit(1) } - // Find migration files - migrationsDir := "migrations" - if _, err := os.Stat(migrationsDir); os.IsNotExist(err) { - // Try from server/ directory - migrationsDir = "server/migrations" - } - - suffix := "." + direction + ".sql" - files, err := filepath.Glob(filepath.Join(migrationsDir, "*"+suffix)) + files, err := migrations.Files(direction) if err != nil { slog.Error("failed to find migration files", "error", err) os.Exit(1) } - if direction == "up" { - sort.Strings(files) - } else { - sort.Sort(sort.Reverse(sort.StringSlice(files))) - } - for _, file := range files { - version := extractVersion(file) + version := migrations.ExtractVersion(file) if direction == "up" { // Check if already applied @@ -133,11 +117,3 @@ func main() { fmt.Println("Done.") } - -func extractVersion(filename string) string { - base := filepath.Base(filename) - // Remove .up.sql or .down.sql - base = strings.TrimSuffix(base, ".up.sql") - base = strings.TrimSuffix(base, ".down.sql") - return base -} diff --git a/server/cmd/server/health.go b/server/cmd/server/health.go new file mode 100644 index 000000000..18610ee73 --- /dev/null +++ b/server/cmd/server/health.go @@ -0,0 +1,161 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "sync" + "sync/atomic" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/multica-ai/multica/server/internal/migrations" +) + +const readinessQuery = `SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version = $1)` + +const readinessCacheTTL = 3 * time.Second + +type readinessDB interface { + Ping(ctx context.Context) error + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} + +type serverHealth struct { + db readinessDB + latestMigration string + initErr error + cacheTTL time.Duration + refreshMu sync.Mutex + cache atomic.Pointer[cachedReadiness] +} + +type cachedReadiness struct { + response readinessResponse + statusCode int + expiresAt time.Time +} + +type liveResponse struct { + Status string `json:"status"` +} + +type readinessResponse struct { + Status string `json:"status"` + Checks readinessChecks `json:"checks"` +} + +type readinessChecks struct { + DB string `json:"db"` + Migrations string `json:"migrations"` +} + +func newServerHealth(pool *pgxpool.Pool) *serverHealth { + latestMigration, err := migrations.LatestVersion() + return &serverHealth{ + db: pool, + latestMigration: latestMigration, + initErr: err, + cacheTTL: readinessCacheTTL, + } +} + +func (h *serverHealth) liveHandler(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, liveResponse{Status: "ok"}) +} + +func (h *serverHealth) readyHandler(w http.ResponseWriter, r *http.Request) { + resp, status := h.readiness(r.Context()) + writeJSON(w, status, resp) +} + +func (h *serverHealth) readiness(parent context.Context) (readinessResponse, int) { + if h.cacheTTL <= 0 { + return h.computeReadiness(parent) + } + + now := time.Now() + if cached := h.loadCachedReadiness(now); cached != nil { + return cached.response, cached.statusCode + } + + h.refreshMu.Lock() + defer h.refreshMu.Unlock() + + now = time.Now() + if cached := h.loadCachedReadiness(now); cached != nil { + return cached.response, cached.statusCode + } + + resp, status := h.computeReadiness(parent) + h.cache.Store(&cachedReadiness{ + response: resp, + statusCode: status, + expiresAt: now.Add(h.cacheTTL), + }) + return resp, status +} + +func (h *serverHealth) loadCachedReadiness(now time.Time) *cachedReadiness { + cached := h.cache.Load() + if cached == nil || !now.Before(cached.expiresAt) { + return nil + } + return cached +} + +func (h *serverHealth) computeReadiness(parent context.Context) (readinessResponse, int) { + resp := readinessResponse{ + Status: "ok", + Checks: readinessChecks{ + DB: "ok", + Migrations: "ok", + }, + } + + if h.db == nil { + resp.Status = "not_ready" + resp.Checks.DB = "error" + resp.Checks.Migrations = "unknown" + return resp, http.StatusServiceUnavailable + } + + ctx, cancel := context.WithTimeout(parent, 2*time.Second) + defer cancel() + + if err := h.db.Ping(ctx); err != nil { + resp.Status = "not_ready" + resp.Checks.DB = "error" + resp.Checks.Migrations = "unknown" + return resp, http.StatusServiceUnavailable + } + + if h.initErr != nil || h.latestMigration == "" { + resp.Status = "not_ready" + resp.Checks.Migrations = "error" + return resp, http.StatusServiceUnavailable + } + + var applied bool + if err := h.db.QueryRow(ctx, readinessQuery, h.latestMigration).Scan(&applied); err != nil { + resp.Status = "not_ready" + resp.Checks.Migrations = "error" + return resp, http.StatusServiceUnavailable + } + + if !applied { + resp.Status = "not_ready" + resp.Checks.Migrations = "out_of_date" + return resp, http.StatusServiceUnavailable + } + + return resp, http.StatusOK +} + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} diff --git a/server/cmd/server/health_test.go b/server/cmd/server/health_test.go new file mode 100644 index 000000000..01b9bea74 --- /dev/null +++ b/server/cmd/server/health_test.go @@ -0,0 +1,132 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/jackc/pgx/v5" +) + +type stubReadinessDB struct { + pingErr error + queryErr error + applied bool + pingCalls atomic.Int32 + queryCalls atomic.Int32 +} + +func (s *stubReadinessDB) Ping(context.Context) error { + s.pingCalls.Add(1) + return s.pingErr +} + +func (s *stubReadinessDB) QueryRow(context.Context, string, ...any) pgx.Row { + s.queryCalls.Add(1) + return stubRow{applied: s.applied, err: s.queryErr} +} + +type stubRow struct { + applied bool + err error +} + +func (r stubRow) Scan(dest ...any) error { + if r.err != nil { + return r.err + } + *(dest[0].(*bool)) = r.applied + return nil +} + +func TestServerHealthReadyHandlerDBPingFailure(t *testing.T) { + db := &stubReadinessDB{pingErr: errors.New("db unavailable")} + h := &serverHealth{ + db: db, + latestMigration: "056_example", + } + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/readyz", nil) + h.readyHandler(rec, req) + + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("expected 503, got %d", rec.Code) + } + + var resp readinessResponse + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + + if resp.Status != "not_ready" { + t.Fatalf("status = %q, want %q", resp.Status, "not_ready") + } + if resp.Checks.DB != "error" { + t.Fatalf("db check = %q, want %q", resp.Checks.DB, "error") + } + if resp.Checks.Migrations != "unknown" { + t.Fatalf("migrations check = %q, want %q", resp.Checks.Migrations, "unknown") + } +} + +func TestServerHealthReadyHandlerMigrationOutOfDate(t *testing.T) { + db := &stubReadinessDB{applied: false} + h := &serverHealth{ + db: db, + latestMigration: "056_example", + } + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/readyz", nil) + h.readyHandler(rec, req) + + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("expected 503, got %d", rec.Code) + } + + var resp readinessResponse + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + + if resp.Status != "not_ready" { + t.Fatalf("status = %q, want %q", resp.Status, "not_ready") + } + if resp.Checks.DB != "ok" { + t.Fatalf("db check = %q, want %q", resp.Checks.DB, "ok") + } + if resp.Checks.Migrations != "out_of_date" { + t.Fatalf("migrations check = %q, want %q", resp.Checks.Migrations, "out_of_date") + } +} + +func TestServerHealthReadinessCachesResult(t *testing.T) { + db := &stubReadinessDB{applied: true} + h := &serverHealth{ + db: db, + latestMigration: "056_example", + cacheTTL: time.Minute, + } + + resp1, status1 := h.readiness(context.Background()) + resp2, status2 := h.readiness(context.Background()) + + if status1 != http.StatusOK || status2 != http.StatusOK { + t.Fatalf("expected cached readiness status 200, got %d and %d", status1, status2) + } + if resp1.Status != "ok" || resp2.Status != "ok" { + t.Fatalf("expected cached readiness status ok, got %q and %q", resp1.Status, resp2.Status) + } + if got := db.pingCalls.Load(); got != 1 { + t.Fatalf("Ping calls = %d, want 1", got) + } + if got := db.queryCalls.Load(); got != 1 { + t.Fatalf("QueryRow calls = %d, want 1", got) + } +} diff --git a/server/cmd/server/integration_test.go b/server/cmd/server/integration_test.go index ce3136c99..73e608a0d 100644 --- a/server/cmd/server/integration_test.go +++ b/server/cmd/server/integration_test.go @@ -222,6 +222,39 @@ func TestHealth(t *testing.T) { } } +func TestReadinessEndpoints(t *testing.T) { + for _, path := range []string{"/readyz", "/healthz"} { + t.Run(path, func(t *testing.T) { + resp, err := http.Get(testServer.URL + path) + if err != nil { + t.Fatalf("readiness check failed: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + var result struct { + Status string `json:"status"` + Checks map[string]string `json:"checks"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + t.Fatalf("decode response: %v", err) + } + if result.Status != "ok" { + t.Fatalf("expected status ok, got %s", result.Status) + } + if result.Checks["db"] != "ok" { + t.Fatalf("expected db check ok, got %s", result.Checks["db"]) + } + if result.Checks["migrations"] != "ok" { + t.Fatalf("expected migrations check ok, got %s", result.Checks["migrations"]) + } + }) + } +} + func TestConfigRouteIsPublic(t *testing.T) { resp, err := http.Get(testServer.URL + "/api/config") if err != nil { diff --git a/server/cmd/server/router.go b/server/cmd/server/router.go index c935e908c..5fc065709 100644 --- a/server/cmd/server/router.go +++ b/server/cmd/server/router.go @@ -88,6 +88,7 @@ func NewRouter(pool *pgxpool.Pool, hub *realtime.Hub, bus *events.Bus, analytics h.LocalSkillListStore = handler.NewRedisLocalSkillListStore(rdb) h.LocalSkillImportStore = handler.NewRedisLocalSkillImportStore(rdb) } + health := newServerHealth(pool) r := chi.NewRouter() @@ -110,11 +111,10 @@ func NewRouter(pool *pgxpool.Pool, hub *realtime.Hub, bus *events.Bus, analytics MaxAge: 300, })) - // Health check - r.Get("/health", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.Write([]byte(`{"status":"ok"}`)) - }) + // Health / readiness checks + r.Get("/health", health.liveHandler) + r.Get("/readyz", health.readyHandler) + r.Get("/healthz", health.readyHandler) // Realtime subsystem metrics — connection counts, slow-client evictions, // and per-event-type send QPS counters. Exposed as JSON so it can be diff --git a/server/internal/handler/workspace_reserved_slugs.go b/server/internal/handler/workspace_reserved_slugs.go index ca25913ca..1307ded8e 100644 --- a/server/internal/handler/workspace_reserved_slugs.go +++ b/server/internal/handler/workspace_reserved_slugs.go @@ -84,10 +84,13 @@ var reservedSlugs = map[string]bool{ "tokens": true, "cli": true, - // Backend ops / observability. `/health` and `/ws` exist on the backend + // Backend ops / observability. `/health`, `/readyz`, `/healthz`, and `/ws` + // exist on the backend // host; reserving them on the workspace slug space prevents naming // confusion if/when these paths are ever proxied through the web origin. "health": true, + "readyz": true, + "healthz": true, "ws": true, "metrics": true, "ping": true, diff --git a/server/internal/middleware/request_logger.go b/server/internal/middleware/request_logger.go index b985a2a87..e1ae83d7a 100644 --- a/server/internal/middleware/request_logger.go +++ b/server/internal/middleware/request_logger.go @@ -12,7 +12,7 @@ import ( // It replaces Chi's built-in chimw.Logger with colored, structured output. func RequestLogger(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Skip noisy endpoints. + // Skip the hot liveness endpoint to keep logs readable. if r.URL.Path == "/health" { next.ServeHTTP(w, r) return diff --git a/server/internal/migrations/migrations.go b/server/internal/migrations/migrations.go new file mode 100644 index 000000000..e91b74ac1 --- /dev/null +++ b/server/internal/migrations/migrations.go @@ -0,0 +1,90 @@ +package migrations + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +const maxSearchDepth = 4 + +var candidateLeaves = []string{ + "migrations", + filepath.Join("server", "migrations"), +} + +// ResolveDir returns the first migrations directory that exists from the +// current working directory. +func ResolveDir() (string, error) { + seen := make(map[string]bool) + for _, root := range searchRoots() { + base := root + for range maxSearchDepth + 1 { + for _, leaf := range candidateLeaves { + dir := filepath.Clean(filepath.Join(base, leaf)) + if seen[dir] { + continue + } + seen[dir] = true + info, err := os.Stat(dir) + if err == nil && info.IsDir() { + return dir, nil + } + } + base = filepath.Join(base, "..") + } + } + return "", fmt.Errorf("migrations directory not found") +} + +func searchRoots() []string { + roots := []string{"."} + if exe, err := os.Executable(); err == nil { + roots = append(roots, filepath.Dir(exe)) + } + return roots +} + +// Files returns sorted migration files for the given direction ("up" or +// "down"). +func Files(direction string) ([]string, error) { + dir, err := ResolveDir() + if err != nil { + return nil, err + } + + suffix := "." + direction + ".sql" + files, err := filepath.Glob(filepath.Join(dir, "*"+suffix)) + if err != nil { + return nil, err + } + + if direction == "down" { + sort.Sort(sort.Reverse(sort.StringSlice(files))) + } else { + sort.Strings(files) + } + return files, nil +} + +// LatestVersion returns the latest "up" migration version found on disk. +func LatestVersion() (string, error) { + files, err := Files("up") + if err != nil { + return "", err + } + if len(files) == 0 { + return "", fmt.Errorf("no up migrations found") + } + return ExtractVersion(files[len(files)-1]), nil +} + +// ExtractVersion strips the .up.sql / .down.sql suffix from a migration file. +func ExtractVersion(filename string) string { + base := filepath.Base(filename) + base = strings.TrimSuffix(base, ".up.sql") + base = strings.TrimSuffix(base, ".down.sql") + return base +}