MUL-4938: support configurable shutdown hold (#5586)

* feat(server): support configurable shutdown hold

Co-authored-by: multica-agent <github@multica.ai>

* fix(server): address shutdown hold review nits

Co-authored-by: multica-agent <github@multica.ai>

---------

Co-authored-by: Eve <eve@multica-ai.local>
Co-authored-by: multica-agent <github@multica.ai>

.env.example

@@ -52,6 +52,11 @@ MULTICA_APP_URL=${FRONTEND_ORIGIN}
 # avoid Host / X-Forwarded-Host spoofing when a self-hosted reverse proxy
 # is not hardened.
 MULTICA_PUBLIC_URL=
+# Optional delay after SIGTERM/SIGINT before the existing graceful shutdown
+# begins. Accepts a non-negative Go duration such as 300s or 5m. Empty or 0
+# preserves the default behavior and starts shutdown immediately. The platform
+# termination grace period must cover this hold plus the later shutdown work.
+MULTICA_SHUTDOWN_HOLD_DURATION=
 # Comma-separated CIDR list of reverse proxies whose X-Forwarded-For /
 # X-Real-IP headers the per-IP webhook rate limiter is allowed to trust.
 # Empty (the default) means "trust no headers" — the limiter uses

apps/docs/content/docs/environment-variables.ja.mdx

@@ -19,6 +19,9 @@ import { Callout } from "fumadocs-ui/components/callout";
 | `APP_ENV` | 空 | **はい**（`production` である必要があります） |
 | `FRONTEND_ORIGIN` | 空 | **はい**（セルフホストは自身のドメインを設定する必要があります） |
 | `MULTICA_DEV_VERIFICATION_CODE` | 空 | いいえ（プロダクションでは必ず空のままにしてください） |
+| `MULTICA_SHUTDOWN_HOLD_DURATION` | `0` | いいえ。`SIGTERM`/`SIGINT` の受信後、グレースフルシャットダウンを開始する前の任意の待機時間。`300s` や `5m` などの非負の Go duration を受け付けます。空または `0` の場合は直ちにシャットダウンを開始します |
+
+Kubernetes で `MULTICA_SHUTDOWN_HOLD_DURATION` を使用する場合、`terminationGracePeriodSeconds` は hold 時間とその後のグレースフルシャットダウン時間の合計より長く設定してください。両方を同じ時間にすると、クリーンアップが完了する前に kubelet が `SIGKILL` を送信する可能性があります。
 
 <Callout type="warning">
 **プロダクションでは `MULTICA_DEV_VERIFICATION_CODE` を空のままにしてください。** 固定のローカルテストコードはデフォルトで無効になっていますが、`MULTICA_DEV_VERIFICATION_CODE=888888` で有効にすると、`APP_ENV` が production 以外の間は、コードを要求できる誰もがその固定値でサインインできてしまいます。このショートカットは `APP_ENV=production` のときには無視されます。

apps/docs/content/docs/environment-variables.ko.mdx

@@ -19,6 +19,9 @@ import { Callout } from "fumadocs-ui/components/callout";
 | `APP_ENV` | 비어 있음 | **예** (`production`이어야 함) |
 | `FRONTEND_ORIGIN` | 비어 있음 | **예** (자체 호스팅은 자체 도메인을 설정해야 함) |
 | `MULTICA_DEV_VERIFICATION_CODE` | 비어 있음 | 아니오 (프로덕션에서는 반드시 비워 두어야 함) |
+| `MULTICA_SHUTDOWN_HOLD_DURATION` | `0` | 아니오. `SIGTERM`/`SIGINT` 수신 후 정상 종료를 시작하기 전의 선택적 대기 시간입니다. `300s`, `5m` 같은 음수가 아닌 Go duration을 허용하며, 비어 있거나 `0`이면 즉시 종료를 시작합니다 |
+
+Kubernetes에서 `MULTICA_SHUTDOWN_HOLD_DURATION`을 사용할 때는 `terminationGracePeriodSeconds`를 hold 시간과 이후 정상 종료 시간의 합보다 길게 설정하세요. 두 값을 같은 시간으로 설정하면 정리가 끝나기 전에 kubelet이 `SIGKILL`을 보낼 수 있습니다.
 
 <Callout type="warning">
 **프로덕션에서는 `MULTICA_DEV_VERIFICATION_CODE`를 비워 두세요.** 고정된 로컬 테스트 코드는 기본적으로 비활성화되어 있지만, `MULTICA_DEV_VERIFICATION_CODE=888888`로 활성화하면 `APP_ENV`가 production이 아닌 동안에는 코드를 요청할 수 있는 누구나 그 고정 값으로 로그인할 수 있습니다. 이 단축 코드는 `APP_ENV=production`일 때 무시됩니다.

apps/docs/content/docs/environment-variables.mdx

@@ -19,6 +19,9 @@ These are the core variables you must think about before deploying — some have
 | `APP_ENV` | empty | **Yes** (must be `production`) |
 | `FRONTEND_ORIGIN` | empty | **Yes** (self-host must set its own domain) |
 | `MULTICA_DEV_VERIFICATION_CODE` | empty | No (must stay empty in production) |
+| `MULTICA_SHUTDOWN_HOLD_DURATION` | `0` | No. Optional delay after `SIGTERM`/`SIGINT` before graceful shutdown begins. Accepts non-negative Go duration strings such as `300s` or `5m`; empty or `0` starts shutdown immediately |
+
+When using `MULTICA_SHUTDOWN_HOLD_DURATION` under Kubernetes, set `terminationGracePeriodSeconds` longer than the hold plus the server's subsequent graceful-shutdown budget. Setting both to the same duration can make kubelet send `SIGKILL` before cleanup finishes.
 
 <Callout type="warning">
 **Keep `MULTICA_DEV_VERIFICATION_CODE` empty in production.** A fixed local test code is disabled by default, but if you opt in with `MULTICA_DEV_VERIFICATION_CODE=888888`, anyone who can request a code can sign in with that fixed value while `APP_ENV` is non-production. The shortcut is ignored when `APP_ENV=production`.

apps/docs/content/docs/environment-variables.zh.mdx

@@ -19,6 +19,9 @@ Multica 的 [自部署](/self-host-quickstart) 服务器启动时从环境变量
 | `APP_ENV` | 空 | **是**（必须 `production`）|
 | `FRONTEND_ORIGIN` | 空 | **是**（self-host 要填你自己的域名）|
 | `MULTICA_DEV_VERIFICATION_CODE` | 空 | 否（生产必须保持为空）|
+| `MULTICA_SHUTDOWN_HOLD_DURATION` | `0` | 否。收到 `SIGTERM`/`SIGINT` 后、开始优雅退出前的可选等待时间。接受 `300s`、`5m` 等非负 Go duration；留空或设为 `0` 时立即开始退出 |
+
+在 Kubernetes 中使用 `MULTICA_SHUTDOWN_HOLD_DURATION` 时，`terminationGracePeriodSeconds` 必须大于 hold 时长与后续优雅退出预算之和。如果两者配置成相同时长，kubelet 可能在清理完成前发送 `SIGKILL`。
 
 <Callout type="warning">
 **生产环境保持 `MULTICA_DEV_VERIFICATION_CODE` 为空。** 固定本地测试验证码默认关闭；如果你设置 `MULTICA_DEV_VERIFICATION_CODE=888888`，在 `APP_ENV` 非 production 时，任何能请求验证码的人都能用这个固定值登录。`APP_ENV=production` 时该快捷码会被忽略。

docker-compose.selfhost.yml

@@ -81,6 +81,7 @@ services:
       APP_ENV: ${APP_ENV:-production}
       MULTICA_DEV_VERIFICATION_CODE: ${MULTICA_DEV_VERIFICATION_CODE:-}
       MULTICA_APP_URL: ${MULTICA_APP_URL:-http://localhost:3000}
+      MULTICA_SHUTDOWN_HOLD_DURATION: ${MULTICA_SHUTDOWN_HOLD_DURATION:-}
       ALLOW_SIGNUP: ${ALLOW_SIGNUP:-true}
       ALLOWED_EMAILS: ${ALLOWED_EMAILS:-}
       ALLOWED_EMAIL_DOMAINS: ${ALLOWED_EMAIL_DOMAINS:-}

server/cmd/server/main.go

@@ -124,6 +124,41 @@ func envDuration(name string, def time.Duration) time.Duration {
 	return v
 }
 
+func envNonNegativeDuration(name string, def time.Duration) time.Duration {
+	raw := os.Getenv(name)
+	if raw == "" {
+		return def
+	}
+	v, err := time.ParseDuration(raw)
+	if err != nil || v < 0 {
+		slog.Warn("invalid env var, using default", "name", name, "value", raw, "default", def.String(), "error", err)
+		return def
+	}
+	return v
+}
+
+func holdBeforeShutdown(sig os.Signal, signals <-chan os.Signal, duration time.Duration) {
+	if duration <= 0 {
+		return
+	}
+	slog.Info("termination signal received; holding before shutdown",
+		"signal", sig.String(),
+		"duration", duration.String(),
+	)
+	timer := time.NewTimer(duration)
+	defer timer.Stop()
+
+	select {
+	case <-timer.C:
+		slog.Info("shutdown hold complete", "duration", duration.String())
+	case interruptSig := <-signals:
+		slog.Info("shutdown hold interrupted by signal",
+			"signal", interruptSig.String(),
+			"configured_duration", duration.String(),
+		)
+	}
+}
+
 func envBool(name string, def bool) bool {
 	raw := os.Getenv(name)
 	if raw == "" {
@@ -159,6 +194,7 @@ func main() {
 	if port == "" {
 		port = "8080"
 	}
+	shutdownHoldDuration := envNonNegativeDuration("MULTICA_SHUTDOWN_HOLD_DURATION", 0)
 
 	// Feature flags: loaded once at startup from MULTICA_FEATURE_FLAGS_FILE
 	// (a YAML rule set) with FF_<KEY> env overrides layered on top.
@@ -454,7 +490,11 @@ func main() {
 
 	quit := make(chan os.Signal, 1)
 	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
-	<-quit
+	sig := <-quit
+	holdBeforeShutdown(sig, quit, shutdownHoldDuration)
+	// Restore the default behavior so another signal during graceful shutdown
+	// can still terminate the process instead of being left unread in quit.
+	signal.Stop(quit)
 
 	slog.Info("shutting down server")
 	autopilotCancel()

server/cmd/server/main_test.go

@@ -2,7 +2,10 @@ package main
 
 import (
 	"os"
+	"strconv"
+	"syscall"
 	"testing"
+	"time"
 
 	"github.com/redis/go-redis/v9"
 )
@@ -133,3 +136,72 @@ func TestEnvBool(t *testing.T) {
 		})
 	}
 }
+
+func TestEnvNonNegativeDuration(t *testing.T) {
+	tests := []struct {
+		name  string
+		value string
+		def   time.Duration
+		want  time.Duration
+	}{
+		{name: "unset returns default", def: 3 * time.Second, want: 3 * time.Second},
+		{name: "empty returns default", value: "", def: 2 * time.Second, want: 2 * time.Second},
+		{name: "bare zero disables hold", value: "0", def: time.Second, want: 0},
+		{name: "zero duration disables hold", value: "0s", def: time.Second, want: 0},
+		{name: "positive duration", value: "5m", want: 5 * time.Minute},
+		{name: "invalid returns default", value: "later", def: 4 * time.Second, want: 4 * time.Second},
+		{name: "negative returns default", value: "-1s", def: 4 * time.Second, want: 4 * time.Second},
+	}
+
+	for i, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			key := "TEST_NON_NEGATIVE_DURATION_" + strconv.Itoa(i)
+			if tt.name == "unset returns default" {
+				os.Unsetenv(key)
+			} else {
+				t.Setenv(key, tt.value)
+			}
+			if got := envNonNegativeDuration(key, tt.def); got != tt.want {
+				t.Fatalf("envNonNegativeDuration(%q, %s) = %s, want %s", key, tt.def, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestHoldBeforeShutdown(t *testing.T) {
+	const hold = 10 * time.Millisecond
+	started := time.Now()
+	holdBeforeShutdown(syscall.SIGTERM, nil, hold)
+	if elapsed := time.Since(started); elapsed < hold {
+		t.Fatalf("holdBeforeShutdown returned after %s, before configured hold %s", elapsed, hold)
+	}
+}
+
+func TestHoldBeforeShutdownInterruptedBySecondSignal(t *testing.T) {
+	signals := make(chan os.Signal, 1)
+	signals <- syscall.SIGINT
+	done := make(chan struct{})
+
+	go func() {
+		holdBeforeShutdown(syscall.SIGTERM, signals, time.Minute)
+		close(done)
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("holdBeforeShutdown did not return after a second signal")
+	}
+	if len(signals) != 0 {
+		t.Fatal("holdBeforeShutdown did not consume the second signal")
+	}
+}
+
+func TestHoldBeforeShutdownDisabled(t *testing.T) {
+	signals := make(chan os.Signal, 1)
+	signals <- syscall.SIGINT
+	holdBeforeShutdown(syscall.SIGTERM, signals, 0)
+	if len(signals) != 1 {
+		t.Fatal("disabled hold should not consume another signal")
+	}
+}