From 36533bbc2b6bb85eaba7d047e512efdf3a6516f8 Mon Sep 17 00:00:00 2001 From: YYClaw <197375+yyclaw@users.noreply.github.com> Date: Thu, 23 Jul 2026 01:59:06 +0800 Subject: [PATCH] fix(test): prevent agent CLI execution in default tests (#5789) --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- CLAUDE.md | 4 + Makefile | 10 +-- scripts/agent-cli-command-names.txt | 17 ++++ scripts/check.sh | 2 +- scripts/go-test-with-agent-cli-guard.sh | 60 +++++++++++++ scripts/test-go.sh | 41 +++++++++ scripts/test-go.test.sh | 78 +++++++++++++++++ .../daemon/agent_command_names_test.go | 84 +++++++++++++++++++ .../pkg/agent/agent_test_executable_test.go | 11 +++ server/pkg/agent/grok_integration_test.go | 64 ++++++++++++++ server/pkg/agent/grok_test.go | 57 ------------- server/pkg/agent/models_test.go | 38 +++++---- .../real_agent_smoke_integration_test.go | 16 ++++ server/pkg/agent/traecli_integration_test.go | 59 +++++++++++++ server/pkg/agent/traecli_test.go | 59 ------------- 17 files changed, 458 insertions(+), 146 deletions(-) create mode 100644 scripts/agent-cli-command-names.txt create mode 100755 scripts/go-test-with-agent-cli-guard.sh create mode 100644 scripts/test-go.sh create mode 100644 scripts/test-go.test.sh create mode 100644 server/pkg/agent/agent_test_executable_test.go create mode 100644 server/pkg/agent/grok_integration_test.go create mode 100644 server/pkg/agent/real_agent_smoke_integration_test.go create mode 100644 server/pkg/agent/traecli_integration_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d98e815fc..e140756a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,7 +170,7 @@ jobs: run: cd server && go run ./cmd/migrate up - name: Test - run: cd server && go test -race ./... + run: bash scripts/test-go.sh --race windows-execenv: # The environment-preparation deadline owns a process tree, not just a Go diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f65267380..abf703431 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,7 +52,7 @@ jobs: cache-dependency-path: server/go.sum - name: Run tests - run: cd server && go test -race ./... + run: bash scripts/test-go.sh --race release: needs: verify diff --git a/CLAUDE.md b/CLAUDE.md index e05fd21bd..cb3c48490 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,6 +206,10 @@ Rules: - Mock `@multica/core/api` for API calls. - E2E tests should use `TestApiClient` for setup/teardown. - Prefer writing the failing test in the correct package before implementation when the change is behavioral. +- Default tests must never resolve or execute user-installed agent CLIs. Pass a test-created fake executable path or a test-created missing path to agent subprocess code. +- Real-agent smoke tests belong behind the `agentintegration` build tag and must check `MULTICA_RUN_REAL_AGENT_SMOKE=1` before executable lookup or account access. +- Run an explicitly authorized real-agent smoke test with `(cd server && MULTICA_RUN_REAL_AGENT_SMOKE=1 go test -tags=agentintegration ./pkg/agent -run '' -count=1 -v)`. This command may access an authenticated account and consume quota. +- When adding a default agent command, add it to `scripts/agent-cli-command-names.txt`; the normal Linux/macOS test entry points fail on ambient agent CLI execution. ## Verification diff --git a/Makefile b/Makefile index b15164708..d6cd26839 100644 --- a/Makefile +++ b/Makefile @@ -322,15 +322,7 @@ test: ## Run Go tests after ensuring the target DB exists and migrations are app $(REQUIRE_ENV) @bash scripts/ensure-postgres.sh "$(ENV_FILE)" cd server && go run ./cmd/migrate up - # pkg/agent spawns many subprocess-backed tests with hard 5s deadlines; under - # -race on high-core machines the default GOMAXPROCS fan-out starves their - # parent event loops so they miss the deadline. Run the rest of the suite at - # full concurrency and cap only pkg/agent's package- and within-package - # parallelism, keeping those tests within budget without slowing the whole suite. - # Build the package list via explicit assignments so a go list failure - # fails this target instead of being swallowed by command substitution. - cd server && pkgs="$$(go list ./...)" && pkgs="$$(printf '%s\n' "$$pkgs" | grep -vE '/pkg/agent(/|$$)')" && go test -race $$pkgs - cd server && go test -race -p 2 -parallel 2 ./pkg/agent/... + bash scripts/test-go.sh --race # Database ##@ Database diff --git a/scripts/agent-cli-command-names.txt b/scripts/agent-cli-command-names.txt new file mode 100644 index 000000000..3645659db --- /dev/null +++ b/scripts/agent-cli-command-names.txt @@ -0,0 +1,17 @@ +agy +claude +codebuddy +codex +copilot +cursor-agent +deveco +grok +hermes +kimi +kiro-cli +openclaw +opencode +pi +qwen +qodercli +traecli diff --git a/scripts/check.sh b/scripts/check.sh index 23421767c..894e1b7b6 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -97,7 +97,7 @@ echo "" echo "==> [3/5] Go tests..." echo "==> Running database migrations..." (cd server && go run ./cmd/migrate up) || { EXIT_CODE=1; exit 1; } -(cd server && go test ./...) || { EXIT_CODE=1; exit 1; } +bash scripts/test-go.sh || { EXIT_CODE=1; exit 1; } # -------------------------------------------------------------------------- # Step 4: Start services for E2E (only if not already running) diff --git a/scripts/go-test-with-agent-cli-guard.sh b/scripts/go-test-with-agent-cli-guard.sh new file mode 100755 index 000000000..22be56be1 --- /dev/null +++ b/scripts/go-test-with-agent-cli-guard.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +NAMES_FILE="$SCRIPT_DIR/agent-cli-command-names.txt" +GUARD_DIR=$(mktemp -d "${TMPDIR:-/tmp}/multica-agent-cli-guard.XXXXXX") +BIN_DIR="$GUARD_DIR/bin" +MARKER_FILE="$GUARD_DIR/invocations.log" + +cleanup() { + rm -rf "$GUARD_DIR" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +mkdir -p "$BIN_DIR" + +while IFS= read -r name || [ -n "$name" ]; do + case "$name" in + ""|'#'*) continue ;; + *[!A-Za-z0-9._-]*) + echo "invalid agent CLI command name in $NAMES_FILE: $name" >&2 + exit 2 + ;; + esac + sentinel="$BIN_DIR/$name" + cat >"$sentinel" <<'EOF' +#!/bin/sh +line=$(basename "$0") +if [ "$#" -gt 0 ]; then + line="$line [arguments redacted]" +fi +printf '%s\n' "$line" >>"$MULTICA_AGENT_CLI_GUARD_MARKER" +exit 126 +EOF + chmod 755 "$sentinel" +done <"$NAMES_FILE" + +if [ "${1:-}" = "--" ]; then + shift +fi +if [ "$#" -eq 0 ]; then + echo "usage: $0 [--] command [args...]" >&2 + exit 2 +fi + +set +e +PATH="$BIN_DIR:$PATH" MULTICA_AGENT_CLI_GUARD_MARKER="$MARKER_FILE" "$@" +command_status=$? +set -e + +if [ -s "$MARKER_FILE" ]; then + while IFS= read -r invocation; do + echo "unexpected agent CLI invocation: $invocation" >&2 + done <"$MARKER_FILE" + exit 1 +fi + +exit "$command_status" diff --git a/scripts/test-go.sh b/scripts/test-go.sh new file mode 100644 index 000000000..ee88c144f --- /dev/null +++ b/scripts/test-go.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd) +GUARD_SCRIPT="$SCRIPT_DIR/go-test-with-agent-cli-guard.sh" + +usage() { + echo "usage: $0 [--race]" >&2 +} + +go_test_args=(test) +case "$#" in + 0) ;; + 1) + if [ "$1" != "--race" ]; then + usage + exit 2 + fi + go_test_args+=(-race) + ;; + *) + usage + exit 2 + ;; +esac + +cd "$REPO_ROOT/server" +packages=$(go list ./...) +regular_packages=() +for package in $packages; do + case "$package" in + */pkg/agent|*/pkg/agent/*) ;; + *) regular_packages+=("$package") ;; + esac +done + +"$GUARD_SCRIPT" -- go "${go_test_args[@]}" "${regular_packages[@]}" +# Subprocess-backed agent tests have hard deadlines. Limit both package and +# within-package parallelism so race builds do not starve their parent loops. +"$GUARD_SCRIPT" -- go "${go_test_args[@]}" -p 2 -parallel 2 ./pkg/agent/... diff --git a/scripts/test-go.test.sh b/scripts/test-go.test.sh new file mode 100644 index 000000000..10a48adb8 --- /dev/null +++ b/scripts/test-go.test.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +TEST_DIR=$(mktemp -d "${TMPDIR:-/tmp}/multica-test-go.XXXXXX") +BIN_DIR="$TEST_DIR/bin" +CALLS_FILE="$TEST_DIR/go-calls.log" +OUTPUT_FILE="$TEST_DIR/output.log" + +cleanup() { + rm -rf "$TEST_DIR" +} +trap cleanup EXIT + +mkdir -p "$BIN_DIR" +export MULTICA_TEST_GO_CALLS="$CALLS_FILE" + +cat >"$BIN_DIR/go" <<'EOF' +#!/usr/bin/env bash +set -eu + +case "${1:-}" in + list) + if [ "$#" -ne 2 ] || [ "$2" != "./..." ]; then + echo "unexpected go list arguments: $*" >&2 + exit 2 + fi + printf '%s\n' \ + github.com/multica-ai/multica/server \ + github.com/multica-ai/multica/server/internal/daemon \ + github.com/multica-ai/multica/server/pkg/agent \ + github.com/multica-ai/multica/server/pkg/agent/internal/testutil + ;; + test) + printf '%s\n' "$*" >>"$MULTICA_TEST_GO_CALLS" + ;; + *) + echo "unexpected go command: $*" >&2 + exit 2 + ;; +esac +EOF +chmod 755 "$BIN_DIR/go" + +PATH="$BIN_DIR:$PATH" bash "$SCRIPT_DIR/test-go.sh" --race + +expected_calls='test -race github.com/multica-ai/multica/server github.com/multica-ai/multica/server/internal/daemon +test -race -p 2 -parallel 2 ./pkg/agent/...' +actual_calls=$(cat "$CALLS_FILE") +if [ "$actual_calls" != "$expected_calls" ]; then + echo "unexpected go test calls:" >&2 + printf '%s\n' "$actual_calls" >&2 + exit 1 +fi + +: >"$CALLS_FILE" +set +e +PATH="$BIN_DIR:$PATH" bash "$SCRIPT_DIR/test-go.sh" --unknown >"$OUTPUT_FILE" 2>&1 +status=$? +set -e + +if [ "$status" -ne 2 ]; then + echo "unknown option returned status $status, want 2" >&2 + cat "$OUTPUT_FILE" >&2 + exit 1 +fi +if [ -s "$CALLS_FILE" ]; then + echo "unknown option invoked go:" >&2 + cat "$CALLS_FILE" >&2 + exit 1 +fi +if ! grep -q '^usage: .*test-go.sh \[--race\]$' "$OUTPUT_FILE"; then + echo "unknown option did not print usage" >&2 + cat "$OUTPUT_FILE" >&2 + exit 1 +fi + +echo "test-go.test.sh: PASS" diff --git a/server/internal/daemon/agent_command_names_test.go b/server/internal/daemon/agent_command_names_test.go index 000afc6ac..ac966a473 100644 --- a/server/internal/daemon/agent_command_names_test.go +++ b/server/internal/daemon/agent_command_names_test.go @@ -4,7 +4,12 @@ import ( "go/ast" "go/parser" "go/token" + "os" + "os/exec" + "path/filepath" + "runtime" "sort" + "strings" "testing" ) @@ -63,3 +68,82 @@ func TestDefaultAgentCommandNamesCoversAllProbes(t *testing.T) { "add them so GUI-launched daemons can resolve these agents via the login shell", missing) } } + +func TestAgentCLIGuardCoversDefaultCommands(t *testing.T) { + data, err := os.ReadFile(filepath.Join("..", "..", "..", "scripts", "agent-cli-command-names.txt")) + if err != nil { + t.Fatalf("read agent CLI guard names: %v", err) + } + guarded := map[string]bool{} + for lineNumber, line := range strings.Split(string(data), "\n") { + if line != strings.TrimSpace(line) { + t.Fatalf("agent CLI guard name on line %d has surrounding whitespace", lineNumber+1) + } + if line != "" && !strings.HasPrefix(line, "#") { + if !isSafeAgentCLICommandName(line) { + t.Fatalf("agent CLI guard name on line %d contains unsafe characters: %q", lineNumber+1, line) + } + guarded[line] = true + } + } + for _, name := range defaultAgentCommandNames { + if !guarded[name] { + t.Errorf("default agent command %q is not covered by the test guard", name) + } + } + if !guarded["qodercli"] { + t.Error("default qoder command \"qodercli\" is not covered by the test guard") + } +} + +func isSafeAgentCLICommandName(name string) bool { + for _, char := range name { + if (char >= 'a' && char <= 'z') || + (char >= 'A' && char <= 'Z') || + (char >= '0' && char <= '9') || + char == '.' || char == '_' || char == '-' { + continue + } + return false + } + return name != "" +} + +func TestAgentCLIGuardDetectsSwallowedFailure(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the full guarded backend suite runs on Linux/macOS") + } + script := filepath.Join("..", "..", "..", "scripts", "go-test-with-agent-cli-guard.sh") + cmd := exec.Command(script, "--", "/bin/sh", "-c", "claude --version --token super-secret >/dev/null 2>&1 || true") + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("guard succeeded after a swallowed agent CLI failure: %s", out) + } + if !strings.Contains(string(out), "unexpected agent CLI invocation: claude [arguments redacted]") { + t.Fatalf("guard diagnostic missing invocation: %s", out) + } + if strings.Contains(string(out), "super-secret") { + t.Fatalf("guard diagnostic exposed command arguments: %s", out) + } +} + +func TestAgentCLIGuardFailsClosedWhenSetupFails(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the full guarded backend suite runs on Linux/macOS") + } + invalidTempDir := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(invalidTempDir, []byte("fixture"), 0o600); err != nil { + t.Fatalf("write invalid temp directory fixture: %v", err) + } + executedMarker := filepath.Join(t.TempDir(), "executed") + script := filepath.Join("..", "..", "..", "scripts", "go-test-with-agent-cli-guard.sh") + cmd := exec.Command(script, "--", "/bin/sh", "-c", "printf ran >\"$1\"", "sh", executedMarker) + cmd.Env = append(os.Environ(), "TMPDIR="+invalidTempDir) + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("guard succeeded after setup failure: %s", out) + } + if _, statErr := os.Stat(executedMarker); !os.IsNotExist(statErr) { + t.Fatalf("wrapped command ran after guard setup failure: %v", statErr) + } +} diff --git a/server/pkg/agent/agent_test_executable_test.go b/server/pkg/agent/agent_test_executable_test.go new file mode 100644 index 000000000..1e584a127 --- /dev/null +++ b/server/pkg/agent/agent_test_executable_test.go @@ -0,0 +1,11 @@ +package agent + +import ( + "path/filepath" + "testing" +) + +func missingAgentExecutable(tb testing.TB, name string) string { + tb.Helper() + return filepath.Join(tb.TempDir(), name) +} diff --git a/server/pkg/agent/grok_integration_test.go b/server/pkg/agent/grok_integration_test.go new file mode 100644 index 000000000..bab70723a --- /dev/null +++ b/server/pkg/agent/grok_integration_test.go @@ -0,0 +1,64 @@ +//go:build agentintegration + +package agent + +import ( + "context" + "log/slog" + "os/exec" + "strings" + "testing" + "time" +) + +// TestGrokRealACPSmoke drives the real `grok agent stdio` binary end-to-end. +func TestGrokRealACPSmoke(t *testing.T) { + requireRealAgentSmoke(t) + if testing.Short() { + t.Skip("skipping real-binary smoke test in -short mode") + } + path, err := exec.LookPath("grok") + if err != nil { + t.Skip("grok not on PATH; skipping real-binary smoke test") + } + if version, err := exec.Command(path, "--version").CombinedOutput(); err == nil { + t.Logf("grok CLI version: %s", strings.TrimSpace(string(version))) + } else { + t.Logf("grok CLI version unavailable: %v (%s)", err, strings.TrimSpace(string(version))) + } + + backend, err := New("grok", Config{ExecutablePath: path, Logger: slog.Default()}) + if err != nil { + t.Fatalf("new grok backend: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + session, err := backend.Execute(ctx, "Reply with exactly one word: pong. Do not use any tools.", ExecOptions{ + Cwd: t.TempDir(), + Timeout: 80 * time.Second, + }) + if err != nil { + t.Fatalf("execute: %v", err) + } + go func() { + for range session.Messages { + } + }() + + select { + case result := <-session.Result: + if result.Status != "completed" { + t.Fatalf("real grok run did not complete: status=%q error=%q", result.Status, result.Error) + } + if !strings.Contains(strings.ToLower(result.Output), "pong") { + t.Fatalf("expected real grok output to contain 'pong', got %q", result.Output) + } + if result.SessionID == "" { + t.Error("expected a non-empty session id from real grok") + } + t.Logf("real grok smoke OK: session=%s output=%q", result.SessionID, result.Output) + case <-time.After(90 * time.Second): + t.Fatal("timeout waiting for real grok result") + } +} diff --git a/server/pkg/agent/grok_test.go b/server/pkg/agent/grok_test.go index 3e85d31d2..877c12dd7 100644 --- a/server/pkg/agent/grok_test.go +++ b/server/pkg/agent/grok_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "log/slog" "os" - "os/exec" "path/filepath" "strings" "testing" @@ -835,59 +834,3 @@ func TestGrokIsKnownThinkingValue(t *testing.T) { } } } - -// TestGrokRealACPSmoke drives the REAL `grok agent stdio` binary end-to-end -// when it is installed and authenticated. Skipped automatically when grok is -// not on PATH or the session cannot be created, so CI stays green. -func TestGrokRealACPSmoke(t *testing.T) { - if testing.Short() { - t.Skip("skipping real-binary smoke test in -short mode") - } - path, err := exec.LookPath("grok") - if err != nil { - t.Skip("grok not on PATH; skipping real-binary smoke test") - } - if version, err := exec.Command(path, "--version").CombinedOutput(); err == nil { - t.Logf("grok CLI version: %s", strings.TrimSpace(string(version))) - } else { - t.Logf("grok CLI version unavailable: %v (%s)", err, strings.TrimSpace(string(version))) - } - - backend, err := New("grok", Config{ExecutablePath: path, Logger: slog.Default()}) - if err != nil { - t.Fatalf("new grok backend: %v", err) - } - ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) - defer cancel() - - session, err := backend.Execute(ctx, "Reply with exactly one word: pong. Do not use any tools.", ExecOptions{ - Cwd: t.TempDir(), - Timeout: 80 * time.Second, - }) - if err != nil { - t.Fatalf("execute: %v", err) - } - go func() { - for range session.Messages { - } - }() - - select { - case result := <-session.Result: - if result.Status == "failed" && (strings.Contains(result.Error, "session/new") || strings.Contains(result.Error, "initialize")) { - t.Skipf("grok not authenticated or ACP unavailable: %v", result.Error) - } - if result.Status != "completed" { - t.Fatalf("real grok run did not complete: status=%q error=%q", result.Status, result.Error) - } - if !strings.Contains(strings.ToLower(result.Output), "pong") { - t.Fatalf("expected real grok output to contain 'pong', got %q", result.Output) - } - if result.SessionID == "" { - t.Error("expected a non-empty session id from real grok") - } - t.Logf("real grok smoke OK: session=%s output=%q", result.SessionID, result.Output) - case <-time.After(90 * time.Second): - t.Fatal("timeout waiting for real grok result") - } -} diff --git a/server/pkg/agent/models_test.go b/server/pkg/agent/models_test.go index 9f3f5fab2..49cbc6c90 100644 --- a/server/pkg/agent/models_test.go +++ b/server/pkg/agent/models_test.go @@ -9,28 +9,30 @@ import ( "testing" ) -func TestListModelsStaticProviders(t *testing.T) { - ctx := context.Background() - for _, provider := range []string{"claude", "codex", "cursor"} { - got, err := ListModels(ctx, provider, "") - if err != nil { - t.Fatalf("ListModels(%q) error: %v", provider, err) +func TestStaticModelCatalogsHaveValidEntries(t *testing.T) { + t.Parallel() + catalogs := map[string][]Model{ + "claude": claudeStaticModels(), + "codex": codexStaticModels(), + "cursor": cursorStaticModels(), + } + for provider, models := range catalogs { + if len(models) == 0 { + t.Errorf("%s static catalog returned no models", provider) } - if len(got) == 0 { - t.Errorf("ListModels(%q) returned no models", provider) - } - for i, m := range got { - if m.ID == "" { - t.Errorf("ListModels(%q)[%d] has empty ID", provider, i) + for i, model := range models { + if model.ID == "" { + t.Errorf("%s static catalog[%d] has empty ID", provider, i) } - if m.Label == "" { - t.Errorf("ListModels(%q)[%d] has empty Label", provider, i) + if model.Label == "" { + t.Errorf("%s static catalog[%d] has empty Label", provider, i) } } } } func TestListModelsQwenUsesRuntimeDefaultAndManualEntry(t *testing.T) { + // Qwen returns its manual-entry catalog without resolving or executing a CLI. got, err := ListModels(context.Background(), "qwen", "") if err != nil { t.Fatalf("ListModels(qwen) error: %v", err) @@ -51,7 +53,7 @@ func TestListModelsCopilotFallsBackToStatic(t *testing.T) { delete(modelCache, "copilot") modelCacheMu.Unlock() - got, err := ListModels(ctx, "copilot", "/nonexistent/copilot-cli") + got, err := ListModels(ctx, "copilot", missingAgentExecutable(t, "copilot")) if err != nil { t.Fatalf("ListModels(copilot) error: %v", err) } @@ -321,7 +323,7 @@ func TestListModelsHermesWithoutBinary(t *testing.T) { delete(modelCache, "hermes") modelCacheMu.Unlock() - got, err := ListModels(ctx, "hermes", "/nonexistent/hermes") + got, err := ListModels(ctx, "hermes", missingAgentExecutable(t, "hermes")) if err != nil { t.Fatalf("ListModels(hermes) error: %v", err) } @@ -336,7 +338,7 @@ func TestListModelsKiroWithoutBinary(t *testing.T) { delete(modelCache, "kiro") modelCacheMu.Unlock() - got, err := ListModels(ctx, "kiro", "/nonexistent/kiro-cli") + got, err := ListModels(ctx, "kiro", missingAgentExecutable(t, "kiro-cli")) if err != nil { t.Fatalf("ListModels(kiro) error: %v", err) } @@ -351,7 +353,7 @@ func TestListModelsQoderWithoutBinary(t *testing.T) { delete(modelCache, "qoder") modelCacheMu.Unlock() - got, err := ListModels(ctx, "qoder", "/nonexistent/qodercli") + got, err := ListModels(ctx, "qoder", missingAgentExecutable(t, "qodercli")) if err != nil { t.Fatalf("ListModels(qoder) error: %v", err) } diff --git a/server/pkg/agent/real_agent_smoke_integration_test.go b/server/pkg/agent/real_agent_smoke_integration_test.go new file mode 100644 index 000000000..aa7a8fe70 --- /dev/null +++ b/server/pkg/agent/real_agent_smoke_integration_test.go @@ -0,0 +1,16 @@ +//go:build agentintegration + +package agent + +import ( + "os" + "testing" +) + +func requireRealAgentSmoke(t *testing.T) { + t.Helper() + if os.Getenv("MULTICA_RUN_REAL_AGENT_SMOKE") != "1" { + t.Skip("set MULTICA_RUN_REAL_AGENT_SMOKE=1 to allow real agent CLI and account access") + } + t.Log("REAL AGENT SMOKE TEST: this test may access an authenticated account and consume quota") +} diff --git a/server/pkg/agent/traecli_integration_test.go b/server/pkg/agent/traecli_integration_test.go new file mode 100644 index 000000000..adeca53c1 --- /dev/null +++ b/server/pkg/agent/traecli_integration_test.go @@ -0,0 +1,59 @@ +//go:build agentintegration + +package agent + +import ( + "context" + "log/slog" + "os/exec" + "strings" + "testing" + "time" +) + +// TestTraecliRealACPSmoke drives the real `traecli acp serve` binary end-to-end. +func TestTraecliRealACPSmoke(t *testing.T) { + requireRealAgentSmoke(t) + if testing.Short() { + t.Skip("skipping real-binary smoke test in -short mode") + } + path, err := exec.LookPath("traecli") + if err != nil { + t.Skip("traecli not on PATH; skipping real-binary smoke test") + } + + backend, err := New("traecli", Config{ExecutablePath: path, Logger: slog.Default()}) + if err != nil { + t.Fatalf("new traecli backend: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + session, err := backend.Execute(ctx, "Reply with exactly one word: pong. Do not use any tools.", ExecOptions{ + Cwd: t.TempDir(), + Timeout: 80 * time.Second, + }) + if err != nil { + t.Fatalf("execute: %v", err) + } + go func() { + for range session.Messages { + } + }() + + select { + case result := <-session.Result: + if result.Status != "completed" { + t.Fatalf("real traecli run did not complete: status=%q error=%q", result.Status, result.Error) + } + if !strings.Contains(strings.ToLower(result.Output), "pong") { + t.Fatalf("expected real traecli output to contain 'pong', got %q", result.Output) + } + if result.SessionID == "" { + t.Error("expected a non-empty session id from real traecli") + } + t.Logf("real traecli smoke OK: session=%s output=%q", result.SessionID, result.Output) + case <-time.After(90 * time.Second): + t.Fatal("timeout waiting for real traecli result") + } +} diff --git a/server/pkg/agent/traecli_test.go b/server/pkg/agent/traecli_test.go index 79ec768c5..6528bffe4 100644 --- a/server/pkg/agent/traecli_test.go +++ b/server/pkg/agent/traecli_test.go @@ -3,7 +3,6 @@ package agent import ( "context" "os" - "os/exec" "path/filepath" "strings" "testing" @@ -288,61 +287,3 @@ func TestTraecliUsesSessionLoadForResume(t *testing.T) { t.Fatalf("traecli must use session/load (loadSession:true), not session/resume:\n%s", requests) } } - -// TestTraecliRealACPSmoke drives the REAL official `traecli acp serve` binary -// end-to-end when it is installed and logged in. It is the live counterpart to -// the fake-ACP tests above: it proves the backend's initialize → session/new → -// session/prompt flow works against the actual binary and the user's account. -// -// Skipped automatically when traecli is not on PATH or the session cannot be -// created (not logged in), so CI — which has neither — stays green. Run locally -// with a logged-in traecli to exercise it. -func TestTraecliRealACPSmoke(t *testing.T) { - if testing.Short() { - t.Skip("skipping real-binary smoke test in -short mode") - } - path, err := exec.LookPath("traecli") - if err != nil { - t.Skip("traecli not on PATH; skipping real-binary smoke test") - } - - backend, err := New("traecli", Config{ExecutablePath: path, Logger: slog.Default()}) - if err != nil { - t.Fatalf("new traecli backend: %v", err) - } - ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) - defer cancel() - - session, err := backend.Execute(ctx, "Reply with exactly one word: pong. Do not use any tools.", ExecOptions{ - Cwd: t.TempDir(), - Timeout: 80 * time.Second, - }) - if err != nil { - t.Fatalf("execute: %v", err) - } - go func() { - for range session.Messages { - } - }() - - select { - case result := <-session.Result: - // "session/new" panics on a NOT-logged-in traecli (no models); treat - // that as a skip so the test only fails for real protocol regressions. - if result.Status == "failed" && strings.Contains(result.Error, "session/new") { - t.Skipf("traecli not logged in (session/new failed): %v", result.Error) - } - if result.Status != "completed" { - t.Fatalf("real traecli run did not complete: status=%q error=%q", result.Status, result.Error) - } - if !strings.Contains(strings.ToLower(result.Output), "pong") { - t.Fatalf("expected real traecli output to contain 'pong', got %q", result.Output) - } - if result.SessionID == "" { - t.Error("expected a non-empty session id from real traecli") - } - t.Logf("real traecli smoke OK: session=%s output=%q", result.SessionID, result.Output) - case <-time.After(90 * time.Second): - t.Fatal("timeout waiting for real traecli result") - } -}