mirror of
https://github.com/multica-ai/multica.git
synced 2026-08-03 11:10:23 +02:00
feat(daemon): discover Codex model and reasoning catalog dynamically (#5198)
Discover the Codex model list and per-model reasoning efforts from the installed CLI (codex debug models --bundled), with a verified static fallback for old/offline installs. Server gates token syntax; the daemon validates the exact (model, effort) pair. Closes #5197 MUL-4354
This commit is contained in:
@@ -163,7 +163,7 @@ func init() {
|
||||
agentCreateCmd.Flags().String("runtime-id", "", "Runtime ID (required)")
|
||||
agentCreateCmd.Flags().String("runtime-config", "", "Runtime config as JSON string")
|
||||
agentCreateCmd.Flags().String("model", "", "Model identifier (e.g. claude-sonnet-4-6, openai/gpt-4o). Prefer this over passing --model in --custom-args.")
|
||||
agentCreateCmd.Flags().String("thinking-level", "", "Reasoning/effort level for the agent's runtime (e.g. Claude: low|medium|high|xhigh|max; Codex: none|minimal|low|medium|high|xhigh). The set is runtime/model-specific and validated server-side — an unknown value is rejected. Empty = runtime default.")
|
||||
agentCreateCmd.Flags().String("thinking-level", "", "Reasoning/effort level for the agent's runtime (e.g. Claude: low|medium|high|xhigh|max; Codex values come from the runtime model catalog). The set is runtime/model-specific; malformed values are rejected server-side and the daemon validates the exact model/level pair. Empty = runtime default.")
|
||||
agentCreateCmd.Flags().String("custom-args", "", "Custom CLI arguments as JSON array. For model selection prefer --model; some providers (codex app-server, openclaw) reject --model in custom_args.")
|
||||
agentCreateCmd.Flags().String("custom-env", "", "Custom environment variables as JSON object, e.g. '{\"KEY\":\"value\"}'. Treated as secret material — never logged by the CLI, but values passed on the command line are visible to shell history and 'ps'; prefer --custom-env-stdin or --custom-env-file for real secrets. Pass '{}' to set an empty map.")
|
||||
agentCreateCmd.Flags().Bool("custom-env-stdin", false, "Read the --custom-env JSON object from stdin. Keeps secrets out of shell history and 'ps'. Mutually exclusive with --custom-env and --custom-env-file.")
|
||||
@@ -185,7 +185,7 @@ func init() {
|
||||
agentUpdateCmd.Flags().String("runtime-id", "", "New runtime ID")
|
||||
agentUpdateCmd.Flags().String("runtime-config", "", "New runtime config as JSON string")
|
||||
agentUpdateCmd.Flags().String("model", "", "New model identifier. Pass an empty string to clear and fall back to the runtime default.")
|
||||
agentUpdateCmd.Flags().String("thinking-level", "", "New reasoning/effort level for the agent's runtime (e.g. Claude: low|medium|high|xhigh|max; Codex: none|minimal|low|medium|high|xhigh). The set is runtime/model-specific and validated server-side. Pass an empty string to clear and fall back to the runtime default.")
|
||||
agentUpdateCmd.Flags().String("thinking-level", "", "New reasoning/effort level for the agent's runtime (e.g. Claude: low|medium|high|xhigh|max; Codex values come from the runtime model catalog). The set is runtime/model-specific; malformed values are rejected server-side and the daemon validates the exact model/level pair. Pass an empty string to clear and fall back to the runtime default.")
|
||||
agentUpdateCmd.Flags().String("custom-args", "", "New custom CLI arguments as JSON array. For model selection prefer --model; some providers (codex app-server, openclaw) reject --model in custom_args.")
|
||||
// custom_env is intentionally NOT part of `agent update`. Use
|
||||
// `multica agent env set <id>` — that path is owner/admin-only,
|
||||
|
||||
@@ -895,10 +895,10 @@ func (h *Handler) CreateAgent(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// thinking_level validation: provider-level enum only. Per-model gaps
|
||||
// are enforced by the daemon at execution time (MUL-2339, Trump's
|
||||
// review note — keep API behaviour consistent: literal-invalid →
|
||||
// always 400; combination-invalid → daemon-side task error).
|
||||
// thinking_level validation: fixed-enum providers reject unknown literals;
|
||||
// dynamic-catalog providers (Codex/OpenCode) reject malformed tokens here.
|
||||
// Per-model gaps are enforced by the daemon at execution time (MUL-2339):
|
||||
// combination-invalid values are logged and omitted from the invocation.
|
||||
if !agent.IsKnownThinkingValue(runtime.Provider, req.ThinkingLevel) {
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("thinking_level %q is not a recognised value for runtime %q", req.ThinkingLevel, runtime.Provider))
|
||||
return
|
||||
@@ -1441,15 +1441,12 @@ func (h *Handler) UpdateAgent(w http.ResponseWriter, r *http.Request) {
|
||||
// thinking_level handling (MUL-2339). Tri-state semantics:
|
||||
// - field omitted → leave column alone (COALESCE narg), but if a
|
||||
// runtime change in this same request would make the *existing*
|
||||
// value literal-invalid for the new provider, reject 400. This
|
||||
// closes the gap Elon's review flagged: previously, switching a
|
||||
// Claude agent storing `max` to a Codex runtime would silently
|
||||
// keep `max` and forward it to the daemon.
|
||||
// value invalid for the new provider's fixed enum or token syntax,
|
||||
// reject 400. Exact dynamic-catalog compatibility is daemon-owned.
|
||||
// - field set to "" → explicit clear (run ClearAgentThinkingLevel post-update)
|
||||
// - field set to value → validate against the target runtime's provider
|
||||
// enum; reject literal-invalid with 400. Per-model combination checks
|
||||
// run in the daemon at execution time, not here — see Trump's review
|
||||
// constraint that API behaviour stays consistent across change paths.
|
||||
// - field set to value → validate against the target runtime's fixed enum
|
||||
// or dynamic-token syntax; reject literal-invalid with 400. Per-model
|
||||
// combination checks run in the daemon at execution time, not here.
|
||||
shouldClearThinkingLevel := false
|
||||
if req.ThinkingLevel != nil {
|
||||
value := *req.ThinkingLevel
|
||||
|
||||
@@ -88,20 +88,22 @@ Defaults when omitted: `runtime_config` → `{}`, `custom_env` → `{}`,
|
||||
are typed `[]string`/`any` and marshaled as-is — the JSON-shape rejection
|
||||
happens in the CLI, not the create handler.
|
||||
|
||||
`thinking_level` is validated only at the provider level: an unrecognized
|
||||
literal returns 400, but a value that is valid for the provider yet
|
||||
unsupported for the chosen model is NOT rejected here — that gap surfaces as a
|
||||
daemon-side task error at execution time.
|
||||
`thinking_level` is validated only at the provider level: fixed-catalog
|
||||
providers reject an unrecognized literal, while dynamic-catalog providers such
|
||||
as Codex/OpenCode accept a syntactically safe token. A value unsupported for
|
||||
the chosen model is NOT rejected here — the daemon checks its local model
|
||||
catalog at execution time, logs a warning, and omits the incompatible override.
|
||||
|
||||
Set it from the CLI with `--thinking-level` on `agent create` and `agent
|
||||
update`, mirroring `--model`: the flag is a thin pass-through to the top-level
|
||||
`thinking_level` field, and on update an empty string (`--thinking-level ""`)
|
||||
clears it back to the runtime default. The CLI deliberately does not enumerate
|
||||
the valid levels — they are runtime/model-specific (Claude
|
||||
`low|medium|high|xhigh|max`, Codex `none|minimal|low|medium|high|xhigh`, and
|
||||
others), so it forwards whatever you pass and lets the server's provider
|
||||
catalog accept or reject it. A runtime whose provider has no thinking concept
|
||||
rejects any non-empty value with a 400.
|
||||
the valid levels — they are runtime/model-specific (Claude currently uses
|
||||
`low|medium|high|xhigh|max`; Codex values are discovered from the runtime's
|
||||
model catalog). It forwards the token, the server applies the provider's
|
||||
fixed-enum or safe-token gate, and the daemon performs the exact model/level
|
||||
check. A runtime whose provider has no thinking concept rejects any non-empty
|
||||
value with a 400.
|
||||
|
||||
### model vs custom_args
|
||||
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
# Creating agents — source map
|
||||
|
||||
Evidence layer for `SKILL.md`. Every contract maps to `file:line` on the
|
||||
current tree (branch `feat/builtin-skills`, latest `main` merged), the runtime
|
||||
effect, and a safe read-only check. Line numbers were re-derived against this
|
||||
tree — re-derive again if the files move, the surrounding context (not the
|
||||
number) is the anchor.
|
||||
current tree, the runtime effect, and a safe read-only check. Line numbers were
|
||||
re-derived against this tree — re-derive again if the files move, the
|
||||
surrounding context (not the number) is the anchor.
|
||||
|
||||
## Verification
|
||||
|
||||
@@ -19,11 +18,11 @@ go test ./internal/service -run TestBuiltinSkillsConformToTemplate
|
||||
| Contract | Line | Behavior | Safe check |
|
||||
|---|---|---|---|
|
||||
| Create flags: `name`, `description`, `instructions`, `runtime-id` | 159–162 | Registered create flags; `name`/`runtime-id` enforced in `runAgentCreate` | `multica agent create --help` |
|
||||
| `runtime-config`, `model`, `thinking-level`, `custom-args` flags | 163–166 | `model` help: "Prefer this over passing --model in --custom-args"; `thinking-level` is a thin pass-through (server validates the provider enum, empty = runtime default); `custom-args` help names codex/openclaw rejecting `--model` (CLI help only, not server-enforced) | `multica agent create --help` |
|
||||
| `runtime-config`, `model`, `thinking-level`, `custom-args` flags | 163–166 | `model` help: "Prefer this over passing --model in --custom-args"; `thinking-level` is a thin pass-through (Codex values come from the runtime catalog; server applies a safe-token gate and the daemon checks the exact pair; empty = runtime default); `custom-args` help names codex/openclaw rejecting `--model` (CLI help only, not server-enforced) | `multica agent create --help` |
|
||||
| Secret-safe env input: `custom-env`, `custom-env-stdin`, `custom-env-file` | 167–169 | `--custom-env` warns about shell history / `ps`; stdin and file modes keep secrets off the command line; mutually exclusive | `multica agent create --help` |
|
||||
| Secret-safe MCP input: `mcp-config`, `mcp-config-stdin`, `mcp-config-file` (create) | 170–172 | Same three-channel pattern as `custom-env`; `--mcp-config` warns about shell history / `ps`; value must be a JSON object or `null` | `multica agent create --help` |
|
||||
| MCP flags on `agent update` | 194–196 | Same three channels on update; `--mcp-config null` clears. Unlike `custom_env`, `mcp_config` IS settable via update | `multica agent update --help` |
|
||||
| `thinking-level` flag on `agent update` | 184 | New reasoning/effort level; thin pass-through; `--thinking-level ""` clears to runtime default (mirrors `--model`) | `multica agent update --help` |
|
||||
| `thinking-level` flag on `agent update` | 184 | New reasoning/effort level; Codex values come from the runtime catalog; thin pass-through; `--thinking-level ""` clears to runtime default (mirrors `--model`) | `multica agent update --help` |
|
||||
| `runAgentCreate` builds body + `POST /api/agents` | 419 | Only sets a body key when the flag `Changed`; posts to `/api/agents` (line 495) | read 419–496 |
|
||||
| Body assembly: description/instructions/runtime-config/custom-args/custom-env/mcp-config/model/thinking-level | 438–488 | `resolveCustomEnv` (460) and `resolveMcpConfig` (465) gate their secret channels; `model` (470) and `thinking_level` (478) are `Changed`-gated pass-throughs; omitted flags are not sent | read 438–488 |
|
||||
| `runAgentUpdate` sends `thinking_level` / `mcp_config` | 508 | `thinking_level` added when `--thinking-level` is `Changed` (556); `resolveMcpConfig` adds `mcp_config` (570); `PUT /api/agents/{id}` at 584; `custom_env` is intentionally not a flag here | read 508–585 |
|
||||
@@ -56,7 +55,7 @@ only.
|
||||
| `description` ≤ 255 code points | 627–629 | `utf8.RuneCountInString(req.Description) > maxAgentDescriptionLength` → 400 |
|
||||
| `runtime_id` required | 631–633 | `if req.RuntimeID == ""` → 400 "runtime_id is required" |
|
||||
| `runtime_id` must resolve in workspace | 642–658 | parsed + `GetAgentRuntimeForWorkspace`; unknown → 400 "invalid runtime_id" |
|
||||
| `thinking_level` provider-level validation | 673–676 | `!agent.IsKnownThinkingValue(runtime.Provider, req.ThinkingLevel)` → 400; per-model gaps deferred to daemon (comment 669–672, MUL-2339) |
|
||||
| `thinking_level` provider-level validation | 896–903 | `!agent.IsKnownThinkingValue(runtime.Provider, req.ThinkingLevel)` → 400; fixed providers use an enum, Codex/OpenCode use safe-token syntax, and per-model gaps are deferred to daemon (MUL-2339) |
|
||||
| Defaults: `{}` config/env, `[]` args | 688–701 | `RuntimeConfig`→`{}`, `CustomEnv`→`{}`, `CustomArgs`→`[]` when nil, before insert |
|
||||
| `visibility` default | 635–636 | `if req.Visibility == "" { req.Visibility = "private" }` — access-control field, not the runtime prompt |
|
||||
| `max_concurrent_tasks` default | 638–639 | `if req.MaxConcurrentTasks == 0 { req.MaxConcurrentTasks = 6 }` — scheduler cap |
|
||||
@@ -67,6 +66,18 @@ only.
|
||||
| `UpdateAgent` persists / clears `mcp_config` | 944–948, 1060–1061 | Tri-state from the raw body: key omitted → no change; literal `null` → `ClearAgentMcpConfig`; object → replace. No 400 like `custom_env` — `mcp_config` IS updatable here |
|
||||
| `description` ≤ 255 on update too | 921–924 | same cap re-checked on update |
|
||||
|
||||
## Runtime model/thinking discovery — `server/pkg/agent/{models,thinking}.go`
|
||||
|
||||
| Contract | Line | Behavior |
|
||||
|---|---|---|
|
||||
| Codex model-list entry point | `models.go` 94–103 | `ListModels("codex")` uses cached daemon-local discovery instead of returning the fallback catalog unconditionally |
|
||||
| Codex fallback catalog | `models.go` 301–354 | Used for Codex <0.122.0 and failed/malformed discovery; includes current verified visible models plus legacy `gpt-5.3-codex`, with a separate `Thinking` catalog on every model |
|
||||
| Codex discovery version gate | `thinking.go` 280, 306–337 | `codex debug models --bundled` is used only for parseable versions ≥0.122.0; unsupported versions and command/parse/empty failures return the static model + thinking fallback |
|
||||
| Codex catalog projection | `thinking.go` 355–409 | Hidden models are excluded; visible `slug`/`display_name` rows and each row's `supported_reasoning_levels`/`default_reasoning_level` are preserved |
|
||||
| Per-model thinking validation | `thinking.go` 547–640 | `ValidateThinkingLevel` accepts only values in the explicit model's `Thinking.SupportedLevels`; an empty Codex model fails closed because its effective `config.toml` model is unknown |
|
||||
| Dynamic Codex token gate | `thinking.go` 642–710 | Server persistence accepts syntactically safe Codex tokens so new catalog values do not require a Multica release; exact support remains a daemon-local per-model check |
|
||||
| Daemon invalid-combination handling | `internal/daemon/daemon.go` 3860–3892 | Before execution, invalid `(provider, model, thinking_level)` combinations log a warning and omit the override rather than failing the task |
|
||||
|
||||
## Env endpoint — `server/internal/handler/agent_env.go`
|
||||
|
||||
| Contract | Line | Behavior |
|
||||
|
||||
@@ -41,8 +41,8 @@ type Model struct {
|
||||
}
|
||||
|
||||
// ModelThinking carries the per-model reasoning/effort catalog
|
||||
// surfaced by an agent runtime. Values are runtime-native — Codex
|
||||
// emits "none|minimal|low|medium|high|xhigh"; Claude emits
|
||||
// surfaced by an agent runtime. Values are runtime-native — Codex can emit
|
||||
// "none|minimal|low|medium|high|xhigh|max|ultra"; Claude emits
|
||||
// "low|medium|high|xhigh|max". The frontend renders SupportedLevels
|
||||
// as-is so what users see matches each CLI's own UI.
|
||||
type ModelThinking struct {
|
||||
@@ -80,14 +80,14 @@ const modelCacheTTL = 60 * time.Second
|
||||
|
||||
// ListModels returns the models supported by the given agent provider.
|
||||
// For providers with a known static catalog it returns the baked-in
|
||||
// list; for providers with a CLI discovery mechanism (opencode, pi,
|
||||
// openclaw) it shells out with caching and falls back to the static
|
||||
// list on failure.
|
||||
// list; for providers with a CLI discovery mechanism (codex, opencode,
|
||||
// pi, openclaw) it shells out with caching and falls back where the
|
||||
// provider has a safe static catalog.
|
||||
//
|
||||
// For claude, codex, and opencode, the catalog is augmented with per-model
|
||||
// thinking-level options discovered from the local CLI. Discovery failures
|
||||
// silently leave Thinking == nil on each entry, which the UI treats as
|
||||
// "no picker for this model" rather than blocking model selection.
|
||||
// thinking-level options discovered from the local CLI. Codex discovery
|
||||
// failures fall back to a model + thinking snapshot; providers without a safe
|
||||
// fallback leave Thinking nil, which makes the UI hide the thinking picker.
|
||||
//
|
||||
// executablePath lets the caller point at a non-default binary; pass
|
||||
// "" to use the provider's default name on PATH.
|
||||
@@ -98,9 +98,9 @@ func ListModels(ctx context.Context, providerType, executablePath string) ([]Mod
|
||||
annotateClaudeThinking(ctx, models, executablePath)
|
||||
return models, nil
|
||||
case "codex":
|
||||
models := codexStaticModels()
|
||||
annotateCodexThinking(ctx, models, executablePath)
|
||||
return models, nil
|
||||
return cachedDiscovery(discoveryCacheKey(providerType, executablePath), func() ([]Model, error) {
|
||||
return discoverCodexModels(ctx, executablePath), nil
|
||||
})
|
||||
case "antigravity":
|
||||
// agy 1.0.6 added a `--model` flag plus an `agy models` catalog
|
||||
// command (MUL-3125). Enumerate it on demand like the other
|
||||
@@ -298,28 +298,58 @@ func claudeStaticModels() []Model {
|
||||
}
|
||||
}
|
||||
|
||||
// codexStaticModels is the fallback for Codex versions older than 0.122.0
|
||||
// and for failed/malformed `codex debug models --bundled` calls. Keep it in
|
||||
// sync with the visible entries in the newest locally verified bundled
|
||||
// catalog, plus still-common models from older Codex releases. Each entry
|
||||
// carries its own reasoning catalog so old/offline CLIs retain the same model
|
||||
// + thinking picker contract as dynamic discovery.
|
||||
func codexStaticModels() []Model {
|
||||
// `Default` here is NOT a user-facing "default model" badge — the picker
|
||||
// stopped rendering that (Multica follows the CLI config when the model is
|
||||
// unset). It only marks the current flagship for the "default must track
|
||||
// the latest release" catalog guard (TestCodexStaticModelsExposesLatest,
|
||||
// the latest release" catalog guard
|
||||
// (TestCodexStaticModelsMatchVerifiedFallbackCatalog,
|
||||
// multica#2009). It is deliberately NOT used to validate effort for an
|
||||
// empty (follow-CLI-config) model: that config can resolve to any model,
|
||||
// so ValidateThinkingLevel fails an empty codex model closed rather than
|
||||
// borrowing this entry's catalog (which alone advertises `ultra`) — see
|
||||
// ValidateThinkingLevel and MUL-4347. Keep exactly one entry flagged.
|
||||
standardThinking := func(defaultLevel string, includeMax, includeUltra bool) *ModelThinking {
|
||||
levels := []ThinkingLevel{
|
||||
{Value: "low", Label: "Low", Description: "Fast responses with lighter reasoning"},
|
||||
{Value: "medium", Label: "Medium", Description: "Balances speed and reasoning depth for everyday tasks"},
|
||||
{Value: "high", Label: "High", Description: "Greater reasoning depth for complex problems"},
|
||||
{Value: "xhigh", Label: "Extra high", Description: "Extra high reasoning depth for complex problems"},
|
||||
}
|
||||
if includeMax {
|
||||
levels = append(levels, ThinkingLevel{Value: "max", Label: "Max", Description: "Maximum reasoning depth for the hardest problems"})
|
||||
}
|
||||
if includeUltra {
|
||||
levels = append(levels, ThinkingLevel{Value: "ultra", Label: "Ultra", Description: "Maximum reasoning with automatic task delegation"})
|
||||
}
|
||||
return &ModelThinking{DefaultLevel: defaultLevel, SupportedLevels: levels}
|
||||
}
|
||||
gpt52Thinking := func() *ModelThinking {
|
||||
return &ModelThinking{
|
||||
DefaultLevel: "medium",
|
||||
SupportedLevels: []ThinkingLevel{
|
||||
{Value: "low", Label: "Low", Description: "Balances speed with some reasoning; useful for straightforward queries and short explanations"},
|
||||
{Value: "medium", Label: "Medium", Description: "Provides a solid balance of reasoning depth and latency for general-purpose tasks"},
|
||||
{Value: "high", Label: "High", Description: "Maximizes reasoning depth for complex or ambiguous problems"},
|
||||
{Value: "xhigh", Label: "Extra high", Description: "Extra high reasoning for complex problems"},
|
||||
},
|
||||
}
|
||||
}
|
||||
return []Model{
|
||||
{ID: "gpt-5.6-sol", Label: "GPT-5.6 Sol", Provider: "openai", Default: true},
|
||||
{ID: "gpt-5.6-terra", Label: "GPT-5.6 Terra", Provider: "openai"},
|
||||
{ID: "gpt-5.6-luna", Label: "GPT-5.6 Luna", Provider: "openai"},
|
||||
{ID: "gpt-5.5", Label: "GPT-5.5", Provider: "openai"},
|
||||
{ID: "gpt-5.5-mini", Label: "GPT-5.5 mini", Provider: "openai"},
|
||||
{ID: "gpt-5.4", Label: "GPT-5.4", Provider: "openai"},
|
||||
{ID: "gpt-5.4-mini", Label: "GPT-5.4 mini", Provider: "openai"},
|
||||
{ID: "gpt-5.3-codex", Label: "GPT-5.3 Codex", Provider: "openai"},
|
||||
{ID: "gpt-5", Label: "GPT-5", Provider: "openai"},
|
||||
{ID: "o3", Label: "o3", Provider: "openai"},
|
||||
{ID: "o3-mini", Label: "o3-mini", Provider: "openai"},
|
||||
{ID: "gpt-5.6-sol", Label: "GPT-5.6-Sol", Provider: "openai", Default: true, Thinking: standardThinking("low", true, true)},
|
||||
{ID: "gpt-5.6-terra", Label: "GPT-5.6-Terra", Provider: "openai", Thinking: standardThinking("medium", true, true)},
|
||||
{ID: "gpt-5.6-luna", Label: "GPT-5.6-Luna", Provider: "openai", Thinking: standardThinking("medium", true, false)},
|
||||
{ID: "gpt-5.5", Label: "GPT-5.5", Provider: "openai", Thinking: standardThinking("medium", false, false)},
|
||||
{ID: "gpt-5.4", Label: "GPT-5.4", Provider: "openai", Thinking: standardThinking("medium", false, false)},
|
||||
{ID: "gpt-5.4-mini", Label: "GPT-5.4-Mini", Provider: "openai", Thinking: standardThinking("medium", false, false)},
|
||||
{ID: "gpt-5.3-codex", Label: "GPT-5.3-Codex", Provider: "openai", Thinking: standardThinking("medium", false, false)},
|
||||
{ID: "gpt-5.2", Label: "GPT-5.2", Provider: "openai", Thinking: gpt52Thinking()},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -103,13 +103,10 @@ func TestClaudeStaticModelsExposesSonnet5(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCodexStaticModelsExposesLatest(t *testing.T) {
|
||||
// Codex CLI has no `models list` subcommand so the catalog is
|
||||
// hand-maintained. Regression guard for multica-ai/multica#2009 —
|
||||
// the current frontier models must be selectable, and the badge
|
||||
// default must point at the latest release rather than lagging a
|
||||
// version behind. Codex's default moved to the gpt-5.6 series
|
||||
// (sol/terra/luna), so the default must track gpt-5.6-sol.
|
||||
func TestCodexStaticModelsMatchVerifiedFallbackCatalog(t *testing.T) {
|
||||
// This fallback is used for Codex <0.122.0 and whenever dynamic bundled
|
||||
// discovery fails. Keep the latest verified visible models plus 5.3 Codex
|
||||
// for older installations, but do not resurrect guessed/nonexistent IDs.
|
||||
models := codexStaticModels()
|
||||
ids := map[string]Model{}
|
||||
for _, m := range models {
|
||||
@@ -117,15 +114,18 @@ func TestCodexStaticModelsExposesLatest(t *testing.T) {
|
||||
}
|
||||
for _, want := range []string{
|
||||
"gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna",
|
||||
"gpt-5.5", "gpt-5.5-mini",
|
||||
"gpt-5.4", "gpt-5.4-mini",
|
||||
"gpt-5.3-codex", "gpt-5",
|
||||
"o3", "o3-mini",
|
||||
"gpt-5.5", "gpt-5.4", "gpt-5.4-mini",
|
||||
"gpt-5.3-codex", "gpt-5.2",
|
||||
} {
|
||||
if _, ok := ids[want]; !ok {
|
||||
t.Errorf("missing expected Codex model %q in: %+v", want, models)
|
||||
}
|
||||
}
|
||||
for _, unwanted := range []string{"gpt-5.5-mini", "gpt-5", "o3", "o3-mini"} {
|
||||
if _, ok := ids[unwanted]; ok {
|
||||
t.Errorf("unexpected stale/invalid Codex model %q in fallback: %+v", unwanted, models)
|
||||
}
|
||||
}
|
||||
latest, ok := ids["gpt-5.6-sol"]
|
||||
if !ok || !latest.Default {
|
||||
t.Errorf("expected `gpt-5.6-sol` to be the default Codex entry, got %+v", latest)
|
||||
@@ -142,6 +142,15 @@ func TestCodexStaticModelsExposesLatest(t *testing.T) {
|
||||
if defaults != 1 {
|
||||
t.Errorf("expected exactly one default Codex entry, got %d", defaults)
|
||||
}
|
||||
if got := ids["gpt-5.6-sol"].Thinking; got == nil || got.DefaultLevel != "low" || !hasThinkingLevel(got, "max") || !hasThinkingLevel(got, "ultra") {
|
||||
t.Errorf("unexpected gpt-5.6-sol thinking catalog: %+v", got)
|
||||
}
|
||||
if got := ids["gpt-5.6-luna"].Thinking; got == nil || !hasThinkingLevel(got, "max") || hasThinkingLevel(got, "ultra") {
|
||||
t.Errorf("unexpected gpt-5.6-luna thinking catalog: %+v", got)
|
||||
}
|
||||
if got := ids["gpt-5.3-codex"].Thinking; got == nil || !hasThinkingLevel(got, "xhigh") || hasThinkingLevel(got, "max") || hasThinkingLevel(got, "ultra") {
|
||||
t.Errorf("unexpected gpt-5.3-codex thinking catalog: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelKnownIncompatibleWithProvider(t *testing.T) {
|
||||
|
||||
@@ -15,8 +15,8 @@ import (
|
||||
// UI without hard-coding (and getting wrong) what's installed locally.
|
||||
//
|
||||
// MUL-2339: we deliberately do not flatten Claude's `low|medium|high|
|
||||
// xhigh|max` and Codex's `none|minimal|low|medium|high|xhigh` onto a
|
||||
// shared enum. OpenCode exposes provider-specific model variants through
|
||||
// xhigh|max` and Codex's `none|minimal|low|medium|high|xhigh|max|ultra`
|
||||
// onto a shared enum. OpenCode exposes provider-specific model variants through
|
||||
// `opencode run --variant`, and those names can be extended by local
|
||||
// opencode.json config. What users pick must round-trip exactly through
|
||||
// each CLI's own value vocabulary.
|
||||
@@ -240,12 +240,15 @@ func projectClaudeLevels(superset []string, allow map[string]bool) []ThinkingLev
|
||||
|
||||
// ── Codex ────────────────────────────────────────────────────────────
|
||||
//
|
||||
// `codex debug models` is the structured discovery hook Elon's review
|
||||
// flagged. It returns the per-model reasoning catalog directly,
|
||||
// including the model's documented default. We prefer this over the
|
||||
// older config-error probe trick because:
|
||||
// `codex debug models --bundled` is the structured discovery hook for both
|
||||
// the visible model catalog and each model's reasoning catalog. OpenAI added
|
||||
// the command and `--bundled` flag together in Codex 0.122.0 (openai/codex
|
||||
// #18625). Older versions, failed invocations, and malformed/empty payloads
|
||||
// use codexStaticModels so the picker remains usable.
|
||||
//
|
||||
// We prefer this over the older config-error probe trick because:
|
||||
// 1. It gives us per-model subsets without hand-maintained tables.
|
||||
// 2. The schema is stable across CLI versions (Codex 0.131.0+).
|
||||
// 2. The schema is structured and has been stable since its 0.122.0 debut.
|
||||
// 3. It doesn't pollute stderr with an intentional misconfiguration.
|
||||
//
|
||||
// The subcommand emits JSON on stdout by default — there is no
|
||||
@@ -257,8 +260,8 @@ func projectClaudeLevels(superset []string, allow map[string]bool) []ThinkingLev
|
||||
// tokens the local binary actually accepts, which is the only thing we
|
||||
// need for validation.
|
||||
//
|
||||
// On older Codex versions / failures, the picker just disappears for
|
||||
// that model rather than offering a wrong list.
|
||||
// The static fallback deliberately mirrors a recently verified bundled
|
||||
// catalog, including thinking metadata, rather than guessing model IDs.
|
||||
|
||||
// codexEffortLabel is the human display string for each Codex effort
|
||||
// value, matching Codex's own TUI (`Extra high`, `Minimal`, …) so
|
||||
@@ -270,61 +273,66 @@ var codexEffortLabel = map[string]string{
|
||||
"medium": "Medium",
|
||||
"high": "High",
|
||||
"xhigh": "Extra high",
|
||||
// Codex 0.144.1 added these for the gpt-5.6 series (sol/terra advertise
|
||||
// `max`+`ultra`, luna advertises `max`). They must stay in sync with the
|
||||
// server enum in providerThinkingEnums["codex"], or the picker shows a
|
||||
// level that 400s on save. See TestCodexAdvertisedLevelsArePersistable.
|
||||
"max": "Max",
|
||||
"ultra": "Ultra",
|
||||
"max": "Max",
|
||||
"ultra": "Ultra",
|
||||
}
|
||||
|
||||
const minCodexDebugModelsVersion = "0.122.0"
|
||||
|
||||
// codexDebugModelsResponse mirrors the JSON shape emitted by
|
||||
// `codex debug models` (Codex 0.131.0+). Only the fields we
|
||||
// `codex debug models --bundled` (Codex 0.122.0+). Only the fields we
|
||||
// consume are typed; unknown keys are ignored.
|
||||
type codexDebugModelsResponse struct {
|
||||
Models []struct {
|
||||
Slug string `json:"slug"`
|
||||
DefaultReasoningLevel string `json:"default_reasoning_level"`
|
||||
SupportedReasoningLevel []struct {
|
||||
Effort string `json:"effort"`
|
||||
Description string `json:"description"`
|
||||
} `json:"supported_reasoning_levels"`
|
||||
} `json:"models"`
|
||||
Models []codexDebugModel `json:"models"`
|
||||
}
|
||||
|
||||
// annotateCodexThinking decorates each model entry with its reasoning
|
||||
// catalog. Models the CLI doesn't know about (older codex install,
|
||||
// brand-new ID we haven't shipped) get Thinking=nil — the UI hides
|
||||
// the picker for those rows rather than guessing.
|
||||
func annotateCodexThinking(ctx context.Context, models []Model, executablePath string) {
|
||||
mapping := loadCodexThinkingByModel(ctx, executablePath)
|
||||
for i := range models {
|
||||
if t, ok := mapping[models[i].ID]; ok && t != nil {
|
||||
models[i].Thinking = t
|
||||
}
|
||||
}
|
||||
type codexDebugModel struct {
|
||||
Slug string `json:"slug"`
|
||||
DisplayName string `json:"display_name"`
|
||||
Visibility string `json:"visibility"`
|
||||
DefaultReasoningLevel string `json:"default_reasoning_level"`
|
||||
SupportedReasoningLevel []codexDebugReasoningLevel `json:"supported_reasoning_levels"`
|
||||
}
|
||||
|
||||
func loadCodexThinkingByModel(ctx context.Context, executablePath string) map[string]*ModelThinking {
|
||||
type codexDebugReasoningLevel struct {
|
||||
Effort string `json:"effort"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
// discoverCodexModels returns the installed Codex binary's bundled visible
|
||||
// catalog, including reasoning metadata. Version detection happens before the
|
||||
// debug command so old binaries do not log a predictable "unknown command"
|
||||
// failure on every cache refresh.
|
||||
func discoverCodexModels(ctx context.Context, executablePath string) []Model {
|
||||
if executablePath == "" {
|
||||
executablePath = "codex"
|
||||
}
|
||||
version, _ := DetectVersion(ctx, executablePath)
|
||||
key := thinkingCacheKey{provider: "codex", executablePath: executablePath, cliVersion: version}
|
||||
if cached, ok := thinkingCacheGet(key); ok {
|
||||
return cached
|
||||
version, err := DetectVersion(ctx, executablePath)
|
||||
if err != nil || !codexSupportsDebugModels(version) {
|
||||
return codexStaticModels()
|
||||
}
|
||||
|
||||
raw, err := runCodexDebugModels(ctx, executablePath)
|
||||
if err != nil {
|
||||
// Cache the empty result so repeated UI polls don't re-shell
|
||||
// the missing binary; TTL eventually retries.
|
||||
thinkingCachePut(key, map[string]*ModelThinking{})
|
||||
return map[string]*ModelThinking{}
|
||||
return codexStaticModels()
|
||||
}
|
||||
parsed := parseCodexDebugModels(raw)
|
||||
thinkingCachePut(key, parsed)
|
||||
return parsed
|
||||
models, err := parseCodexModelCatalog(raw)
|
||||
if err != nil || len(models) == 0 {
|
||||
return codexStaticModels()
|
||||
}
|
||||
return models
|
||||
}
|
||||
|
||||
func codexSupportsDebugModels(version string) bool {
|
||||
parsed, err := parseSemver(version)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
minimum, err := parseSemver(minCodexDebugModelsVersion)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return !parsed.lessThan(minimum)
|
||||
}
|
||||
|
||||
// codexDebugModelsArgs is the argv we pass to discover the local Codex
|
||||
@@ -341,53 +349,62 @@ func runCodexDebugModels(ctx context.Context, executablePath string) ([]byte, er
|
||||
return cmd.Output()
|
||||
}
|
||||
|
||||
// parseCodexDebugModels takes the JSON payload from `codex debug
|
||||
// models` and projects it into a per-model thinking catalog.
|
||||
// Returns an empty map (never nil) so callers can compose safely
|
||||
// without nil-checking the result.
|
||||
func parseCodexDebugModels(raw []byte) map[string]*ModelThinking {
|
||||
out := map[string]*ModelThinking{}
|
||||
// parseCodexModelCatalog projects the CLI's raw catalog into the daemon wire
|
||||
// model. Hidden entries are intentionally excluded to match Codex's own model
|
||||
// picker; the first visible entry is the bundled catalog's preferred default.
|
||||
func parseCodexModelCatalog(raw []byte) ([]Model, error) {
|
||||
var resp codexDebugModelsResponse
|
||||
if err := json.Unmarshal(raw, &resp); err != nil {
|
||||
return out
|
||||
return nil, err
|
||||
}
|
||||
models := make([]Model, 0, len(resp.Models))
|
||||
for _, m := range resp.Models {
|
||||
if m.Slug == "" || len(m.SupportedReasoningLevel) == 0 {
|
||||
if m.Slug == "" || m.Visibility == "hide" {
|
||||
continue
|
||||
}
|
||||
levels := make([]ThinkingLevel, 0, len(m.SupportedReasoningLevel))
|
||||
for _, lvl := range m.SupportedReasoningLevel {
|
||||
if lvl.Effort == "" {
|
||||
continue
|
||||
}
|
||||
// Only surface efforts we have a label for. codexEffortLabel is
|
||||
// the single source of truth for "a Codex effort Multica knows",
|
||||
// and TestCodexAdvertisedLevelsArePersistable guarantees every key
|
||||
// here is also in providerThinkingEnums["codex"] — so a labelled
|
||||
// effort is always persistable. Dropping unlabelled tokens (a
|
||||
// future Codex release advertising a new level we haven't taught
|
||||
// the server yet) keeps the picker from ever offering a level the
|
||||
// Create/Update enum gate would 400 on save. Fail closed until the
|
||||
// maps learn it, rather than showing an unsaveable option. (MUL-4347)
|
||||
label, ok := codexEffortLabel[lvl.Effort]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
levels = append(levels, ThinkingLevel{
|
||||
Value: lvl.Effort,
|
||||
Label: label,
|
||||
Description: lvl.Description,
|
||||
})
|
||||
}
|
||||
if len(levels) == 0 {
|
||||
continue
|
||||
}
|
||||
out[m.Slug] = &ModelThinking{
|
||||
SupportedLevels: levels,
|
||||
DefaultLevel: m.DefaultReasoningLevel,
|
||||
label := m.DisplayName
|
||||
if label == "" {
|
||||
label = m.Slug
|
||||
}
|
||||
models = append(models, Model{
|
||||
ID: m.Slug,
|
||||
Label: label,
|
||||
Provider: "openai",
|
||||
Thinking: codexThinkingFromDebugModel(m),
|
||||
})
|
||||
}
|
||||
if len(models) > 0 {
|
||||
models[0].Default = true
|
||||
}
|
||||
return models, nil
|
||||
}
|
||||
|
||||
func codexThinkingFromDebugModel(m codexDebugModel) *ModelThinking {
|
||||
levels := make([]ThinkingLevel, 0, len(m.SupportedReasoningLevel))
|
||||
for _, lvl := range m.SupportedReasoningLevel {
|
||||
if lvl.Effort == "" {
|
||||
continue
|
||||
}
|
||||
label, ok := codexEffortLabel[lvl.Effort]
|
||||
if !ok {
|
||||
// Codex effort tokens are catalog-owned. Surface new safe tokens
|
||||
// immediately; the server accepts their syntax and the daemon uses
|
||||
// this exact per-model catalog for compatibility validation.
|
||||
label = strings.Title(lvl.Effort) //nolint:staticcheck
|
||||
}
|
||||
levels = append(levels, ThinkingLevel{
|
||||
Value: lvl.Effort,
|
||||
Label: label,
|
||||
Description: lvl.Description,
|
||||
})
|
||||
}
|
||||
if len(levels) == 0 {
|
||||
return nil
|
||||
}
|
||||
return &ModelThinking{
|
||||
SupportedLevels: levels,
|
||||
DefaultLevel: m.DefaultReasoningLevel,
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ── CodeBuddy ────────────────────────────────────────────────────────
|
||||
@@ -623,23 +640,18 @@ func anyModelSupportsThinkingValue(models []Model, value string) bool {
|
||||
}
|
||||
|
||||
// providerThinkingEnums is the server-side accept-list for runtimes with a
|
||||
// fixed reasoning-effort vocabulary. OpenCode is deliberately absent because
|
||||
// its `--variant` values come from the local model catalog and custom
|
||||
// opencode.json entries can define additional variant names.
|
||||
// fixed reasoning-effort vocabulary. Codex and OpenCode are deliberately
|
||||
// absent because their values come from daemon-local model catalogs, which can
|
||||
// gain new tokens without a Multica release.
|
||||
//
|
||||
// The server doesn't have local CLI binaries, so it cannot do per-model
|
||||
// discovery the way the daemon can; what it CAN do is reject values that are
|
||||
// not in any version of the provider's enum at all. Per-model gaps (e.g. user
|
||||
// sets `xhigh` while the chosen model only supports up to `high`) are handled
|
||||
// by the daemon's pre-execution guard, which logs and skips injection rather
|
||||
// than mutating persisted agent state. That split keeps API behaviour
|
||||
// consistent: always 400 on literal-invalid, never auto-clear on
|
||||
// combination-invalid. See MUL-2339 review notes.
|
||||
// discovery the way the daemon can. Fixed-catalog providers use this enum;
|
||||
// dynamic providers take the safe-token path in IsKnownThinkingValue below.
|
||||
// Per-model gaps are handled by the daemon's pre-execution guard, which logs
|
||||
// and skips injection rather than mutating persisted agent state.
|
||||
//
|
||||
// Keep these lists permissive: they're a "is this a known token in this
|
||||
// runtime's universe" check, not an "is this the right level for this
|
||||
// model" check. Adding a new level upstream means adding it here too so
|
||||
// users can persist it before the next discovery refresh.
|
||||
// Keep fixed-provider lists permissive: this is a provider-universe check,
|
||||
// not an "is this right for this model" check.
|
||||
var providerThinkingEnums = map[string]map[string]bool{
|
||||
"claude": {
|
||||
"low": true,
|
||||
@@ -648,19 +660,6 @@ var providerThinkingEnums = map[string]map[string]bool{
|
||||
"xhigh": true,
|
||||
"max": true,
|
||||
},
|
||||
"codex": {
|
||||
"none": true,
|
||||
"minimal": true,
|
||||
"low": true,
|
||||
"medium": true,
|
||||
"high": true,
|
||||
"xhigh": true,
|
||||
// Added for the gpt-5.6 series (Codex 0.144.1). Keep in lockstep with
|
||||
// codexEffortLabel — the daemon advertises these, so the server must
|
||||
// let users persist them.
|
||||
"max": true,
|
||||
"ultra": true,
|
||||
},
|
||||
"codebuddy": {
|
||||
"low": true,
|
||||
"medium": true,
|
||||
@@ -672,8 +671,8 @@ var providerThinkingEnums = map[string]map[string]bool{
|
||||
// IsKnownThinkingValue reports whether `value` is a recognised effort
|
||||
// token for the given provider. Empty string is always accepted (means
|
||||
// "use runtime default"). Unknown providers (no thinking concept) accept
|
||||
// only empty; OpenCode accepts well-formed variant names because its local
|
||||
// catalog can be extended by opencode.json.
|
||||
// only empty; Codex and OpenCode accept well-formed tokens here because their
|
||||
// daemon-local catalogs perform the exact per-model check before execution.
|
||||
//
|
||||
// This is the cheap synchronous gate the server uses on CreateAgent /
|
||||
// UpdateAgent. Unlike ValidateThinkingLevel it does NOT consult the live
|
||||
@@ -682,8 +681,8 @@ func IsKnownThinkingValue(providerType, value string) bool {
|
||||
if value == "" {
|
||||
return true
|
||||
}
|
||||
if providerType == "opencode" {
|
||||
return isValidOpenCodeVariantName(value)
|
||||
if providerType == "codex" || providerType == "opencode" {
|
||||
return isValidDynamicThinkingValue(value)
|
||||
}
|
||||
enum, ok := providerThinkingEnums[providerType]
|
||||
if !ok {
|
||||
@@ -692,7 +691,7 @@ func IsKnownThinkingValue(providerType, value string) bool {
|
||||
return enum[value]
|
||||
}
|
||||
|
||||
func isValidOpenCodeVariantName(value string) bool {
|
||||
func isValidDynamicThinkingValue(value string) bool {
|
||||
if len(value) > 64 {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -199,92 +199,169 @@ func splitNonEmptyLines(s string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// ── Codex debug models JSON parsing ──────────────────────────────────
|
||||
// ── Codex debug models version/catalog discovery ────────────────────
|
||||
|
||||
func TestParseCodexDebugModels(t *testing.T) {
|
||||
func TestCodexSupportsDebugModels(t *testing.T) {
|
||||
t.Parallel()
|
||||
raw := []byte(`{
|
||||
"models": [
|
||||
{
|
||||
"slug": "gpt-5.5",
|
||||
"default_reasoning_level": "medium",
|
||||
"supported_reasoning_levels": [
|
||||
{"effort": "low", "description": "Fast"},
|
||||
{"effort": "medium", "description": "Balanced"},
|
||||
{"effort": "high", "description": "Deeper"},
|
||||
{"effort": "xhigh", "description": "Maximum"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"slug": "gpt-5",
|
||||
"default_reasoning_level": "low",
|
||||
"supported_reasoning_levels": [
|
||||
{"effort": "minimal", "description": "Quick"},
|
||||
{"effort": "low", "description": "Fast"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"slug": "no-reasoning",
|
||||
"supported_reasoning_levels": []
|
||||
}
|
||||
]
|
||||
}`)
|
||||
got := parseCodexDebugModels(raw)
|
||||
|
||||
gpt55, ok := got["gpt-5.5"]
|
||||
if !ok || gpt55 == nil {
|
||||
t.Fatalf("missing gpt-5.5 entry: %+v", got)
|
||||
}
|
||||
if gpt55.DefaultLevel != "medium" {
|
||||
t.Errorf("gpt-5.5 default: got %q, want medium", gpt55.DefaultLevel)
|
||||
}
|
||||
if len(gpt55.SupportedLevels) != 4 {
|
||||
t.Errorf("gpt-5.5 supported count: got %d, want 4", len(gpt55.SupportedLevels))
|
||||
}
|
||||
// Labels should come from codexEffortLabel mapping, not from raw effort.
|
||||
for _, lvl := range gpt55.SupportedLevels {
|
||||
if lvl.Value == "xhigh" && lvl.Label != "Extra high" {
|
||||
t.Errorf("xhigh label: got %q, want Extra high", lvl.Label)
|
||||
for _, tc := range []struct {
|
||||
version string
|
||||
want bool
|
||||
}{
|
||||
{"codex-cli 0.121.0", false},
|
||||
{"codex-cli 0.122.0", true},
|
||||
{"codex-cli 0.144.1", true},
|
||||
{"invalid", false},
|
||||
} {
|
||||
if got := codexSupportsDebugModels(tc.version); got != tc.want {
|
||||
t.Errorf("codexSupportsDebugModels(%q) = %v, want %v", tc.version, got, tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
gpt5, ok := got["gpt-5"]
|
||||
if !ok || gpt5 == nil {
|
||||
t.Fatalf("missing gpt-5 entry: %+v", got)
|
||||
}
|
||||
if gpt5.DefaultLevel != "low" {
|
||||
t.Errorf("gpt-5 default: got %q, want low", gpt5.DefaultLevel)
|
||||
}
|
||||
|
||||
// Models with empty supported_reasoning_levels should be omitted to
|
||||
// keep the wire payload small and avoid rendering empty pickers.
|
||||
if _, ok := got["no-reasoning"]; ok {
|
||||
t.Errorf("no-reasoning should be omitted, got %+v", got["no-reasoning"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCodexDebugModels_Malformed(t *testing.T) {
|
||||
func TestParseCodexModelCatalog(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := parseCodexDebugModels([]byte("not json"))
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("expected empty map on malformed input, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseCodexDebugModels_DropsNonPersistableEfforts drives the REAL parser
|
||||
// against a catalog that mixes known gpt-5.6 levels with a bogus future token,
|
||||
// then asserts every level the parser surfaces is persistable by the server
|
||||
// enum. This is the contract TestCodexAdvertisedLevelsArePersistable checks
|
||||
// statically (two hand-written maps); doing it through parseCodexDebugModels
|
||||
// closes the gap Elon flagged: a future Codex release advertising an effort we
|
||||
// haven't taught the server would otherwise reach the picker and 400 on save.
|
||||
func TestParseCodexDebugModels_DropsNonPersistableEfforts(t *testing.T) {
|
||||
t.Parallel()
|
||||
// sol advertises max+ultra (plus a made-up `hyper`); luna tops out at max.
|
||||
raw := []byte(`{
|
||||
"models": [
|
||||
{
|
||||
"slug": "gpt-5.6-sol",
|
||||
"display_name": "GPT-5.6-Sol",
|
||||
"visibility": "list",
|
||||
"default_reasoning_level": "low",
|
||||
"supported_reasoning_levels": [
|
||||
{"effort": "low", "description": "Fast"},
|
||||
{"effort": "max", "description": "Maximum"},
|
||||
{"effort": "ultra", "description": "Delegates"},
|
||||
{"effort": "future", "description": "New CLI value"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"slug": "hidden-model",
|
||||
"display_name": "Hidden",
|
||||
"visibility": "hide",
|
||||
"supported_reasoning_levels": [{"effort": "low"}]
|
||||
},
|
||||
{
|
||||
"slug": "no-reasoning",
|
||||
"display_name": "No Reasoning",
|
||||
"visibility": "list",
|
||||
"supported_reasoning_levels": []
|
||||
}
|
||||
]
|
||||
}`)
|
||||
got, err := parseCodexModelCatalog(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parseCodexModelCatalog: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("expected two visible models, got %+v", got)
|
||||
}
|
||||
if got[0].ID != "gpt-5.6-sol" || got[0].Label != "GPT-5.6-Sol" || !got[0].Default {
|
||||
t.Errorf("unexpected first model: %+v", got[0])
|
||||
}
|
||||
if got[0].Thinking == nil || got[0].Thinking.DefaultLevel != "low" || !hasThinkingLevel(got[0].Thinking, "max") || !hasThinkingLevel(got[0].Thinking, "ultra") || !hasThinkingLevel(got[0].Thinking, "future") {
|
||||
t.Errorf("unexpected per-model thinking catalog: %+v", got[0].Thinking)
|
||||
}
|
||||
if got[1].ID != "no-reasoning" || got[1].Thinking != nil {
|
||||
t.Errorf("model without reasoning should remain selectable without a thinking picker: %+v", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCodexModelCatalogMalformed(t *testing.T) {
|
||||
t.Parallel()
|
||||
if _, err := parseCodexModelCatalog([]byte("not json")); err == nil {
|
||||
t.Fatal("expected malformed catalog error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverCodexModelsVersionGateAndFallback(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-script fake binary requires a POSIX shell")
|
||||
}
|
||||
|
||||
t.Run("supported version uses bundled catalog", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := filepath.Join(dir, "codex")
|
||||
script := `#!/bin/sh
|
||||
if [ "$1" = "--version" ]; then
|
||||
echo "codex-cli 0.122.0"
|
||||
exit 0
|
||||
fi
|
||||
printf '%s\n' "$@" > "` + filepath.Join(dir, "argv.txt") + `"
|
||||
echo '{"models":[{"slug":"runtime-model","display_name":"Runtime Model","visibility":"list","default_reasoning_level":"high","supported_reasoning_levels":[{"effort":"high","description":"Live"}]}]}'
|
||||
`
|
||||
writeTestExecutable(t, fake, []byte(script))
|
||||
|
||||
got := discoverCodexModels(context.Background(), fake)
|
||||
if len(got) != 1 || got[0].ID != "runtime-model" || got[0].Thinking == nil || !hasThinkingLevel(got[0].Thinking, "high") {
|
||||
t.Fatalf("expected runtime catalog, got %+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("old version uses static fallback", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := filepath.Join(dir, "codex")
|
||||
script := "#!/bin/sh\n" +
|
||||
"if [ \"$1\" = \"--version\" ]; then echo 'codex-cli 0.121.0'; exit 0; fi\n" +
|
||||
"exit 99\n"
|
||||
writeTestExecutable(t, fake, []byte(script))
|
||||
|
||||
got := discoverCodexModels(context.Background(), fake)
|
||||
if len(got) == 0 || got[0].ID != "gpt-5.6-sol" {
|
||||
t.Fatalf("expected static fallback, got %+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("debug command failure uses static fallback", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := filepath.Join(dir, "codex")
|
||||
script := "#!/bin/sh\n" +
|
||||
"if [ \"$1\" = \"--version\" ]; then echo 'codex-cli 0.144.1'; exit 0; fi\n" +
|
||||
"exit 1\n"
|
||||
writeTestExecutable(t, fake, []byte(script))
|
||||
|
||||
got := discoverCodexModels(context.Background(), fake)
|
||||
if len(got) == 0 || got[0].ID != "gpt-5.6-sol" || got[0].Thinking == nil {
|
||||
t.Fatalf("expected model + thinking fallback, got %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidateThinkingLevelCodexPerModelFallbackCatalog(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, tc := range []struct {
|
||||
model string
|
||||
level string
|
||||
want bool
|
||||
}{
|
||||
{model: "gpt-5.6-sol", level: "ultra", want: true},
|
||||
{model: "gpt-5.6-terra", level: "ultra", want: true},
|
||||
{model: "gpt-5.6-luna", level: "max", want: true},
|
||||
{model: "gpt-5.6-luna", level: "ultra", want: false},
|
||||
{model: "gpt-5.3-codex", level: "xhigh", want: true},
|
||||
{model: "gpt-5.3-codex", level: "max", want: false},
|
||||
} {
|
||||
got, err := ValidateThinkingLevel(context.Background(), "codex", "/nonexistent/codex", tc.model, tc.level)
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateThinkingLevel(%q, %q): %v", tc.model, tc.level, err)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Errorf("ValidateThinkingLevel(%q, %q) = %v, want %v", tc.model, tc.level, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseCodexModelCatalog_PreservesFutureEfforts pins the dynamic-catalog
|
||||
// contract: a future Codex effort should reach the picker without a Multica
|
||||
// code update, pass the server's safe-token gate, and remain scoped to the
|
||||
// model that advertised it.
|
||||
func TestParseCodexModelCatalog_PreservesFutureEfforts(t *testing.T) {
|
||||
t.Parallel()
|
||||
raw := []byte(`{
|
||||
"models": [
|
||||
{
|
||||
"slug": "gpt-5.6-sol",
|
||||
"display_name": "GPT-5.6-Sol",
|
||||
"visibility": "list",
|
||||
"default_reasoning_level": "high",
|
||||
"supported_reasoning_levels": [
|
||||
{"effort": "medium"},
|
||||
@@ -296,6 +373,8 @@ func TestParseCodexDebugModels_DropsNonPersistableEfforts(t *testing.T) {
|
||||
},
|
||||
{
|
||||
"slug": "gpt-5.6-luna",
|
||||
"display_name": "GPT-5.6-Luna",
|
||||
"visibility": "list",
|
||||
"default_reasoning_level": "medium",
|
||||
"supported_reasoning_levels": [
|
||||
{"effort": "medium"},
|
||||
@@ -304,39 +383,31 @@ func TestParseCodexDebugModels_DropsNonPersistableEfforts(t *testing.T) {
|
||||
}
|
||||
]
|
||||
}`)
|
||||
got := parseCodexDebugModels(raw)
|
||||
|
||||
// Contract: nothing the parser surfaces may be unsaveable.
|
||||
for slug, mt := range got {
|
||||
for _, lvl := range mt.SupportedLevels {
|
||||
if !IsKnownThinkingValue("codex", lvl.Value) {
|
||||
t.Errorf("parser surfaced non-persistable effort %q for %q; the picker would 400 it on save", lvl.Value, slug)
|
||||
}
|
||||
}
|
||||
got, err := parseCodexModelCatalog(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parseCodexModelCatalog: %v", err)
|
||||
}
|
||||
byID := make(map[string]Model, len(got))
|
||||
for _, model := range got {
|
||||
byID[model.ID] = model
|
||||
}
|
||||
|
||||
sol := got["gpt-5.6-sol"]
|
||||
if sol == nil {
|
||||
t.Fatalf("missing gpt-5.6-sol entry: %+v", got)
|
||||
sol := byID["gpt-5.6-sol"]
|
||||
if sol.Thinking == nil {
|
||||
t.Fatalf("missing gpt-5.6-sol thinking entry: %+v", got)
|
||||
}
|
||||
// The bogus token is dropped, not Title-cased through to the picker.
|
||||
if hasThinkingLevel(sol, "hyper") {
|
||||
t.Errorf("unknown effort 'hyper' leaked into sol picker: %+v", sol.SupportedLevels)
|
||||
if !hasThinkingLevel(sol.Thinking, "hyper") {
|
||||
t.Errorf("future effort should be preserved for sol: %+v", sol.Thinking.SupportedLevels)
|
||||
}
|
||||
// Known per-model levels survive, with the real per-model gap preserved:
|
||||
// sol keeps ultra, luna must not advertise it.
|
||||
if !hasThinkingLevel(sol, "ultra") {
|
||||
t.Errorf("sol should keep ultra: %+v", sol.SupportedLevels)
|
||||
if !IsKnownThinkingValue("codex", "hyper") {
|
||||
t.Error("future safe Codex effort should pass the server token gate")
|
||||
}
|
||||
luna := got["gpt-5.6-luna"]
|
||||
if luna == nil {
|
||||
t.Fatalf("missing gpt-5.6-luna entry: %+v", got)
|
||||
luna := byID["gpt-5.6-luna"]
|
||||
if luna.Thinking == nil {
|
||||
t.Fatalf("missing gpt-5.6-luna thinking entry: %+v", got)
|
||||
}
|
||||
if hasThinkingLevel(luna, "ultra") {
|
||||
t.Errorf("luna must not advertise ultra (Codex 0.144.1 tops it out at max): %+v", luna.SupportedLevels)
|
||||
}
|
||||
if !hasThinkingLevel(luna, "max") {
|
||||
t.Errorf("luna should keep max: %+v", luna.SupportedLevels)
|
||||
if hasThinkingLevel(luna.Thinking, "hyper") {
|
||||
t.Errorf("future effort must remain model-specific: %+v", luna.Thinking.SupportedLevels)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -367,9 +438,11 @@ func TestIsKnownThinkingValue(t *testing.T) {
|
||||
{"codex", "none", true},
|
||||
{"codex", "minimal", true},
|
||||
{"codex", "xhigh", true},
|
||||
{"codex", "max", true}, // Codex 0.144.1 advertises `max` for gpt-5.6
|
||||
{"codex", "ultra", true}, // ...and `ultra` for sol/terra
|
||||
{"codex", "insane", false}, // still reject tokens outside the enum
|
||||
{"codex", "max", true},
|
||||
{"codex", "ultra", true},
|
||||
{"codex", "future-level", true}, // exact support is checked against the daemon catalog
|
||||
{"codex", ".hidden", false},
|
||||
{"codex", "bad value", false},
|
||||
{"opencode", "", true},
|
||||
{"opencode", "max", true},
|
||||
{"opencode", "fast-mode", true}, // custom opencode.json variant names are valid
|
||||
@@ -396,7 +469,7 @@ func TestCodexAdvertisedLevelsArePersistable(t *testing.T) {
|
||||
for effort := range codexEffortLabel {
|
||||
if !IsKnownThinkingValue("codex", effort) {
|
||||
t.Errorf("Codex advertises effort %q but IsKnownThinkingValue rejects it; "+
|
||||
"add it to providerThinkingEnums[\"codex\"] so it can be saved", effort)
|
||||
"keep the dynamic Codex token gate compatible so it can be saved", effort)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user