highperfocused/multica
1
0
Fork 0
mirror of https://github.com/multica-ai/multica.git synced 2026-08-03 11:10:23 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(daemon): discover Codex model and reasoning catalog dynamically (#5198)

Browse Source 
Discover the Codex model list and per-model reasoning efforts from the installed CLI (codex debug models --bundled), with a verified static fallback for old/offline installs. Server gates token syntax; the daemon validates the exact (model, effort) pair.

Closes #5197
MUL-4354
This commit is contained in:
Wood
2026-07-10 14:32:05 +08:00
committed by GitHub
parent bf161f2f9c
commit f7ca045fb1
9 changed files with 410 additions and 289 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
server/pkg/agent/models.go
View File

@@ -41,8 +41,8 @@ type Model struct {
}
// ModelThinking carries the per-model reasoning/effort catalog
// surfaced by an agent runtime. Values are runtime-native — Codex
// emits "none|minimal|low|medium|high|xhigh"; Claude emits
// surfaced by an agent runtime. Values are runtime-native — Codex can emit
// "none|minimal|low|medium|high|xhigh|max|ultra"; Claude emits
// "low|medium|high|xhigh|max". The frontend renders SupportedLevels
// as-is so what users see matches each CLI's own UI.
type ModelThinking struct {
@@ -80,14 +80,14 @@ const modelCacheTTL = 60 * time.Second
// ListModels returns the models supported by the given agent provider.
// For providers with a known static catalog it returns the baked-in
// list; for providers with a CLI discovery mechanism (opencode, pi,
// openclaw) it shells out with caching and falls back to the static
// list on failure.
// list; for providers with a CLI discovery mechanism (codex, opencode,
// pi, openclaw) it shells out with caching and falls back where the
// provider has a safe static catalog.
//
// For claude, codex, and opencode, the catalog is augmented with per-model
// thinking-level options discovered from the local CLI. Discovery failures
// silently leave Thinking == nil on each entry, which the UI treats as
// "no picker for this model" rather than blocking model selection.
// thinking-level options discovered from the local CLI. Codex discovery
// failures fall back to a model + thinking snapshot; providers without a safe
// fallback leave Thinking nil, which makes the UI hide the thinking picker.
//
// executablePath lets the caller point at a non-default binary; pass
// "" to use the provider's default name on PATH.
@@ -98,9 +98,9 @@ func ListModels(ctx context.Context, providerType, executablePath string) ([]Mod
annotateClaudeThinking(ctx, models, executablePath)
return models, nil
case "codex":
models := codexStaticModels()
annotateCodexThinking(ctx, models, executablePath)
return models, nil
return cachedDiscovery(discoveryCacheKey(providerType, executablePath), func() ([]Model, error) {
return discoverCodexModels(ctx, executablePath), nil
})
case "antigravity":
// agy 1.0.6 added a `--model` flag plus an `agy models` catalog
// command (MUL-3125). Enumerate it on demand like the other
@@ -298,28 +298,58 @@ func claudeStaticModels() []Model {
}
}
// codexStaticModels is the fallback for Codex versions older than 0.122.0
// and for failed/malformed `codex debug models --bundled` calls. Keep it in
// sync with the visible entries in the newest locally verified bundled
// catalog, plus still-common models from older Codex releases. Each entry
// carries its own reasoning catalog so old/offline CLIs retain the same model
// + thinking picker contract as dynamic discovery.
func codexStaticModels() []Model {
// `Default` here is NOT a user-facing "default model" badge — the picker
// stopped rendering that (Multica follows the CLI config when the model is
// unset). It only marks the current flagship for the "default must track
// the latest release" catalog guard (TestCodexStaticModelsExposesLatest,
// the latest release" catalog guard
// (TestCodexStaticModelsMatchVerifiedFallbackCatalog,
// multica#2009). It is deliberately NOT used to validate effort for an
// empty (follow-CLI-config) model: that config can resolve to any model,
// so ValidateThinkingLevel fails an empty codex model closed rather than
// borrowing this entry's catalog (which alone advertises `ultra`) — see
// ValidateThinkingLevel and MUL-4347. Keep exactly one entry flagged.
standardThinking := func(defaultLevel string, includeMax, includeUltra bool) *ModelThinking {
levels := []ThinkingLevel{
{Value: "low", Label: "Low", Description: "Fast responses with lighter reasoning"},
{Value: "medium", Label: "Medium", Description: "Balances speed and reasoning depth for everyday tasks"},
{Value: "high", Label: "High", Description: "Greater reasoning depth for complex problems"},
{Value: "xhigh", Label: "Extra high", Description: "Extra high reasoning depth for complex problems"},
}
if includeMax {
levels = append(levels, ThinkingLevel{Value: "max", Label: "Max", Description: "Maximum reasoning depth for the hardest problems"})
}
if includeUltra {
levels = append(levels, ThinkingLevel{Value: "ultra", Label: "Ultra", Description: "Maximum reasoning with automatic task delegation"})
}
return &ModelThinking{DefaultLevel: defaultLevel, SupportedLevels: levels}
}
gpt52Thinking := func() *ModelThinking {
return &ModelThinking{
DefaultLevel: "medium",
SupportedLevels: []ThinkingLevel{
{Value: "low", Label: "Low", Description: "Balances speed with some reasoning; useful for straightforward queries and short explanations"},
{Value: "medium", Label: "Medium", Description: "Provides a solid balance of reasoning depth and latency for general-purpose tasks"},
{Value: "high", Label: "High", Description: "Maximizes reasoning depth for complex or ambiguous problems"},
{Value: "xhigh", Label: "Extra high", Description: "Extra high reasoning for complex problems"},
},
}
}
return []Model{
{ID: "gpt-5.6-sol", Label: "GPT-5.6 Sol", Provider: "openai", Default: true},
{ID: "gpt-5.6-terra", Label: "GPT-5.6 Terra", Provider: "openai"},
{ID: "gpt-5.6-luna", Label: "GPT-5.6 Luna", Provider: "openai"},
{ID: "gpt-5.5", Label: "GPT-5.5", Provider: "openai"},
{ID: "gpt-5.5-mini", Label: "GPT-5.5 mini", Provider: "openai"},
{ID: "gpt-5.4", Label: "GPT-5.4", Provider: "openai"},
{ID: "gpt-5.4-mini", Label: "GPT-5.4 mini", Provider: "openai"},
{ID: "gpt-5.3-codex", Label: "GPT-5.3 Codex", Provider: "openai"},
{ID: "gpt-5", Label: "GPT-5", Provider: "openai"},
{ID: "o3", Label: "o3", Provider: "openai"},
{ID: "o3-mini", Label: "o3-mini", Provider: "openai"},
{ID: "gpt-5.6-sol", Label: "GPT-5.6-Sol", Provider: "openai", Default: true, Thinking: standardThinking("low", true, true)},
{ID: "gpt-5.6-terra", Label: "GPT-5.6-Terra", Provider: "openai", Thinking: standardThinking("medium", true, true)},
{ID: "gpt-5.6-luna", Label: "GPT-5.6-Luna", Provider: "openai", Thinking: standardThinking("medium", true, false)},
{ID: "gpt-5.5", Label: "GPT-5.5", Provider: "openai", Thinking: standardThinking("medium", false, false)},
{ID: "gpt-5.4", Label: "GPT-5.4", Provider: "openai", Thinking: standardThinking("medium", false, false)},
{ID: "gpt-5.4-mini", Label: "GPT-5.4-Mini", Provider: "openai", Thinking: standardThinking("medium", false, false)},
{ID: "gpt-5.3-codex", Label: "GPT-5.3-Codex", Provider: "openai", Thinking: standardThinking("medium", false, false)},
{ID: "gpt-5.2", Label: "GPT-5.2", Provider: "openai", Thinking: gpt52Thinking()},
}
}

31
server/pkg/agent/models_test.go
View File

@@ -103,13 +103,10 @@ func TestClaudeStaticModelsExposesSonnet5(t *testing.T) {
}
}
func TestCodexStaticModelsExposesLatest(t *testing.T) {
// Codex CLI has no `models list` subcommand so the catalog is
// hand-maintained. Regression guard for multica-ai/multica#2009 —
// the current frontier models must be selectable, and the badge
// default must point at the latest release rather than lagging a
// version behind. Codex's default moved to the gpt-5.6 series
// (sol/terra/luna), so the default must track gpt-5.6-sol.
func TestCodexStaticModelsMatchVerifiedFallbackCatalog(t *testing.T) {
// This fallback is used for Codex <0.122.0 and whenever dynamic bundled
// discovery fails. Keep the latest verified visible models plus 5.3 Codex
// for older installations, but do not resurrect guessed/nonexistent IDs.
models := codexStaticModels()
ids := map[string]Model{}
for _, m := range models {
@@ -117,15 +114,18 @@ func TestCodexStaticModelsExposesLatest(t *testing.T) {
}
for _, want := range []string{
"gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna",
"gpt-5.5", "gpt-5.5-mini",
"gpt-5.4", "gpt-5.4-mini",
"gpt-5.3-codex", "gpt-5",
"o3", "o3-mini",
"gpt-5.5", "gpt-5.4", "gpt-5.4-mini",
"gpt-5.3-codex", "gpt-5.2",
} {
if _, ok := ids[want]; !ok {
t.Errorf("missing expected Codex model %q in: %+v", want, models)
}
}
for _, unwanted := range []string{"gpt-5.5-mini", "gpt-5", "o3", "o3-mini"} {
if _, ok := ids[unwanted]; ok {
t.Errorf("unexpected stale/invalid Codex model %q in fallback: %+v", unwanted, models)
}
}
latest, ok := ids["gpt-5.6-sol"]
if !ok || !latest.Default {
t.Errorf("expected `gpt-5.6-sol` to be the default Codex entry, got %+v", latest)
@@ -142,6 +142,15 @@ func TestCodexStaticModelsExposesLatest(t *testing.T) {
if defaults != 1 {
t.Errorf("expected exactly one default Codex entry, got %d", defaults)
}
if got := ids["gpt-5.6-sol"].Thinking; got == nil || got.DefaultLevel != "low" || !hasThinkingLevel(got, "max") || !hasThinkingLevel(got, "ultra") {
t.Errorf("unexpected gpt-5.6-sol thinking catalog: %+v", got)
}
if got := ids["gpt-5.6-luna"].Thinking; got == nil || !hasThinkingLevel(got, "max") || hasThinkingLevel(got, "ultra") {
t.Errorf("unexpected gpt-5.6-luna thinking catalog: %+v", got)
}
if got := ids["gpt-5.3-codex"].Thinking; got == nil || !hasThinkingLevel(got, "xhigh") || hasThinkingLevel(got, "max") || hasThinkingLevel(got, "ultra") {
t.Errorf("unexpected gpt-5.3-codex thinking catalog: %+v", got)
}
}
func TestModelKnownIncompatibleWithProvider(t *testing.T) {

235
server/pkg/agent/thinking.go
View File

@@ -15,8 +15,8 @@ import (
// UI without hard-coding (and getting wrong) what's installed locally.
//
// MUL-2339: we deliberately do not flatten Claude's `low|medium|high|
// xhigh|max` and Codex's `none|minimal|low|medium|high|xhigh` onto a
// shared enum. OpenCode exposes provider-specific model variants through
// xhigh|max` and Codex's `none|minimal|low|medium|high|xhigh|max|ultra`
// onto a shared enum. OpenCode exposes provider-specific model variants through
// `opencode run --variant`, and those names can be extended by local
// opencode.json config. What users pick must round-trip exactly through
// each CLI's own value vocabulary.
@@ -240,12 +240,15 @@ func projectClaudeLevels(superset []string, allow map[string]bool) []ThinkingLev
// ── Codex ────────────────────────────────────────────────────────────
//
// `codex debug models` is the structured discovery hook Elon's review
// flagged. It returns the per-model reasoning catalog directly,
// including the model's documented default. We prefer this over the
// older config-error probe trick because:
// `codex debug models --bundled` is the structured discovery hook for both
// the visible model catalog and each model's reasoning catalog. OpenAI added
// the command and `--bundled` flag together in Codex 0.122.0 (openai/codex
// #18625). Older versions, failed invocations, and malformed/empty payloads
// use codexStaticModels so the picker remains usable.
//
// We prefer this over the older config-error probe trick because:
// 1. It gives us per-model subsets without hand-maintained tables.
// 2. The schema is stable across CLI versions (Codex 0.131.0+).
// 2. The schema is structured and has been stable since its 0.122.0 debut.
// 3. It doesn't pollute stderr with an intentional misconfiguration.
//
// The subcommand emits JSON on stdout by default — there is no
@@ -257,8 +260,8 @@ func projectClaudeLevels(superset []string, allow map[string]bool) []ThinkingLev
// tokens the local binary actually accepts, which is the only thing we
// need for validation.
//
// On older Codex versions / failures, the picker just disappears for
// that model rather than offering a wrong list.
// The static fallback deliberately mirrors a recently verified bundled
// catalog, including thinking metadata, rather than guessing model IDs.
// codexEffortLabel is the human display string for each Codex effort
// value, matching Codex's own TUI (`Extra high`, `Minimal`, …) so
@@ -270,61 +273,66 @@ var codexEffortLabel = map[string]string{
"medium": "Medium",
"high": "High",
"xhigh": "Extra high",
// Codex 0.144.1 added these for the gpt-5.6 series (sol/terra advertise
// `max`+`ultra`, luna advertises `max`). They must stay in sync with the
// server enum in providerThinkingEnums["codex"], or the picker shows a
// level that 400s on save. See TestCodexAdvertisedLevelsArePersistable.
"max": "Max",
"ultra": "Ultra",
"max": "Max",
"ultra": "Ultra",
}
const minCodexDebugModelsVersion = "0.122.0"
// codexDebugModelsResponse mirrors the JSON shape emitted by
// `codex debug models` (Codex 0.131.0+). Only the fields we
// `codex debug models --bundled` (Codex 0.122.0+). Only the fields we
// consume are typed; unknown keys are ignored.
type codexDebugModelsResponse struct {
Models []struct {
Slug string `json:"slug"`
DefaultReasoningLevel string `json:"default_reasoning_level"`
SupportedReasoningLevel []struct {
Effort string `json:"effort"`
Description string `json:"description"`
} `json:"supported_reasoning_levels"`
} `json:"models"`
Models []codexDebugModel `json:"models"`
}
// annotateCodexThinking decorates each model entry with its reasoning
// catalog. Models the CLI doesn't know about (older codex install,
// brand-new ID we haven't shipped) get Thinking=nil — the UI hides
// the picker for those rows rather than guessing.
func annotateCodexThinking(ctx context.Context, models []Model, executablePath string) {
mapping := loadCodexThinkingByModel(ctx, executablePath)
for i := range models {
if t, ok := mapping[models[i].ID]; ok && t != nil {
models[i].Thinking = t
}
}
type codexDebugModel struct {
Slug string `json:"slug"`
DisplayName string `json:"display_name"`
Visibility string `json:"visibility"`
DefaultReasoningLevel string `json:"default_reasoning_level"`
SupportedReasoningLevel []codexDebugReasoningLevel `json:"supported_reasoning_levels"`
}
func loadCodexThinkingByModel(ctx context.Context, executablePath string) map[string]*ModelThinking {
type codexDebugReasoningLevel struct {
Effort string `json:"effort"`
Description string `json:"description"`
}
// discoverCodexModels returns the installed Codex binary's bundled visible
// catalog, including reasoning metadata. Version detection happens before the
// debug command so old binaries do not log a predictable "unknown command"
// failure on every cache refresh.
func discoverCodexModels(ctx context.Context, executablePath string) []Model {
if executablePath == "" {
executablePath = "codex"
}
version, _ := DetectVersion(ctx, executablePath)
key := thinkingCacheKey{provider: "codex", executablePath: executablePath, cliVersion: version}
if cached, ok := thinkingCacheGet(key); ok {
return cached
version, err := DetectVersion(ctx, executablePath)
if err != nil || !codexSupportsDebugModels(version) {
return codexStaticModels()
}
raw, err := runCodexDebugModels(ctx, executablePath)
if err != nil {
// Cache the empty result so repeated UI polls don't re-shell
// the missing binary; TTL eventually retries.
thinkingCachePut(key, map[string]*ModelThinking{})
return map[string]*ModelThinking{}
return codexStaticModels()
}
parsed := parseCodexDebugModels(raw)
thinkingCachePut(key, parsed)
return parsed
models, err := parseCodexModelCatalog(raw)
if err != nil || len(models) == 0 {
return codexStaticModels()
}
return models
}
func codexSupportsDebugModels(version string) bool {
parsed, err := parseSemver(version)
if err != nil {
return false
}
minimum, err := parseSemver(minCodexDebugModelsVersion)
if err != nil {
return false
}
return !parsed.lessThan(minimum)
}
// codexDebugModelsArgs is the argv we pass to discover the local Codex
@@ -341,53 +349,62 @@ func runCodexDebugModels(ctx context.Context, executablePath string) ([]byte, er
return cmd.Output()
}
// parseCodexDebugModels takes the JSON payload from `codex debug
// models` and projects it into a per-model thinking catalog.
// Returns an empty map (never nil) so callers can compose safely
// without nil-checking the result.
func parseCodexDebugModels(raw []byte) map[string]*ModelThinking {
out := map[string]*ModelThinking{}
// parseCodexModelCatalog projects the CLI's raw catalog into the daemon wire
// model. Hidden entries are intentionally excluded to match Codex's own model
// picker; the first visible entry is the bundled catalog's preferred default.
func parseCodexModelCatalog(raw []byte) ([]Model, error) {
var resp codexDebugModelsResponse
if err := json.Unmarshal(raw, &resp); err != nil {
return out
return nil, err
}
models := make([]Model, 0, len(resp.Models))
for _, m := range resp.Models {
if m.Slug == "" || len(m.SupportedReasoningLevel) == 0 {
if m.Slug == "" || m.Visibility == "hide" {
continue
}
levels := make([]ThinkingLevel, 0, len(m.SupportedReasoningLevel))
for _, lvl := range m.SupportedReasoningLevel {
if lvl.Effort == "" {
continue
}
// Only surface efforts we have a label for. codexEffortLabel is
// the single source of truth for "a Codex effort Multica knows",
// and TestCodexAdvertisedLevelsArePersistable guarantees every key
// here is also in providerThinkingEnums["codex"] — so a labelled
// effort is always persistable. Dropping unlabelled tokens (a
// future Codex release advertising a new level we haven't taught
// the server yet) keeps the picker from ever offering a level the
// Create/Update enum gate would 400 on save. Fail closed until the
// maps learn it, rather than showing an unsaveable option. (MUL-4347)
label, ok := codexEffortLabel[lvl.Effort]
if !ok {
continue
}
levels = append(levels, ThinkingLevel{
Value: lvl.Effort,
Label: label,
Description: lvl.Description,
})
}
if len(levels) == 0 {
continue
}
out[m.Slug] = &ModelThinking{
SupportedLevels: levels,
DefaultLevel: m.DefaultReasoningLevel,
label := m.DisplayName
if label == "" {
label = m.Slug
}
models = append(models, Model{
ID: m.Slug,
Label: label,
Provider: "openai",
Thinking: codexThinkingFromDebugModel(m),
})
}
if len(models) > 0 {
models[0].Default = true
}
return models, nil
}
func codexThinkingFromDebugModel(m codexDebugModel) *ModelThinking {
levels := make([]ThinkingLevel, 0, len(m.SupportedReasoningLevel))
for _, lvl := range m.SupportedReasoningLevel {
if lvl.Effort == "" {
continue
}
label, ok := codexEffortLabel[lvl.Effort]
if !ok {
// Codex effort tokens are catalog-owned. Surface new safe tokens
// immediately; the server accepts their syntax and the daemon uses
// this exact per-model catalog for compatibility validation.
label = strings.Title(lvl.Effort) //nolint:staticcheck
}
levels = append(levels, ThinkingLevel{
Value: lvl.Effort,
Label: label,
Description: lvl.Description,
})
}
if len(levels) == 0 {
return nil
}
return &ModelThinking{
SupportedLevels: levels,
DefaultLevel: m.DefaultReasoningLevel,
}
return out
}
// ── CodeBuddy ────────────────────────────────────────────────────────
@@ -623,23 +640,18 @@ func anyModelSupportsThinkingValue(models []Model, value string) bool {
}
// providerThinkingEnums is the server-side accept-list for runtimes with a
// fixed reasoning-effort vocabulary. OpenCode is deliberately absent because
// its `--variant` values come from the local model catalog and custom
// opencode.json entries can define additional variant names.
// fixed reasoning-effort vocabulary. Codex and OpenCode are deliberately
// absent because their values come from daemon-local model catalogs, which can
// gain new tokens without a Multica release.
//
// The server doesn't have local CLI binaries, so it cannot do per-model
// discovery the way the daemon can; what it CAN do is reject values that are
// not in any version of the provider's enum at all. Per-model gaps (e.g. user
// sets `xhigh` while the chosen model only supports up to `high`) are handled
// by the daemon's pre-execution guard, which logs and skips injection rather
// than mutating persisted agent state. That split keeps API behaviour
// consistent: always 400 on literal-invalid, never auto-clear on
// combination-invalid. See MUL-2339 review notes.
// discovery the way the daemon can. Fixed-catalog providers use this enum;
// dynamic providers take the safe-token path in IsKnownThinkingValue below.
// Per-model gaps are handled by the daemon's pre-execution guard, which logs
// and skips injection rather than mutating persisted agent state.
//
// Keep these lists permissive: they're a "is this a known token in this
// runtime's universe" check, not an "is this the right level for this
// model" check. Adding a new level upstream means adding it here too so
// users can persist it before the next discovery refresh.
// Keep fixed-provider lists permissive: this is a provider-universe check,
// not an "is this right for this model" check.
var providerThinkingEnums = map[string]map[string]bool{
"claude": {
"low": true,
@@ -648,19 +660,6 @@ var providerThinkingEnums = map[string]map[string]bool{
"xhigh": true,
"max": true,
},
"codex": {
"none": true,
"minimal": true,
"low": true,
"medium": true,
"high": true,
"xhigh": true,
// Added for the gpt-5.6 series (Codex 0.144.1). Keep in lockstep with
// codexEffortLabel — the daemon advertises these, so the server must
// let users persist them.
"max": true,
"ultra": true,
},
"codebuddy": {
"low": true,
"medium": true,
@@ -672,8 +671,8 @@ var providerThinkingEnums = map[string]map[string]bool{
// IsKnownThinkingValue reports whether `value` is a recognised effort
// token for the given provider. Empty string is always accepted (means
// "use runtime default"). Unknown providers (no thinking concept) accept
// only empty; OpenCode accepts well-formed variant names because its local
// catalog can be extended by opencode.json.
// only empty; Codex and OpenCode accept well-formed tokens here because their
// daemon-local catalogs perform the exact per-model check before execution.
//
// This is the cheap synchronous gate the server uses on CreateAgent /
// UpdateAgent. Unlike ValidateThinkingLevel it does NOT consult the live
@@ -682,8 +681,8 @@ func IsKnownThinkingValue(providerType, value string) bool {
if value == "" {
return true
}
if providerType == "opencode" {
return isValidOpenCodeVariantName(value)
if providerType == "codex" || providerType == "opencode" {
return isValidDynamicThinkingValue(value)
}
enum, ok := providerThinkingEnums[providerType]
if !ok {
@@ -692,7 +691,7 @@ func IsKnownThinkingValue(providerType, value string) bool {
return enum[value]
}
func isValidOpenCodeVariantName(value string) bool {
func isValidDynamicThinkingValue(value string) bool {
if len(value) > 64 {
return false
}

285
server/pkg/agent/thinking_test.go
View File

@@ -199,92 +199,169 @@ func splitNonEmptyLines(s string) []string {
return out
}
// ── Codex debug models JSON parsing ──────────────────────────────────
// ── Codex debug models version/catalog discovery ────────────────────
func TestParseCodexDebugModels(t *testing.T) {
func TestCodexSupportsDebugModels(t *testing.T) {
t.Parallel()
raw := []byte(`{
"models": [
{
"slug": "gpt-5.5",
"default_reasoning_level": "medium",
"supported_reasoning_levels": [
{"effort": "low", "description": "Fast"},
{"effort": "medium", "description": "Balanced"},
{"effort": "high", "description": "Deeper"},
{"effort": "xhigh", "description": "Maximum"}
]
},
{
"slug": "gpt-5",
"default_reasoning_level": "low",
"supported_reasoning_levels": [
{"effort": "minimal", "description": "Quick"},
{"effort": "low", "description": "Fast"}
]
},
{
"slug": "no-reasoning",
"supported_reasoning_levels": []
}
]
}`)
got := parseCodexDebugModels(raw)
gpt55, ok := got["gpt-5.5"]
if !ok || gpt55 == nil {
t.Fatalf("missing gpt-5.5 entry: %+v", got)
}
if gpt55.DefaultLevel != "medium" {
t.Errorf("gpt-5.5 default: got %q, want medium", gpt55.DefaultLevel)
}
if len(gpt55.SupportedLevels) != 4 {
t.Errorf("gpt-5.5 supported count: got %d, want 4", len(gpt55.SupportedLevels))
}
// Labels should come from codexEffortLabel mapping, not from raw effort.
for _, lvl := range gpt55.SupportedLevels {
if lvl.Value == "xhigh" && lvl.Label != "Extra high" {
t.Errorf("xhigh label: got %q, want Extra high", lvl.Label)
for _, tc := range []struct {
version string
want bool
}{
{"codex-cli 0.121.0", false},
{"codex-cli 0.122.0", true},
{"codex-cli 0.144.1", true},
{"invalid", false},
} {
if got := codexSupportsDebugModels(tc.version); got != tc.want {
t.Errorf("codexSupportsDebugModels(%q) = %v, want %v", tc.version, got, tc.want)
}
}
gpt5, ok := got["gpt-5"]
if !ok || gpt5 == nil {
t.Fatalf("missing gpt-5 entry: %+v", got)
}
if gpt5.DefaultLevel != "low" {
t.Errorf("gpt-5 default: got %q, want low", gpt5.DefaultLevel)
}
// Models with empty supported_reasoning_levels should be omitted to
// keep the wire payload small and avoid rendering empty pickers.
if _, ok := got["no-reasoning"]; ok {
t.Errorf("no-reasoning should be omitted, got %+v", got["no-reasoning"])
}
}
func TestParseCodexDebugModels_Malformed(t *testing.T) {
func TestParseCodexModelCatalog(t *testing.T) {
t.Parallel()
got := parseCodexDebugModels([]byte("not json"))
if len(got) != 0 {
t.Fatalf("expected empty map on malformed input, got %+v", got)
}
}
// TestParseCodexDebugModels_DropsNonPersistableEfforts drives the REAL parser
// against a catalog that mixes known gpt-5.6 levels with a bogus future token,
// then asserts every level the parser surfaces is persistable by the server
// enum. This is the contract TestCodexAdvertisedLevelsArePersistable checks
// statically (two hand-written maps); doing it through parseCodexDebugModels
// closes the gap Elon flagged: a future Codex release advertising an effort we
// haven't taught the server would otherwise reach the picker and 400 on save.
func TestParseCodexDebugModels_DropsNonPersistableEfforts(t *testing.T) {
t.Parallel()
// sol advertises max+ultra (plus a made-up `hyper`); luna tops out at max.
raw := []byte(`{
"models": [
{
"slug": "gpt-5.6-sol",
"display_name": "GPT-5.6-Sol",
"visibility": "list",
"default_reasoning_level": "low",
"supported_reasoning_levels": [
{"effort": "low", "description": "Fast"},
{"effort": "max", "description": "Maximum"},
{"effort": "ultra", "description": "Delegates"},
{"effort": "future", "description": "New CLI value"}
]
},
{
"slug": "hidden-model",
"display_name": "Hidden",
"visibility": "hide",
"supported_reasoning_levels": [{"effort": "low"}]
},
{
"slug": "no-reasoning",
"display_name": "No Reasoning",
"visibility": "list",
"supported_reasoning_levels": []
}
]
}`)
got, err := parseCodexModelCatalog(raw)
if err != nil {
t.Fatalf("parseCodexModelCatalog: %v", err)
}
if len(got) != 2 {
t.Fatalf("expected two visible models, got %+v", got)
}
if got[0].ID != "gpt-5.6-sol" || got[0].Label != "GPT-5.6-Sol" || !got[0].Default {
t.Errorf("unexpected first model: %+v", got[0])
}
if got[0].Thinking == nil || got[0].Thinking.DefaultLevel != "low" || !hasThinkingLevel(got[0].Thinking, "max") || !hasThinkingLevel(got[0].Thinking, "ultra") || !hasThinkingLevel(got[0].Thinking, "future") {
t.Errorf("unexpected per-model thinking catalog: %+v", got[0].Thinking)
}
if got[1].ID != "no-reasoning" || got[1].Thinking != nil {
t.Errorf("model without reasoning should remain selectable without a thinking picker: %+v", got[1])
}
}
func TestParseCodexModelCatalogMalformed(t *testing.T) {
t.Parallel()
if _, err := parseCodexModelCatalog([]byte("not json")); err == nil {
t.Fatal("expected malformed catalog error")
}
}
func TestDiscoverCodexModelsVersionGateAndFallback(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("shell-script fake binary requires a POSIX shell")
}
t.Run("supported version uses bundled catalog", func(t *testing.T) {
dir := t.TempDir()
fake := filepath.Join(dir, "codex")
script := `#!/bin/sh
if [ "$1" = "--version" ]; then
echo "codex-cli 0.122.0"
exit 0
fi
printf '%s\n' "$@" > "` + filepath.Join(dir, "argv.txt") + `"
echo '{"models":[{"slug":"runtime-model","display_name":"Runtime Model","visibility":"list","default_reasoning_level":"high","supported_reasoning_levels":[{"effort":"high","description":"Live"}]}]}'
`
writeTestExecutable(t, fake, []byte(script))
got := discoverCodexModels(context.Background(), fake)
if len(got) != 1 || got[0].ID != "runtime-model" || got[0].Thinking == nil || !hasThinkingLevel(got[0].Thinking, "high") {
t.Fatalf("expected runtime catalog, got %+v", got)
}
})
t.Run("old version uses static fallback", func(t *testing.T) {
dir := t.TempDir()
fake := filepath.Join(dir, "codex")
script := "#!/bin/sh\n" +
"if [ \"$1\" = \"--version\" ]; then echo 'codex-cli 0.121.0'; exit 0; fi\n" +
"exit 99\n"
writeTestExecutable(t, fake, []byte(script))
got := discoverCodexModels(context.Background(), fake)
if len(got) == 0 || got[0].ID != "gpt-5.6-sol" {
t.Fatalf("expected static fallback, got %+v", got)
}
})
t.Run("debug command failure uses static fallback", func(t *testing.T) {
dir := t.TempDir()
fake := filepath.Join(dir, "codex")
script := "#!/bin/sh\n" +
"if [ \"$1\" = \"--version\" ]; then echo 'codex-cli 0.144.1'; exit 0; fi\n" +
"exit 1\n"
writeTestExecutable(t, fake, []byte(script))
got := discoverCodexModels(context.Background(), fake)
if len(got) == 0 || got[0].ID != "gpt-5.6-sol" || got[0].Thinking == nil {
t.Fatalf("expected model + thinking fallback, got %+v", got)
}
})
}
func TestValidateThinkingLevelCodexPerModelFallbackCatalog(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
model string
level string
want bool
}{
{model: "gpt-5.6-sol", level: "ultra", want: true},
{model: "gpt-5.6-terra", level: "ultra", want: true},
{model: "gpt-5.6-luna", level: "max", want: true},
{model: "gpt-5.6-luna", level: "ultra", want: false},
{model: "gpt-5.3-codex", level: "xhigh", want: true},
{model: "gpt-5.3-codex", level: "max", want: false},
} {
got, err := ValidateThinkingLevel(context.Background(), "codex", "/nonexistent/codex", tc.model, tc.level)
if err != nil {
t.Fatalf("ValidateThinkingLevel(%q, %q): %v", tc.model, tc.level, err)
}
if got != tc.want {
t.Errorf("ValidateThinkingLevel(%q, %q) = %v, want %v", tc.model, tc.level, got, tc.want)
}
}
}
// TestParseCodexModelCatalog_PreservesFutureEfforts pins the dynamic-catalog
// contract: a future Codex effort should reach the picker without a Multica
// code update, pass the server's safe-token gate, and remain scoped to the
// model that advertised it.
func TestParseCodexModelCatalog_PreservesFutureEfforts(t *testing.T) {
t.Parallel()
raw := []byte(`{
"models": [
{
"slug": "gpt-5.6-sol",
"display_name": "GPT-5.6-Sol",
"visibility": "list",
"default_reasoning_level": "high",
"supported_reasoning_levels": [
{"effort": "medium"},
@@ -296,6 +373,8 @@ func TestParseCodexDebugModels_DropsNonPersistableEfforts(t *testing.T) {
},
{
"slug": "gpt-5.6-luna",
"display_name": "GPT-5.6-Luna",
"visibility": "list",
"default_reasoning_level": "medium",
"supported_reasoning_levels": [
{"effort": "medium"},
@@ -304,39 +383,31 @@ func TestParseCodexDebugModels_DropsNonPersistableEfforts(t *testing.T) {
}
]
}`)
got := parseCodexDebugModels(raw)
// Contract: nothing the parser surfaces may be unsaveable.
for slug, mt := range got {
for _, lvl := range mt.SupportedLevels {
if !IsKnownThinkingValue("codex", lvl.Value) {
t.Errorf("parser surfaced non-persistable effort %q for %q; the picker would 400 it on save", lvl.Value, slug)
}
}
got, err := parseCodexModelCatalog(raw)
if err != nil {
t.Fatalf("parseCodexModelCatalog: %v", err)
}
byID := make(map[string]Model, len(got))
for _, model := range got {
byID[model.ID] = model
}
sol := got["gpt-5.6-sol"]
if sol == nil {
t.Fatalf("missing gpt-5.6-sol entry: %+v", got)
sol := byID["gpt-5.6-sol"]
if sol.Thinking == nil {
t.Fatalf("missing gpt-5.6-sol thinking entry: %+v", got)
}
// The bogus token is dropped, not Title-cased through to the picker.
if hasThinkingLevel(sol, "hyper") {
t.Errorf("unknown effort 'hyper' leaked into sol picker: %+v", sol.SupportedLevels)
if !hasThinkingLevel(sol.Thinking, "hyper") {
t.Errorf("future effort should be preserved for sol: %+v", sol.Thinking.SupportedLevels)
}
// Known per-model levels survive, with the real per-model gap preserved:
// sol keeps ultra, luna must not advertise it.
if !hasThinkingLevel(sol, "ultra") {
t.Errorf("sol should keep ultra: %+v", sol.SupportedLevels)
if !IsKnownThinkingValue("codex", "hyper") {
t.Error("future safe Codex effort should pass the server token gate")
}
luna := got["gpt-5.6-luna"]
if luna == nil {
t.Fatalf("missing gpt-5.6-luna entry: %+v", got)
luna := byID["gpt-5.6-luna"]
if luna.Thinking == nil {
t.Fatalf("missing gpt-5.6-luna thinking entry: %+v", got)
}
if hasThinkingLevel(luna, "ultra") {
t.Errorf("luna must not advertise ultra (Codex 0.144.1 tops it out at max): %+v", luna.SupportedLevels)
}
if !hasThinkingLevel(luna, "max") {
t.Errorf("luna should keep max: %+v", luna.SupportedLevels)
if hasThinkingLevel(luna.Thinking, "hyper") {
t.Errorf("future effort must remain model-specific: %+v", luna.Thinking.SupportedLevels)
}
}
@@ -367,9 +438,11 @@ func TestIsKnownThinkingValue(t *testing.T) {
{"codex", "none", true},
{"codex", "minimal", true},
{"codex", "xhigh", true},
{"codex", "max", true}, // Codex 0.144.1 advertises `max` for gpt-5.6
{"codex", "ultra", true}, // ...and `ultra` for sol/terra
{"codex", "insane", false}, // still reject tokens outside the enum
{"codex", "max", true},
{"codex", "ultra", true},
{"codex", "future-level", true}, // exact support is checked against the daemon catalog
{"codex", ".hidden", false},
{"codex", "bad value", false},
{"opencode", "", true},
{"opencode", "max", true},
{"opencode", "fast-mode", true}, // custom opencode.json variant names are valid
@@ -396,7 +469,7 @@ func TestCodexAdvertisedLevelsArePersistable(t *testing.T) {
for effort := range codexEffortLabel {
if !IsKnownThinkingValue("codex", effort) {
t.Errorf("Codex advertises effort %q but IsKnownThinkingValue rejects it; "+
"add it to providerThinkingEnums[\"codex\"] so it can be saved", effort)
"keep the dynamic Codex token gate compatible so it can be saved", effort)
}
}
}