openai_codex_model_metadata_test.go 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744
  1. package service
  2. import (
  3. "context"
  4. "encoding/json"
  5. "errors"
  6. "testing"
  7. "github.com/stretchr/testify/require"
  8. )
  9. func TestAstraUltraCatalogPreservesWorkflowMetadata(t *testing.T) {
  10. // Ultra is a Codex workflow. Its inference effort must survive catalog sync,
  11. // aliases and group generation instead of silently falling back to max.
  12. _, metadata, err := extractUpstreamModelCatalog([]byte(`{"models":[{
  13. "slug":"gpt-6-astra","supported_reasoning_levels":[{"effort":"high"},{"effort":"ultra"}],
  14. "multi_agent_reasoning_effort":"high","multi_agent_version":"v2"
  15. }]}`), false)
  16. require.NoError(t, err)
  17. account := Account{Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Credentials: map[string]any{
  18. "base_url": "https://relay.example/v1", "model_mapping": map[string]any{"public-astra": "gpt-6-astra"},
  19. }}
  20. account.SetUpstreamModelMetadataSnapshot(UpstreamModelMetadataSnapshot{Models: metadata})
  21. body, err := buildCodexModelsManifestForAccounts(PlatformOpenAI, []string{"public-astra"}, []Account{account}, nil, nil, true)
  22. require.NoError(t, err)
  23. model := decodeCodexManifestModels(t, body)[0]
  24. require.Equal(t, "high", model["multi_agent_reasoning_effort"])
  25. require.Equal(t, "v2", model["multi_agent_version"])
  26. require.Equal(t, []string{"high", "ultra"}, effortsFromManifestModel(t, model))
  27. peer := Account{Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Credentials: account.Credentials}
  28. body, err = buildCodexModelsManifestForAccounts(PlatformOpenAI, []string{"public-astra"}, []Account{account, peer}, nil, nil, true)
  29. require.NoError(t, err)
  30. model = decodeCodexManifestModels(t, body)[0]
  31. require.Nil(t, model["multi_agent_reasoning_effort"], "do not advertise one account's override for all peers")
  32. require.Nil(t, model["multi_agent_version"])
  33. }
  34. func TestAstraUltraCatalogPreservesExplicitWorkflowOverrides(t *testing.T) {
  35. account := &Account{Platform: PlatformOpenAI, Type: AccountTypeAPIKey,
  36. Credentials: map[string]any{"base_url": "https://relay.example/v1"}}
  37. for _, fields := range []string{
  38. `"multi_agent_reasoning_effort":"high","multi_agent_version":"v1"`,
  39. `"multi_agent_reasoning_effort":null,"multi_agent_version":null`,
  40. } {
  41. for _, source := range []string{
  42. `{"models":[{"slug":"gpt-6-astra",` + fields + `}]}`,
  43. `{"data":[{"id":"gpt-6-astra",` + fields + `}]}`,
  44. } {
  45. converted := convertOpenAIModelListToCodexManifestForAccount([]byte(source), account)
  46. body, err := completeAPIKeyCodexModelsManifestMetadata(converted, true, account)
  47. require.NoError(t, err)
  48. model := decodeCodexManifestModels(t, body)[0]
  49. var expected map[string]any
  50. require.NoError(t, json.Unmarshal([]byte(`{`+fields+`}`), &expected))
  51. for key, value := range expected {
  52. require.Contains(t, model, key)
  53. require.Equal(t, value, model[key])
  54. }
  55. }
  56. }
  57. }
  58. func TestAstraCodexToolCapabilitiesUseAccountScopeAndSharedDeclarations(t *testing.T) {
  59. newAccount := func(baseURL string) Account {
  60. return Account{Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Credentials: map[string]any{
  61. "base_url": baseURL, "model_mapping": map[string]any{"public-astra": "gpt-6-astra"},
  62. }}
  63. }
  64. official := newAccount("https://api.openai.com/v1")
  65. custom := newAccount("https://relay.example/v1")
  66. bridge := newAccount("https://bridge.example/v1")
  67. bridge.Extra = map[string]any{"openai_responses_supported": false}
  68. for _, tt := range []struct {
  69. name string
  70. accounts []Account
  71. search bool
  72. }{
  73. {"official fallback", []Account{official}, true},
  74. {"custom host has no guessed capability", []Account{custom}, false},
  75. {"missing peer capability", []Account{official, custom}, false},
  76. {"implemented chat bridge", []Account{bridge}, true},
  77. } {
  78. t.Run(tt.name, func(t *testing.T) {
  79. body, err := buildCodexModelsManifestForAccounts(PlatformOpenAI, []string{"public-astra"}, tt.accounts, nil, nil, true)
  80. require.NoError(t, err)
  81. model := decodeCodexManifestModels(t, body)[0]
  82. require.Equal(t, "public-astra", model["slug"])
  83. require.Equal(t, tt.search, model["supports_search_tool"])
  84. require.Equal(t, false, model["use_responses_lite"])
  85. if tt.name == "official fallback" {
  86. require.Equal(t, "freeform", model["apply_patch_tool_type"])
  87. require.Equal(t, "3000", model["comp_hash"])
  88. }
  89. })
  90. }
  91. custom.SetUpstreamModelMetadataSnapshot(UpstreamModelMetadataSnapshot{Models: map[string]UpstreamModelMetadata{
  92. "gpt-6-astra": {CodexToolCapabilities: map[string]json.RawMessage{
  93. "supports_search_tool": json.RawMessage("true"), "apply_patch_tool_type": json.RawMessage("null"),
  94. "comp_hash": json.RawMessage(`"3000"`), "tool_mode": json.RawMessage("null"), "use_responses_lite": json.RawMessage("false"),
  95. }},
  96. }})
  97. body, err := buildCodexModelsManifestForAccounts(PlatformOpenAI, []string{"public-astra"}, []Account{official, custom}, nil, nil, true)
  98. require.NoError(t, err)
  99. model := decodeCodexManifestModels(t, body)[0]
  100. require.Equal(t, true, model["supports_search_tool"])
  101. require.Nil(t, model["apply_patch_tool_type"], "conflicting tool protocols must not be advertised")
  102. require.Equal(t, "3000", model["comp_hash"])
  103. }
  104. func TestAstraCodexToolCapabilitiesPreserveLiveNullAndFalse(t *testing.T) {
  105. account := &Account{Platform: PlatformOpenAI, Type: AccountTypeAPIKey,
  106. Credentials: map[string]any{"base_url": "https://api.openai.com/v1"}}
  107. account.SetUpstreamModelMetadataSnapshot(UpstreamModelMetadataSnapshot{Models: map[string]UpstreamModelMetadata{
  108. "gpt-6-astra": {CodexToolCapabilities: map[string]json.RawMessage{
  109. "supports_search_tool": json.RawMessage("true"), "apply_patch_tool_type": json.RawMessage(`"freeform"`),
  110. "comp_hash": json.RawMessage(`"3000"`), "tool_mode": json.RawMessage(`"code_mode_only"`),
  111. }},
  112. }})
  113. for _, source := range []string{
  114. `{"models":[{"slug":"gpt-6-astra","supports_search_tool":false,"apply_patch_tool_type":null,"tool_mode":null}]}`,
  115. `{"data":[{"id":"gpt-6-astra","supports_search_tool":false,"apply_patch_tool_type":null,"tool_mode":null}]}`,
  116. } {
  117. converted := convertOpenAIModelListToCodexManifestForAccount([]byte(source), account)
  118. body, err := applySyncedAPIKeyCodexModelMetadata(converted, account, source[2:6] == "data")
  119. require.NoError(t, err)
  120. body, err = completeAPIKeyCodexModelsManifestMetadata(body, true, account)
  121. require.NoError(t, err)
  122. model := decodeCodexManifestModels(t, body)[0]
  123. require.Equal(t, false, model["supports_search_tool"])
  124. require.Nil(t, model["apply_patch_tool_type"])
  125. require.Nil(t, model["tool_mode"])
  126. require.Equal(t, "3000", model["comp_hash"])
  127. }
  128. fields := make(map[string]json.RawMessage)
  129. applyCodexToolCapabilities(fields, map[string]json.RawMessage{
  130. "supports_search_tool": json.RawMessage(`"true"`), "comp_hash": json.RawMessage(`{"unexpected":true}`),
  131. "model_messages": json.RawMessage(`"do not persist prompts"`),
  132. }, true)
  133. require.Empty(t, fields)
  134. }
  135. // Scenario: mixed groups prefer capability metadata synced for the routed account.
  136. func TestBuildCodexModelsManifestForGroupUsesSyncedAccountMetadata(t *testing.T) {
  137. t.Parallel()
  138. const groupID int64 = 735
  139. account := Account{
  140. ID: 25,
  141. Platform: PlatformOpenAI,
  142. Type: AccountTypeAPIKey,
  143. Credentials: map[string]any{
  144. "base_url": "https://opencode.ai/zen/v1",
  145. "model_mapping": map[string]any{"x-preview-f-free": "x-preview-f-free"},
  146. },
  147. Extra: map[string]any{
  148. UpstreamModelMetadataExtraKey: map[string]any{
  149. "source": "models.dev",
  150. "models": map[string]any{
  151. "x-preview-f-free": map[string]any{
  152. "id": "x-preview-f-free",
  153. "display_name": "Ox Alpha Free (Unlimited)",
  154. "description": "Stealth reasoning model",
  155. "reasoning": true,
  156. "supported_reasoning_levels": []any{"low", "high", "max"},
  157. "input_modalities": []any{"text", "image"},
  158. "context_window": float64(1_000_000),
  159. "max_output_tokens": float64(131_072),
  160. },
  161. },
  162. },
  163. },
  164. }
  165. svc := &GatewayService{accountRepo: codexModelsVisibilityAccountRepo{byGroup: map[int64][]Account{
  166. groupID: {account},
  167. }}}
  168. body, err := svc.BuildCodexModelsManifestForGroup(
  169. context.Background(),
  170. &Group{ID: groupID, Platform: PlatformComposite},
  171. "",
  172. []string{"x-preview-f-free"},
  173. )
  174. require.NoError(t, err)
  175. models := decodeCodexManifestModels(t, body)
  176. require.Len(t, models, 1)
  177. require.Equal(t, "Ox Alpha Free (Unlimited)", models[0]["display_name"])
  178. require.Equal(t, "low", models[0]["default_reasoning_level"])
  179. require.Equal(t, []string{"low", "high", "max"}, effortsFromManifestModel(t, models[0]))
  180. require.Equal(t, []any{"text", "image"}, models[0]["input_modalities"])
  181. require.EqualValues(t, 1_000_000, models[0]["context_window"])
  182. }
  183. // Scenario: an explicitly non-reasoning model remains directly selectable in Codex.
  184. func TestBuildCodexModelsManifestForGroupUsesNoneForExplicitNonReasoningMetadata(t *testing.T) {
  185. t.Parallel()
  186. const groupID int64 = 737
  187. reasoning := false
  188. account := Account{
  189. ID: 28, Platform: PlatformOpenAI, Type: AccountTypeAPIKey,
  190. Credentials: map[string]any{
  191. "base_url": "https://provider.example/v1",
  192. "model_mapping": map[string]any{"company-coding-model": "company-coding-model"},
  193. },
  194. }
  195. account.SetUpstreamModelMetadataSnapshot(UpstreamModelMetadataSnapshot{Models: map[string]UpstreamModelMetadata{
  196. "company-coding-model": {
  197. ID: "company-coding-model", Reasoning: &reasoning,
  198. InputModalities: []string{"text"}, ContextWindow: 64_000,
  199. },
  200. }})
  201. svc := &GatewayService{accountRepo: codexModelsVisibilityAccountRepo{byGroup: map[int64][]Account{
  202. groupID: {account},
  203. }}}
  204. body, err := svc.BuildCodexModelsManifestForGroup(
  205. context.Background(), &Group{ID: groupID, Platform: PlatformComposite}, "", []string{"company-coding-model"},
  206. )
  207. require.NoError(t, err)
  208. models := decodeCodexManifestModels(t, body)
  209. require.Len(t, models, 1)
  210. require.Equal(t, "none", models[0]["default_reasoning_level"])
  211. require.Equal(t, []string{"none"}, effortsFromManifestModel(t, models[0]))
  212. }
  213. func TestBuildCodexModelsManifestForGroupAdvertisesSearchOnlyForChatBridgeRoutes(t *testing.T) {
  214. t.Parallel()
  215. newAccount := func(id int64, nativeResponses bool) Account {
  216. return Account{
  217. ID: id, Platform: PlatformOpenAI, Type: AccountTypeAPIKey,
  218. Credentials: map[string]any{
  219. "base_url": "https://provider.example/v1",
  220. "model_mapping": map[string]any{"company-coding-model": "company-coding-model"},
  221. },
  222. Extra: map[string]any{"openai_responses_supported": nativeResponses},
  223. }
  224. }
  225. for _, tc := range []struct {
  226. name string
  227. accounts []Account
  228. want bool
  229. }{
  230. {name: "chat bridge", accounts: []Account{newAccount(80, false)}, want: true},
  231. {name: "native responses", accounts: []Account{newAccount(81, true)}, want: false},
  232. {name: "mixed routes", accounts: []Account{newAccount(82, false), newAccount(83, true)}, want: false},
  233. } {
  234. tc := tc
  235. t.Run(tc.name, func(t *testing.T) {
  236. t.Parallel()
  237. body, err := buildCodexModelsManifestForAccounts(
  238. PlatformOpenAI, []string{"company-coding-model"}, tc.accounts, nil, nil, true,
  239. )
  240. require.NoError(t, err)
  241. models := decodeCodexManifestModels(t, body)
  242. require.Len(t, models, 1)
  243. require.Equal(t, tc.want, models[0]["supports_search_tool"])
  244. })
  245. }
  246. }
  247. func TestCompleteAPIKeyCodexManifestSearchCapabilityPreservesUpstreamAndFailsClosed(t *testing.T) {
  248. t.Parallel()
  249. nativeAccount := &Account{
  250. Platform: PlatformOpenAI,
  251. Type: AccountTypeAPIKey,
  252. Credentials: map[string]any{
  253. "base_url": "https://provider.example/v1",
  254. },
  255. Extra: map[string]any{"openai_responses_supported": true},
  256. }
  257. body, err := completeAPIKeyCodexModelsManifestMetadata([]byte(`{"models":[
  258. {"slug":"explicit","supports_search_tool":true},
  259. {"slug":"missing"}
  260. ]}`), true, nativeAccount)
  261. require.NoError(t, err)
  262. models := decodeCodexManifestModels(t, body)
  263. require.Equal(t, true, models[0]["supports_search_tool"])
  264. require.Equal(t, false, models[1]["supports_search_tool"])
  265. chatAccount := &Account{
  266. Platform: PlatformOpenAI,
  267. Type: AccountTypeAPIKey,
  268. Credentials: map[string]any{
  269. "base_url": "https://provider.example/v1",
  270. },
  271. Extra: map[string]any{"openai_responses_supported": false},
  272. }
  273. body, err = completeAPIKeyCodexModelsManifestMetadata(
  274. []byte(`{"models":[
  275. {"slug":"explicit-disabled","supports_search_tool":false},
  276. {"slug":"generated"}
  277. ]}`), true, chatAccount,
  278. )
  279. require.NoError(t, err)
  280. models = decodeCodexManifestModels(t, body)
  281. require.Equal(t, false, models[0]["supports_search_tool"])
  282. require.Equal(t, true, models[1]["supports_search_tool"])
  283. }
  284. // Scenario: multiple schedulable accounts advertise only their shared capabilities.
  285. func TestBuildCodexModelsManifestForGroupIntersectsSyncedAccountMetadata(t *testing.T) {
  286. t.Parallel()
  287. const groupID int64 = 736
  288. reasoning := true
  289. newAccount := func(id int64, levels, modalities []string, contextWindow int64) Account {
  290. account := Account{
  291. ID: id, Platform: PlatformOpenAI, Type: AccountTypeAPIKey,
  292. Credentials: map[string]any{
  293. "base_url": "https://provider.example/v1",
  294. "model_mapping": map[string]any{"shared-model": "shared-model"},
  295. },
  296. }
  297. account.SetUpstreamModelMetadataSnapshot(UpstreamModelMetadataSnapshot{Models: map[string]UpstreamModelMetadata{
  298. "shared-model": {
  299. ID: "shared-model", Reasoning: &reasoning,
  300. SupportedReasoningLevels: levels,
  301. InputModalities: modalities,
  302. ContextWindow: contextWindow,
  303. },
  304. }})
  305. return account
  306. }
  307. svc := &GatewayService{accountRepo: codexModelsVisibilityAccountRepo{byGroup: map[int64][]Account{
  308. groupID: {
  309. newAccount(26, []string{"low", "high"}, []string{"text", "image"}, 256_000),
  310. newAccount(27, []string{"high", "max"}, []string{"text"}, 128_000),
  311. },
  312. }}}
  313. body, err := svc.BuildCodexModelsManifestForGroup(
  314. context.Background(), &Group{ID: groupID, Platform: PlatformComposite}, "", []string{"shared-model"},
  315. )
  316. require.NoError(t, err)
  317. models := decodeCodexManifestModels(t, body)
  318. require.Len(t, models, 1)
  319. require.Equal(t, []string{"high"}, effortsFromManifestModel(t, models[0]))
  320. require.Equal(t, "high", models[0]["default_reasoning_level"])
  321. require.Equal(t, []any{"text"}, models[0]["input_modalities"])
  322. require.EqualValues(t, 128_000, models[0]["context_window"])
  323. }
  324. // Scenario: the same public alias may target different models on one platform when complete snapshots can be intersected.
  325. func TestBuildCodexModelsManifestForGroupIntersectsDifferentMappedTargetsWithoutLeakingAlias(t *testing.T) {
  326. t.Parallel()
  327. const groupID int64 = 739
  328. reasoning := true
  329. newAccount := func(id int64, target, displayName, description string, levels, modalities []string, contextWindow int64) Account {
  330. account := Account{
  331. ID: id, Platform: PlatformOpenAI, Type: AccountTypeAPIKey,
  332. Credentials: map[string]any{
  333. "base_url": "https://provider.example/v1",
  334. "model_mapping": map[string]any{"my-coder": target},
  335. },
  336. }
  337. account.SetUpstreamModelMetadataSnapshot(UpstreamModelMetadataSnapshot{Models: map[string]UpstreamModelMetadata{
  338. target: {
  339. ID: target, DisplayName: displayName, Description: description, Reasoning: &reasoning,
  340. SupportedReasoningLevels: levels,
  341. InputModalities: modalities,
  342. ContextWindow: contextWindow,
  343. },
  344. }})
  345. return account
  346. }
  347. openAIAccount := newAccount(
  348. 31,
  349. "gpt-5.6-sol",
  350. "GPT-5.6 Sol",
  351. "OpenAI upstream model",
  352. []string{"low", "medium", "high", "xhigh"},
  353. []string{"text", "image"},
  354. 272_000,
  355. )
  356. arkAccount := newAccount(
  357. 32,
  358. "glm-5.3",
  359. "GLM 5.3",
  360. "Ark upstream model",
  361. []string{"low", "medium", "high"},
  362. []string{"text"},
  363. 1_000_000,
  364. )
  365. for _, accounts := range [][]Account{{openAIAccount, arkAccount}, {arkAccount, openAIAccount}} {
  366. svc := &GatewayService{accountRepo: codexModelsVisibilityAccountRepo{byGroup: map[int64][]Account{
  367. groupID: accounts,
  368. }}}
  369. body, err := svc.BuildCodexModelsManifestForGroup(
  370. context.Background(), &Group{ID: groupID, Platform: PlatformOpenAI}, "", []string{"my-coder"},
  371. )
  372. require.NoError(t, err)
  373. models := decodeCodexManifestModels(t, body)
  374. require.Len(t, models, 1)
  375. require.Equal(t, "my-coder", models[0]["slug"])
  376. require.Equal(t, "my-coder", models[0]["display_name"])
  377. require.Equal(t, "Custom model routed through Sub2API.", models[0]["description"])
  378. require.Equal(t, []string{"low", "medium", "high"}, effortsFromManifestModel(t, models[0]))
  379. require.Equal(t, []any{"text"}, models[0]["input_modalities"])
  380. require.EqualValues(t, 272_000, models[0]["context_window"])
  381. }
  382. }
  383. // Scenario: accounts that leave the current scheduling pool only because of
  384. // transient state still participate in capability intersection.
  385. func TestBuildCodexModelsManifestForGroupIntersectsTransientlyUnschedulableMappedAccounts(t *testing.T) {
  386. t.Parallel()
  387. const groupID int64 = 741
  388. schedulable := newCodexCatalogMappedAccount(
  389. 41,
  390. "gpt-5.6-sol",
  391. "GPT-5.6 Sol",
  392. []string{"low", "medium", "high", "xhigh"},
  393. []string{"text", "image"},
  394. 1_000_000,
  395. true,
  396. nil,
  397. )
  398. transientlyUnschedulable := newCodexCatalogMappedAccount(
  399. 42,
  400. "glm-5.3",
  401. "GLM 5.3",
  402. []string{"low", "medium", "high"},
  403. []string{"text"},
  404. 272_000,
  405. true,
  406. map[string]any{"exclusive-model": "exclusive-upstream"},
  407. )
  408. svc := &GatewayService{accountRepo: splitCodexModelsAccountRepo{
  409. schedulable: map[int64][]Account{groupID: {schedulable}},
  410. catalog: map[int64][]Account{groupID: {schedulable, transientlyUnschedulable}},
  411. }}
  412. body, err := svc.BuildCodexModelsManifestForGroup(
  413. context.Background(), &Group{ID: groupID, Platform: PlatformOpenAI}, "", []string{"my-coder"},
  414. )
  415. require.NoError(t, err)
  416. models := decodeCodexManifestModels(t, body)
  417. require.Len(t, models, 1)
  418. require.Equal(t, "my-coder", models[0]["slug"])
  419. require.Equal(t, "my-coder", models[0]["display_name"])
  420. require.Equal(t, []string{"low", "medium", "high"}, effortsFromManifestModel(t, models[0]))
  421. require.Equal(t, []any{"text"}, models[0]["input_modalities"])
  422. require.EqualValues(t, 272_000, models[0]["context_window"])
  423. }
  424. // Scenario: a persistently disabled account cannot narrow the advertised contract.
  425. func TestBuildCodexModelsManifestForGroupIgnoresPersistentlyDisabledMappedAccounts(t *testing.T) {
  426. t.Parallel()
  427. const groupID int64 = 742
  428. remaining := newCodexCatalogMappedAccount(
  429. 41,
  430. "gpt-5.6-sol",
  431. "GPT-5.6 Sol",
  432. []string{"low", "medium", "high", "xhigh"},
  433. []string{"text", "image"},
  434. 1_000_000,
  435. true,
  436. nil,
  437. )
  438. disabled := newCodexCatalogMappedAccount(
  439. 42,
  440. "glm-5.3",
  441. "GLM 5.3",
  442. []string{"low", "medium", "high"},
  443. []string{"text"},
  444. 272_000,
  445. false,
  446. nil,
  447. )
  448. svc := &GatewayService{accountRepo: splitCodexModelsAccountRepo{
  449. schedulable: map[int64][]Account{groupID: {remaining}},
  450. catalog: map[int64][]Account{groupID: {remaining}},
  451. all: map[int64][]Account{groupID: {remaining, disabled}},
  452. }}
  453. body, err := svc.BuildCodexModelsManifestForGroup(
  454. context.Background(), &Group{ID: groupID, Platform: PlatformOpenAI}, "", []string{"my-coder"},
  455. )
  456. require.NoError(t, err)
  457. models := decodeCodexManifestModels(t, body)
  458. require.Len(t, models, 1)
  459. require.Equal(t, []any{"text", "image"}, models[0]["input_modalities"])
  460. require.EqualValues(t, 1_000_000, models[0]["context_window"])
  461. }
  462. func TestBuildCodexModelsManifestForGroupFallsBackToSchedulableWhenAvailabilityLookupFails(t *testing.T) {
  463. t.Parallel()
  464. const groupID int64 = 743
  465. repo := &countingCodexModelsAccountRepo{
  466. accounts: []Account{newCodexCatalogMappedAccount(
  467. 41,
  468. "gpt-5.6-sol",
  469. "GPT-5.6 Sol",
  470. []string{"low", "medium", "high", "xhigh"},
  471. []string{"text", "image"},
  472. 1_000_000,
  473. true,
  474. nil,
  475. )},
  476. availabilityErr: errors.New("group listing unavailable"),
  477. }
  478. svc := &GatewayService{accountRepo: repo}
  479. body, err := svc.BuildCodexModelsManifestForGroup(
  480. context.Background(), &Group{ID: groupID, Platform: PlatformOpenAI}, "", []string{"my-coder"},
  481. )
  482. require.NoError(t, err)
  483. require.Equal(t, int32(1), repo.calls.Load())
  484. models := decodeCodexManifestModels(t, body)
  485. require.Len(t, models, 1)
  486. require.Equal(t, []any{"text", "image"}, models[0]["input_modalities"])
  487. require.EqualValues(t, 1_000_000, models[0]["context_window"])
  488. }
  489. // Scenario: a Composite alias claimed across platforms remains ambiguous and fails closed.
  490. func TestBuildCodexModelsManifestForGroupKeepsCrossPlatformAliasAmbiguityClosed(t *testing.T) {
  491. t.Parallel()
  492. const groupID int64 = 740
  493. reasoning := true
  494. newAccount := func(id int64, platform, target string) Account {
  495. account := Account{
  496. ID: id, Platform: platform, Type: AccountTypeAPIKey,
  497. Credentials: map[string]any{"model_mapping": map[string]any{"shared-alias": target}},
  498. }
  499. account.SetUpstreamModelMetadataSnapshot(UpstreamModelMetadataSnapshot{Models: map[string]UpstreamModelMetadata{
  500. target: {
  501. ID: target, DisplayName: target, Reasoning: &reasoning,
  502. SupportedReasoningLevels: []string{"low", "high"},
  503. InputModalities: []string{"text", "image"},
  504. ContextWindow: 128_000,
  505. },
  506. }})
  507. return account
  508. }
  509. svc := &GatewayService{accountRepo: codexModelsVisibilityAccountRepo{byGroup: map[int64][]Account{
  510. groupID: {
  511. newAccount(33, PlatformOpenAI, "gpt-5.6-sol"),
  512. newAccount(34, PlatformGrok, "grok-4.6"),
  513. },
  514. }}}
  515. body, err := svc.BuildCodexModelsManifestForGroup(
  516. context.Background(), &Group{ID: groupID, Platform: PlatformComposite}, "", []string{"shared-alias"},
  517. )
  518. require.NoError(t, err)
  519. models := decodeCodexManifestModels(t, body)
  520. require.Len(t, models, 1)
  521. require.Equal(t, "shared-alias", models[0]["display_name"])
  522. require.Empty(t, effortsFromManifestModel(t, models[0]))
  523. require.Equal(t, []any{"text"}, models[0]["input_modalities"])
  524. }
  525. func TestBuildCodexModelsManifestForGroupDoesNotAdvertiseNoneWhenAccountReasoningConflicts(t *testing.T) {
  526. t.Parallel()
  527. const groupID int64 = 738
  528. reasoning := true
  529. noReasoning := false
  530. newAccount := func(id int64, metadata UpstreamModelMetadata) Account {
  531. account := Account{
  532. ID: id, Platform: PlatformOpenAI, Type: AccountTypeAPIKey,
  533. Credentials: map[string]any{
  534. "base_url": "https://provider.example/v1",
  535. "model_mapping": map[string]any{"shared-model": "shared-model"},
  536. },
  537. }
  538. account.SetUpstreamModelMetadataSnapshot(UpstreamModelMetadataSnapshot{Models: map[string]UpstreamModelMetadata{
  539. "shared-model": metadata,
  540. }})
  541. return account
  542. }
  543. svc := &GatewayService{accountRepo: codexModelsVisibilityAccountRepo{byGroup: map[int64][]Account{
  544. groupID: {
  545. newAccount(29, UpstreamModelMetadata{
  546. ID: "shared-model", Reasoning: &reasoning,
  547. SupportedReasoningLevels: []string{"low", "high"},
  548. InputModalities: []string{"text"}, ContextWindow: 128_000,
  549. }),
  550. newAccount(30, UpstreamModelMetadata{
  551. ID: "shared-model", Reasoning: &noReasoning,
  552. InputModalities: []string{"text"}, ContextWindow: 128_000,
  553. }),
  554. },
  555. }}}
  556. body, err := svc.BuildCodexModelsManifestForGroup(
  557. context.Background(), &Group{ID: groupID, Platform: PlatformComposite}, "", []string{"shared-model"},
  558. )
  559. require.NoError(t, err)
  560. models := decodeCodexManifestModels(t, body)
  561. require.Len(t, models, 1)
  562. _, hasDefault := models[0]["default_reasoning_level"]
  563. require.False(t, hasDefault)
  564. require.Empty(t, models[0]["supported_reasoning_levels"])
  565. }
  566. func TestAstraCodexToolCapabilitiesFollowAPIKeyAlias(t *testing.T) {
  567. account := &Account{Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Credentials: map[string]any{
  568. "base_url": "https://relay.example/v1", "model_mapping": map[string]any{"my-astra": "gpt-6-astra"},
  569. }}
  570. account.SetUpstreamModelMetadataSnapshot(UpstreamModelMetadataSnapshot{Models: map[string]UpstreamModelMetadata{
  571. "gpt-6-astra": {CodexToolCapabilities: map[string]json.RawMessage{
  572. "supports_search_tool": json.RawMessage("true"), "apply_patch_tool_type": json.RawMessage(`"freeform"`),
  573. "use_responses_lite": json.RawMessage("false"),
  574. }},
  575. }})
  576. for _, source := range []string{`{"data":[{"id":"my-astra"}]}`, `{"models":[{"slug":"my-astra"}]}`} {
  577. converted := convertOpenAIModelListToCodexManifestForAccount([]byte(source), account)
  578. body, err := completeAPIKeyCodexModelsManifestMetadata(converted, true, account)
  579. require.NoError(t, err)
  580. models := decodeCodexManifestModels(t, body)
  581. require.Len(t, models, 1)
  582. model := models[0]
  583. require.Equal(t, "my-astra", model["slug"])
  584. require.Equal(t, "my-astra", model["display_name"])
  585. require.Equal(t, true, model["supports_search_tool"])
  586. require.Equal(t, "freeform", model["apply_patch_tool_type"])
  587. require.Equal(t, false, model["use_responses_lite"])
  588. }
  589. }
  590. func TestAstraCodexToolCapabilitiesKeepAPIKeyResponsesLiteGuard(t *testing.T) {
  591. account := Account{Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Credentials: map[string]any{
  592. "base_url": "https://relay.example/v1", "model_mapping": map[string]any{"my-astra": "gpt-6-astra"},
  593. }}
  594. account.SetUpstreamModelMetadataSnapshot(UpstreamModelMetadataSnapshot{Models: map[string]UpstreamModelMetadata{
  595. "gpt-6-astra": {CodexToolCapabilities: map[string]json.RawMessage{"use_responses_lite": json.RawMessage("true")}},
  596. }})
  597. body, err := buildCodexModelsManifestForAccounts(PlatformOpenAI, []string{"my-astra"}, []Account{account}, nil, nil, true)
  598. require.NoError(t, err)
  599. require.Equal(t, false, decodeCodexManifestModels(t, body)[0]["use_responses_lite"])
  600. body, err = adjustAPIKeyCodexModelsManifest([]byte(`{"models":[{"slug":"my-astra","use_responses_lite":true}]}`), &account)
  601. require.NoError(t, err)
  602. require.Equal(t, false, decodeCodexManifestModels(t, body)[0]["use_responses_lite"])
  603. }
  604. // Scenario: an OpenAI group keeps a failover account whose mapping points the
  605. // shared public alias at a different upstream model. Without an explicit routing
  606. // rule the alias target is ambiguous and capabilities must fail closed; with one,
  607. // the operator has declared who serves the alias, so the reasoning slider and
  608. // image input must survive.
  609. func TestCodexAliasFailoverMappingHonorsModelRouting(t *testing.T) {
  610. newAccount := func(id int64, target string, priority int) Account {
  611. return Account{
  612. ID: id,
  613. Platform: PlatformOpenAI,
  614. Type: AccountTypeAPIKey,
  615. Priority: priority,
  616. Credentials: map[string]any{
  617. "base_url": "https://relay.example/v1",
  618. "model_mapping": map[string]any{"gpt-6-astra": target},
  619. },
  620. }
  621. }
  622. // Account 16-alike serves the alias natively; account 1-alike is the failover
  623. // and can only reach a different upstream model.
  624. primary := newAccount(16, "gpt-6-astra", 0)
  625. failover := newAccount(1, "gpt-5.6-sol", 10)
  626. accounts := []Account{primary, failover}
  627. // supported_reasoning_levels entries are {effort, description} objects; the
  628. // slider only cares about the effort names.
  629. manifestFieldsOf := func(t *testing.T, group *Group) ([]string, any, any) {
  630. t.Helper()
  631. body, err := buildCodexModelsManifestForAccounts(
  632. PlatformOpenAI, []string{"gpt-6-astra"}, accounts, group, nil, true,
  633. )
  634. require.NoError(t, err)
  635. models := decodeCodexManifestModels(t, body)
  636. require.Len(t, models, 1)
  637. model := models[0]
  638. require.Equal(t, "gpt-6-astra", model["slug"])
  639. var efforts []string
  640. levels, _ := model["supported_reasoning_levels"].([]any)
  641. for _, level := range levels {
  642. entry, ok := level.(map[string]any)
  643. require.True(t, ok, "reasoning level entry must be an object")
  644. effort, ok := entry["effort"].(string)
  645. require.True(t, ok, "reasoning level entry must carry an effort name")
  646. efforts = append(efforts, effort)
  647. }
  648. return efforts, model["default_reasoning_level"], model["input_modalities"]
  649. }
  650. t.Run("no routing rule still fails closed", func(t *testing.T) {
  651. levels, def, modalities := manifestFieldsOf(t, nil)
  652. require.Empty(t, levels, "ambiguous alias target must not advertise reasoning levels")
  653. require.Nil(t, def)
  654. require.Equal(t, []any{"text", "image"}, modalities,
  655. "image input is resolved per-account independently of routing rules")
  656. })
  657. t.Run("routing rule resolves the alias", func(t *testing.T) {
  658. group := &Group{
  659. ID: 2,
  660. Platform: PlatformOpenAI,
  661. ModelRoutingEnabled: true,
  662. ModelRouting: map[string][]int64{"gpt-6-astra": {16, 1}},
  663. }
  664. levels, def, modalities := manifestFieldsOf(t, group)
  665. require.Equal(t,
  666. []string{"low", "medium", "high", "xhigh", "max", "ultra"},
  667. levels,
  668. "routed alias must keep a movable reasoning slider",
  669. )
  670. require.Equal(t, "medium", def)
  671. require.Contains(t, modalities, "image")
  672. })
  673. t.Run("routing disabled falls back to failing closed", func(t *testing.T) {
  674. group := &Group{
  675. ID: 2,
  676. Platform: PlatformOpenAI,
  677. ModelRoutingEnabled: false,
  678. ModelRouting: map[string][]int64{"gpt-6-astra": {16, 1}},
  679. }
  680. levels, _, _ := manifestFieldsOf(t, group)
  681. require.Empty(t, levels, "a disabled routing rule must not resolve the alias")
  682. })
  683. t.Run("routing rule for another alias does not leak", func(t *testing.T) {
  684. group := &Group{
  685. ID: 2,
  686. Platform: PlatformOpenAI,
  687. ModelRoutingEnabled: true,
  688. ModelRouting: map[string][]int64{"gpt-5.6-sol": {1}},
  689. }
  690. levels, _, _ := manifestFieldsOf(t, group)
  691. require.Empty(t, levels, "unrelated routing rules must not resolve this alias")
  692. })
  693. }
  694. // Scenario: mixed groups prefer capability metadata synced for the routed account.