ratelimit_service_model_not_found_test.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527
  1. //go:build unit
  2. package service
  3. import (
  4. "context"
  5. "encoding/json"
  6. "errors"
  7. "net/http"
  8. "testing"
  9. "time"
  10. "github.com/stretchr/testify/require"
  11. )
  12. type modelNotFoundRateLimitCall struct {
  13. accountID int64
  14. scope string
  15. resetAt time.Time
  16. reason string
  17. }
  18. type modelNotFoundAccountRepoStub struct {
  19. mockAccountRepoForGemini
  20. tempCalls int
  21. modelRateLimitCalls []modelNotFoundRateLimitCall
  22. modelRateLimitErr error
  23. }
  24. func (r *modelNotFoundAccountRepoStub) SetTempUnschedulable(ctx context.Context, id int64, until time.Time, reason string) error {
  25. r.tempCalls++
  26. return nil
  27. }
  28. func (r *modelNotFoundAccountRepoStub) SetModelRateLimit(ctx context.Context, id int64, scope string, resetAt time.Time, reason ...string) error {
  29. call := modelNotFoundRateLimitCall{
  30. accountID: id,
  31. scope: scope,
  32. resetAt: resetAt,
  33. }
  34. if len(reason) > 0 {
  35. call.reason = reason[0]
  36. }
  37. r.modelRateLimitCalls = append(r.modelRateLimitCalls, call)
  38. return r.modelRateLimitErr
  39. }
  40. func TestRateLimitService_HandleUpstreamError_ModelNotFoundUsesModelRateLimit(t *testing.T) {
  41. repo := &modelNotFoundAccountRepoStub{}
  42. svc := &RateLimitService{accountRepo: repo}
  43. account := openAIModelNotFoundTempAccount()
  44. handled := svc.HandleUpstreamError(
  45. context.Background(),
  46. account,
  47. http.StatusNotFound,
  48. http.Header{},
  49. []byte(`{"error":{"code":"model_not_found","message":"model not found"}}`),
  50. "gpt-5.4",
  51. )
  52. require.True(t, handled)
  53. require.Zero(t, repo.tempCalls)
  54. require.Len(t, repo.modelRateLimitCalls, 1)
  55. call := repo.modelRateLimitCalls[0]
  56. require.Equal(t, account.ID, call.accountID)
  57. require.Equal(t, "gpt-5.4", call.scope)
  58. require.Equal(t, upstreamModelNotFoundReason, call.reason)
  59. require.WithinDuration(t, time.Now().Add(upstreamModelNotFoundCooldown), call.resetAt, 5*time.Second)
  60. }
  61. func TestRateLimitService_HandleUpstreamError_ModelNotFoundWriteFailureDoesNotTempUnschedule(t *testing.T) {
  62. repo := &modelNotFoundAccountRepoStub{modelRateLimitErr: errors.New("write failed")}
  63. svc := &RateLimitService{accountRepo: repo}
  64. account := openAIModelNotFoundTempAccount()
  65. handled := svc.HandleUpstreamError(
  66. context.Background(),
  67. account,
  68. http.StatusNotFound,
  69. http.Header{},
  70. []byte(`{"error":{"code":"model_not_found","message":"model not found"}}`),
  71. "gpt-5.4",
  72. )
  73. require.True(t, handled)
  74. require.Zero(t, repo.tempCalls)
  75. require.Len(t, repo.modelRateLimitCalls, 1)
  76. }
  77. func TestRateLimitService_HandleUpstreamError_Bare404UsesModelScopedTempUnschedulableWhenModelKnown(t *testing.T) {
  78. repo := &modelNotFoundAccountRepoStub{}
  79. svc := &RateLimitService{accountRepo: repo}
  80. account := openAIModelNotFoundTempAccount()
  81. handled := svc.HandleUpstreamError(
  82. context.Background(),
  83. account,
  84. http.StatusNotFound,
  85. http.Header{},
  86. []byte(`{"error":{"message":"endpoint not found"}}`),
  87. "gpt-5.4",
  88. )
  89. require.True(t, handled)
  90. require.Zero(t, repo.tempCalls)
  91. require.Len(t, repo.modelRateLimitCalls, 1)
  92. call := repo.modelRateLimitCalls[0]
  93. require.Equal(t, account.ID, call.accountID)
  94. require.Equal(t, "gpt-5.4", call.scope)
  95. require.WithinDuration(t, time.Now().Add(10*time.Minute), call.resetAt, 5*time.Second)
  96. var state TempUnschedState
  97. require.NoError(t, json.Unmarshal([]byte(call.reason), &state))
  98. require.Equal(t, http.StatusNotFound, state.StatusCode)
  99. require.Equal(t, "not found", state.MatchedKeyword)
  100. }
  101. func TestRateLimitService_HandleUpstreamError_Bare404WithoutModelKeepsAccountTempUnschedulable(t *testing.T) {
  102. repo := &modelNotFoundAccountRepoStub{}
  103. svc := &RateLimitService{accountRepo: repo}
  104. account := openAIModelNotFoundTempAccount()
  105. handled := svc.HandleUpstreamError(
  106. context.Background(),
  107. account,
  108. http.StatusNotFound,
  109. http.Header{},
  110. []byte(`{"error":{"message":"endpoint not found"}}`),
  111. )
  112. require.True(t, handled)
  113. require.Equal(t, 1, repo.tempCalls)
  114. require.Empty(t, repo.modelRateLimitCalls)
  115. }
  116. func TestRateLimitService_HandleUpstreamError_ModelTempWriteFailureNeverWidensToAccount(t *testing.T) {
  117. repo := &modelNotFoundAccountRepoStub{modelRateLimitErr: errors.New("write failed")}
  118. svc := &RateLimitService{accountRepo: repo}
  119. account := openAIModelNotFoundTempAccount()
  120. handled := svc.HandleUpstreamError(
  121. context.Background(),
  122. account,
  123. http.StatusNotFound,
  124. http.Header{},
  125. []byte(`{"error":{"message":"endpoint not found"}}`),
  126. "gpt-5.4",
  127. )
  128. require.True(t, handled)
  129. require.Zero(t, repo.tempCalls)
  130. require.Len(t, repo.modelRateLimitCalls, 1)
  131. }
  132. func TestRateLimitService_HandleTempUnschedulable_PoolModeWithoutCustomPolicySkipsState(t *testing.T) {
  133. repo := &modelNotFoundAccountRepoStub{}
  134. svc := &RateLimitService{accountRepo: repo}
  135. account := openAIModelNotFoundTempAccount()
  136. account.Credentials["pool_mode"] = true
  137. handled := svc.HandleTempUnschedulable(
  138. context.Background(),
  139. account,
  140. http.StatusNotFound,
  141. []byte(`{"error":{"message":"endpoint not found"}}`),
  142. "gpt-5.4",
  143. )
  144. require.False(t, handled)
  145. require.Zero(t, repo.tempCalls)
  146. require.Empty(t, repo.modelRateLimitCalls)
  147. }
  148. func TestRateLimitService_HandleTempUnschedulable_PoolModeCustomPolicyUsesModelScope(t *testing.T) {
  149. repo := &modelNotFoundAccountRepoStub{}
  150. svc := &RateLimitService{accountRepo: repo}
  151. account := openAIModelNotFoundTempAccount()
  152. account.Credentials["pool_mode"] = true
  153. account.Credentials["custom_error_codes_enabled"] = true
  154. account.Credentials["custom_error_codes"] = []any{float64(http.StatusNotFound)}
  155. handled := svc.HandleTempUnschedulable(
  156. context.Background(),
  157. account,
  158. http.StatusNotFound,
  159. []byte(`{"error":{"message":"endpoint not found"}}`),
  160. "gpt-5.4",
  161. )
  162. require.True(t, handled)
  163. require.Zero(t, repo.tempCalls)
  164. require.Len(t, repo.modelRateLimitCalls, 1)
  165. require.Equal(t, "gpt-5.4", repo.modelRateLimitCalls[0].scope)
  166. }
  167. func TestRateLimitService_TempUnschedulableContextPreservesModelForPoolDependency(t *testing.T) {
  168. repo := &modelNotFoundAccountRepoStub{}
  169. svc := &RateLimitService{accountRepo: repo}
  170. account := openAIModelNotFoundTempAccount()
  171. account.Credentials["pool_mode"] = true
  172. ctx := withTempUnschedulableModel(context.Background(), []string{"gpt-5.4"})
  173. // #4496 calls tryTempUnschedulable from the pool-mode branch without an
  174. // explicit model argument. The request context must preserve the canonical
  175. // model so that combined behavior remains model-scoped after it lands.
  176. handled := svc.tryTempUnschedulable(
  177. ctx,
  178. account,
  179. http.StatusNotFound,
  180. []byte(`{"error":{"message":"endpoint not found"}}`),
  181. )
  182. require.True(t, handled)
  183. require.Zero(t, repo.tempCalls)
  184. require.Len(t, repo.modelRateLimitCalls, 1)
  185. require.Equal(t, "gpt-5.4", repo.modelRateLimitCalls[0].scope)
  186. }
  187. func TestRateLimitService_HandleUpstreamError_CustomPolicyExclusionSkipsAllState(t *testing.T) {
  188. repo := &modelNotFoundAccountRepoStub{}
  189. svc := &RateLimitService{accountRepo: repo}
  190. account := openAIModelNotFoundTempAccount()
  191. account.Credentials["custom_error_codes_enabled"] = true
  192. account.Credentials["custom_error_codes"] = []any{float64(http.StatusServiceUnavailable)}
  193. handled := svc.HandleUpstreamError(
  194. context.Background(),
  195. account,
  196. http.StatusNotFound,
  197. http.Header{},
  198. []byte(`{"error":{"message":"endpoint not found"}}`),
  199. "gpt-5.4",
  200. )
  201. require.False(t, handled)
  202. require.Zero(t, repo.tempCalls)
  203. require.Empty(t, repo.modelRateLimitCalls)
  204. }
  205. func TestRateLimitService_HandleTempUnschedulable_AuthenticationFailureStaysAccountScoped(t *testing.T) {
  206. repo := &modelNotFoundAccountRepoStub{}
  207. svc := &RateLimitService{accountRepo: repo}
  208. account := openAIModelNotFoundTempAccount()
  209. account.TempUnschedulableReason = "legacy non-JSON reason"
  210. account.Credentials["temp_unschedulable_rules"] = []any{
  211. map[string]any{
  212. "error_code": float64(http.StatusUnauthorized),
  213. "keywords": []any{"unauthorized"},
  214. "duration_minutes": float64(10),
  215. },
  216. }
  217. handled := svc.HandleTempUnschedulable(
  218. context.Background(),
  219. account,
  220. http.StatusUnauthorized,
  221. []byte(`{"error":{"message":"unauthorized"}}`),
  222. "gpt-5.4",
  223. )
  224. require.True(t, handled)
  225. require.Equal(t, 1, repo.tempCalls)
  226. require.Empty(t, repo.modelRateLimitCalls)
  227. }
  228. func TestRateLimitService_ModelTempUnschedulableIsolatesSchedulerByModel(t *testing.T) {
  229. repo := &modelNotFoundAccountRepoStub{}
  230. svc := &RateLimitService{accountRepo: repo}
  231. account := openAIModelNotFoundTempAccount()
  232. account.Credentials["model_mapping"] = map[string]any{
  233. "public-a": "upstream-a",
  234. "upstream-a": "upstream-b",
  235. }
  236. handled := svc.HandleUpstreamError(
  237. context.Background(),
  238. account,
  239. http.StatusNotFound,
  240. http.Header{},
  241. []byte(`{"error":{"message":"endpoint not found"}}`),
  242. "upstream-a",
  243. )
  244. require.True(t, handled)
  245. require.Len(t, repo.modelRateLimitCalls, 1)
  246. call := repo.modelRateLimitCalls[0]
  247. require.Equal(t, "upstream-a", call.scope, "canonical upstream model must not be mapped a second time")
  248. account.Extra = map[string]any{
  249. modelRateLimitsKey: map[string]any{
  250. call.scope: map[string]any{
  251. "rate_limit_reset_at": call.resetAt.UTC().Format(time.RFC3339),
  252. },
  253. },
  254. }
  255. require.False(t, account.IsSchedulableForModelWithContext(context.Background(), "public-a"))
  256. require.True(t, account.IsSchedulableForModelWithContext(context.Background(), "gpt-5.6-sol"))
  257. }
  258. func openAIModelNotFoundTempAccount() *Account {
  259. return &Account{
  260. ID: 101,
  261. Platform: PlatformOpenAI,
  262. Type: AccountTypeAPIKey,
  263. Status: StatusActive,
  264. Schedulable: true,
  265. Credentials: map[string]any{
  266. "temp_unschedulable_enabled": true,
  267. "temp_unschedulable_rules": []any{
  268. map[string]any{
  269. "error_code": float64(http.StatusNotFound),
  270. "keywords": []any{"not found"},
  271. "duration_minutes": float64(10),
  272. },
  273. },
  274. },
  275. }
  276. }
  277. func TestRateLimitService_HandleUpstreamError_CodexPlanGatedModelUsesModelRateLimit(t *testing.T) {
  278. repo := &modelNotFoundAccountRepoStub{}
  279. svc := &RateLimitService{accountRepo: repo}
  280. account := openAICodexPlanGatedOAuthAccount()
  281. handled := svc.HandleUpstreamError(
  282. context.Background(),
  283. account,
  284. http.StatusBadRequest,
  285. http.Header{},
  286. []byte(`{"detail":"The 'gpt-5.6-sol' model is not supported when using Codex with a ChatGPT account."}`),
  287. "gpt-5.6-sol",
  288. )
  289. require.True(t, handled)
  290. require.Zero(t, repo.tempCalls)
  291. require.Len(t, repo.modelRateLimitCalls, 1)
  292. call := repo.modelRateLimitCalls[0]
  293. require.Equal(t, account.ID, call.accountID)
  294. require.Equal(t, "gpt-5.6-sol", call.scope)
  295. require.Equal(t, upstreamCodexPlanGatedModelReason, call.reason)
  296. require.WithinDuration(t, time.Now().Add(upstreamCodexPlanGatedModelCooldown), call.resetAt, 5*time.Second)
  297. }
  298. func TestRateLimitService_HandleUpstreamError_CodexPlanGatedModelRespectsModelMapping(t *testing.T) {
  299. repo := &modelNotFoundAccountRepoStub{}
  300. svc := &RateLimitService{accountRepo: repo}
  301. account := openAICodexPlanGatedOAuthAccount()
  302. account.Credentials["model_mapping"] = map[string]any{"gpt-5.6-sol": "gpt-5.6-sol-upstream"}
  303. handled := svc.HandleUpstreamError(
  304. context.Background(),
  305. account,
  306. http.StatusBadRequest,
  307. http.Header{},
  308. []byte(`{"detail":"The 'gpt-5.6-sol-upstream' model is not supported when using Codex with a ChatGPT account."}`),
  309. "gpt-5.6-sol",
  310. )
  311. require.True(t, handled)
  312. require.Len(t, repo.modelRateLimitCalls, 1)
  313. require.Equal(t, "gpt-5.6-sol-upstream", repo.modelRateLimitCalls[0].scope)
  314. }
  315. func TestRateLimitService_HandleUpstreamError_CodexPlanGatedModelIgnoresAPIKeyAccount(t *testing.T) {
  316. repo := &modelNotFoundAccountRepoStub{}
  317. svc := &RateLimitService{accountRepo: repo}
  318. account := openAICodexPlanGatedOAuthAccount()
  319. account.Type = AccountTypeAPIKey
  320. handled := svc.HandleUpstreamError(
  321. context.Background(),
  322. account,
  323. http.StatusBadRequest,
  324. http.Header{},
  325. []byte(`{"detail":"The 'gpt-5.6-sol' model is not supported when using Codex with a ChatGPT account."}`),
  326. "gpt-5.6-sol",
  327. )
  328. require.False(t, handled)
  329. require.Empty(t, repo.modelRateLimitCalls)
  330. }
  331. func TestRateLimitService_HandleUpstreamError_CodexPlanGatedImageModelSkipsCooldown(t *testing.T) {
  332. for _, model := range []string{"gpt-image-1", "gpt-image-1.5", "gpt-image-2"} {
  333. t.Run(model, func(t *testing.T) {
  334. repo := &modelNotFoundAccountRepoStub{}
  335. svc := &RateLimitService{accountRepo: repo}
  336. account := openAICodexPlanGatedOAuthAccount()
  337. handled := svc.HandleUpstreamError(
  338. context.Background(),
  339. account,
  340. http.StatusBadRequest,
  341. http.Header{},
  342. []byte(`{"detail":"The '`+model+`' model is not supported when using Codex with a ChatGPT account."}`),
  343. model,
  344. )
  345. require.True(t, handled, "attempt should still fail over")
  346. require.Empty(t, repo.modelRateLimitCalls,
  347. "image models must not be cooled down: the account still serves them over /v1/images/*")
  348. require.Zero(t, repo.tempCalls)
  349. })
  350. }
  351. }
  352. func TestRateLimitService_HandleUpstreamError_CodexPlanGatedTextModelStillCoolsDown(t *testing.T) {
  353. repo := &modelNotFoundAccountRepoStub{}
  354. svc := &RateLimitService{accountRepo: repo}
  355. account := openAICodexPlanGatedOAuthAccount()
  356. handled := svc.HandleUpstreamError(
  357. context.Background(),
  358. account,
  359. http.StatusBadRequest,
  360. http.Header{},
  361. []byte(`{"detail":"The 'gpt-5.6-sol' model is not supported when using Codex with a ChatGPT account."}`),
  362. "gpt-5.6-sol",
  363. )
  364. require.True(t, handled)
  365. require.Len(t, repo.modelRateLimitCalls, 1, "non-image plan-gated models keep the existing cooldown")
  366. require.Equal(t, upstreamCodexPlanGatedModelReason, repo.modelRateLimitCalls[0].reason)
  367. }
  368. func openAICodexPlanGatedOAuthAccount() *Account {
  369. return &Account{
  370. ID: 202,
  371. Platform: PlatformOpenAI,
  372. Type: AccountTypeOAuth,
  373. Status: StatusActive,
  374. Schedulable: true,
  375. Credentials: map[string]any{},
  376. }
  377. }
  378. // 请求本身就走 /v1/images/* 时必须保留冷却。
  379. //
  380. // OAuth 账号的 /v1/images/* 上游同样是 Codex Responses(openai_images_responses.go
  381. // → handleOpenAIImagesErrorResponse → handleOpenAIAccountUpstreamError →
  382. // HandleUpstreamModelNotFound),所以这条路径也会命中 plan-gated 分支。账号确实
  383. // 不具备生图能力时,冷却是唯一的刹车:调度层靠 model_rate_limits 跳过该账号后
  384. // 快速 503;一旦跳过冷却,每个请求都会完整走一遍号池,对上游形成无上界的 400 放大。
  385. func TestRateLimitService_HandleUpstreamError_CodexPlanGatedImageModelKeepsCooldownOnImagesEndpoint(t *testing.T) {
  386. repo := &modelNotFoundAccountRepoStub{}
  387. svc := &RateLimitService{accountRepo: repo}
  388. account := openAICodexPlanGatedOAuthAccount()
  389. handled := svc.HandleUpstreamError(
  390. WithOpenAIImagesEndpoint(context.Background()),
  391. account,
  392. http.StatusBadRequest,
  393. http.Header{},
  394. []byte(`{"detail":"The 'gpt-image-2' model is not supported when using Codex with a ChatGPT account."}`),
  395. "gpt-image-2",
  396. )
  397. require.True(t, handled)
  398. require.Len(t, repo.modelRateLimitCalls, 1,
  399. "/v1/images/* 上的 plan-gated 拒绝是真实的能力缺失,必须保留冷却刹车")
  400. require.Equal(t, "gpt-image-2", repo.modelRateLimitCalls[0].scope)
  401. require.Equal(t, upstreamCodexPlanGatedModelReason, repo.modelRateLimitCalls[0].reason)
  402. }
  403. // 仅 WithOpenAIImageGenerationIntent(/v1/responses 因模型名自动置位)不算专用生图
  404. // 端点,仍按"用错端点"处理。
  405. func TestRateLimitService_HandleUpstreamError_CodexPlanGatedImageModelSkipsCooldownOnIntentOnly(t *testing.T) {
  406. repo := &modelNotFoundAccountRepoStub{}
  407. svc := &RateLimitService{accountRepo: repo}
  408. account := openAICodexPlanGatedOAuthAccount()
  409. handled := svc.HandleUpstreamError(
  410. WithOpenAIImageGenerationIntent(context.Background()),
  411. account,
  412. http.StatusBadRequest,
  413. http.Header{},
  414. []byte(`{"detail":"The 'gpt-image-2' model is not supported when using Codex with a ChatGPT account."}`),
  415. "gpt-image-2",
  416. )
  417. require.True(t, handled)
  418. require.Empty(t, repo.modelRateLimitCalls)
  419. }
  420. // 守卫口径必须与冷却键一致:冷却键走 account.GetMappedModel,账号可以把文本别名
  421. // 映射到 gpt-image-*,只判请求模型会漏掉这种形态,原 bug 原样复现。
  422. func TestRateLimitService_HandleUpstreamError_CodexPlanGatedImageModelSkipsCooldownViaModelMapping(t *testing.T) {
  423. repo := &modelNotFoundAccountRepoStub{}
  424. svc := &RateLimitService{accountRepo: repo}
  425. account := openAICodexPlanGatedOAuthAccount()
  426. account.Credentials["model_mapping"] = map[string]any{"my-draw-alias": "gpt-image-2"}
  427. handled := svc.HandleUpstreamError(
  428. context.Background(),
  429. account,
  430. http.StatusBadRequest,
  431. http.Header{},
  432. []byte(`{"detail":"The 'gpt-image-2' model is not supported when using Codex with a ChatGPT account."}`),
  433. "my-draw-alias",
  434. )
  435. require.True(t, handled)
  436. require.Empty(t, repo.modelRateLimitCalls,
  437. "映射后的上游模型是图片模型,冷却键会写到 gpt-image-2 上,守卫必须一并识别")
  438. }
  439. // 404 model-not-found 分支不受守卫影响:即使是图片模型也照常冷却。
  440. func TestRateLimitService_HandleUpstreamError_ModelNotFoundImageModelStillCoolsDown(t *testing.T) {
  441. repo := &modelNotFoundAccountRepoStub{}
  442. svc := &RateLimitService{accountRepo: repo}
  443. account := openAICodexPlanGatedOAuthAccount()
  444. handled := svc.HandleUpstreamError(
  445. context.Background(),
  446. account,
  447. http.StatusNotFound,
  448. http.Header{},
  449. []byte(`{"error":{"message":"The model 'gpt-image-2' does not exist","code":"model_not_found"}}`),
  450. "gpt-image-2",
  451. )
  452. require.True(t, handled)
  453. require.Len(t, repo.modelRateLimitCalls, 1, "守卫只作用于 codex plan-gated 分支")
  454. require.Equal(t, upstreamModelNotFoundReason, repo.modelRateLimitCalls[0].reason)
  455. }