antigravity_single_account_retry_test.go 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911
  1. //go:build unit
  2. package service
  3. import (
  4. "bytes"
  5. "context"
  6. "io"
  7. "net/http"
  8. "strings"
  9. "testing"
  10. "time"
  11. "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey"
  12. "github.com/stretchr/testify/require"
  13. )
  14. // ---------------------------------------------------------------------------
  15. // 辅助函数:构造带 SingleAccountRetry 标记的 context
  16. // ---------------------------------------------------------------------------
  17. func ctxWithSingleAccountRetry() context.Context {
  18. return context.WithValue(context.Background(), ctxkey.SingleAccountRetry, true)
  19. }
  20. // ---------------------------------------------------------------------------
  21. // 1. isSingleAccountRetry 测试
  22. // ---------------------------------------------------------------------------
  23. func TestIsSingleAccountRetry_True(t *testing.T) {
  24. ctx := context.WithValue(context.Background(), ctxkey.SingleAccountRetry, true)
  25. require.True(t, isSingleAccountRetry(ctx))
  26. }
  27. func TestIsSingleAccountRetry_False_NoValue(t *testing.T) {
  28. require.False(t, isSingleAccountRetry(context.Background()))
  29. }
  30. func TestIsSingleAccountRetry_False_ExplicitFalse(t *testing.T) {
  31. ctx := context.WithValue(context.Background(), ctxkey.SingleAccountRetry, false)
  32. require.False(t, isSingleAccountRetry(ctx))
  33. }
  34. func TestIsSingleAccountRetry_False_WrongType(t *testing.T) {
  35. ctx := context.WithValue(context.Background(), ctxkey.SingleAccountRetry, "true")
  36. require.False(t, isSingleAccountRetry(ctx))
  37. }
  38. // ---------------------------------------------------------------------------
  39. // 2. 常量验证
  40. // ---------------------------------------------------------------------------
  41. func TestSingleAccountRetryConstants(t *testing.T) {
  42. require.Equal(t, 3, antigravitySingleAccountSmartRetryMaxAttempts,
  43. "单账号原地重试最多 3 次")
  44. require.Equal(t, 15*time.Second, antigravitySingleAccountSmartRetryMaxWait,
  45. "单次最大等待 15s")
  46. require.Equal(t, 30*time.Second, antigravitySingleAccountSmartRetryTotalMaxWait,
  47. "总累计等待不超过 30s")
  48. }
  49. // ---------------------------------------------------------------------------
  50. // 3. handleSmartRetry + 503 + SingleAccountRetry → 走 handleSingleAccountRetryInPlace
  51. // (而非设模型限流 + 切换账号)
  52. // ---------------------------------------------------------------------------
  53. // TestHandleSmartRetry_503_LongDelay_SingleAccountRetry_RetryInPlace
  54. // 核心场景:503 + retryDelay >= 7s + SingleAccountRetry 标记
  55. // → 不设模型限流、不切换账号,改为原地重试
  56. func TestHandleSmartRetry_503_LongDelay_SingleAccountRetry_RetryInPlace(t *testing.T) {
  57. // 原地重试成功
  58. successResp := &http.Response{
  59. StatusCode: http.StatusOK,
  60. Header: http.Header{},
  61. Body: io.NopCloser(strings.NewReader(`{"result":"ok"}`)),
  62. }
  63. upstream := &mockSmartRetryUpstream{
  64. responses: []*http.Response{successResp},
  65. errors: []error{nil},
  66. }
  67. repo := &stubAntigravityAccountRepo{}
  68. account := &Account{
  69. ID: 1,
  70. Name: "acc-single",
  71. Type: AccountTypeOAuth,
  72. Platform: PlatformAntigravity,
  73. Concurrency: 1,
  74. }
  75. // 503 + 39s >= 7s 阈值 + MODEL_CAPACITY_EXHAUSTED
  76. respBody := []byte(`{
  77. "error": {
  78. "code": 503,
  79. "status": "UNAVAILABLE",
  80. "details": [
  81. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-pro-high"}, "reason": "MODEL_CAPACITY_EXHAUSTED"},
  82. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "39s"}
  83. ],
  84. "message": "No capacity available for model gemini-3-pro-high on the server"
  85. }
  86. }`)
  87. resp := &http.Response{
  88. StatusCode: http.StatusServiceUnavailable,
  89. Header: http.Header{},
  90. Body: io.NopCloser(bytes.NewReader(respBody)),
  91. }
  92. params := antigravityRetryLoopParams{
  93. ctx: ctxWithSingleAccountRetry(), // 关键:设置单账号标记
  94. prefix: "[test]",
  95. account: account,
  96. accessToken: "token",
  97. action: "generateContent",
  98. body: []byte(`{"input":"test"}`),
  99. httpUpstream: upstream,
  100. accountRepo: repo,
  101. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  102. return nil
  103. },
  104. }
  105. availableURLs := []string{"https://ag-1.test"}
  106. svc := &AntigravityGatewayService{}
  107. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  108. require.NotNil(t, result)
  109. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  110. // 关键断言:返回 resp(原地重试成功),而非 switchError(切换账号)
  111. require.NotNil(t, result.resp, "should return successful response from in-place retry")
  112. require.Equal(t, http.StatusOK, result.resp.StatusCode)
  113. require.Nil(t, result.switchError, "should NOT return switchError in single account mode")
  114. require.Nil(t, result.err)
  115. // 验证未设模型限流(单账号模式不应设限流)
  116. require.Len(t, repo.modelRateLimitCalls, 0,
  117. "should NOT set model rate limit in single account retry mode")
  118. // 验证确实调用了 upstream(原地重试)
  119. require.GreaterOrEqual(t, len(upstream.calls), 1, "should have made at least one retry call")
  120. }
  121. // TestHandleSmartRetry_503_LongDelay_NoSingleAccountRetry_StillSwitches
  122. // 对照组:503 + retryDelay >= 7s + 无 SingleAccountRetry 标记
  123. // → 照常设模型限流 + 切换账号
  124. func TestHandleSmartRetry_503_LongDelay_NoSingleAccountRetry_StillSwitches(t *testing.T) {
  125. repo := &stubAntigravityAccountRepo{}
  126. account := &Account{
  127. ID: 2,
  128. Name: "acc-multi",
  129. Type: AccountTypeOAuth,
  130. Platform: PlatformAntigravity,
  131. }
  132. // 503 + 39s >= 7s 阈值(使用 RATE_LIMIT_EXCEEDED 而非 MODEL_CAPACITY_EXHAUSTED,
  133. // 因为 MODEL_CAPACITY_EXHAUSTED 走独立的重试路径,不触发 shouldRateLimitModel)
  134. respBody := []byte(`{
  135. "error": {
  136. "code": 503,
  137. "status": "RESOURCE_EXHAUSTED",
  138. "details": [
  139. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-pro-high"}, "reason": "RATE_LIMIT_EXCEEDED"},
  140. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "39s"}
  141. ]
  142. }
  143. }`)
  144. resp := &http.Response{
  145. StatusCode: http.StatusServiceUnavailable,
  146. Header: http.Header{},
  147. Body: io.NopCloser(bytes.NewReader(respBody)),
  148. }
  149. params := antigravityRetryLoopParams{
  150. ctx: context.Background(), // 关键:无单账号标记
  151. prefix: "[test]",
  152. account: account,
  153. accessToken: "token",
  154. action: "generateContent",
  155. body: []byte(`{"input":"test"}`),
  156. accountRepo: repo,
  157. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  158. return nil
  159. },
  160. }
  161. availableURLs := []string{"https://ag-1.test"}
  162. svc := &AntigravityGatewayService{}
  163. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  164. require.NotNil(t, result)
  165. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  166. // 对照:多账号模式返回 switchError
  167. require.NotNil(t, result.switchError, "multi-account mode should return switchError for 503")
  168. require.Nil(t, result.resp, "should not return resp when switchError is set")
  169. // 对照:多账号模式应设模型限流
  170. require.Len(t, repo.modelRateLimitCalls, 2,
  171. "multi-account mode SHOULD set model rate limit")
  172. require.Equal(t, "gemini-3-pro-high", repo.modelRateLimitCalls[0].modelKey)
  173. require.Equal(t, antigravityGeminiModelRateLimitKey, repo.modelRateLimitCalls[1].modelKey)
  174. }
  175. // TestHandleSmartRetry_429_LongDelay_SingleAccountRetry_StillSwitches
  176. // 边界情况:429(非 503)+ SingleAccountRetry 标记
  177. // → 单账号原地重试仅针对 503,429 依然走切换账号逻辑
  178. func TestHandleSmartRetry_429_LongDelay_SingleAccountRetry_StillSwitches(t *testing.T) {
  179. repo := &stubAntigravityAccountRepo{}
  180. account := &Account{
  181. ID: 3,
  182. Name: "acc-429",
  183. Type: AccountTypeOAuth,
  184. Platform: PlatformAntigravity,
  185. }
  186. // 429 + 15s >= 7s 阈值
  187. respBody := []byte(`{
  188. "error": {
  189. "status": "RESOURCE_EXHAUSTED",
  190. "details": [
  191. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"},
  192. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "15s"}
  193. ]
  194. }
  195. }`)
  196. resp := &http.Response{
  197. StatusCode: http.StatusTooManyRequests, // 429,不是 503
  198. Header: http.Header{},
  199. Body: io.NopCloser(bytes.NewReader(respBody)),
  200. }
  201. params := antigravityRetryLoopParams{
  202. ctx: ctxWithSingleAccountRetry(), // 有单账号标记
  203. prefix: "[test]",
  204. account: account,
  205. accessToken: "token",
  206. action: "generateContent",
  207. body: []byte(`{"input":"test"}`),
  208. accountRepo: repo,
  209. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  210. return nil
  211. },
  212. }
  213. availableURLs := []string{"https://ag-1.test"}
  214. svc := &AntigravityGatewayService{}
  215. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  216. require.NotNil(t, result)
  217. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  218. // 429 即使有单账号标记,也应走切换账号
  219. require.NotNil(t, result.switchError, "429 should still return switchError even with SingleAccountRetry")
  220. require.Len(t, repo.modelRateLimitCalls, 1,
  221. "429 should still set model rate limit even with SingleAccountRetry")
  222. }
  223. // ---------------------------------------------------------------------------
  224. // 4. handleSmartRetry + 503 + 短延迟 + SingleAccountRetry → 智能重试耗尽后不设限流
  225. // ---------------------------------------------------------------------------
  226. // TestHandleSmartRetry_503_ShortDelay_SingleAccountRetry_NoRateLimit
  227. // 503 + retryDelay < 7s + SingleAccountRetry → 智能重试耗尽后直接返回 503,不设限流
  228. // 使用 RATE_LIMIT_EXCEEDED(走 1 次智能重试),避免 MODEL_CAPACITY_EXHAUSTED 的 60 次重试导致测试超时
  229. func TestHandleSmartRetry_503_ShortDelay_SingleAccountRetry_NoRateLimit(t *testing.T) {
  230. // 智能重试也返回 503
  231. failRespBody := `{
  232. "error": {
  233. "code": 503,
  234. "status": "RESOURCE_EXHAUSTED",
  235. "details": [
  236. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-flash"}, "reason": "RATE_LIMIT_EXCEEDED"},
  237. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  238. ]
  239. }
  240. }`
  241. failResp := &http.Response{
  242. StatusCode: http.StatusServiceUnavailable,
  243. Header: http.Header{},
  244. Body: io.NopCloser(strings.NewReader(failRespBody)),
  245. }
  246. upstream := &mockSmartRetryUpstream{
  247. responses: []*http.Response{failResp},
  248. errors: []error{nil},
  249. repeatLast: true,
  250. }
  251. repo := &stubAntigravityAccountRepo{}
  252. account := &Account{
  253. ID: 4,
  254. Name: "acc-short-503",
  255. Type: AccountTypeOAuth,
  256. Platform: PlatformAntigravity,
  257. }
  258. // 0.1s < 7s 阈值
  259. respBody := []byte(`{
  260. "error": {
  261. "code": 503,
  262. "status": "RESOURCE_EXHAUSTED",
  263. "details": [
  264. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-flash"}, "reason": "RATE_LIMIT_EXCEEDED"},
  265. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  266. ]
  267. }
  268. }`)
  269. resp := &http.Response{
  270. StatusCode: http.StatusServiceUnavailable,
  271. Header: http.Header{},
  272. Body: io.NopCloser(bytes.NewReader(respBody)),
  273. }
  274. params := antigravityRetryLoopParams{
  275. ctx: ctxWithSingleAccountRetry(),
  276. prefix: "[test]",
  277. account: account,
  278. accessToken: "token",
  279. action: "generateContent",
  280. body: []byte(`{"input":"test"}`),
  281. httpUpstream: upstream,
  282. accountRepo: repo,
  283. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  284. return nil
  285. },
  286. }
  287. availableURLs := []string{"https://ag-1.test"}
  288. svc := &AntigravityGatewayService{}
  289. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  290. require.NotNil(t, result)
  291. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  292. // 关键断言:单账号 503 模式下,智能重试耗尽后直接返回 503 响应,不切换
  293. require.NotNil(t, result.resp, "should return 503 response directly for single account mode")
  294. require.Equal(t, http.StatusServiceUnavailable, result.resp.StatusCode)
  295. require.Nil(t, result.switchError, "should NOT switch account in single account mode")
  296. // 关键断言:不设模型限流
  297. require.Len(t, repo.modelRateLimitCalls, 0,
  298. "should NOT set model rate limit for 503 in single account mode")
  299. }
  300. // TestHandleSmartRetry_503_ShortDelay_NoSingleAccountRetry_SetsRateLimit
  301. // 对照组:503 + retryDelay < 7s + 无 SingleAccountRetry → 智能重试耗尽后照常设限流
  302. // 使用 RATE_LIMIT_EXCEEDED 而非 MODEL_CAPACITY_EXHAUSTED,因为后者走独立的 60 次重试路径
  303. func TestHandleSmartRetry_503_ShortDelay_NoSingleAccountRetry_SetsRateLimit(t *testing.T) {
  304. failRespBody := `{
  305. "error": {
  306. "code": 503,
  307. "status": "RESOURCE_EXHAUSTED",
  308. "details": [
  309. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-flash"}, "reason": "RATE_LIMIT_EXCEEDED"},
  310. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  311. ]
  312. }
  313. }`
  314. failResp := &http.Response{
  315. StatusCode: http.StatusServiceUnavailable,
  316. Header: http.Header{},
  317. Body: io.NopCloser(strings.NewReader(failRespBody)),
  318. }
  319. upstream := &mockSmartRetryUpstream{
  320. responses: []*http.Response{failResp},
  321. errors: []error{nil},
  322. }
  323. repo := &stubAntigravityAccountRepo{}
  324. account := &Account{
  325. ID: 5,
  326. Name: "acc-multi-503",
  327. Type: AccountTypeOAuth,
  328. Platform: PlatformAntigravity,
  329. }
  330. respBody := []byte(`{
  331. "error": {
  332. "code": 503,
  333. "status": "RESOURCE_EXHAUSTED",
  334. "details": [
  335. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-flash"}, "reason": "RATE_LIMIT_EXCEEDED"},
  336. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  337. ]
  338. }
  339. }`)
  340. resp := &http.Response{
  341. StatusCode: http.StatusServiceUnavailable,
  342. Header: http.Header{},
  343. Body: io.NopCloser(bytes.NewReader(respBody)),
  344. }
  345. params := antigravityRetryLoopParams{
  346. ctx: context.Background(), // 无单账号标记
  347. prefix: "[test]",
  348. account: account,
  349. accessToken: "token",
  350. action: "generateContent",
  351. body: []byte(`{"input":"test"}`),
  352. httpUpstream: upstream,
  353. accountRepo: repo,
  354. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  355. return nil
  356. },
  357. }
  358. availableURLs := []string{"https://ag-1.test"}
  359. svc := &AntigravityGatewayService{}
  360. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  361. require.NotNil(t, result)
  362. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  363. // 对照:多账号模式应返回 switchError
  364. require.NotNil(t, result.switchError, "multi-account mode should return switchError for 503")
  365. // 对照:多账号模式应设模型限流
  366. require.Len(t, repo.modelRateLimitCalls, 2,
  367. "multi-account mode should set model rate limit")
  368. require.Equal(t, "gemini-3-flash", repo.modelRateLimitCalls[0].modelKey)
  369. require.Equal(t, antigravityGeminiModelRateLimitKey, repo.modelRateLimitCalls[1].modelKey)
  370. }
  371. // ---------------------------------------------------------------------------
  372. // 5. handleSingleAccountRetryInPlace 直接测试
  373. // ---------------------------------------------------------------------------
  374. // TestHandleSingleAccountRetryInPlace_Success 原地重试成功
  375. func TestHandleSingleAccountRetryInPlace_Success(t *testing.T) {
  376. successResp := &http.Response{
  377. StatusCode: http.StatusOK,
  378. Header: http.Header{},
  379. Body: io.NopCloser(strings.NewReader(`{"result":"ok"}`)),
  380. }
  381. upstream := &mockSmartRetryUpstream{
  382. responses: []*http.Response{successResp},
  383. errors: []error{nil},
  384. }
  385. account := &Account{
  386. ID: 10,
  387. Name: "acc-inplace-ok",
  388. Type: AccountTypeOAuth,
  389. Platform: PlatformAntigravity,
  390. Concurrency: 1,
  391. }
  392. resp := &http.Response{
  393. StatusCode: http.StatusServiceUnavailable,
  394. Header: http.Header{},
  395. }
  396. params := antigravityRetryLoopParams{
  397. ctx: ctxWithSingleAccountRetry(),
  398. prefix: "[test]",
  399. account: account,
  400. accessToken: "token",
  401. action: "generateContent",
  402. body: []byte(`{"input":"test"}`),
  403. httpUpstream: upstream,
  404. }
  405. svc := &AntigravityGatewayService{}
  406. result := svc.handleSingleAccountRetryInPlace(params, resp, nil, "https://ag-1.test", 1*time.Second, "gemini-3-pro")
  407. require.NotNil(t, result)
  408. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  409. require.NotNil(t, result.resp, "should return successful response")
  410. require.Equal(t, http.StatusOK, result.resp.StatusCode)
  411. require.Nil(t, result.switchError, "should not switch account on success")
  412. require.Nil(t, result.err)
  413. }
  414. // TestHandleSingleAccountRetryInPlace_AllRetriesFail 所有重试都失败,返回 503(不设限流)
  415. func TestHandleSingleAccountRetryInPlace_AllRetriesFail(t *testing.T) {
  416. // 构造 3 个 503 响应(对应 3 次原地重试)
  417. var responses []*http.Response
  418. var errors []error
  419. for i := 0; i < antigravitySingleAccountSmartRetryMaxAttempts; i++ {
  420. responses = append(responses, &http.Response{
  421. StatusCode: http.StatusServiceUnavailable,
  422. Header: http.Header{},
  423. Body: io.NopCloser(strings.NewReader(`{
  424. "error": {
  425. "code": 503,
  426. "status": "UNAVAILABLE",
  427. "details": [
  428. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-pro"}, "reason": "MODEL_CAPACITY_EXHAUSTED"},
  429. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  430. ]
  431. }
  432. }`)),
  433. })
  434. errors = append(errors, nil)
  435. }
  436. upstream := &mockSmartRetryUpstream{
  437. responses: responses,
  438. errors: errors,
  439. }
  440. account := &Account{
  441. ID: 11,
  442. Name: "acc-inplace-fail",
  443. Type: AccountTypeOAuth,
  444. Platform: PlatformAntigravity,
  445. Concurrency: 1,
  446. }
  447. origBody := []byte(`{"error":{"code":503,"status":"UNAVAILABLE"}}`)
  448. resp := &http.Response{
  449. StatusCode: http.StatusServiceUnavailable,
  450. Header: http.Header{"X-Test": {"original"}},
  451. }
  452. params := antigravityRetryLoopParams{
  453. ctx: ctxWithSingleAccountRetry(),
  454. prefix: "[test]",
  455. account: account,
  456. accessToken: "token",
  457. action: "generateContent",
  458. body: []byte(`{"input":"test"}`),
  459. httpUpstream: upstream,
  460. }
  461. svc := &AntigravityGatewayService{}
  462. result := svc.handleSingleAccountRetryInPlace(params, resp, origBody, "https://ag-1.test", 1*time.Second, "gemini-3-pro")
  463. require.NotNil(t, result)
  464. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  465. // 关键:返回 503 resp,不返回 switchError
  466. require.NotNil(t, result.resp, "should return 503 response directly")
  467. require.Equal(t, http.StatusServiceUnavailable, result.resp.StatusCode)
  468. require.Nil(t, result.switchError, "should NOT return switchError - let Handler handle it")
  469. require.Nil(t, result.err)
  470. // 验证确实重试了指定次数
  471. require.Len(t, upstream.calls, antigravitySingleAccountSmartRetryMaxAttempts,
  472. "should have made exactly maxAttempts retry calls")
  473. }
  474. // TestHandleSingleAccountRetryInPlace_WaitDurationClamped 等待时间被限制在 [min, max] 范围
  475. func TestHandleSingleAccountRetryInPlace_WaitDurationClamped(t *testing.T) {
  476. // 用短延迟的成功响应,只验证不 panic
  477. successResp := &http.Response{
  478. StatusCode: http.StatusOK,
  479. Header: http.Header{},
  480. Body: io.NopCloser(strings.NewReader(`{"result":"ok"}`)),
  481. }
  482. upstream := &mockSmartRetryUpstream{
  483. responses: []*http.Response{successResp},
  484. errors: []error{nil},
  485. }
  486. account := &Account{
  487. ID: 12,
  488. Name: "acc-clamp",
  489. Type: AccountTypeOAuth,
  490. Platform: PlatformAntigravity,
  491. Concurrency: 1,
  492. }
  493. resp := &http.Response{
  494. StatusCode: http.StatusServiceUnavailable,
  495. Header: http.Header{},
  496. }
  497. params := antigravityRetryLoopParams{
  498. ctx: ctxWithSingleAccountRetry(),
  499. prefix: "[test]",
  500. account: account,
  501. accessToken: "token",
  502. action: "generateContent",
  503. body: []byte(`{"input":"test"}`),
  504. httpUpstream: upstream,
  505. }
  506. svc := &AntigravityGatewayService{}
  507. // waitDuration=0 会被 clamp 到 antigravitySmartRetryMinWait=1s。
  508. // 首次重试即成功(200),总耗时 ~1s。
  509. result := svc.handleSingleAccountRetryInPlace(params, resp, nil, "https://ag-1.test", 0, "gemini-3-pro")
  510. require.NotNil(t, result)
  511. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  512. require.NotNil(t, result.resp)
  513. require.Equal(t, http.StatusOK, result.resp.StatusCode)
  514. }
  515. // TestHandleSingleAccountRetryInPlace_ContextCanceled context 取消时立即返回
  516. func TestHandleSingleAccountRetryInPlace_ContextCanceled(t *testing.T) {
  517. upstream := &mockSmartRetryUpstream{
  518. responses: []*http.Response{nil},
  519. errors: []error{nil},
  520. }
  521. account := &Account{
  522. ID: 13,
  523. Name: "acc-cancel",
  524. Type: AccountTypeOAuth,
  525. Platform: PlatformAntigravity,
  526. Concurrency: 1,
  527. }
  528. resp := &http.Response{
  529. StatusCode: http.StatusServiceUnavailable,
  530. Header: http.Header{},
  531. }
  532. ctx, cancel := context.WithCancel(context.Background())
  533. ctx = context.WithValue(ctx, ctxkey.SingleAccountRetry, true)
  534. cancel() // 立即取消
  535. params := antigravityRetryLoopParams{
  536. ctx: ctx,
  537. prefix: "[test]",
  538. account: account,
  539. accessToken: "token",
  540. action: "generateContent",
  541. body: []byte(`{"input":"test"}`),
  542. httpUpstream: upstream,
  543. }
  544. svc := &AntigravityGatewayService{}
  545. result := svc.handleSingleAccountRetryInPlace(params, resp, nil, "https://ag-1.test", 1*time.Second, "gemini-3-pro")
  546. require.NotNil(t, result)
  547. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  548. require.Error(t, result.err, "should return context error")
  549. // 不应调用 upstream(因为在等待阶段就被取消了)
  550. require.Len(t, upstream.calls, 0, "should not call upstream when context is canceled")
  551. }
  552. // TestHandleSingleAccountRetryInPlace_NetworkError_ContinuesRetry 网络错误时继续重试
  553. func TestHandleSingleAccountRetryInPlace_NetworkError_ContinuesRetry(t *testing.T) {
  554. successResp := &http.Response{
  555. StatusCode: http.StatusOK,
  556. Header: http.Header{},
  557. Body: io.NopCloser(strings.NewReader(`{"result":"ok"}`)),
  558. }
  559. upstream := &mockSmartRetryUpstream{
  560. // 第1次网络错误(nil resp),第2次成功
  561. responses: []*http.Response{nil, successResp},
  562. errors: []error{nil, nil},
  563. }
  564. account := &Account{
  565. ID: 14,
  566. Name: "acc-net-retry",
  567. Type: AccountTypeOAuth,
  568. Platform: PlatformAntigravity,
  569. Concurrency: 1,
  570. }
  571. resp := &http.Response{
  572. StatusCode: http.StatusServiceUnavailable,
  573. Header: http.Header{},
  574. }
  575. params := antigravityRetryLoopParams{
  576. ctx: ctxWithSingleAccountRetry(),
  577. prefix: "[test]",
  578. account: account,
  579. accessToken: "token",
  580. action: "generateContent",
  581. body: []byte(`{"input":"test"}`),
  582. httpUpstream: upstream,
  583. }
  584. svc := &AntigravityGatewayService{}
  585. result := svc.handleSingleAccountRetryInPlace(params, resp, nil, "https://ag-1.test", 1*time.Second, "gemini-3-pro")
  586. require.NotNil(t, result)
  587. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  588. require.NotNil(t, result.resp, "should return successful response after network error recovery")
  589. require.Equal(t, http.StatusOK, result.resp.StatusCode)
  590. require.Len(t, upstream.calls, 2, "first call fails (network error), second succeeds")
  591. }
  592. // ---------------------------------------------------------------------------
  593. // 6. antigravityRetryLoop 预检查:单账号模式跳过限流
  594. // ---------------------------------------------------------------------------
  595. // TestAntigravityRetryLoop_PreCheck_SingleAccountRetry_SkipsRateLimit
  596. // 预检查中,如果有 SingleAccountRetry 标记,即使账号已限流也跳过直接发请求
  597. func TestAntigravityRetryLoop_PreCheck_SingleAccountRetry_SkipsRateLimit(t *testing.T) {
  598. // 创建一个已设模型限流的账号
  599. upstream := &recordingOKUpstream{}
  600. account := &Account{
  601. ID: 20,
  602. Name: "acc-rate-limited",
  603. Type: AccountTypeOAuth,
  604. Platform: PlatformAntigravity,
  605. Schedulable: true,
  606. Status: StatusActive,
  607. Concurrency: 1,
  608. Extra: map[string]any{
  609. modelRateLimitsKey: map[string]any{
  610. "claude-sonnet-4-5": map[string]any{
  611. "rate_limit_reset_at": time.Now().Add(30 * time.Second).Format(time.RFC3339),
  612. },
  613. },
  614. },
  615. }
  616. svc := &AntigravityGatewayService{}
  617. result, err := svc.antigravityRetryLoop(antigravityRetryLoopParams{
  618. ctx: ctxWithSingleAccountRetry(),
  619. prefix: "[test]",
  620. account: account,
  621. accessToken: "token",
  622. action: "generateContent",
  623. body: []byte(`{"input":"test"}`),
  624. httpUpstream: upstream,
  625. requestedModel: "claude-sonnet-4-5",
  626. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  627. return nil
  628. },
  629. })
  630. require.NoError(t, err, "should not return error")
  631. require.NotNil(t, result, "should return result")
  632. require.NotNil(t, result.resp, "should have response")
  633. require.Equal(t, http.StatusOK, result.resp.StatusCode)
  634. // 关键:尽管限流了,有 SingleAccountRetry 标记时仍然到达了 upstream
  635. require.Equal(t, 1, upstream.calls, "should have reached upstream despite rate limit")
  636. }
  637. // TestAntigravityRetryLoop_PreCheck_NoSingleAccountRetry_SwitchesOnRateLimit
  638. // 对照组:无 SingleAccountRetry + 已限流 → 预检查返回 switchError
  639. func TestAntigravityRetryLoop_PreCheck_NoSingleAccountRetry_SwitchesOnRateLimit(t *testing.T) {
  640. upstream := &recordingOKUpstream{}
  641. account := &Account{
  642. ID: 21,
  643. Name: "acc-rate-limited-multi",
  644. Type: AccountTypeOAuth,
  645. Platform: PlatformAntigravity,
  646. Schedulable: true,
  647. Status: StatusActive,
  648. Concurrency: 1,
  649. Extra: map[string]any{
  650. modelRateLimitsKey: map[string]any{
  651. "claude-sonnet-4-5": map[string]any{
  652. "rate_limit_reset_at": time.Now().Add(30 * time.Second).Format(time.RFC3339),
  653. },
  654. },
  655. },
  656. }
  657. svc := &AntigravityGatewayService{}
  658. result, err := svc.antigravityRetryLoop(antigravityRetryLoopParams{
  659. ctx: context.Background(), // 无单账号标记
  660. prefix: "[test]",
  661. account: account,
  662. accessToken: "token",
  663. action: "generateContent",
  664. body: []byte(`{"input":"test"}`),
  665. httpUpstream: upstream,
  666. requestedModel: "claude-sonnet-4-5",
  667. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  668. return nil
  669. },
  670. })
  671. require.Nil(t, result, "should not return result on rate limit switch")
  672. require.NotNil(t, err, "should return error")
  673. var switchErr *AntigravityAccountSwitchError
  674. require.ErrorAs(t, err, &switchErr, "should return AntigravityAccountSwitchError")
  675. require.Equal(t, account.ID, switchErr.OriginalAccountID)
  676. require.Equal(t, "claude-sonnet-4-5", switchErr.RateLimitedModel)
  677. // upstream 不应被调用(预检查就短路了)
  678. require.Equal(t, 0, upstream.calls, "upstream should NOT be called when pre-check blocks")
  679. }
  680. // ---------------------------------------------------------------------------
  681. // 7. 端到端集成场景测试
  682. // ---------------------------------------------------------------------------
  683. // TestHandleSmartRetry_503_SingleAccount_RetryInPlace_ThenSuccess_E2E
  684. // 端到端场景:503 + 单账号 + 原地重试第2次成功
  685. func TestHandleSmartRetry_503_SingleAccount_RetryInPlace_ThenSuccess_E2E(t *testing.T) {
  686. // 第1次原地重试仍返回 503,第2次成功
  687. fail503Body := `{
  688. "error": {
  689. "code": 503,
  690. "status": "UNAVAILABLE",
  691. "details": [
  692. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-pro"}, "reason": "MODEL_CAPACITY_EXHAUSTED"},
  693. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  694. ]
  695. }
  696. }`
  697. resp503 := &http.Response{
  698. StatusCode: http.StatusServiceUnavailable,
  699. Header: http.Header{},
  700. Body: io.NopCloser(strings.NewReader(fail503Body)),
  701. }
  702. successResp := &http.Response{
  703. StatusCode: http.StatusOK,
  704. Header: http.Header{},
  705. Body: io.NopCloser(strings.NewReader(`{"result":"ok"}`)),
  706. }
  707. upstream := &mockSmartRetryUpstream{
  708. responses: []*http.Response{resp503, successResp},
  709. errors: []error{nil, nil},
  710. }
  711. account := &Account{
  712. ID: 30,
  713. Name: "acc-e2e",
  714. Type: AccountTypeOAuth,
  715. Platform: PlatformAntigravity,
  716. Concurrency: 1,
  717. }
  718. resp := &http.Response{
  719. StatusCode: http.StatusServiceUnavailable,
  720. Header: http.Header{},
  721. }
  722. params := antigravityRetryLoopParams{
  723. ctx: ctxWithSingleAccountRetry(),
  724. prefix: "[test]",
  725. account: account,
  726. accessToken: "token",
  727. action: "generateContent",
  728. body: []byte(`{"input":"test"}`),
  729. httpUpstream: upstream,
  730. }
  731. svc := &AntigravityGatewayService{}
  732. result := svc.handleSingleAccountRetryInPlace(params, resp, nil, "https://ag-1.test", 1*time.Second, "gemini-3-pro")
  733. require.NotNil(t, result)
  734. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  735. require.NotNil(t, result.resp, "should return successful response after 2nd attempt")
  736. require.Equal(t, http.StatusOK, result.resp.StatusCode)
  737. require.Nil(t, result.switchError)
  738. require.Len(t, upstream.calls, 2, "first 503, second OK")
  739. }
  740. // TestAntigravityRetryLoop_503_SingleAccount_InPlaceRetryUsed_E2E
  741. // 通过 antigravityRetryLoop → handleSmartRetry → handleSingleAccountRetryInPlace 完整链路
  742. func TestAntigravityRetryLoop_503_SingleAccount_InPlaceRetryUsed_E2E(t *testing.T) {
  743. // 初始请求返回 503 + 长延迟
  744. initial503Body := []byte(`{
  745. "error": {
  746. "code": 503,
  747. "status": "UNAVAILABLE",
  748. "details": [
  749. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-pro"}, "reason": "MODEL_CAPACITY_EXHAUSTED"},
  750. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "10s"}
  751. ],
  752. "message": "No capacity available"
  753. }
  754. }`)
  755. initial503Resp := &http.Response{
  756. StatusCode: http.StatusServiceUnavailable,
  757. Header: http.Header{},
  758. Body: io.NopCloser(bytes.NewReader(initial503Body)),
  759. }
  760. // 原地重试成功
  761. successResp := &http.Response{
  762. StatusCode: http.StatusOK,
  763. Header: http.Header{},
  764. Body: io.NopCloser(strings.NewReader(`{"result":"ok"}`)),
  765. }
  766. upstream := &mockSmartRetryUpstream{
  767. // 第1次调用(retryLoop 主循环)返回 503
  768. // 第2次调用(handleSingleAccountRetryInPlace 原地重试)返回 200
  769. responses: []*http.Response{initial503Resp, successResp},
  770. errors: []error{nil, nil},
  771. }
  772. repo := &stubAntigravityAccountRepo{}
  773. account := &Account{
  774. ID: 31,
  775. Name: "acc-e2e-loop",
  776. Type: AccountTypeOAuth,
  777. Platform: PlatformAntigravity,
  778. Schedulable: true,
  779. Status: StatusActive,
  780. Concurrency: 1,
  781. }
  782. svc := &AntigravityGatewayService{}
  783. result, err := svc.antigravityRetryLoop(antigravityRetryLoopParams{
  784. ctx: ctxWithSingleAccountRetry(),
  785. prefix: "[test]",
  786. account: account,
  787. accessToken: "token",
  788. action: "generateContent",
  789. body: []byte(`{"input":"test"}`),
  790. httpUpstream: upstream,
  791. accountRepo: repo,
  792. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  793. return nil
  794. },
  795. })
  796. require.NoError(t, err, "should not return error on successful retry")
  797. require.NotNil(t, result, "should return result")
  798. require.NotNil(t, result.resp, "should return response")
  799. require.Equal(t, http.StatusOK, result.resp.StatusCode)
  800. // 验证未设模型限流
  801. require.Len(t, repo.modelRateLimitCalls, 0,
  802. "should NOT set model rate limit in single account retry mode")
  803. }