antigravity_smart_retry_test.go 50 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404
  1. //go:build unit
  2. package service
  3. import (
  4. "bytes"
  5. "context"
  6. "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint"
  7. "github.com/stretchr/testify/require"
  8. "io"
  9. "net/http"
  10. "strings"
  11. "testing"
  12. )
  13. // stubSmartRetryCache 用于 handleSmartRetry 测试的 GatewayCache mock
  14. // 仅关注 DeleteSessionAccountID 的调用记录
  15. type stubSmartRetryCache struct {
  16. GatewayCache // 嵌入接口,未实现的方法 panic(确保只调用预期方法)
  17. deleteCalls []deleteSessionCall
  18. }
  19. type deleteSessionCall struct {
  20. groupID int64
  21. sessionHash string
  22. }
  23. func (c *stubSmartRetryCache) DeleteSessionAccountID(_ context.Context, groupID int64, sessionHash string) error {
  24. c.deleteCalls = append(c.deleteCalls, deleteSessionCall{groupID: groupID, sessionHash: sessionHash})
  25. return nil
  26. }
  27. // mockSmartRetryUpstream 用于 handleSmartRetry 测试的 mock upstream
  28. type mockSmartRetryUpstream struct {
  29. responses []*http.Response
  30. responseBodies [][]byte // 缓存的 response body 字节(用于 repeatLast 重建)
  31. errors []error
  32. callIdx int
  33. calls []string
  34. requestBodies [][]byte
  35. repeatLast bool // 超出范围时重复最后一个响应
  36. }
  37. func (m *mockSmartRetryUpstream) Do(req *http.Request, proxyURL string, accountID int64, accountConcurrency int) (*http.Response, error) {
  38. idx := m.callIdx
  39. m.calls = append(m.calls, req.URL.String())
  40. if req != nil && req.Body != nil {
  41. body, _ := io.ReadAll(req.Body)
  42. m.requestBodies = append(m.requestBodies, body)
  43. req.Body = io.NopCloser(bytes.NewReader(body))
  44. } else {
  45. m.requestBodies = append(m.requestBodies, nil)
  46. }
  47. m.callIdx++
  48. // 确定使用哪个索引
  49. respIdx := idx
  50. if respIdx >= len(m.responses) {
  51. if !m.repeatLast || len(m.responses) == 0 {
  52. return nil, nil
  53. }
  54. respIdx = len(m.responses) - 1
  55. }
  56. resp := m.responses[respIdx]
  57. respErr := m.errors[respIdx]
  58. if resp == nil {
  59. return nil, respErr
  60. }
  61. // 首次调用时缓存 body 字节
  62. if respIdx >= len(m.responseBodies) {
  63. for len(m.responseBodies) <= respIdx {
  64. m.responseBodies = append(m.responseBodies, nil)
  65. }
  66. }
  67. if m.responseBodies[respIdx] == nil && resp.Body != nil {
  68. bodyBytes, _ := io.ReadAll(resp.Body)
  69. _ = resp.Body.Close()
  70. m.responseBodies[respIdx] = bodyBytes
  71. }
  72. // 用缓存的 body 重建 reader(支持重试场景多次读取)
  73. cloned := *resp
  74. if m.responseBodies[respIdx] != nil {
  75. cloned.Body = io.NopCloser(bytes.NewReader(m.responseBodies[respIdx]))
  76. }
  77. return &cloned, respErr
  78. }
  79. func (m *mockSmartRetryUpstream) DoWithTLS(req *http.Request, proxyURL string, accountID int64, accountConcurrency int, profile *tlsfingerprint.Profile) (*http.Response, error) {
  80. return m.Do(req, proxyURL, accountID, accountConcurrency)
  81. }
  82. // TestHandleSmartRetry_URLLevelRateLimit 测试 URL 级别限流切换
  83. func TestHandleSmartRetry_URLLevelRateLimit(t *testing.T) {
  84. account := &Account{
  85. ID: 1,
  86. Name: "acc-1",
  87. Type: AccountTypeOAuth,
  88. Platform: PlatformAntigravity,
  89. }
  90. respBody := []byte(`{"error":{"message":"Resource has been exhausted"}}`)
  91. resp := &http.Response{
  92. StatusCode: http.StatusTooManyRequests,
  93. Header: http.Header{},
  94. Body: io.NopCloser(bytes.NewReader(respBody)),
  95. }
  96. params := antigravityRetryLoopParams{
  97. ctx: context.Background(),
  98. prefix: "[test]",
  99. account: account,
  100. accessToken: "token",
  101. action: "generateContent",
  102. body: []byte(`{"input":"test"}`),
  103. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  104. return nil
  105. },
  106. }
  107. availableURLs := []string{"https://ag-1.test", "https://ag-2.test"}
  108. svc := &AntigravityGatewayService{}
  109. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  110. require.NotNil(t, result)
  111. require.Equal(t, smartRetryActionContinueURL, result.action)
  112. require.Nil(t, result.resp)
  113. require.Nil(t, result.err)
  114. require.Nil(t, result.switchError)
  115. }
  116. // TestHandleSmartRetry_LongDelay_ReturnsSwitchError 测试 retryDelay >= 阈值时返回 switchError
  117. func TestHandleSmartRetry_LongDelay_ReturnsSwitchError(t *testing.T) {
  118. repo := &stubAntigravityAccountRepo{}
  119. account := &Account{
  120. ID: 1,
  121. Name: "acc-1",
  122. Type: AccountTypeOAuth,
  123. Platform: PlatformAntigravity,
  124. }
  125. // 15s >= 7s 阈值,应该返回 switchError
  126. respBody := []byte(`{
  127. "error": {
  128. "status": "RESOURCE_EXHAUSTED",
  129. "details": [
  130. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"},
  131. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "15s"}
  132. ]
  133. }
  134. }`)
  135. resp := &http.Response{
  136. StatusCode: http.StatusTooManyRequests,
  137. Header: http.Header{},
  138. Body: io.NopCloser(bytes.NewReader(respBody)),
  139. }
  140. params := antigravityRetryLoopParams{
  141. ctx: context.Background(),
  142. prefix: "[test]",
  143. account: account,
  144. accessToken: "token",
  145. action: "generateContent",
  146. body: []byte(`{"input":"test"}`),
  147. accountRepo: repo,
  148. isStickySession: true,
  149. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  150. return nil
  151. },
  152. }
  153. availableURLs := []string{"https://ag-1.test"}
  154. svc := &AntigravityGatewayService{}
  155. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  156. require.NotNil(t, result)
  157. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  158. require.Nil(t, result.resp, "should not return resp when switchError is set")
  159. require.Nil(t, result.err)
  160. require.NotNil(t, result.switchError, "should return switchError for long delay")
  161. require.Equal(t, account.ID, result.switchError.OriginalAccountID)
  162. require.Equal(t, "claude-sonnet-4-5", result.switchError.RateLimitedModel)
  163. require.True(t, result.switchError.IsStickySession)
  164. // 验证模型限流已设置
  165. require.Len(t, repo.modelRateLimitCalls, 1)
  166. require.Equal(t, "claude-sonnet-4-5", repo.modelRateLimitCalls[0].modelKey)
  167. }
  168. // TestHandleSmartRetry_ShortDelay_SmartRetrySuccess 测试智能重试成功
  169. func TestHandleSmartRetry_ShortDelay_SmartRetrySuccess(t *testing.T) {
  170. successResp := &http.Response{
  171. StatusCode: http.StatusOK,
  172. Header: http.Header{},
  173. Body: io.NopCloser(strings.NewReader(`{"result":"ok"}`)),
  174. }
  175. upstream := &mockSmartRetryUpstream{
  176. responses: []*http.Response{successResp},
  177. errors: []error{nil},
  178. }
  179. account := &Account{
  180. ID: 1,
  181. Name: "acc-1",
  182. Type: AccountTypeOAuth,
  183. Platform: PlatformAntigravity,
  184. }
  185. // 0.5s < 7s 阈值,应该触发智能重试
  186. respBody := []byte(`{
  187. "error": {
  188. "status": "RESOURCE_EXHAUSTED",
  189. "details": [
  190. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-opus-4"}, "reason": "RATE_LIMIT_EXCEEDED"},
  191. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.5s"}
  192. ]
  193. }
  194. }`)
  195. resp := &http.Response{
  196. StatusCode: http.StatusTooManyRequests,
  197. Header: http.Header{},
  198. Body: io.NopCloser(bytes.NewReader(respBody)),
  199. }
  200. params := antigravityRetryLoopParams{
  201. ctx: context.Background(),
  202. prefix: "[test]",
  203. account: account,
  204. accessToken: "token",
  205. action: "generateContent",
  206. body: []byte(`{"input":"test"}`),
  207. httpUpstream: upstream,
  208. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  209. return nil
  210. },
  211. }
  212. availableURLs := []string{"https://ag-1.test"}
  213. svc := &AntigravityGatewayService{}
  214. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  215. require.NotNil(t, result)
  216. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  217. require.NotNil(t, result.resp, "should return successful response")
  218. require.Equal(t, http.StatusOK, result.resp.StatusCode)
  219. require.Nil(t, result.err)
  220. require.Nil(t, result.switchError, "should not return switchError on success")
  221. require.Len(t, upstream.calls, 1, "should have made one retry call")
  222. }
  223. // TestHandleSmartRetry_ShortDelay_SmartRetryFailed_ReturnsSwitchError 测试智能重试失败后返回 switchError
  224. func TestHandleSmartRetry_ShortDelay_SmartRetryFailed_ReturnsSwitchError(t *testing.T) {
  225. // 智能重试后仍然返回 429(需要提供 1 个响应,因为智能重试最多 1 次)
  226. failRespBody := `{
  227. "error": {
  228. "status": "RESOURCE_EXHAUSTED",
  229. "details": [
  230. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-flash"}, "reason": "RATE_LIMIT_EXCEEDED"},
  231. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  232. ]
  233. }
  234. }`
  235. failResp1 := &http.Response{
  236. StatusCode: http.StatusTooManyRequests,
  237. Header: http.Header{},
  238. Body: io.NopCloser(strings.NewReader(failRespBody)),
  239. }
  240. upstream := &mockSmartRetryUpstream{
  241. responses: []*http.Response{failResp1},
  242. errors: []error{nil},
  243. }
  244. repo := &stubAntigravityAccountRepo{}
  245. account := &Account{
  246. ID: 2,
  247. Name: "acc-2",
  248. Type: AccountTypeOAuth,
  249. Platform: PlatformAntigravity,
  250. }
  251. // 3s < 7s 阈值,应该触发智能重试(最多 1 次)
  252. respBody := []byte(`{
  253. "error": {
  254. "status": "RESOURCE_EXHAUSTED",
  255. "details": [
  256. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-flash"}, "reason": "RATE_LIMIT_EXCEEDED"},
  257. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  258. ]
  259. }
  260. }`)
  261. resp := &http.Response{
  262. StatusCode: http.StatusTooManyRequests,
  263. Header: http.Header{},
  264. Body: io.NopCloser(bytes.NewReader(respBody)),
  265. }
  266. params := antigravityRetryLoopParams{
  267. ctx: context.Background(),
  268. prefix: "[test]",
  269. account: account,
  270. accessToken: "token",
  271. action: "generateContent",
  272. body: []byte(`{"input":"test"}`),
  273. httpUpstream: upstream,
  274. accountRepo: repo,
  275. isStickySession: false,
  276. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  277. return nil
  278. },
  279. }
  280. availableURLs := []string{"https://ag-1.test"}
  281. svc := &AntigravityGatewayService{}
  282. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  283. require.NotNil(t, result)
  284. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  285. require.Nil(t, result.resp, "should not return resp when switchError is set")
  286. require.Nil(t, result.err)
  287. require.NotNil(t, result.switchError, "should return switchError after smart retry failed")
  288. require.Equal(t, account.ID, result.switchError.OriginalAccountID)
  289. require.Equal(t, "gemini-3-flash", result.switchError.RateLimitedModel)
  290. require.False(t, result.switchError.IsStickySession)
  291. // 验证模型限流已设置:Gemini 同时写入精确模型和家族级 scope
  292. require.Len(t, repo.modelRateLimitCalls, 2)
  293. require.Equal(t, "gemini-3-flash", repo.modelRateLimitCalls[0].modelKey)
  294. require.Equal(t, antigravityGeminiModelRateLimitKey, repo.modelRateLimitCalls[1].modelKey)
  295. require.Len(t, upstream.calls, 1, "should have made one retry call (max attempts)")
  296. }
  297. // TestHandleSmartRetry_503_ModelCapacityExhausted_RetrySuccess 测试 503 MODEL_CAPACITY_EXHAUSTED 重试成功
  298. // MODEL_CAPACITY_EXHAUSTED 使用固定 1s 间隔重试,不切换账号
  299. func TestHandleSmartRetry_503_ModelCapacityExhausted_RetrySuccess(t *testing.T) {
  300. repo := &stubAntigravityAccountRepo{}
  301. account := &Account{
  302. ID: 3,
  303. Name: "acc-3",
  304. Type: AccountTypeOAuth,
  305. Platform: PlatformAntigravity,
  306. }
  307. // 503 + MODEL_CAPACITY_EXHAUSTED + 39s(上游 retryDelay 应被忽略,使用固定 1s)
  308. respBody := []byte(`{
  309. "error": {
  310. "code": 503,
  311. "status": "UNAVAILABLE",
  312. "details": [
  313. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-pro-high"}, "reason": "MODEL_CAPACITY_EXHAUSTED"},
  314. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "39s"}
  315. ],
  316. "message": "No capacity available for model gemini-3-pro-high on the server"
  317. }
  318. }`)
  319. resp := &http.Response{
  320. StatusCode: http.StatusServiceUnavailable,
  321. Header: http.Header{},
  322. Body: io.NopCloser(bytes.NewReader(respBody)),
  323. }
  324. // mock: 第 1 次重试返回 200 成功
  325. upstream := &mockSmartRetryUpstream{
  326. responses: []*http.Response{
  327. {StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"ok":true}`))},
  328. },
  329. errors: []error{nil},
  330. }
  331. params := antigravityRetryLoopParams{
  332. ctx: context.Background(),
  333. prefix: "[test]",
  334. account: account,
  335. accessToken: "token",
  336. action: "generateContent",
  337. body: []byte(`{"input":"test"}`),
  338. accountRepo: repo,
  339. httpUpstream: upstream,
  340. isStickySession: true,
  341. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  342. return nil
  343. },
  344. }
  345. availableURLs := []string{"https://ag-1.test"}
  346. svc := &AntigravityGatewayService{}
  347. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  348. require.NotNil(t, result)
  349. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  350. require.NotNil(t, result.resp, "should return successful response")
  351. require.Equal(t, http.StatusOK, result.resp.StatusCode)
  352. require.Nil(t, result.err)
  353. require.Nil(t, result.switchError, "MODEL_CAPACITY_EXHAUSTED should not return switchError")
  354. // 不应设置模型限流
  355. require.Empty(t, repo.modelRateLimitCalls, "MODEL_CAPACITY_EXHAUSTED should not set model rate limit")
  356. require.Len(t, upstream.calls, 1, "should have made one retry call before success")
  357. }
  358. // TestHandleSmartRetry_503_ModelCapacityExhausted_ContextCancel 测试 MODEL_CAPACITY_EXHAUSTED 上下文取消
  359. func TestHandleSmartRetry_503_ModelCapacityExhausted_ContextCancel(t *testing.T) {
  360. repo := &stubAntigravityAccountRepo{}
  361. account := &Account{
  362. ID: 3,
  363. Name: "acc-3",
  364. Type: AccountTypeOAuth,
  365. Platform: PlatformAntigravity,
  366. }
  367. respBody := []byte(`{
  368. "error": {
  369. "code": 503,
  370. "status": "UNAVAILABLE",
  371. "details": [
  372. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-pro-high"}, "reason": "MODEL_CAPACITY_EXHAUSTED"},
  373. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "39s"}
  374. ]
  375. }
  376. }`)
  377. resp := &http.Response{
  378. StatusCode: http.StatusServiceUnavailable,
  379. Header: http.Header{},
  380. Body: io.NopCloser(bytes.NewReader(respBody)),
  381. }
  382. // 立即取消上下文,验证重试循环能正确退出
  383. ctx, cancel := context.WithCancel(context.Background())
  384. cancel()
  385. params := antigravityRetryLoopParams{
  386. ctx: ctx,
  387. prefix: "[test]",
  388. account: account,
  389. accessToken: "token",
  390. action: "generateContent",
  391. body: []byte(`{"input":"test"}`),
  392. accountRepo: repo,
  393. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  394. return nil
  395. },
  396. }
  397. svc := &AntigravityGatewayService{}
  398. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, []string{"https://ag-1.test"})
  399. require.NotNil(t, result)
  400. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  401. require.Error(t, result.err, "should return context error")
  402. require.Nil(t, result.switchError, "should not return switchError on context cancel")
  403. require.Empty(t, repo.modelRateLimitCalls, "should not set model rate limit on context cancel")
  404. }
  405. // TestHandleSmartRetry_NonAntigravityAccount_ContinuesDefaultLogic 测试非 Antigravity 平台账号走默认逻辑
  406. func TestHandleSmartRetry_NonAntigravityAccount_ContinuesDefaultLogic(t *testing.T) {
  407. account := &Account{
  408. ID: 4,
  409. Name: "acc-4",
  410. Type: AccountTypeAPIKey, // 非 Antigravity 平台账号
  411. Platform: PlatformAnthropic,
  412. }
  413. // 即使是模型限流响应,非 OAuth 账号也应该走默认逻辑
  414. respBody := []byte(`{
  415. "error": {
  416. "status": "RESOURCE_EXHAUSTED",
  417. "details": [
  418. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"},
  419. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "15s"}
  420. ]
  421. }
  422. }`)
  423. resp := &http.Response{
  424. StatusCode: http.StatusTooManyRequests,
  425. Header: http.Header{},
  426. Body: io.NopCloser(bytes.NewReader(respBody)),
  427. }
  428. params := antigravityRetryLoopParams{
  429. ctx: context.Background(),
  430. prefix: "[test]",
  431. account: account,
  432. accessToken: "token",
  433. action: "generateContent",
  434. body: []byte(`{"input":"test"}`),
  435. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  436. return nil
  437. },
  438. }
  439. availableURLs := []string{"https://ag-1.test"}
  440. svc := &AntigravityGatewayService{}
  441. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  442. require.NotNil(t, result)
  443. require.Equal(t, smartRetryActionContinue, result.action, "non-Antigravity platform account should continue default logic")
  444. require.Nil(t, result.resp)
  445. require.Nil(t, result.err)
  446. require.Nil(t, result.switchError)
  447. }
  448. // TestHandleSmartRetry_NonModelRateLimit_ContinuesDefaultLogic 测试非模型限流响应走默认逻辑
  449. func TestHandleSmartRetry_NonModelRateLimit_ContinuesDefaultLogic(t *testing.T) {
  450. account := &Account{
  451. ID: 5,
  452. Name: "acc-5",
  453. Type: AccountTypeOAuth,
  454. Platform: PlatformAntigravity,
  455. }
  456. // 429 但没有 RATE_LIMIT_EXCEEDED 或 MODEL_CAPACITY_EXHAUSTED
  457. respBody := []byte(`{
  458. "error": {
  459. "status": "RESOURCE_EXHAUSTED",
  460. "details": [
  461. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "5s"}
  462. ],
  463. "message": "Quota exceeded"
  464. }
  465. }`)
  466. resp := &http.Response{
  467. StatusCode: http.StatusTooManyRequests,
  468. Header: http.Header{},
  469. Body: io.NopCloser(bytes.NewReader(respBody)),
  470. }
  471. params := antigravityRetryLoopParams{
  472. ctx: context.Background(),
  473. prefix: "[test]",
  474. account: account,
  475. accessToken: "token",
  476. action: "generateContent",
  477. body: []byte(`{"input":"test"}`),
  478. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  479. return nil
  480. },
  481. }
  482. availableURLs := []string{"https://ag-1.test"}
  483. svc := &AntigravityGatewayService{}
  484. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  485. require.NotNil(t, result)
  486. require.Equal(t, smartRetryActionContinue, result.action, "non-model rate limit should continue default logic")
  487. require.Nil(t, result.resp)
  488. require.Nil(t, result.err)
  489. require.Nil(t, result.switchError)
  490. }
  491. // TestHandleSmartRetry_ExactlyAtThreshold_ReturnsSwitchError 测试刚好等于阈值时返回 switchError
  492. func TestHandleSmartRetry_ExactlyAtThreshold_ReturnsSwitchError(t *testing.T) {
  493. repo := &stubAntigravityAccountRepo{}
  494. account := &Account{
  495. ID: 6,
  496. Name: "acc-6",
  497. Type: AccountTypeOAuth,
  498. Platform: PlatformAntigravity,
  499. }
  500. // 刚好 7s = 7s 阈值,应该返回 switchError
  501. respBody := []byte(`{
  502. "error": {
  503. "status": "RESOURCE_EXHAUSTED",
  504. "details": [
  505. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-pro"}, "reason": "RATE_LIMIT_EXCEEDED"},
  506. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "7s"}
  507. ]
  508. }
  509. }`)
  510. resp := &http.Response{
  511. StatusCode: http.StatusTooManyRequests,
  512. Header: http.Header{},
  513. Body: io.NopCloser(bytes.NewReader(respBody)),
  514. }
  515. params := antigravityRetryLoopParams{
  516. ctx: context.Background(),
  517. prefix: "[test]",
  518. account: account,
  519. accessToken: "token",
  520. action: "generateContent",
  521. body: []byte(`{"input":"test"}`),
  522. accountRepo: repo,
  523. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  524. return nil
  525. },
  526. }
  527. availableURLs := []string{"https://ag-1.test"}
  528. svc := &AntigravityGatewayService{}
  529. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  530. require.NotNil(t, result)
  531. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  532. require.Nil(t, result.resp)
  533. require.NotNil(t, result.switchError, "exactly at threshold should return switchError")
  534. require.Equal(t, "gemini-pro", result.switchError.RateLimitedModel)
  535. }
  536. // TestAntigravityRetryLoop_HandleSmartRetry_SwitchError_Propagates 测试 switchError 正确传播到上层
  537. func TestAntigravityRetryLoop_HandleSmartRetry_SwitchError_Propagates(t *testing.T) {
  538. // 模拟 429 + 长延迟的响应
  539. respBody := []byte(`{
  540. "error": {
  541. "status": "RESOURCE_EXHAUSTED",
  542. "details": [
  543. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-opus-4-6"}, "reason": "RATE_LIMIT_EXCEEDED"},
  544. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "30s"}
  545. ]
  546. }
  547. }`)
  548. rateLimitResp := &http.Response{
  549. StatusCode: http.StatusTooManyRequests,
  550. Header: http.Header{},
  551. Body: io.NopCloser(bytes.NewReader(respBody)),
  552. }
  553. upstream := &mockSmartRetryUpstream{
  554. responses: []*http.Response{rateLimitResp},
  555. errors: []error{nil},
  556. }
  557. repo := &stubAntigravityAccountRepo{}
  558. account := &Account{
  559. ID: 7,
  560. Name: "acc-7",
  561. Type: AccountTypeOAuth,
  562. Platform: PlatformAntigravity,
  563. Schedulable: true,
  564. Status: StatusActive,
  565. Concurrency: 1,
  566. }
  567. svc := &AntigravityGatewayService{}
  568. result, err := svc.antigravityRetryLoop(antigravityRetryLoopParams{
  569. ctx: context.Background(),
  570. prefix: "[test]",
  571. account: account,
  572. accessToken: "token",
  573. action: "generateContent",
  574. body: []byte(`{"input":"test"}`),
  575. httpUpstream: upstream,
  576. accountRepo: repo,
  577. isStickySession: true,
  578. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  579. return nil
  580. },
  581. })
  582. require.Nil(t, result, "should not return result when switchError")
  583. require.NotNil(t, err, "should return error")
  584. var switchErr *AntigravityAccountSwitchError
  585. require.ErrorAs(t, err, &switchErr, "error should be AntigravityAccountSwitchError")
  586. require.Equal(t, account.ID, switchErr.OriginalAccountID)
  587. require.Equal(t, "claude-opus-4-6", switchErr.RateLimitedModel)
  588. require.True(t, switchErr.IsStickySession)
  589. }
  590. // TestHandleSmartRetry_NetworkError_ExhaustsRetry 测试网络错误时(maxAttempts=1)直接耗尽重试并切换账号
  591. func TestHandleSmartRetry_NetworkError_ExhaustsRetry(t *testing.T) {
  592. // 唯一一次重试遇到网络错误(nil response)
  593. upstream := &mockSmartRetryUpstream{
  594. responses: []*http.Response{nil}, // 返回 nil(模拟网络错误)
  595. errors: []error{nil}, // mock 不返回 error,靠 nil response 触发
  596. }
  597. repo := &stubAntigravityAccountRepo{}
  598. account := &Account{
  599. ID: 8,
  600. Name: "acc-8",
  601. Type: AccountTypeOAuth,
  602. Platform: PlatformAntigravity,
  603. }
  604. // 0.1s < 7s 阈值,应该触发智能重试
  605. respBody := []byte(`{
  606. "error": {
  607. "status": "RESOURCE_EXHAUSTED",
  608. "details": [
  609. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"},
  610. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  611. ]
  612. }
  613. }`)
  614. resp := &http.Response{
  615. StatusCode: http.StatusTooManyRequests,
  616. Header: http.Header{},
  617. Body: io.NopCloser(bytes.NewReader(respBody)),
  618. }
  619. params := antigravityRetryLoopParams{
  620. ctx: context.Background(),
  621. prefix: "[test]",
  622. account: account,
  623. accessToken: "token",
  624. action: "generateContent",
  625. body: []byte(`{"input":"test"}`),
  626. httpUpstream: upstream,
  627. accountRepo: repo,
  628. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  629. return nil
  630. },
  631. }
  632. availableURLs := []string{"https://ag-1.test"}
  633. svc := &AntigravityGatewayService{}
  634. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  635. require.NotNil(t, result)
  636. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  637. require.Nil(t, result.resp, "should not return resp when switchError is set")
  638. require.NotNil(t, result.switchError, "should return switchError after network error exhausted retry")
  639. require.Equal(t, account.ID, result.switchError.OriginalAccountID)
  640. require.Equal(t, "claude-sonnet-4-5", result.switchError.RateLimitedModel)
  641. require.Len(t, upstream.calls, 1, "should have made one retry call")
  642. // 验证模型限流已设置
  643. require.Len(t, repo.modelRateLimitCalls, 1)
  644. require.Equal(t, "claude-sonnet-4-5", repo.modelRateLimitCalls[0].modelKey)
  645. }
  646. // TestHandleSmartRetry_NoRetryDelay_UsesDefaultRateLimit 测试无 retryDelay 时使用默认 1 分钟限流
  647. func TestHandleSmartRetry_NoRetryDelay_UsesDefaultRateLimit(t *testing.T) {
  648. repo := &stubAntigravityAccountRepo{}
  649. account := &Account{
  650. ID: 9,
  651. Name: "acc-9",
  652. Type: AccountTypeOAuth,
  653. Platform: PlatformAntigravity,
  654. }
  655. // 429 + RATE_LIMIT_EXCEEDED + 无 retryDelay → 使用默认 1 分钟限流
  656. respBody := []byte(`{
  657. "error": {
  658. "status": "RESOURCE_EXHAUSTED",
  659. "details": [
  660. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"}
  661. ],
  662. "message": "You have exhausted your capacity on this model."
  663. }
  664. }`)
  665. resp := &http.Response{
  666. StatusCode: http.StatusTooManyRequests,
  667. Header: http.Header{},
  668. Body: io.NopCloser(bytes.NewReader(respBody)),
  669. }
  670. params := antigravityRetryLoopParams{
  671. ctx: context.Background(),
  672. prefix: "[test]",
  673. account: account,
  674. accessToken: "token",
  675. action: "generateContent",
  676. body: []byte(`{"input":"test"}`),
  677. accountRepo: repo,
  678. isStickySession: true,
  679. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  680. return nil
  681. },
  682. }
  683. availableURLs := []string{"https://ag-1.test"}
  684. svc := &AntigravityGatewayService{}
  685. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  686. require.NotNil(t, result)
  687. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  688. require.Nil(t, result.resp, "should not return resp when switchError is set")
  689. require.NotNil(t, result.switchError, "should return switchError for no retryDelay")
  690. require.Equal(t, "claude-sonnet-4-5", result.switchError.RateLimitedModel)
  691. require.True(t, result.switchError.IsStickySession)
  692. // 验证模型限流已设置
  693. require.Len(t, repo.modelRateLimitCalls, 1)
  694. require.Equal(t, "claude-sonnet-4-5", repo.modelRateLimitCalls[0].modelKey)
  695. }
  696. // ---------------------------------------------------------------------------
  697. // 以下测试覆盖本次改动:
  698. // 1. antigravitySmartRetryMaxAttempts = 1(仅重试 1 次)
  699. // 2. 智能重试失败后清除粘性会话绑定(DeleteSessionAccountID)
  700. // ---------------------------------------------------------------------------
  701. // TestSmartRetryMaxAttempts_VerifyConstant 验证常量值为 1
  702. func TestSmartRetryMaxAttempts_VerifyConstant(t *testing.T) {
  703. require.Equal(t, 1, antigravitySmartRetryMaxAttempts,
  704. "antigravitySmartRetryMaxAttempts should be 1 to prevent repeated rate limiting")
  705. }
  706. // TestHandleSmartRetry_ShortDelay_StickySession_FailedRetry_ClearsSession
  707. // 核心场景:粘性会话 + 短延迟重试失败 → 必须清除粘性绑定
  708. func TestHandleSmartRetry_ShortDelay_StickySession_FailedRetry_ClearsSession(t *testing.T) {
  709. failRespBody := `{
  710. "error": {
  711. "status": "RESOURCE_EXHAUSTED",
  712. "details": [
  713. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"},
  714. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  715. ]
  716. }
  717. }`
  718. failResp := &http.Response{
  719. StatusCode: http.StatusTooManyRequests,
  720. Header: http.Header{},
  721. Body: io.NopCloser(strings.NewReader(failRespBody)),
  722. }
  723. upstream := &mockSmartRetryUpstream{
  724. responses: []*http.Response{failResp},
  725. errors: []error{nil},
  726. }
  727. repo := &stubAntigravityAccountRepo{}
  728. cache := &stubSmartRetryCache{}
  729. account := &Account{
  730. ID: 10,
  731. Name: "acc-10",
  732. Type: AccountTypeOAuth,
  733. Platform: PlatformAntigravity,
  734. }
  735. respBody := []byte(`{
  736. "error": {
  737. "status": "RESOURCE_EXHAUSTED",
  738. "details": [
  739. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"},
  740. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  741. ]
  742. }
  743. }`)
  744. resp := &http.Response{
  745. StatusCode: http.StatusTooManyRequests,
  746. Header: http.Header{},
  747. Body: io.NopCloser(bytes.NewReader(respBody)),
  748. }
  749. params := antigravityRetryLoopParams{
  750. ctx: context.Background(),
  751. prefix: "[test]",
  752. account: account,
  753. accessToken: "token",
  754. action: "generateContent",
  755. body: []byte(`{"input":"test"}`),
  756. httpUpstream: upstream,
  757. accountRepo: repo,
  758. isStickySession: true,
  759. groupID: 42,
  760. sessionHash: "sticky-hash-abc",
  761. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  762. return nil
  763. },
  764. }
  765. availableURLs := []string{"https://ag-1.test"}
  766. svc := &AntigravityGatewayService{cache: cache}
  767. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  768. // 验证返回 switchError
  769. require.NotNil(t, result)
  770. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  771. require.NotNil(t, result.switchError)
  772. require.True(t, result.switchError.IsStickySession, "switchError should carry IsStickySession=true")
  773. require.Equal(t, account.ID, result.switchError.OriginalAccountID)
  774. // 核心断言:DeleteSessionAccountID 被调用,且参数正确
  775. require.Len(t, cache.deleteCalls, 1, "should call DeleteSessionAccountID exactly once")
  776. require.Equal(t, int64(42), cache.deleteCalls[0].groupID)
  777. require.Equal(t, "sticky-hash-abc", cache.deleteCalls[0].sessionHash)
  778. // 验证仅重试 1 次
  779. require.Len(t, upstream.calls, 1, "should make exactly 1 retry call (maxAttempts=1)")
  780. // 验证模型限流已设置
  781. require.Len(t, repo.modelRateLimitCalls, 1)
  782. require.Equal(t, "claude-sonnet-4-5", repo.modelRateLimitCalls[0].modelKey)
  783. }
  784. // TestHandleSmartRetry_ShortDelay_NonStickySession_FailedRetry_NoDeleteSession
  785. // 非粘性会话 + 短延迟重试失败 → 不应调用 DeleteSessionAccountID(sessionHash 为空)
  786. func TestHandleSmartRetry_ShortDelay_NonStickySession_FailedRetry_NoDeleteSession(t *testing.T) {
  787. failRespBody := `{
  788. "error": {
  789. "status": "RESOURCE_EXHAUSTED",
  790. "details": [
  791. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-flash"}, "reason": "RATE_LIMIT_EXCEEDED"},
  792. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  793. ]
  794. }
  795. }`
  796. failResp := &http.Response{
  797. StatusCode: http.StatusTooManyRequests,
  798. Header: http.Header{},
  799. Body: io.NopCloser(strings.NewReader(failRespBody)),
  800. }
  801. upstream := &mockSmartRetryUpstream{
  802. responses: []*http.Response{failResp},
  803. errors: []error{nil},
  804. }
  805. repo := &stubAntigravityAccountRepo{}
  806. cache := &stubSmartRetryCache{}
  807. account := &Account{
  808. ID: 11,
  809. Name: "acc-11",
  810. Type: AccountTypeOAuth,
  811. Platform: PlatformAntigravity,
  812. }
  813. respBody := []byte(`{
  814. "error": {
  815. "status": "RESOURCE_EXHAUSTED",
  816. "details": [
  817. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-flash"}, "reason": "RATE_LIMIT_EXCEEDED"},
  818. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  819. ]
  820. }
  821. }`)
  822. resp := &http.Response{
  823. StatusCode: http.StatusTooManyRequests,
  824. Header: http.Header{},
  825. Body: io.NopCloser(bytes.NewReader(respBody)),
  826. }
  827. params := antigravityRetryLoopParams{
  828. ctx: context.Background(),
  829. prefix: "[test]",
  830. account: account,
  831. accessToken: "token",
  832. action: "generateContent",
  833. body: []byte(`{"input":"test"}`),
  834. httpUpstream: upstream,
  835. accountRepo: repo,
  836. isStickySession: false,
  837. groupID: 42,
  838. sessionHash: "", // 非粘性会话,sessionHash 为空
  839. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  840. return nil
  841. },
  842. }
  843. availableURLs := []string{"https://ag-1.test"}
  844. svc := &AntigravityGatewayService{cache: cache}
  845. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  846. require.NotNil(t, result)
  847. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  848. require.NotNil(t, result.switchError)
  849. require.False(t, result.switchError.IsStickySession)
  850. // 核心断言:sessionHash 为空时不应调用 DeleteSessionAccountID
  851. require.Len(t, cache.deleteCalls, 0, "should NOT call DeleteSessionAccountID when sessionHash is empty")
  852. }
  853. // TestHandleSmartRetry_ShortDelay_StickySession_FailedRetry_NilCache_NoPanic
  854. // 边界:cache 为 nil 时不应 panic
  855. func TestHandleSmartRetry_ShortDelay_StickySession_FailedRetry_NilCache_NoPanic(t *testing.T) {
  856. failRespBody := `{
  857. "error": {
  858. "status": "RESOURCE_EXHAUSTED",
  859. "details": [
  860. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"},
  861. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  862. ]
  863. }
  864. }`
  865. failResp := &http.Response{
  866. StatusCode: http.StatusTooManyRequests,
  867. Header: http.Header{},
  868. Body: io.NopCloser(strings.NewReader(failRespBody)),
  869. }
  870. upstream := &mockSmartRetryUpstream{
  871. responses: []*http.Response{failResp},
  872. errors: []error{nil},
  873. }
  874. repo := &stubAntigravityAccountRepo{}
  875. account := &Account{
  876. ID: 12,
  877. Name: "acc-12",
  878. Type: AccountTypeOAuth,
  879. Platform: PlatformAntigravity,
  880. }
  881. respBody := []byte(`{
  882. "error": {
  883. "status": "RESOURCE_EXHAUSTED",
  884. "details": [
  885. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"},
  886. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  887. ]
  888. }
  889. }`)
  890. resp := &http.Response{
  891. StatusCode: http.StatusTooManyRequests,
  892. Header: http.Header{},
  893. Body: io.NopCloser(bytes.NewReader(respBody)),
  894. }
  895. params := antigravityRetryLoopParams{
  896. ctx: context.Background(),
  897. prefix: "[test]",
  898. account: account,
  899. accessToken: "token",
  900. action: "generateContent",
  901. body: []byte(`{"input":"test"}`),
  902. httpUpstream: upstream,
  903. accountRepo: repo,
  904. isStickySession: true,
  905. groupID: 42,
  906. sessionHash: "sticky-hash-nil-cache",
  907. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  908. return nil
  909. },
  910. }
  911. availableURLs := []string{"https://ag-1.test"}
  912. // cache 为 nil,不应 panic
  913. svc := &AntigravityGatewayService{cache: nil}
  914. require.NotPanics(t, func() {
  915. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  916. require.NotNil(t, result)
  917. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  918. require.NotNil(t, result.switchError)
  919. require.True(t, result.switchError.IsStickySession)
  920. })
  921. }
  922. // TestHandleSmartRetry_ShortDelay_StickySession_SuccessRetry_NoDeleteSession
  923. // 重试成功时不应清除粘性会话(只有失败才清除)
  924. func TestHandleSmartRetry_ShortDelay_StickySession_SuccessRetry_NoDeleteSession(t *testing.T) {
  925. successResp := &http.Response{
  926. StatusCode: http.StatusOK,
  927. Header: http.Header{},
  928. Body: io.NopCloser(strings.NewReader(`{"result":"ok"}`)),
  929. }
  930. upstream := &mockSmartRetryUpstream{
  931. responses: []*http.Response{successResp},
  932. errors: []error{nil},
  933. }
  934. cache := &stubSmartRetryCache{}
  935. account := &Account{
  936. ID: 13,
  937. Name: "acc-13",
  938. Type: AccountTypeOAuth,
  939. Platform: PlatformAntigravity,
  940. }
  941. respBody := []byte(`{
  942. "error": {
  943. "status": "RESOURCE_EXHAUSTED",
  944. "details": [
  945. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-opus-4"}, "reason": "RATE_LIMIT_EXCEEDED"},
  946. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.5s"}
  947. ]
  948. }
  949. }`)
  950. resp := &http.Response{
  951. StatusCode: http.StatusTooManyRequests,
  952. Header: http.Header{},
  953. Body: io.NopCloser(bytes.NewReader(respBody)),
  954. }
  955. params := antigravityRetryLoopParams{
  956. ctx: context.Background(),
  957. prefix: "[test]",
  958. account: account,
  959. accessToken: "token",
  960. action: "generateContent",
  961. body: []byte(`{"input":"test"}`),
  962. httpUpstream: upstream,
  963. isStickySession: true,
  964. groupID: 42,
  965. sessionHash: "sticky-hash-success",
  966. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  967. return nil
  968. },
  969. }
  970. availableURLs := []string{"https://ag-1.test"}
  971. svc := &AntigravityGatewayService{cache: cache}
  972. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  973. require.NotNil(t, result)
  974. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  975. require.NotNil(t, result.resp, "should return successful response")
  976. require.Equal(t, http.StatusOK, result.resp.StatusCode)
  977. require.Nil(t, result.switchError, "should not return switchError on success")
  978. // 核心断言:重试成功时不应清除粘性会话
  979. require.Len(t, cache.deleteCalls, 0, "should NOT call DeleteSessionAccountID on successful retry")
  980. }
  981. // TestHandleSmartRetry_LongDelay_StickySession_ClearsSession
  982. // 长延迟路径(情况1)应立即清除 sticky 绑定,避免下一次请求继续命中已限流账号。
  983. func TestHandleSmartRetry_LongDelay_StickySession_ClearsSession(t *testing.T) {
  984. repo := &stubAntigravityAccountRepo{}
  985. cache := &stubSmartRetryCache{}
  986. account := &Account{
  987. ID: 14,
  988. Name: "acc-14",
  989. Type: AccountTypeOAuth,
  990. Platform: PlatformAntigravity,
  991. }
  992. // 15s >= 7s 阈值 → 走长延迟路径
  993. respBody := []byte(`{
  994. "error": {
  995. "status": "RESOURCE_EXHAUSTED",
  996. "details": [
  997. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"},
  998. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "15s"}
  999. ]
  1000. }
  1001. }`)
  1002. resp := &http.Response{
  1003. StatusCode: http.StatusTooManyRequests,
  1004. Header: http.Header{},
  1005. Body: io.NopCloser(bytes.NewReader(respBody)),
  1006. }
  1007. params := antigravityRetryLoopParams{
  1008. ctx: context.Background(),
  1009. prefix: "[test]",
  1010. account: account,
  1011. accessToken: "token",
  1012. action: "generateContent",
  1013. body: []byte(`{"input":"test"}`),
  1014. accountRepo: repo,
  1015. isStickySession: true,
  1016. groupID: 42,
  1017. sessionHash: "sticky-hash-long-delay",
  1018. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  1019. return nil
  1020. },
  1021. }
  1022. availableURLs := []string{"https://ag-1.test"}
  1023. svc := &AntigravityGatewayService{cache: cache}
  1024. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  1025. require.NotNil(t, result)
  1026. require.Equal(t, smartRetryActionBreakWithResp, result.action)
  1027. require.NotNil(t, result.switchError)
  1028. require.True(t, result.switchError.IsStickySession)
  1029. require.Len(t, cache.deleteCalls, 1, "long delay path should clear sticky session in handleSmartRetry")
  1030. require.Equal(t, int64(42), cache.deleteCalls[0].groupID)
  1031. require.Equal(t, "sticky-hash-long-delay", cache.deleteCalls[0].sessionHash)
  1032. }
  1033. // TestHandleSmartRetry_ShortDelay_NetworkError_StickySession_ClearsSession
  1034. // 网络错误耗尽重试 + 粘性会话 → 也应清除粘性绑定
  1035. func TestHandleSmartRetry_ShortDelay_NetworkError_StickySession_ClearsSession(t *testing.T) {
  1036. upstream := &mockSmartRetryUpstream{
  1037. responses: []*http.Response{nil}, // 网络错误
  1038. errors: []error{nil},
  1039. }
  1040. repo := &stubAntigravityAccountRepo{}
  1041. cache := &stubSmartRetryCache{}
  1042. account := &Account{
  1043. ID: 15,
  1044. Name: "acc-15",
  1045. Type: AccountTypeOAuth,
  1046. Platform: PlatformAntigravity,
  1047. }
  1048. respBody := []byte(`{
  1049. "error": {
  1050. "status": "RESOURCE_EXHAUSTED",
  1051. "details": [
  1052. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-flash"}, "reason": "RATE_LIMIT_EXCEEDED"},
  1053. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  1054. ]
  1055. }
  1056. }`)
  1057. resp := &http.Response{
  1058. StatusCode: http.StatusTooManyRequests,
  1059. Header: http.Header{},
  1060. Body: io.NopCloser(bytes.NewReader(respBody)),
  1061. }
  1062. params := antigravityRetryLoopParams{
  1063. ctx: context.Background(),
  1064. prefix: "[test]",
  1065. account: account,
  1066. accessToken: "token",
  1067. action: "generateContent",
  1068. body: []byte(`{"input":"test"}`),
  1069. httpUpstream: upstream,
  1070. accountRepo: repo,
  1071. isStickySession: true,
  1072. groupID: 99,
  1073. sessionHash: "sticky-net-error",
  1074. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  1075. return nil
  1076. },
  1077. }
  1078. availableURLs := []string{"https://ag-1.test"}
  1079. svc := &AntigravityGatewayService{cache: cache}
  1080. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  1081. require.NotNil(t, result)
  1082. require.NotNil(t, result.switchError)
  1083. require.True(t, result.switchError.IsStickySession)
  1084. // 核心断言:网络错误耗尽重试后也应清除粘性绑定
  1085. require.Len(t, cache.deleteCalls, 1, "should call DeleteSessionAccountID after network error exhausts retry")
  1086. require.Equal(t, int64(99), cache.deleteCalls[0].groupID)
  1087. require.Equal(t, "sticky-net-error", cache.deleteCalls[0].sessionHash)
  1088. require.Len(t, repo.modelRateLimitCalls, 2)
  1089. require.Equal(t, "gemini-3-flash", repo.modelRateLimitCalls[0].modelKey)
  1090. require.Equal(t, antigravityGeminiModelRateLimitKey, repo.modelRateLimitCalls[1].modelKey)
  1091. }
  1092. // TestHandleSmartRetry_ShortDelay_503_StickySession_FailedRetry_ClearsSession
  1093. // 429 + 短延迟 + 粘性会话 + 重试失败 → 清除粘性绑定
  1094. func TestHandleSmartRetry_ShortDelay_503_StickySession_FailedRetry_ClearsSession(t *testing.T) {
  1095. failRespBody := `{
  1096. "error": {
  1097. "code": 429,
  1098. "status": "RESOURCE_EXHAUSTED",
  1099. "details": [
  1100. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-pro"}, "reason": "RATE_LIMIT_EXCEEDED"},
  1101. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.5s"}
  1102. ]
  1103. }
  1104. }`
  1105. failResp := &http.Response{
  1106. StatusCode: http.StatusTooManyRequests,
  1107. Header: http.Header{},
  1108. Body: io.NopCloser(strings.NewReader(failRespBody)),
  1109. }
  1110. upstream := &mockSmartRetryUpstream{
  1111. responses: []*http.Response{failResp},
  1112. errors: []error{nil},
  1113. }
  1114. repo := &stubAntigravityAccountRepo{}
  1115. cache := &stubSmartRetryCache{}
  1116. account := &Account{
  1117. ID: 16,
  1118. Name: "acc-16",
  1119. Type: AccountTypeOAuth,
  1120. Platform: PlatformAntigravity,
  1121. }
  1122. respBody := []byte(`{
  1123. "error": {
  1124. "code": 429,
  1125. "status": "RESOURCE_EXHAUSTED",
  1126. "details": [
  1127. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-pro"}, "reason": "RATE_LIMIT_EXCEEDED"},
  1128. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.5s"}
  1129. ]
  1130. }
  1131. }`)
  1132. resp := &http.Response{
  1133. StatusCode: http.StatusTooManyRequests,
  1134. Header: http.Header{},
  1135. Body: io.NopCloser(bytes.NewReader(respBody)),
  1136. }
  1137. params := antigravityRetryLoopParams{
  1138. ctx: context.Background(),
  1139. prefix: "[test]",
  1140. account: account,
  1141. accessToken: "token",
  1142. action: "generateContent",
  1143. body: []byte(`{"input":"test"}`),
  1144. httpUpstream: upstream,
  1145. accountRepo: repo,
  1146. isStickySession: true,
  1147. groupID: 77,
  1148. sessionHash: "sticky-503-short",
  1149. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  1150. return nil
  1151. },
  1152. }
  1153. availableURLs := []string{"https://ag-1.test"}
  1154. svc := &AntigravityGatewayService{cache: cache}
  1155. result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs)
  1156. require.NotNil(t, result)
  1157. require.NotNil(t, result.switchError)
  1158. require.True(t, result.switchError.IsStickySession)
  1159. // 验证粘性绑定被清除
  1160. require.Len(t, cache.deleteCalls, 1)
  1161. require.Equal(t, int64(77), cache.deleteCalls[0].groupID)
  1162. require.Equal(t, "sticky-503-short", cache.deleteCalls[0].sessionHash)
  1163. // 验证模型限流已设置:Gemini 同时写入精确模型和家族级 scope
  1164. require.Len(t, repo.modelRateLimitCalls, 2)
  1165. require.Equal(t, "gemini-3-pro", repo.modelRateLimitCalls[0].modelKey)
  1166. require.Equal(t, antigravityGeminiModelRateLimitKey, repo.modelRateLimitCalls[1].modelKey)
  1167. }
  1168. // TestAntigravityRetryLoop_SmartRetryFailed_StickySession_SwitchErrorPropagates
  1169. // 集成测试:antigravityRetryLoop → handleSmartRetry → switchError 传播
  1170. // 验证 IsStickySession 正确传递到上层,且粘性绑定被清除
  1171. func TestAntigravityRetryLoop_SmartRetryFailed_StickySession_SwitchErrorPropagates(t *testing.T) {
  1172. // 初始 429 响应
  1173. initialRespBody := []byte(`{
  1174. "error": {
  1175. "status": "RESOURCE_EXHAUSTED",
  1176. "details": [
  1177. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-opus-4-6"}, "reason": "RATE_LIMIT_EXCEEDED"},
  1178. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  1179. ]
  1180. }
  1181. }`)
  1182. initialResp := &http.Response{
  1183. StatusCode: http.StatusTooManyRequests,
  1184. Header: http.Header{},
  1185. Body: io.NopCloser(bytes.NewReader(initialRespBody)),
  1186. }
  1187. // 智能重试也返回 429
  1188. retryRespBody := `{
  1189. "error": {
  1190. "status": "RESOURCE_EXHAUSTED",
  1191. "details": [
  1192. {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-opus-4-6"}, "reason": "RATE_LIMIT_EXCEEDED"},
  1193. {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"}
  1194. ]
  1195. }
  1196. }`
  1197. retryResp := &http.Response{
  1198. StatusCode: http.StatusTooManyRequests,
  1199. Header: http.Header{},
  1200. Body: io.NopCloser(strings.NewReader(retryRespBody)),
  1201. }
  1202. upstream := &mockSmartRetryUpstream{
  1203. responses: []*http.Response{initialResp, retryResp},
  1204. errors: []error{nil, nil},
  1205. }
  1206. repo := &stubAntigravityAccountRepo{}
  1207. cache := &stubSmartRetryCache{}
  1208. account := &Account{
  1209. ID: 17,
  1210. Name: "acc-17",
  1211. Type: AccountTypeOAuth,
  1212. Platform: PlatformAntigravity,
  1213. Schedulable: true,
  1214. Status: StatusActive,
  1215. Concurrency: 1,
  1216. }
  1217. svc := &AntigravityGatewayService{cache: cache}
  1218. result, err := svc.antigravityRetryLoop(antigravityRetryLoopParams{
  1219. ctx: context.Background(),
  1220. prefix: "[test]",
  1221. account: account,
  1222. accessToken: "token",
  1223. action: "generateContent",
  1224. body: []byte(`{"input":"test"}`),
  1225. httpUpstream: upstream,
  1226. accountRepo: repo,
  1227. isStickySession: true,
  1228. groupID: 55,
  1229. sessionHash: "sticky-loop-test",
  1230. handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
  1231. return nil
  1232. },
  1233. })
  1234. require.Nil(t, result, "should not return result when switchError")
  1235. require.NotNil(t, err, "should return error")
  1236. var switchErr *AntigravityAccountSwitchError
  1237. require.ErrorAs(t, err, &switchErr, "error should be AntigravityAccountSwitchError")
  1238. require.Equal(t, account.ID, switchErr.OriginalAccountID)
  1239. require.Equal(t, "claude-opus-4-6", switchErr.RateLimitedModel)
  1240. require.True(t, switchErr.IsStickySession, "IsStickySession must propagate through retryLoop")
  1241. // 验证粘性绑定被清除
  1242. require.Len(t, cache.deleteCalls, 1, "should clear sticky session in handleSmartRetry")
  1243. require.Equal(t, int64(55), cache.deleteCalls[0].groupID)
  1244. require.Equal(t, "sticky-loop-test", cache.deleteCalls[0].sessionHash)
  1245. }