ratelimit_service_401_test.go 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336
  1. //go:build unit
  2. package service
  3. import (
  4. "context"
  5. "errors"
  6. "net/http"
  7. "testing"
  8. "time"
  9. "github.com/Wei-Shaw/sub2api/internal/config"
  10. "github.com/stretchr/testify/require"
  11. )
  12. type rateLimitAccountRepoStub struct {
  13. mockAccountRepoForGemini
  14. setErrorCalls int
  15. tempCalls int
  16. updateCredentialsCalls int
  17. updateExtraCalls int
  18. lastCredentials map[string]any
  19. lastExtraUpdates map[string]any
  20. lastErrorMsg string
  21. lastTempReason string
  22. lastErrorID int64
  23. lastTempID int64
  24. tempErr error
  25. }
  26. func (r *rateLimitAccountRepoStub) SetError(ctx context.Context, id int64, errorMsg string) error {
  27. r.setErrorCalls++
  28. r.lastErrorID = id
  29. r.lastErrorMsg = errorMsg
  30. return nil
  31. }
  32. func (r *rateLimitAccountRepoStub) SetTempUnschedulable(ctx context.Context, id int64, until time.Time, reason string) error {
  33. r.tempCalls++
  34. r.lastTempID = id
  35. r.lastTempReason = reason
  36. return r.tempErr
  37. }
  38. func (r *rateLimitAccountRepoStub) UpdateCredentials(ctx context.Context, id int64, credentials map[string]any) error {
  39. r.updateCredentialsCalls++
  40. r.lastCredentials = shallowCopyMap(credentials)
  41. return nil
  42. }
  43. func (r *rateLimitAccountRepoStub) UpdateExtra(ctx context.Context, id int64, updates map[string]any) error {
  44. r.updateExtraCalls++
  45. r.lastExtraUpdates = shallowCopyMap(updates)
  46. return nil
  47. }
  48. type tokenCacheInvalidatorRecorder struct {
  49. accounts []*Account
  50. err error
  51. }
  52. type openAI403CounterCacheStub struct {
  53. counts []int64
  54. resetCalls []int64
  55. err error
  56. }
  57. func (s *openAI403CounterCacheStub) IncrementOpenAI403Count(_ context.Context, _ int64, _ int) (int64, error) {
  58. if s.err != nil {
  59. return 0, s.err
  60. }
  61. if len(s.counts) == 0 {
  62. return 1, nil
  63. }
  64. count := s.counts[0]
  65. s.counts = s.counts[1:]
  66. return count, nil
  67. }
  68. func (s *openAI403CounterCacheStub) ResetOpenAI403Count(_ context.Context, accountID int64) error {
  69. s.resetCalls = append(s.resetCalls, accountID)
  70. return nil
  71. }
  72. func (r *tokenCacheInvalidatorRecorder) InvalidateToken(ctx context.Context, account *Account) error {
  73. r.accounts = append(r.accounts, account)
  74. return r.err
  75. }
  76. func TestRateLimitService_HandleUpstreamError_OAuth401SetsTempUnschedulable(t *testing.T) {
  77. t.Run("gemini", func(t *testing.T) {
  78. repo := &rateLimitAccountRepoStub{}
  79. invalidator := &tokenCacheInvalidatorRecorder{}
  80. service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
  81. service.SetTokenCacheInvalidator(invalidator)
  82. account := &Account{
  83. ID: 100,
  84. Platform: PlatformGemini,
  85. Type: AccountTypeOAuth,
  86. Credentials: map[string]any{
  87. "refresh_token": "rt-100",
  88. "temp_unschedulable_enabled": true,
  89. "temp_unschedulable_rules": []any{
  90. map[string]any{
  91. "error_code": 401,
  92. "keywords": []any{"unauthorized"},
  93. "duration_minutes": 30,
  94. "description": "custom rule",
  95. },
  96. },
  97. },
  98. }
  99. shouldDisable := service.HandleUpstreamError(context.Background(), account, 401, http.Header{}, []byte("unauthorized"))
  100. require.True(t, shouldDisable)
  101. require.Equal(t, 0, repo.setErrorCalls)
  102. require.Equal(t, 1, repo.tempCalls)
  103. require.Len(t, invalidator.accounts, 1)
  104. })
  105. t.Run("antigravity_401_sets_temp_unschedulable", func(t *testing.T) {
  106. repo := &rateLimitAccountRepoStub{}
  107. invalidator := &tokenCacheInvalidatorRecorder{}
  108. service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
  109. service.SetTokenCacheInvalidator(invalidator)
  110. account := &Account{
  111. ID: 100,
  112. Platform: PlatformAntigravity,
  113. Type: AccountTypeOAuth,
  114. Status: StatusActive,
  115. Credentials: map[string]any{
  116. "access_token": "expired-at",
  117. "refresh_token": "rt-100",
  118. },
  119. }
  120. shouldDisable := service.HandleUpstreamError(context.Background(), account, 401, http.Header{}, []byte("unauthorized"))
  121. require.True(t, shouldDisable)
  122. require.Equal(t, 0, repo.setErrorCalls, "Antigravity OAuth 401 must keep status=active so refresh worker can recover it")
  123. require.Equal(t, 1, repo.tempCalls)
  124. require.Equal(t, int64(100), repo.lastTempID)
  125. require.Contains(t, repo.lastTempReason, "invalid or expired credentials")
  126. require.Equal(t, 1, repo.updateExtraCalls)
  127. require.Equal(t, true, repo.lastExtraUpdates[antigravityForceTokenRefreshExtraKey])
  128. require.Equal(t, "401_invalid", repo.lastExtraUpdates[antigravityForceTokenRefreshReasonExtraKey])
  129. require.Equal(t, true, account.Extra[antigravityForceTokenRefreshExtraKey])
  130. require.Len(t, invalidator.accounts, 1)
  131. require.Equal(t, int64(100), invalidator.accounts[0].ID)
  132. })
  133. }
  134. // TestRateLimitService_HandleUpstreamError_SparkShadow401RedirectsToParent 外审第9轮:影子无独立凭据,
  135. // 401(母账号 token 问题)必须重定向到凭据 owner(母账号)——母账号 temp-unschedulable + token cache 失效,
  136. // 影子不得被永久禁用(否则母账号可恢复的 token 问题会把影子永久打死)。
  137. func TestRateLimitService_HandleUpstreamError_SparkShadow401RedirectsToParent(t *testing.T) {
  138. repo := &rateLimitAccountRepoStub{}
  139. repo.accountsByID = map[int64]*Account{}
  140. invalidator := &tokenCacheInvalidatorRecorder{}
  141. service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
  142. service.SetTokenCacheInvalidator(invalidator)
  143. const parentID = int64(500)
  144. mother := &Account{
  145. ID: parentID,
  146. Platform: PlatformOpenAI,
  147. Type: AccountTypeOAuth,
  148. Credentials: map[string]any{"refresh_token": "rt-mother"},
  149. }
  150. repo.accountsByID[parentID] = mother
  151. shadowParent := parentID
  152. shadow := &Account{
  153. ID: 501,
  154. Platform: PlatformOpenAI,
  155. Type: AccountTypeOAuth,
  156. ParentAccountID: &shadowParent,
  157. QuotaDimension: QuotaDimensionSpark,
  158. // 影子不持凭据:GetCredential("refresh_token") == ""
  159. }
  160. shouldDisable := service.HandleUpstreamError(context.Background(), shadow, 401, http.Header{}, []byte("unauthorized"))
  161. require.True(t, shouldDisable)
  162. require.Equal(t, 0, repo.setErrorCalls, "spark shadow must not be permanently disabled on a parent-token 401")
  163. require.Equal(t, 1, repo.tempCalls)
  164. require.Equal(t, parentID, repo.lastTempID, "temp-unschedulable must target the credential owner (parent)")
  165. require.Len(t, invalidator.accounts, 1)
  166. require.Equal(t, parentID, invalidator.accounts[0].ID, "token cache invalidation must target the parent")
  167. }
  168. // TestRateLimitService_HandleUpstreamError_OAuth401InvalidatorError
  169. // OpenAI OAuth 401 缓存失效出错时仍走 temp_unschedulable。
  170. // 注意:401 handler 不再回写 credentials(避免请求开始时的快照整列覆盖 DB
  171. // 把另一个 worker 刚刷新出来的新 refresh_token 回滚为旧值),
  172. // 因此 updateCredentialsCalls 应当为 0。
  173. func TestRateLimitService_HandleUpstreamError_OAuth401InvalidatorError(t *testing.T) {
  174. repo := &rateLimitAccountRepoStub{}
  175. invalidator := &tokenCacheInvalidatorRecorder{err: errors.New("boom")}
  176. service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
  177. service.SetTokenCacheInvalidator(invalidator)
  178. account := &Account{
  179. ID: 101,
  180. Platform: PlatformOpenAI,
  181. Type: AccountTypeOAuth,
  182. Credentials: map[string]any{
  183. "refresh_token": "rt-101",
  184. },
  185. }
  186. shouldDisable := service.HandleUpstreamError(context.Background(), account, 401, http.Header{}, []byte("unauthorized"))
  187. require.True(t, shouldDisable)
  188. require.Equal(t, 0, repo.setErrorCalls)
  189. require.Equal(t, 1, repo.tempCalls)
  190. require.Equal(t, 0, repo.updateCredentialsCalls)
  191. require.Len(t, invalidator.accounts, 1)
  192. }
  193. func TestRateLimitService_HandleUpstreamError_NonOAuth401(t *testing.T) {
  194. repo := &rateLimitAccountRepoStub{}
  195. invalidator := &tokenCacheInvalidatorRecorder{}
  196. service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
  197. service.SetTokenCacheInvalidator(invalidator)
  198. account := &Account{
  199. ID: 102,
  200. Platform: PlatformOpenAI,
  201. Type: AccountTypeAPIKey,
  202. }
  203. shouldDisable := service.HandleUpstreamError(context.Background(), account, 401, http.Header{}, []byte("unauthorized"))
  204. require.True(t, shouldDisable)
  205. require.Equal(t, 1, repo.setErrorCalls)
  206. require.Empty(t, invalidator.accounts)
  207. }
  208. // TestRateLimitService_HandleUpstreamError_OAuth401DoesNotOverwriteCredentials
  209. // 回归测试:确保 401 handler 不再使用请求开始时的 account 快照写回 credentials。
  210. // 原实现会通过 persistAccountCredentials → UpdateCredentials → SetCredentials
  211. // 整列覆盖 credentials JSONB,在另一个 worker 刚刷新完 refresh_token 的窄窗口内
  212. // 会把新 refresh_token 回滚为快照中的旧值,导致下一周期拿 invalid_grant 被错误 disable。
  213. func TestRateLimitService_HandleUpstreamError_OAuth401DoesNotOverwriteCredentials(t *testing.T) {
  214. repo := &rateLimitAccountRepoStub{}
  215. service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
  216. account := &Account{
  217. ID: 103,
  218. Platform: PlatformOpenAI,
  219. Type: AccountTypeOAuth,
  220. Credentials: map[string]any{
  221. "access_token": "token",
  222. "refresh_token": "rt-103",
  223. },
  224. }
  225. shouldDisable := service.HandleUpstreamError(context.Background(), account, 401, http.Header{}, []byte("unauthorized"))
  226. require.True(t, shouldDisable)
  227. require.Equal(t, 0, repo.updateCredentialsCalls, "401 handler must not write credentials back from the request-start snapshot")
  228. require.Equal(t, 0, repo.updateExtraCalls, "OpenAI 401 must not set Antigravity force-refresh marker")
  229. require.Equal(t, 1, repo.tempCalls, "401 handler should still set temp-unschedulable cooldown")
  230. require.Nil(t, repo.lastCredentials, "no credentials should have been persisted")
  231. }
  232. // 缺少 refresh_token 的 OAuth 账号 401 应直接 SetError 永久禁用,
  233. // 不再走 10 分钟冷却(冷却期内无人能刷新它,结束后还会被选中再 502 一次)。
  234. func TestRateLimitService_HandleUpstreamError_OAuth401NoRefreshTokenSetsError(t *testing.T) {
  235. t.Run("openai_no_refresh_token", func(t *testing.T) {
  236. repo := &rateLimitAccountRepoStub{}
  237. invalidator := &tokenCacheInvalidatorRecorder{}
  238. service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
  239. service.SetTokenCacheInvalidator(invalidator)
  240. account := &Account{
  241. ID: 2881,
  242. Platform: PlatformOpenAI,
  243. Type: AccountTypeOAuth,
  244. Credentials: map[string]any{
  245. "access_token": "expired-at",
  246. // no refresh_token
  247. },
  248. }
  249. shouldDisable := service.HandleUpstreamError(context.Background(), account, 401, http.Header{}, []byte("unauthorized"))
  250. require.True(t, shouldDisable)
  251. require.Equal(t, 1, repo.setErrorCalls, "AT-only OAuth 401 must SetError")
  252. require.Equal(t, 0, repo.tempCalls, "AT-only OAuth 401 must NOT temp-unschedule")
  253. require.Equal(t, 0, repo.updateCredentialsCalls, "no point forcing expires_at when refresh is impossible")
  254. require.Contains(t, repo.lastErrorMsg, "refresh_token missing")
  255. require.Len(t, invalidator.accounts, 1, "cache should still be invalidated")
  256. })
  257. t.Run("openai_blank_refresh_token_treated_as_missing", func(t *testing.T) {
  258. repo := &rateLimitAccountRepoStub{}
  259. service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
  260. account := &Account{
  261. ID: 2882,
  262. Platform: PlatformOpenAI,
  263. Type: AccountTypeOAuth,
  264. Credentials: map[string]any{
  265. "access_token": "expired-at",
  266. "refresh_token": " ",
  267. },
  268. }
  269. shouldDisable := service.HandleUpstreamError(context.Background(), account, 401, http.Header{}, []byte("unauthorized"))
  270. require.True(t, shouldDisable)
  271. require.Equal(t, 1, repo.setErrorCalls)
  272. require.Equal(t, 0, repo.tempCalls)
  273. })
  274. t.Run("antigravity_no_refresh_token_sets_error", func(t *testing.T) {
  275. repo := &rateLimitAccountRepoStub{}
  276. invalidator := &tokenCacheInvalidatorRecorder{}
  277. service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
  278. service.SetTokenCacheInvalidator(invalidator)
  279. account := &Account{
  280. ID: 2883,
  281. Platform: PlatformAntigravity,
  282. Type: AccountTypeOAuth,
  283. Credentials: map[string]any{
  284. "access_token": "expired-at",
  285. },
  286. }
  287. shouldDisable := service.HandleUpstreamError(context.Background(), account, 401, http.Header{}, []byte("unauthorized"))
  288. require.True(t, shouldDisable)
  289. require.Equal(t, 1, repo.setErrorCalls, "Antigravity OAuth without refresh_token cannot self-recover")
  290. require.Equal(t, 0, repo.tempCalls)
  291. require.Contains(t, repo.lastErrorMsg, "refresh_token missing")
  292. require.Len(t, invalidator.accounts, 1)
  293. })
  294. }