ratelimit_service_anthropic_window_limit_test.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305
  1. //go:build unit
  2. package service
  3. import (
  4. "context"
  5. "net/http"
  6. "strconv"
  7. "testing"
  8. "time"
  9. "github.com/stretchr/testify/require"
  10. )
  11. type anthropicWindowLimitRepo struct {
  12. mockAccountRepoForGemini
  13. rateLimitCalls int
  14. tempUnschedCalls int
  15. lastRateLimitReset time.Time
  16. modelRateLimitCalls int
  17. lastModelRateLimitScope string
  18. lastModelRateLimitReset time.Time
  19. sessionWindowCalls int
  20. lastExtraUpdates map[string]any
  21. }
  22. func (r *anthropicWindowLimitRepo) SetRateLimited(_ context.Context, _ int64, resetAt time.Time) error {
  23. r.rateLimitCalls++
  24. r.lastRateLimitReset = resetAt
  25. return nil
  26. }
  27. func (r *anthropicWindowLimitRepo) SetTempUnschedulable(_ context.Context, _ int64, _ time.Time, _ string) error {
  28. r.tempUnschedCalls++
  29. return nil
  30. }
  31. func (r *anthropicWindowLimitRepo) SetModelRateLimit(_ context.Context, _ int64, scope string, resetAt time.Time, _ ...string) error {
  32. r.modelRateLimitCalls++
  33. r.lastModelRateLimitScope = scope
  34. r.lastModelRateLimitReset = resetAt
  35. return nil
  36. }
  37. func (r *anthropicWindowLimitRepo) UpdateSessionWindow(_ context.Context, _ int64, _, _ *time.Time, _ string) error {
  38. r.sessionWindowCalls++
  39. return nil
  40. }
  41. func (r *anthropicWindowLimitRepo) UpdateExtra(_ context.Context, _ int64, updates map[string]any) error {
  42. r.lastExtraUpdates = updates
  43. return nil
  44. }
  45. func TestHandleUpstreamError_AnthropicWindowLimitPreemptsTempUnschedRule(t *testing.T) {
  46. resetAt := time.Now().Add(3 * time.Hour).Truncate(time.Second)
  47. headers := http.Header{}
  48. headers.Set("anthropic-ratelimit-unified-5h-utilization", "1.02")
  49. headers.Set("anthropic-ratelimit-unified-5h-reset", strconv.FormatInt(resetAt.Unix(), 10))
  50. repo := &anthropicWindowLimitRepo{}
  51. svc := NewRateLimitService(repo, nil, nil, nil, nil)
  52. account := &Account{
  53. ID: 42,
  54. Type: AccountTypeOAuth,
  55. Platform: PlatformAnthropic,
  56. Credentials: map[string]any{
  57. "temp_unschedulable_enabled": true,
  58. "temp_unschedulable_rules": []any{
  59. map[string]any{
  60. "error_code": float64(http.StatusTooManyRequests),
  61. "keywords": []any{"rate limit"},
  62. "duration_minutes": float64(10),
  63. },
  64. },
  65. },
  66. }
  67. svc.HandleUpstreamError(
  68. context.Background(),
  69. account,
  70. http.StatusTooManyRequests,
  71. headers,
  72. []byte(`{"type":"error","error":{"type":"rate_limit_error","message":"This request would exceed your account's rate limit. Please try again later."}}`),
  73. )
  74. require.Zero(t, repo.tempUnschedCalls, "official Anthropic window limits should not be shortened by local temp-unsched rules")
  75. require.Equal(t, 1, repo.rateLimitCalls)
  76. require.Equal(t, resetAt, repo.lastRateLimitReset)
  77. }
  78. // fable429Headers 构造 7d_oi(Fable 专属 7d 窗口)触发 429 的完整响应头,
  79. // 数值取自真实抓包(5h/7d 均 allowed,仅 7d_oi rejected)。
  80. func fable429Headers(reset5h, resetOI time.Time) http.Header {
  81. headers := http.Header{}
  82. headers.Set("anthropic-ratelimit-unified-5h-reset", strconv.FormatInt(reset5h.Unix(), 10))
  83. headers.Set("anthropic-ratelimit-unified-5h-status", "allowed")
  84. headers.Set("anthropic-ratelimit-unified-5h-utilization", "0.41")
  85. headers.Set("anthropic-ratelimit-unified-7d-reset", strconv.FormatInt(resetOI.Unix(), 10))
  86. headers.Set("anthropic-ratelimit-unified-7d-status", "allowed")
  87. headers.Set("anthropic-ratelimit-unified-7d-utilization", "0.56")
  88. headers.Set("anthropic-ratelimit-unified-7d_oi-reset", strconv.FormatInt(resetOI.Unix(), 10))
  89. headers.Set("anthropic-ratelimit-unified-7d_oi-status", "rejected")
  90. headers.Set("anthropic-ratelimit-unified-7d_oi-surpassed-threshold", "1.0")
  91. headers.Set("anthropic-ratelimit-unified-7d_oi-utilization", "1.0")
  92. headers.Set("anthropic-ratelimit-unified-fallback-percentage", "0.5")
  93. headers.Set("anthropic-ratelimit-unified-overage-disabled-reason", "org_level_disabled")
  94. headers.Set("anthropic-ratelimit-unified-overage-status", "rejected")
  95. headers.Set("anthropic-ratelimit-unified-representative-claim", "seven_day_overage_included")
  96. headers.Set("anthropic-ratelimit-unified-reset", strconv.FormatInt(resetOI.Unix(), 10))
  97. headers.Set("anthropic-ratelimit-unified-status", "rejected")
  98. return headers
  99. }
  100. func TestHandleUpstreamError_Anthropic7dOiOnlyMarksModelRateLimit(t *testing.T) {
  101. now := time.Now()
  102. reset5h := now.Add(2 * time.Hour).Truncate(time.Second)
  103. resetOI := now.Add(80 * time.Hour).Truncate(time.Second)
  104. headers := fable429Headers(reset5h, resetOI)
  105. repo := &anthropicWindowLimitRepo{}
  106. svc := NewRateLimitService(repo, nil, nil, nil, nil)
  107. account := &Account{
  108. ID: 42,
  109. Type: AccountTypeOAuth,
  110. Platform: PlatformAnthropic,
  111. Credentials: map[string]any{
  112. "temp_unschedulable_enabled": true,
  113. "temp_unschedulable_rules": []any{
  114. map[string]any{
  115. "error_code": float64(http.StatusTooManyRequests),
  116. "keywords": []any{"rate limit"},
  117. "duration_minutes": float64(10),
  118. },
  119. },
  120. },
  121. }
  122. shouldDisable := svc.HandleUpstreamError(
  123. context.Background(),
  124. account,
  125. http.StatusTooManyRequests,
  126. headers,
  127. []byte(`{"type":"error","error":{"type":"rate_limit_error","message":"This request would exceed your account's rate limit. Please try again later."}}`),
  128. "claude-fable-5",
  129. )
  130. require.False(t, shouldDisable)
  131. require.Zero(t, repo.rateLimitCalls, "7d_oi (Fable-only) window must not mark the whole account rate limited")
  132. require.Zero(t, repo.tempUnschedCalls, "7d_oi window must not trigger local temp-unsched rules")
  133. require.Zero(t, repo.sessionWindowCalls, "7d_oi window must not rewrite the 5h session window as rejected")
  134. require.Equal(t, 1, repo.modelRateLimitCalls)
  135. require.Equal(t, anthropicFableRateLimitKey, repo.lastModelRateLimitScope)
  136. require.Equal(t, resetOI, repo.lastModelRateLimitReset)
  137. // 429 响应头也要被动采样,避免 7d F 进度条在限流期内冻结在旧值
  138. require.NotNil(t, repo.lastExtraUpdates)
  139. require.Equal(t, 1.0, repo.lastExtraUpdates["passive_usage_7d_oi_utilization"])
  140. require.Equal(t, resetOI.Unix(), repo.lastExtraUpdates["passive_usage_7d_oi_reset"])
  141. require.Equal(t, 0.41, repo.lastExtraUpdates["session_window_utilization"])
  142. }
  143. func TestHandleUpstreamError_AnthropicFableCreditsRequiredOnlyMarksModelRateLimit(t *testing.T) {
  144. resetAt := time.Now().Add(29 * 24 * time.Hour).Truncate(time.Second)
  145. headers := http.Header{}
  146. headers.Set("anthropic-ratelimit-unified-reset", strconv.FormatInt(resetAt.Unix(), 10))
  147. repo := &anthropicWindowLimitRepo{}
  148. svc := NewRateLimitService(repo, nil, nil, nil, nil)
  149. account := &Account{ID: 42, Type: AccountTypeOAuth, Platform: PlatformAnthropic}
  150. body := []byte(`{"type":"error","error":{"details":{"error_code":"credits_required","model":"claude-fable-5","disabled_reason":"org_level_disabled"},"message":"Usage credits are required for this model."}}`)
  151. shouldDisable := svc.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, headers, body, "claude-fable-5")
  152. require.False(t, shouldDisable)
  153. require.Zero(t, repo.rateLimitCalls, "Fable entitlement failure must not rate limit the whole account")
  154. require.Zero(t, repo.sessionWindowCalls, "Fable entitlement failure must not rewrite the shared session window")
  155. require.Equal(t, 1, repo.modelRateLimitCalls)
  156. require.Equal(t, anthropicFableRateLimitKey, repo.lastModelRateLimitScope)
  157. require.Equal(t, resetAt, repo.lastModelRateLimitReset)
  158. }
  159. func TestHandleUpstreamError_AnthropicFableCreditsRequiredFallsBackToRequestedModel(t *testing.T) {
  160. startedAt := time.Now()
  161. repo := &anthropicWindowLimitRepo{}
  162. svc := NewRateLimitService(repo, nil, nil, nil, nil)
  163. account := &Account{ID: 42, Type: AccountTypeOAuth, Platform: PlatformAnthropic}
  164. body := []byte(`{"type":"error","error":{"details":{"error_code":"credits_required"},"message":"Usage credits are required for this model."}}`)
  165. svc.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, http.Header{}, body, "claude-fable-5[1m]")
  166. require.Zero(t, repo.rateLimitCalls)
  167. require.Equal(t, 1, repo.modelRateLimitCalls)
  168. require.Equal(t, anthropicFableRateLimitKey, repo.lastModelRateLimitScope)
  169. require.True(t, repo.lastModelRateLimitReset.After(startedAt))
  170. }
  171. func TestHandleUpstreamError_AnthropicNonFableCreditsRequiredKeepsLegacyBehavior(t *testing.T) {
  172. resetAt := time.Now().Add(2 * time.Hour).Truncate(time.Second)
  173. headers := http.Header{}
  174. headers.Set("anthropic-ratelimit-unified-reset", strconv.FormatInt(resetAt.Unix(), 10))
  175. repo := &anthropicWindowLimitRepo{}
  176. svc := NewRateLimitService(repo, nil, nil, nil, nil)
  177. account := &Account{ID: 42, Type: AccountTypeOAuth, Platform: PlatformAnthropic}
  178. body := []byte(`{"type":"error","error":{"details":{"error_code":"credits_required","model":"claude-opus-5"},"message":"Usage credits are required for this model."}}`)
  179. svc.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, headers, body, "claude-opus-5")
  180. require.Zero(t, repo.modelRateLimitCalls)
  181. require.Equal(t, 1, repo.rateLimitCalls)
  182. require.Equal(t, resetAt, repo.lastRateLimitReset)
  183. }
  184. func TestHandleUpstreamError_AnthropicSharedWindowStillWinsWithFableCreditsRequired(t *testing.T) {
  185. now := time.Now()
  186. reset5h := now.Add(2 * time.Hour).Truncate(time.Second)
  187. reset7d := now.Add(80 * time.Hour).Truncate(time.Second)
  188. headers := http.Header{}
  189. headers.Set("anthropic-ratelimit-unified-5h-status", "rejected")
  190. headers.Set("anthropic-ratelimit-unified-5h-utilization", "1.0")
  191. headers.Set("anthropic-ratelimit-unified-5h-reset", strconv.FormatInt(reset5h.Unix(), 10))
  192. headers.Set("anthropic-ratelimit-unified-7d-status", "allowed")
  193. headers.Set("anthropic-ratelimit-unified-7d-reset", strconv.FormatInt(reset7d.Unix(), 10))
  194. headers.Set("anthropic-ratelimit-unified-reset", strconv.FormatInt(reset5h.Unix(), 10))
  195. repo := &anthropicWindowLimitRepo{}
  196. svc := NewRateLimitService(repo, nil, nil, nil, nil)
  197. account := &Account{ID: 42, Type: AccountTypeOAuth, Platform: PlatformAnthropic}
  198. body := []byte(`{"type":"error","error":{"details":{"error_code":"credits_required","model":"claude-fable-5"}}}`)
  199. svc.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, headers, body, "claude-fable-5")
  200. require.Equal(t, 1, repo.modelRateLimitCalls, "Fable-specific failure should still be recorded")
  201. require.Equal(t, 1, repo.rateLimitCalls, "an explicitly rejected shared window must still rate limit the account")
  202. require.Equal(t, reset5h, repo.lastRateLimitReset)
  203. }
  204. func TestHandleUpstreamError_Anthropic5hWindowStillWinsOver7dOi(t *testing.T) {
  205. // 5h 窗口 rejected 时必须仍按账号级限流处理(用 5h reset),同时记录 Fable 模型限流。
  206. now := time.Now()
  207. reset5h := now.Add(2 * time.Hour).Truncate(time.Second)
  208. resetOI := now.Add(80 * time.Hour).Truncate(time.Second)
  209. headers := fable429Headers(reset5h, resetOI)
  210. headers.Set("anthropic-ratelimit-unified-5h-status", "rejected")
  211. headers.Set("anthropic-ratelimit-unified-5h-utilization", "1.0")
  212. repo := &anthropicWindowLimitRepo{}
  213. svc := NewRateLimitService(repo, nil, nil, nil, nil)
  214. account := &Account{ID: 42, Type: AccountTypeOAuth, Platform: PlatformAnthropic}
  215. svc.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, headers, nil, "claude-fable-5")
  216. require.Equal(t, 1, repo.rateLimitCalls, "exhausted 5h window must still rate limit the account")
  217. require.Equal(t, reset5h, repo.lastRateLimitReset)
  218. require.Equal(t, 1, repo.modelRateLimitCalls)
  219. require.Equal(t, anthropicFableRateLimitKey, repo.lastModelRateLimitScope)
  220. }
  221. func TestHandleUpstreamError_AnthropicAccountWindowStillWinsOver7dOi(t *testing.T) {
  222. // 7d 窗口真超限时必须仍按账号级限流处理,同时记录 Fable 模型限流。
  223. now := time.Now()
  224. reset5h := now.Add(2 * time.Hour).Truncate(time.Second)
  225. resetOI := now.Add(80 * time.Hour).Truncate(time.Second)
  226. headers := fable429Headers(reset5h, resetOI)
  227. headers.Set("anthropic-ratelimit-unified-7d-status", "rejected")
  228. headers.Set("anthropic-ratelimit-unified-7d-utilization", "1.02")
  229. repo := &anthropicWindowLimitRepo{}
  230. svc := NewRateLimitService(repo, nil, nil, nil, nil)
  231. account := &Account{ID: 42, Type: AccountTypeOAuth, Platform: PlatformAnthropic}
  232. svc.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, headers, nil, "claude-fable-5")
  233. require.Equal(t, 1, repo.rateLimitCalls, "exhausted 7d window must still rate limit the account")
  234. require.Equal(t, resetOI, repo.lastRateLimitReset)
  235. require.Equal(t, 1, repo.modelRateLimitCalls, "Fable model rate limit should also be recorded")
  236. require.Equal(t, anthropicFableRateLimitKey, repo.lastModelRateLimitScope)
  237. }
  238. func TestHandleUpstreamError_Anthropic429Without7dOiKeepsLegacyBehavior(t *testing.T) {
  239. // 无 7d_oi 头、5h/7d 均未超限的 429:保持旧行为(按较早 reset 标记账号限流)。
  240. now := time.Now()
  241. reset5h := now.Add(2 * time.Hour).Truncate(time.Second)
  242. reset7d := now.Add(80 * time.Hour).Truncate(time.Second)
  243. headers := http.Header{}
  244. headers.Set("anthropic-ratelimit-unified-5h-reset", strconv.FormatInt(reset5h.Unix(), 10))
  245. headers.Set("anthropic-ratelimit-unified-5h-status", "allowed")
  246. headers.Set("anthropic-ratelimit-unified-5h-utilization", "0.41")
  247. headers.Set("anthropic-ratelimit-unified-7d-reset", strconv.FormatInt(reset7d.Unix(), 10))
  248. headers.Set("anthropic-ratelimit-unified-7d-status", "allowed")
  249. headers.Set("anthropic-ratelimit-unified-7d-utilization", "0.56")
  250. repo := &anthropicWindowLimitRepo{}
  251. svc := NewRateLimitService(repo, nil, nil, nil, nil)
  252. account := &Account{ID: 42, Type: AccountTypeOAuth, Platform: PlatformAnthropic}
  253. svc.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, headers, nil, "claude-fable-5")
  254. require.Zero(t, repo.modelRateLimitCalls, "no 7d_oi signal → no model rate limit")
  255. require.Equal(t, 1, repo.rateLimitCalls)
  256. require.Equal(t, reset5h, repo.lastRateLimitReset, "legacy path picks the sooner reset")
  257. require.Equal(t, 1, repo.sessionWindowCalls)
  258. }