| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637 |
- //go:build unit
- package service
- import (
- "context"
- "net/http"
- "net/http/httptest"
- "testing"
- "time"
- "github.com/Wei-Shaw/sub2api/internal/config"
- "github.com/gin-gonic/gin"
- "github.com/stretchr/testify/require"
- )
- // ---------------------------------------------------------------------------
- // TestShouldFailoverGeminiUpstreamError — verifies the failover decision
- // for the ErrorPolicyNone path (original logic preserved).
- // ---------------------------------------------------------------------------
- func TestShouldFailoverGeminiUpstreamError(t *testing.T) {
- svc := &GeminiMessagesCompatService{}
- tests := []struct {
- name string
- statusCode int
- expected bool
- }{
- {"401_failover", 401, true},
- {"403_failover", 403, true},
- {"429_failover", 429, true},
- {"529_failover", 529, true},
- {"500_failover", 500, true},
- {"502_failover", 502, true},
- {"503_failover", 503, true},
- {"400_no_failover", 400, false},
- {"404_no_failover", 404, false},
- {"422_no_failover", 422, false},
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got := svc.shouldFailoverGeminiUpstreamError(tt.statusCode)
- require.Equal(t, tt.expected, got)
- })
- }
- }
- // ---------------------------------------------------------------------------
- // TestCheckErrorPolicy_GeminiAccounts — verifies CheckErrorPolicy works
- // correctly for Gemini platform accounts (API Key type).
- // ---------------------------------------------------------------------------
- func TestCheckErrorPolicy_GeminiAccounts(t *testing.T) {
- tests := []struct {
- name string
- account *Account
- statusCode int
- body []byte
- expected ErrorPolicyResult
- }{
- {
- name: "gemini_apikey_custom_codes_hit",
- account: &Account{
- ID: 100,
- Type: AccountTypeAPIKey,
- Platform: PlatformGemini,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(429), float64(500)},
- },
- },
- statusCode: 429,
- body: []byte(`{"error":"rate limited"}`),
- expected: ErrorPolicyMatched,
- },
- {
- name: "gemini_apikey_custom_codes_miss",
- account: &Account{
- ID: 101,
- Type: AccountTypeAPIKey,
- Platform: PlatformGemini,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(429)},
- },
- },
- statusCode: 500,
- body: []byte(`{"error":"internal"}`),
- expected: ErrorPolicySkipped,
- },
- {
- name: "gemini_apikey_no_custom_codes_returns_none",
- account: &Account{
- ID: 102,
- Type: AccountTypeAPIKey,
- Platform: PlatformGemini,
- },
- statusCode: 500,
- body: []byte(`{"error":"internal"}`),
- expected: ErrorPolicyNone,
- },
- {
- name: "gemini_apikey_temp_unschedulable_hit",
- account: &Account{
- ID: 103,
- Type: AccountTypeAPIKey,
- Platform: PlatformGemini,
- Credentials: map[string]any{
- "temp_unschedulable_enabled": true,
- "temp_unschedulable_rules": []any{
- map[string]any{
- "error_code": float64(503),
- "keywords": []any{"overloaded"},
- "duration_minutes": float64(10),
- },
- },
- },
- },
- statusCode: 503,
- body: []byte(`overloaded service`),
- expected: ErrorPolicyTempUnscheduled,
- },
- {
- name: "gemini_apikey_temp_unschedulable_401_second_hit_returns_none",
- account: &Account{
- ID: 105,
- Type: AccountTypeAPIKey,
- Platform: PlatformGemini,
- TempUnschedulableReason: `{"status_code":401,"until_unix":1735689600}`,
- Credentials: map[string]any{
- "temp_unschedulable_enabled": true,
- "temp_unschedulable_rules": []any{
- map[string]any{
- "error_code": float64(401),
- "keywords": []any{"unauthorized"},
- "duration_minutes": float64(10),
- },
- },
- },
- },
- statusCode: 401,
- body: []byte(`unauthorized`),
- expected: ErrorPolicyNone,
- },
- {
- name: "gemini_custom_codes_override_temp_unschedulable",
- account: &Account{
- ID: 104,
- Type: AccountTypeAPIKey,
- Platform: PlatformGemini,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(503)},
- "temp_unschedulable_enabled": true,
- "temp_unschedulable_rules": []any{
- map[string]any{
- "error_code": float64(503),
- "keywords": []any{"overloaded"},
- "duration_minutes": float64(10),
- },
- },
- },
- },
- statusCode: 503,
- body: []byte(`overloaded`),
- expected: ErrorPolicyMatched, // custom codes take precedence
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- repo := &errorPolicyRepoStub{}
- svc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
- result := svc.CheckErrorPolicy(context.Background(), tt.account, tt.statusCode, tt.body)
- require.Equal(t, tt.expected, result)
- })
- }
- }
- // ---------------------------------------------------------------------------
- // TestGeminiErrorPolicyIntegration — verifies the Gemini error handling
- // paths produce the correct behavior for each ErrorPolicyResult.
- //
- // These tests simulate the inline error policy switch in handleClaudeCompat
- // and forwardNativeGemini by calling the same methods in the same order.
- // ---------------------------------------------------------------------------
- func TestGeminiErrorPolicyIntegration(t *testing.T) {
- gin.SetMode(gin.TestMode)
- tests := []struct {
- name string
- account *Account
- statusCode int
- respBody []byte
- expectFailover bool // expect UpstreamFailoverError
- expectHandleError bool // expect handleGeminiUpstreamError to be called
- expectShouldFailover bool // for None path, whether shouldFailover triggers
- expectModelScope string
- }{
- {
- name: "custom_codes_matched_429_failover",
- account: &Account{
- ID: 200,
- Type: AccountTypeAPIKey,
- Platform: PlatformGemini,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(429)},
- },
- },
- statusCode: 429,
- respBody: []byte(`{"error":"rate limited"}`),
- expectFailover: true,
- expectHandleError: true,
- },
- {
- name: "custom_codes_skipped_500_failover",
- account: &Account{
- ID: 201,
- Type: AccountTypeAPIKey,
- Platform: PlatformGemini,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(429)},
- },
- },
- statusCode: 500,
- respBody: []byte(`{"error":"internal"}`),
- expectFailover: true,
- expectHandleError: false,
- },
- {
- name: "custom_codes_skipped_400_no_failover",
- account: &Account{
- ID: 205,
- Type: AccountTypeAPIKey,
- Platform: PlatformGemini,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(429)},
- },
- },
- statusCode: 400,
- respBody: []byte(`{"error":"bad request"}`),
- expectFailover: false,
- expectHandleError: false,
- },
- {
- name: "temp_unschedulable_matched_failover",
- account: &Account{
- ID: 202,
- Type: AccountTypeAPIKey,
- Platform: PlatformGemini,
- Credentials: map[string]any{
- "temp_unschedulable_enabled": true,
- "temp_unschedulable_rules": []any{
- map[string]any{
- "error_code": float64(503),
- "keywords": []any{"overloaded"},
- "duration_minutes": float64(10),
- },
- },
- },
- },
- statusCode: 503,
- respBody: []byte(`overloaded`),
- expectFailover: true,
- expectHandleError: false,
- expectModelScope: "gemini-2.5-pro",
- },
- {
- name: "no_policy_429_failover_via_shouldFailover",
- account: &Account{
- ID: 203,
- Type: AccountTypeAPIKey,
- Platform: PlatformGemini,
- },
- statusCode: 429,
- respBody: []byte(`{"error":"rate limited"}`),
- expectFailover: true,
- expectHandleError: true,
- expectShouldFailover: true,
- },
- {
- name: "no_policy_400_no_failover",
- account: &Account{
- ID: 204,
- Type: AccountTypeAPIKey,
- Platform: PlatformGemini,
- },
- statusCode: 400,
- respBody: []byte(`{"error":"bad request"}`),
- expectFailover: false,
- expectHandleError: true,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- repo := &geminiErrorPolicyRepo{}
- rlSvc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
- svc := &GeminiMessagesCompatService{
- accountRepo: repo,
- rateLimitService: rlSvc,
- }
- writer := httptest.NewRecorder()
- c, _ := gin.CreateTestContext(writer)
- c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil)
- // Simulate the Claude compat error handling path (same logic as native).
- // This mirrors the inline switch in handleClaudeCompat.
- var handleErrorCalled bool
- var gotFailover bool
- ctx := context.Background()
- statusCode := tt.statusCode
- respBody := tt.respBody
- account := tt.account
- headers := http.Header{}
- if svc.rateLimitService != nil {
- policy := svc.rateLimitService.CheckErrorPolicy(ctx, account, statusCode, respBody, "gemini-2.5-pro")
- switch policy {
- case ErrorPolicySkipped:
- // Skipped → 不标记账号状态;可 failover 的状态码仍换号
- handleErrorCalled = false
- gotFailover = svc.skippedErrorPolicyFailoverError(c, account, statusCode, respBody, "req-test") != nil
- goto verify
- case ErrorPolicyMatched:
- svc.handleGeminiUpstreamError(ctx, account, statusCode, headers, respBody)
- handleErrorCalled = true
- gotFailover = true
- goto verify
- case ErrorPolicyTempUnscheduled:
- handleErrorCalled = false
- gotFailover = true
- goto verify
- }
- }
- // ErrorPolicyNone → original logic
- svc.handleGeminiUpstreamError(ctx, account, statusCode, headers, respBody)
- handleErrorCalled = true
- if svc.shouldFailoverGeminiUpstreamError(statusCode) {
- gotFailover = true
- }
- verify:
- require.Equal(t, tt.expectFailover, gotFailover, "failover mismatch")
- require.Equal(t, tt.expectHandleError, handleErrorCalled, "handleGeminiUpstreamError call mismatch")
- if tt.expectModelScope != "" {
- require.Equal(t, 1, repo.setModelRateLimitedCalls)
- require.Equal(t, tt.expectModelScope, repo.lastModelScope)
- require.Zero(t, repo.setTempCalls)
- require.Zero(t, repo.setRateLimitedCalls, "model temp rule must not be widened into an account rate limit")
- }
- if tt.expectShouldFailover {
- require.True(t, svc.shouldFailoverGeminiUpstreamError(statusCode),
- "shouldFailoverGeminiUpstreamError should return true for status %d", statusCode)
- }
- })
- }
- }
- // ---------------------------------------------------------------------------
- // TestSkippedErrorPolicyFailoverError — ErrorPolicySkipped(池模式、或自定义
- // 错误码未命中)不豁免换号:可 failover 的状态码返回 UpstreamFailoverError,
- // 仅池模式账号可携带同账号重试标记。
- // ---------------------------------------------------------------------------
- func TestSkippedErrorPolicyFailoverError(t *testing.T) {
- gin.SetMode(gin.TestMode)
- svc := &GeminiMessagesCompatService{}
- poolAccount := func(extra map[string]any) *Account {
- creds := map[string]any{"pool_mode": true}
- for k, v := range extra {
- creds[k] = v
- }
- return &Account{ID: 300, Type: AccountTypeAPIKey, Platform: PlatformGemini, Credentials: creds}
- }
- customCodesAccount := &Account{
- ID: 301, Type: AccountTypeAPIKey, Platform: PlatformGemini,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(429)},
- },
- }
- tests := []struct {
- name string
- account *Account
- statusCode int
- expectFailover bool
- expectSameAccount bool
- }{
- {"pool_500_failover_no_same_account_retry", poolAccount(nil), 500, true, false},
- {"pool_429_failover_with_same_account_retry", poolAccount(nil), 429, true, true},
- {"pool_custom_retry_codes_500", poolAccount(map[string]any{
- "pool_mode_retry_status_codes": []any{float64(500)},
- }), 500, true, true},
- {"pool_400_not_failover_worthy", poolAccount(nil), 400, false, false},
- {"custom_codes_miss_500_failover_no_same_account_retry", customCodesAccount, 500, true, false},
- {"custom_codes_miss_400_not_failover_worthy", customCodesAccount, 400, false, false},
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- writer := httptest.NewRecorder()
- c, _ := gin.CreateTestContext(writer)
- c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil)
- body := []byte(`{"error":{"code":"bad_response_status_code","message":"openai_error"}}`)
- failoverErr := svc.skippedErrorPolicyFailoverError(c, tt.account, tt.statusCode, body, "req-1")
- if !tt.expectFailover {
- require.Nil(t, failoverErr)
- return
- }
- require.NotNil(t, failoverErr)
- require.Equal(t, tt.statusCode, failoverErr.StatusCode)
- require.Equal(t, body, failoverErr.ResponseBody)
- require.Equal(t, tt.expectSameAccount, failoverErr.RetryableOnSameAccount)
- require.True(t, failoverErr.ShouldRetryNextAccount())
- })
- }
- }
- // ---------------------------------------------------------------------------
- // TestGeminiErrorPolicy_NilRateLimitService — verifies nil safety
- // ---------------------------------------------------------------------------
- func TestGeminiErrorPolicy_NilRateLimitService(t *testing.T) {
- svc := &GeminiMessagesCompatService{
- rateLimitService: nil,
- }
- // When rateLimitService is nil, error policy is skipped → falls through to
- // shouldFailoverGeminiUpstreamError (original logic).
- // Verify this doesn't panic and follows expected behavior.
- ctx := context.Background()
- account := &Account{
- ID: 300,
- Type: AccountTypeAPIKey,
- Platform: PlatformGemini,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(429)},
- },
- }
- // The nil check should prevent CheckErrorPolicy from being called
- if svc.rateLimitService != nil {
- t.Fatal("rateLimitService should be nil for this test")
- }
- // shouldFailoverGeminiUpstreamError still works
- require.True(t, svc.shouldFailoverGeminiUpstreamError(429))
- require.False(t, svc.shouldFailoverGeminiUpstreamError(400))
- // handleGeminiUpstreamError should not panic with nil rateLimitService
- require.NotPanics(t, func() {
- svc.handleGeminiUpstreamError(ctx, account, 500, http.Header{}, []byte(`error`))
- })
- }
- // ---------------------------------------------------------------------------
- // geminiErrorPolicyRepo — minimal AccountRepository stub for Gemini error
- // policy tests. Embeds mockAccountRepoForGemini and adds tracking.
- // ---------------------------------------------------------------------------
- func TestHandleGeminiUpstreamError_GoogleOneCapacityExhaustedUsesTierCooldown(t *testing.T) {
- repo := &rateLimit429AccountRepoStub{}
- quotaSvc := NewGeminiQuotaService(&config.Config{}, nil)
- rlSvc := NewRateLimitService(repo, nil, &config.Config{}, quotaSvc, nil)
- svc := &GeminiMessagesCompatService{
- accountRepo: repo,
- rateLimitService: rlSvc,
- }
- account := &Account{
- ID: 511,
- Platform: PlatformGemini,
- Type: AccountTypeOAuth,
- Credentials: map[string]any{
- "oauth_type": "google_one",
- "tier_id": "google_ai_pro",
- },
- }
- body := []byte(`{"error":{"code":429,"details":[{"@type":"type.googleapis.com/google.rpc.ErrorInfo","domain":"cloudcode-pa.googleapis.com","metadata":{"model":"gemini-3.1-pro-preview"},"reason":"MODEL_CAPACITY_EXHAUSTED"}],"message":"No capacity available for model gemini-3.1-pro-preview on the server","status":"RESOURCE_EXHAUSTED"}}`)
- before := time.Now()
- svc.handleGeminiUpstreamError(context.Background(), account, http.StatusTooManyRequests, http.Header{}, body)
- after := time.Now()
- require.Equal(t, 1, repo.rateLimitCalls)
- require.Equal(t, int64(511), repo.lastRateLimitID)
- require.WithinDuration(t, before.Add(5*time.Minute), repo.lastRateLimitReset, 2*time.Second)
- require.True(t, repo.lastRateLimitReset.After(before))
- require.True(t, repo.lastRateLimitReset.Before(after.Add(5*time.Minute).Add(2*time.Second)))
- }
- // ---------------------------------------------------------------------------
- // TestHandleGeminiUpstreamError_PoolMode429 — 池模式账号的 429 不写账号级限流。
- //
- // 429 的标记点在重试循环内(handleClaudeCompat / forwardNativeGemini /
- // chat completions 三条路径),先于 CheckErrorPolicy 执行,池模式豁免只能落在
- // handleGeminiUpstreamError 自身;否则一次上游 429 会把账号锁到 PST 午夜,
- // 即便重试已经成功返回客户端。
- // ---------------------------------------------------------------------------
- func TestHandleGeminiUpstreamError_PoolMode429(t *testing.T) {
- // 中转上游的真实 429 文案:不含 "per day",也没有 quotaResetDelay,
- // 解析失败后 apikey 账号会落到 PST 午夜兜底。
- body := []byte(`{"error":{"code":429,"message":"You have exhausted your capacity on this model. Your quota will reset after 6h53m10s."}}`)
- tests := []struct {
- name string
- account *Account
- expectRateLimited bool
- }{
- {
- name: "pool_mode_apikey_stays_in_pool",
- account: &Account{
- ID: 600,
- Platform: PlatformGemini,
- Type: AccountTypeAPIKey,
- Credentials: map[string]any{"pool_mode": true},
- },
- expectRateLimited: false,
- },
- {
- name: "custom_error_codes_hit_overrides_pool_mode",
- account: &Account{
- ID: 601,
- Platform: PlatformGemini,
- Type: AccountTypeAPIKey,
- Credentials: map[string]any{
- "pool_mode": true,
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(429)},
- },
- },
- expectRateLimited: true,
- },
- {
- name: "custom_error_codes_miss_skips",
- account: &Account{
- ID: 602,
- Platform: PlatformGemini,
- Type: AccountTypeAPIKey,
- Credentials: map[string]any{
- "pool_mode": true,
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(500)},
- },
- },
- expectRateLimited: false,
- },
- {
- name: "non_pool_apikey_still_rate_limited",
- account: &Account{
- ID: 603,
- Platform: PlatformGemini,
- Type: AccountTypeAPIKey,
- },
- expectRateLimited: true,
- },
- {
- name: "oauth_account_ignores_pool_mode_flag",
- account: &Account{
- ID: 604,
- Platform: PlatformGemini,
- Type: AccountTypeOAuth,
- Credentials: map[string]any{"pool_mode": true},
- },
- expectRateLimited: true,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- repo := &rateLimit429AccountRepoStub{}
- svc := &GeminiMessagesCompatService{
- accountRepo: repo,
- rateLimitService: NewRateLimitService(repo, nil, &config.Config{}, nil, nil),
- }
- svc.handleGeminiUpstreamError(context.Background(), tt.account, http.StatusTooManyRequests, http.Header{}, body)
- if !tt.expectRateLimited {
- require.Zero(t, repo.rateLimitCalls, "池模式账号不应被标记账号级限流")
- return
- }
- require.Equal(t, 1, repo.rateLimitCalls)
- require.Equal(t, tt.account.ID, repo.lastRateLimitID)
- require.True(t, repo.lastRateLimitReset.After(time.Now()))
- })
- }
- }
- type geminiErrorPolicyRepo struct {
- mockAccountRepoForGemini
- setErrorCalls int
- setRateLimitedCalls int
- setTempCalls int
- setModelRateLimitedCalls int
- lastModelScope string
- }
- func (r *geminiErrorPolicyRepo) SetError(_ context.Context, _ int64, _ string) error {
- r.setErrorCalls++
- return nil
- }
- func (r *geminiErrorPolicyRepo) SetRateLimited(_ context.Context, _ int64, _ time.Time) error {
- r.setRateLimitedCalls++
- return nil
- }
- func (r *geminiErrorPolicyRepo) SetTempUnschedulable(_ context.Context, _ int64, _ time.Time, _ string) error {
- r.setTempCalls++
- return nil
- }
- func (r *geminiErrorPolicyRepo) SetModelRateLimit(_ context.Context, _ int64, scope string, _ time.Time, _ ...string) error {
- r.setModelRateLimitedCalls++
- r.lastModelScope = scope
- return nil
- }
|