| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648 |
- //go:build unit
- package service
- import (
- "context"
- "net/http"
- "testing"
- "time"
- "github.com/Wei-Shaw/sub2api/internal/config"
- "github.com/stretchr/testify/require"
- )
- // ---------------------------------------------------------------------------
- // TestCheckErrorPolicy — 6 table-driven cases for the pure logic function
- // ---------------------------------------------------------------------------
- func TestCheckErrorPolicy(t *testing.T) {
- tests := []struct {
- name string
- account *Account
- statusCode int
- body []byte
- expected ErrorPolicyResult
- }{
- {
- name: "no_policy_oauth_returns_none",
- account: &Account{
- ID: 1,
- Type: AccountTypeOAuth,
- Platform: PlatformAntigravity,
- // no custom error codes, no temp rules
- },
- statusCode: 500,
- body: []byte(`"error"`),
- expected: ErrorPolicyNone,
- },
- {
- name: "custom_error_codes_hit_returns_matched",
- account: &Account{
- ID: 2,
- Type: AccountTypeAPIKey,
- Platform: PlatformAntigravity,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(429), float64(500)},
- },
- },
- statusCode: 500,
- body: []byte(`"error"`),
- expected: ErrorPolicyMatched,
- },
- {
- name: "custom_error_codes_miss_returns_skipped",
- account: &Account{
- ID: 3,
- Type: AccountTypeAPIKey,
- Platform: PlatformAntigravity,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(429), float64(500)},
- },
- },
- statusCode: 503,
- body: []byte(`"error"`),
- expected: ErrorPolicySkipped,
- },
- {
- name: "custom_error_codes_excluding_529_skip_global_cooldown",
- account: &Account{
- ID: 33,
- Type: AccountTypeAPIKey,
- Platform: PlatformOpenAI,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(429)},
- },
- },
- statusCode: 529,
- body: []byte(`{"error":{"message":"overloaded"}}`),
- expected: ErrorPolicySkipped,
- },
- {
- name: "pool_mode_skips_global_529_cooldown",
- account: &Account{
- ID: 34,
- Type: AccountTypeAPIKey,
- Platform: PlatformOpenAI,
- Credentials: map[string]any{
- "pool_mode": true,
- },
- },
- statusCode: 529,
- body: []byte(`{"error":{"message":"overloaded"}}`),
- expected: ErrorPolicySkipped,
- },
- {
- name: "ordinary_account_uses_global_529_cooldown",
- account: &Account{
- ID: 35,
- Type: AccountTypeAPIKey,
- Platform: PlatformOpenAI,
- },
- statusCode: 529,
- body: []byte(`{"error":{"message":"overloaded"}}`),
- expected: ErrorPolicyMatched,
- },
- {
- name: "custom_error_codes_including_529_take_precedence",
- account: &Account{
- ID: 36,
- Type: AccountTypeAPIKey,
- Platform: PlatformOpenAI,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(529)},
- },
- },
- statusCode: 529,
- body: []byte(`{"error":{"message":"overloaded"}}`),
- expected: ErrorPolicyMatched,
- },
- {
- name: "temp_unschedulable_hit_returns_temp_unscheduled",
- account: &Account{
- ID: 4,
- Type: AccountTypeOAuth,
- Platform: PlatformAntigravity,
- Credentials: map[string]any{
- "temp_unschedulable_enabled": true,
- "temp_unschedulable_rules": []any{
- map[string]any{
- "error_code": float64(503),
- "keywords": []any{"overloaded"},
- "duration_minutes": float64(10),
- "description": "overloaded rule",
- },
- },
- },
- },
- statusCode: 503,
- body: []byte(`overloaded service`),
- expected: ErrorPolicyTempUnscheduled,
- },
- {
- name: "temp_unschedulable_401_first_hit_returns_temp_unscheduled",
- account: &Account{
- ID: 14,
- Type: AccountTypeOAuth,
- Platform: PlatformAntigravity,
- Credentials: map[string]any{
- "temp_unschedulable_enabled": true,
- "temp_unschedulable_rules": []any{
- map[string]any{
- "error_code": float64(401),
- "keywords": []any{"unauthorized"},
- "duration_minutes": float64(10),
- },
- },
- },
- },
- statusCode: 401,
- body: []byte(`unauthorized`),
- expected: ErrorPolicyTempUnscheduled,
- },
- {
- // Antigravity 401 不走升级逻辑(由 applyErrorPolicy 的 temp_unschedulable_rules 自行控制),
- // second hit 仍然返回 TempUnscheduled。
- name: "temp_unschedulable_401_second_hit_antigravity_stays_temp",
- account: &Account{
- ID: 15,
- Type: AccountTypeOAuth,
- Platform: PlatformAntigravity,
- TempUnschedulableReason: `{"status_code":401,"until_unix":1735689600}`,
- Credentials: map[string]any{
- "temp_unschedulable_enabled": true,
- "temp_unschedulable_rules": []any{
- map[string]any{
- "error_code": float64(401),
- "keywords": []any{"unauthorized"},
- "duration_minutes": float64(10),
- },
- },
- },
- },
- statusCode: 401,
- body: []byte(`unauthorized`),
- expected: ErrorPolicyTempUnscheduled,
- },
- {
- name: "temp_unschedulable_body_miss_returns_none",
- account: &Account{
- ID: 5,
- Type: AccountTypeOAuth,
- Platform: PlatformAntigravity,
- Credentials: map[string]any{
- "temp_unschedulable_enabled": true,
- "temp_unschedulable_rules": []any{
- map[string]any{
- "error_code": float64(503),
- "keywords": []any{"overloaded"},
- "duration_minutes": float64(10),
- "description": "overloaded rule",
- },
- },
- },
- },
- statusCode: 503,
- body: []byte(`random msg`),
- expected: ErrorPolicyNone,
- },
- {
- name: "custom_error_codes_override_temp_unschedulable",
- account: &Account{
- ID: 6,
- Type: AccountTypeAPIKey,
- Platform: PlatformAntigravity,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(503)},
- "temp_unschedulable_enabled": true,
- "temp_unschedulable_rules": []any{
- map[string]any{
- "error_code": float64(503),
- "keywords": []any{"overloaded"},
- "duration_minutes": float64(10),
- "description": "overloaded rule",
- },
- },
- },
- },
- statusCode: 503,
- body: []byte(`overloaded`),
- expected: ErrorPolicyMatched, // custom codes take precedence
- },
- {
- name: "pool_mode_custom_error_codes_hit_returns_matched",
- account: &Account{
- ID: 7,
- Type: AccountTypeAPIKey,
- Platform: PlatformOpenAI,
- Credentials: map[string]any{
- "pool_mode": true,
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(401), float64(403)},
- },
- },
- statusCode: 401,
- body: []byte(`unauthorized`),
- expected: ErrorPolicyMatched,
- },
- {
- name: "pool_mode_without_custom_error_codes_returns_skipped",
- account: &Account{
- ID: 8,
- Type: AccountTypeAPIKey,
- Platform: PlatformOpenAI,
- Credentials: map[string]any{
- "pool_mode": true,
- },
- },
- statusCode: 401,
- body: []byte(`unauthorized`),
- expected: ErrorPolicySkipped,
- },
- {
- name: "pool_mode_temp_unschedulable_hit_returns_temp_unscheduled",
- account: &Account{
- ID: 9,
- Type: AccountTypeAPIKey,
- Platform: PlatformOpenAI,
- Credentials: map[string]any{
- "pool_mode": true,
- "temp_unschedulable_enabled": true,
- "temp_unschedulable_rules": []any{
- map[string]any{
- "error_code": float64(http.StatusServiceUnavailable),
- "keywords": []any{"unavailable"},
- "duration_minutes": float64(30),
- },
- },
- },
- },
- statusCode: http.StatusServiceUnavailable,
- body: []byte(`Service temporarily unavailable`),
- expected: ErrorPolicyTempUnscheduled,
- },
- {
- name: "pool_mode_temp_unschedulable_miss_returns_skipped",
- account: &Account{
- ID: 10,
- Type: AccountTypeAPIKey,
- Platform: PlatformOpenAI,
- Credentials: map[string]any{
- "pool_mode": true,
- "temp_unschedulable_enabled": true,
- "temp_unschedulable_rules": []any{
- map[string]any{
- "error_code": float64(http.StatusServiceUnavailable),
- "keywords": []any{"maintenance"},
- "duration_minutes": float64(30),
- },
- },
- },
- },
- statusCode: http.StatusServiceUnavailable,
- body: []byte(`Service temporarily unavailable`),
- expected: ErrorPolicySkipped,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- repo := &errorPolicyRepoStub{}
- svc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
- result := svc.CheckErrorPolicy(context.Background(), tt.account, tt.statusCode, tt.body)
- require.Equal(t, tt.expected, result, "unexpected ErrorPolicyResult")
- })
- }
- }
- func TestHandleUpstreamError_PoolModePolicies(t *testing.T) {
- t.Run("pool_mode_without_custom_error_codes_still_skips", func(t *testing.T) {
- repo := &errorPolicyRepoStub{}
- svc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
- account := &Account{
- ID: 30,
- Type: AccountTypeAPIKey,
- Platform: PlatformOpenAI,
- Credentials: map[string]any{
- "pool_mode": true,
- },
- }
- shouldDisable := svc.HandleUpstreamError(context.Background(), account, 401, http.Header{}, []byte("unauthorized"))
- require.False(t, shouldDisable)
- require.Equal(t, 0, repo.setErrCalls)
- require.Equal(t, 0, repo.tempCalls)
- })
- t.Run("pool_mode_with_custom_error_codes_uses_local_error_policy", func(t *testing.T) {
- repo := &errorPolicyRepoStub{}
- svc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
- account := &Account{
- ID: 31,
- Type: AccountTypeAPIKey,
- Platform: PlatformOpenAI,
- Credentials: map[string]any{
- "pool_mode": true,
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(401)},
- },
- }
- shouldDisable := svc.HandleUpstreamError(context.Background(), account, 401, http.Header{}, []byte("unauthorized"))
- require.True(t, shouldDisable)
- require.Equal(t, 1, repo.setErrCalls)
- require.Equal(t, 0, repo.tempCalls)
- })
- t.Run("pool_mode_explicit_temp_rule_stops_scheduling", func(t *testing.T) {
- repo := &errorPolicyRepoStub{}
- svc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
- account := &Account{
- ID: 32,
- Type: AccountTypeAPIKey,
- Platform: PlatformOpenAI,
- Credentials: map[string]any{
- "pool_mode": true,
- "temp_unschedulable_enabled": true,
- "temp_unschedulable_rules": []any{
- map[string]any{
- "error_code": float64(http.StatusServiceUnavailable),
- "keywords": []any{"unavailable"},
- "duration_minutes": float64(30),
- },
- },
- },
- }
- shouldDisable := svc.HandleUpstreamError(
- context.Background(),
- account,
- http.StatusServiceUnavailable,
- http.Header{},
- []byte("Service temporarily unavailable"),
- )
- require.True(t, shouldDisable)
- require.Equal(t, 0, repo.setErrCalls)
- require.Equal(t, 1, repo.tempCalls)
- })
- t.Run("pool_mode_temp_rule_miss_still_skips", func(t *testing.T) {
- repo := &errorPolicyRepoStub{}
- svc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
- account := &Account{
- ID: 33,
- Type: AccountTypeAPIKey,
- Platform: PlatformOpenAI,
- Credentials: map[string]any{
- "pool_mode": true,
- "temp_unschedulable_enabled": true,
- "temp_unschedulable_rules": []any{
- map[string]any{
- "error_code": float64(http.StatusServiceUnavailable),
- "keywords": []any{"maintenance"},
- "duration_minutes": float64(30),
- },
- },
- },
- }
- shouldDisable := svc.HandleUpstreamError(
- context.Background(),
- account,
- http.StatusServiceUnavailable,
- http.Header{},
- []byte("Service temporarily unavailable"),
- )
- require.False(t, shouldDisable)
- require.Equal(t, 0, repo.setErrCalls)
- require.Equal(t, 0, repo.tempCalls)
- })
- }
- // ---------------------------------------------------------------------------
- // TestApplyErrorPolicy — 4 table-driven cases for the wrapper method
- // ---------------------------------------------------------------------------
- func TestApplyErrorPolicy(t *testing.T) {
- tests := []struct {
- name string
- account *Account
- statusCode int
- body []byte
- expectedHandled bool
- expectedStatus int // expected outStatus
- expectedSwitchErr bool // expect *AntigravityAccountSwitchError
- handleErrorCalls int
- }{
- {
- name: "none_not_handled",
- account: &Account{
- ID: 10,
- Type: AccountTypeOAuth,
- Platform: PlatformAntigravity,
- },
- statusCode: 500,
- body: []byte(`"error"`),
- expectedHandled: false,
- expectedStatus: 500, // passthrough
- handleErrorCalls: 0,
- },
- {
- name: "skipped_handled_no_handleError",
- account: &Account{
- ID: 11,
- Type: AccountTypeAPIKey,
- Platform: PlatformAntigravity,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(429)},
- },
- },
- statusCode: 500, // not in custom codes
- body: []byte(`"error"`),
- expectedHandled: true,
- expectedStatus: http.StatusInternalServerError, // skipped → 500
- handleErrorCalls: 0,
- },
- {
- name: "matched_handled_calls_handleError",
- account: &Account{
- ID: 12,
- Type: AccountTypeAPIKey,
- Platform: PlatformAntigravity,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(500)},
- },
- },
- statusCode: 500,
- body: []byte(`"error"`),
- expectedHandled: true,
- expectedStatus: 500, // matched → original status
- handleErrorCalls: 1,
- },
- {
- name: "temp_unscheduled_returns_switch_error",
- account: &Account{
- ID: 13,
- Type: AccountTypeOAuth,
- Platform: PlatformAntigravity,
- Credentials: map[string]any{
- "model_mapping": map[string]any{
- "claude-sonnet-4-5": "claude-sonnet-4-5",
- },
- "temp_unschedulable_enabled": true,
- "temp_unschedulable_rules": []any{
- map[string]any{
- "error_code": float64(503),
- "keywords": []any{"overloaded"},
- "duration_minutes": float64(10),
- },
- },
- },
- },
- statusCode: 503,
- body: []byte(`overloaded`),
- expectedHandled: true,
- expectedStatus: 503, // temp_unscheduled → original status
- expectedSwitchErr: true,
- handleErrorCalls: 0,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- repo := &errorPolicyRepoStub{}
- rlSvc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
- svc := &AntigravityGatewayService{
- rateLimitService: rlSvc,
- }
- var handleErrorCount int
- p := antigravityRetryLoopParams{
- ctx: context.Background(),
- prefix: "[test]",
- account: tt.account,
- requestedModel: "claude-sonnet-4-5",
- handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult {
- handleErrorCount++
- return nil
- },
- isStickySession: true,
- }
- handled, outStatus, retErr := svc.applyErrorPolicy(p, tt.statusCode, http.Header{}, tt.body)
- require.Equal(t, tt.expectedHandled, handled, "handled mismatch")
- require.Equal(t, tt.expectedStatus, outStatus, "outStatus mismatch")
- require.Equal(t, tt.handleErrorCalls, handleErrorCount, "handleError call count mismatch")
- if tt.expectedSwitchErr {
- var switchErr *AntigravityAccountSwitchError
- require.ErrorAs(t, retErr, &switchErr)
- require.Equal(t, tt.account.ID, switchErr.OriginalAccountID)
- require.Zero(t, repo.tempCalls)
- require.Len(t, repo.modelRateLimitCalls, 1)
- require.Equal(t, "claude-sonnet-4-5", repo.modelRateLimitCalls[0].scope)
- } else {
- require.NoError(t, retErr)
- }
- })
- }
- }
- func TestApplyErrorPolicy_GeminiRateLimitBypassesCustomSkip(t *testing.T) {
- repo := &stubAntigravityAccountRepo{}
- cache := &stubSmartRetryCache{}
- rlSvc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
- svc := &AntigravityGatewayService{
- rateLimitService: rlSvc,
- accountRepo: repo,
- cache: cache,
- }
- account := &Account{
- ID: 31,
- Type: AccountTypeAPIKey,
- Platform: PlatformAntigravity,
- Credentials: map[string]any{
- "custom_error_codes_enabled": true,
- "custom_error_codes": []any{float64(500)},
- },
- }
- body := []byte(`{
- "error": {
- "status": "RESOURCE_EXHAUSTED",
- "details": [
- {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-flash"}, "reason": "RATE_LIMIT_EXCEEDED"},
- {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "15s"}
- ]
- }
- }`)
- p := antigravityRetryLoopParams{
- ctx: context.Background(),
- prefix: "[test]",
- account: account,
- accountRepo: repo,
- groupID: 42,
- sessionHash: "gemini:sticky",
- handleError: func(context.Context, string, *Account, int, http.Header, []byte, string, int64, string, bool) *handleModelRateLimitResult {
- t.Fatal("model rate limit should be handled before custom error fallback")
- return nil
- },
- }
- handled, outStatus, retErr := svc.applyErrorPolicy(p, http.StatusTooManyRequests, http.Header{}, body)
- require.True(t, handled)
- require.Equal(t, http.StatusTooManyRequests, outStatus)
- require.NoError(t, retErr)
- require.Len(t, repo.modelRateLimitCalls, 2)
- require.Equal(t, "gemini-3-flash", repo.modelRateLimitCalls[0].modelKey)
- require.Equal(t, antigravityGeminiModelRateLimitKey, repo.modelRateLimitCalls[1].modelKey)
- require.Len(t, cache.deleteCalls, 1)
- require.Equal(t, int64(42), cache.deleteCalls[0].groupID)
- require.Equal(t, "gemini:sticky", cache.deleteCalls[0].sessionHash)
- }
- // ---------------------------------------------------------------------------
- // errorPolicyRepoStub — minimal AccountRepository stub for error policy tests
- // ---------------------------------------------------------------------------
- type errorPolicyRepoStub struct {
- mockAccountRepoForGemini
- tempCalls int
- setErrCalls int
- lastErrorMsg string
- modelRateLimitCalls []modelNotFoundRateLimitCall
- }
- func (r *errorPolicyRepoStub) SetTempUnschedulable(ctx context.Context, id int64, until time.Time, reason string) error {
- r.tempCalls++
- return nil
- }
- func (r *errorPolicyRepoStub) SetError(ctx context.Context, id int64, errorMsg string) error {
- r.setErrCalls++
- r.lastErrorMsg = errorMsg
- return nil
- }
- func (r *errorPolicyRepoStub) SetModelRateLimit(_ context.Context, id int64, scope string, resetAt time.Time, reason ...string) error {
- call := modelNotFoundRateLimitCall{accountID: id, scope: scope, resetAt: resetAt}
- if len(reason) > 0 {
- call.reason = reason[0]
- }
- r.modelRateLimitCalls = append(r.modelRateLimitCalls, call)
- return nil
- }
|