| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849 |
- package service
- import (
- "bytes"
- "context"
- "encoding/json"
- "fmt"
- "log/slog"
- "net/http"
- "regexp"
- "strconv"
- "strings"
- "sync"
- "time"
- "github.com/Wei-Shaw/sub2api/internal/config"
- "github.com/Wei-Shaw/sub2api/internal/pkg/logger"
- "github.com/tidwall/gjson"
- )
- // RateLimitService 处理限流和过载状态管理
- type RateLimitService struct {
- accountRepo AccountRepository
- usageRepo UsageLogRepository
- cfg *config.Config
- geminiQuotaService *GeminiQuotaService
- tempUnschedCache TempUnschedCache
- openAIAPIKeyHealth OpenAIAPIKeyHealthCache
- timeoutCounterCache TimeoutCounterCache
- openAI403CounterCache OpenAI403CounterCache
- settingService *SettingService
- tokenCacheInvalidator TokenCacheInvalidator
- runtimeBlocker AccountRuntimeBlocker
- usageCacheMu sync.RWMutex
- usageCache map[int64]*geminiUsageCacheEntry
- // OpenAI Team 联动熔断的进程内去重:teamID → 去重窗口截止时间
- openaiTeamLinkedMu sync.Mutex
- openaiTeamLinkedRecent map[string]time.Time
- }
- type AccountRuntimeBlocker interface {
- BlockAccountScheduling(account *Account, until time.Time, reason string)
- ClearAccountSchedulingBlock(accountID int64)
- }
- // SuccessfulTestRecoveryResult 表示测试成功后恢复了哪些运行时状态。
- type SuccessfulTestRecoveryResult struct {
- ClearedError bool
- ClearedRateLimit bool
- }
- // AccountRecoveryOptions 控制账号恢复时的附加行为。
- type AccountRecoveryOptions struct {
- InvalidateToken bool
- }
- type geminiUsageCacheEntry struct {
- windowStart time.Time
- cachedAt time.Time
- totals GeminiUsageTotals
- }
- type geminiUsageTotalsBatchProvider interface {
- GetGeminiUsageTotalsBatch(ctx context.Context, accountIDs []int64, startTime, endTime time.Time) (map[int64]GeminiUsageTotals, error)
- }
- const geminiPrecheckCacheTTL = time.Minute
- const (
- defaultRateLimit429CooldownSeconds = 5
- maxRateLimit429CooldownSeconds = 7200
- )
- const (
- openAIImageRateLimitDefaultCooldown = time.Minute
- openAIImageRateLimitReason = "openai_image_rate_limited"
- openAIImageCapabilityLossCooldown = 30 * time.Minute
- openAIImageCapabilityLossReason = "openai_image_capability_lost"
- )
- var openAIImageTryAgainPattern = regexp.MustCompile(`(?i)try again in\s+([0-9]+(?:\.[0-9]+)?)\s*(ms|s|sec|secs|second|seconds|m|min|mins|minute|minutes)`)
- var openCodeGoUsageLimitResetPattern = regexp.MustCompile(`(?i)\bresets\s+in\s+`)
- var openCodeGoUsageLimitDurationPartPattern = regexp.MustCompile(`(?i)^([0-9]+(?:\.[0-9]+)?)\s*(s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours|d|day|days|w|week|weeks)\b`)
- const (
- openAI403CooldownMinutesDefault = 10
- openAI403DisableThreshold = 3
- openAI403CounterWindowMinutes = 180
- )
- // NewRateLimitService 创建RateLimitService实例
- func NewRateLimitService(accountRepo AccountRepository, usageRepo UsageLogRepository, cfg *config.Config, geminiQuotaService *GeminiQuotaService, tempUnschedCache TempUnschedCache) *RateLimitService {
- return &RateLimitService{
- accountRepo: accountRepo,
- usageRepo: usageRepo,
- cfg: cfg,
- geminiQuotaService: geminiQuotaService,
- tempUnschedCache: tempUnschedCache,
- usageCache: make(map[int64]*geminiUsageCacheEntry),
- }
- }
- // SetTimeoutCounterCache 设置超时计数器缓存(可选依赖)
- func (s *RateLimitService) SetTimeoutCounterCache(cache TimeoutCounterCache) {
- s.timeoutCounterCache = cache
- }
- func (s *RateLimitService) SetOpenAIAPIKeyHealthCache(cache OpenAIAPIKeyHealthCache) {
- s.openAIAPIKeyHealth = cache
- }
- // SetOpenAI403CounterCache 设置 OpenAI 403 连续失败计数器(可选依赖)
- func (s *RateLimitService) SetOpenAI403CounterCache(cache OpenAI403CounterCache) {
- s.openAI403CounterCache = cache
- }
- // SetSettingService 设置系统设置服务(可选依赖)
- func (s *RateLimitService) SetSettingService(settingService *SettingService) {
- s.settingService = settingService
- }
- // SetTokenCacheInvalidator 设置 token 缓存清理器(可选依赖)
- func (s *RateLimitService) SetTokenCacheInvalidator(invalidator TokenCacheInvalidator) {
- s.tokenCacheInvalidator = invalidator
- }
- func (s *RateLimitService) SetAccountRuntimeBlocker(blocker AccountRuntimeBlocker) {
- s.runtimeBlocker = blocker
- }
- func (s *RateLimitService) IsOpenAIAdvancedSchedulerStickyWeightedEnabled(ctx context.Context) bool {
- if s == nil || s.settingService == nil {
- return false
- }
- gateway := &OpenAIGatewayService{rateLimitService: s}
- return gateway.isOpenAIAdvancedSchedulerStickyWeightedEnabled(ctx)
- }
- func (s *RateLimitService) notifyAccountSchedulingBlocked(account *Account, until time.Time, reason string) {
- if s == nil || s.runtimeBlocker == nil || account == nil {
- return
- }
- s.runtimeBlocker.BlockAccountScheduling(account, until, reason)
- }
- func (s *RateLimitService) notifyAccountSchedulingBlockCleared(accountID int64) {
- if s == nil || s.runtimeBlocker == nil || accountID <= 0 {
- return
- }
- s.runtimeBlocker.ClearAccountSchedulingBlock(accountID)
- }
- // ApplyAccountSchedulingThreshold evaluates admin-configured per-platform
- // utilization thresholds and, when breached, parks the account as temp-
- // unschedulable until the winning window resets. Returns true when the account
- // is blocked (either newly or already paused for the same threshold reason).
- func (s *RateLimitService) ApplyAccountSchedulingThreshold(ctx context.Context, account *Account) bool {
- if s == nil || s.settingService == nil || s.accountRepo == nil || account == nil || account.ID <= 0 {
- return false
- }
- if !account.IsActive() || !account.Schedulable {
- return false
- }
- now := time.Now().UTC()
- thresholds := s.settingService.GetAccountSchedulingThresholds(ctx)
- decision := EvaluateAccountSchedulingThreshold(account, thresholds, now)
- if !decision.ShouldPause || decision.Until == nil || !decision.Until.After(now) {
- s.applyAnthropicFableSchedulingThreshold(ctx, account, thresholds, now)
- return false
- }
- reason := BuildDetailedAccountSchedulingThresholdReason(AccountSchedulingThresholdReasonInput{
- Platform: decision.Platform,
- Window: decision.Window,
- Scope: decision.Scope,
- ThresholdPercent: decision.ThresholdPercent,
- UsedPercent: decision.UsedPercent,
- Until: *decision.Until,
- Now: now,
- })
- if accountHasSameSchedulingThresholdPause(account, *decision.Until, reason) {
- return true
- }
- if !account.IsSchedulable() {
- return false
- }
- account.TempUnschedulableUntil = cloneTimePtr(decision.Until)
- account.TempUnschedulableReason = reason
- s.notifyAccountSchedulingBlocked(account, *decision.Until, "account_scheduling_threshold")
- if err := s.accountRepo.SetTempUnschedulable(ctx, account.ID, *decision.Until, reason); err != nil {
- slog.Warn("account_scheduling_threshold_set_temp_unsched_failed",
- "account_id", account.ID,
- "platform", decision.Platform,
- "window", decision.Window,
- "scope", decision.Scope,
- "threshold_percent", decision.ThresholdPercent,
- "used_percent", decision.UsedPercent,
- "until", decision.Until.UTC(),
- "error", err)
- } else if s.tempUnschedCache != nil {
- if state := tempUnschedStateFromStoredReason(reason, decision.Until.Unix()); state != nil {
- if err := s.tempUnschedCache.SetTempUnsched(ctx, account.ID, state); err != nil {
- slog.Warn("account_scheduling_threshold_cache_set_failed", "account_id", account.ID, "error", err)
- }
- }
- }
- slog.Info("account_scheduling_threshold_temp_unschedulable",
- "account_id", account.ID,
- "platform", decision.Platform,
- "window", decision.Window,
- "scope", decision.Scope,
- "threshold_percent", decision.ThresholdPercent,
- "used_percent", decision.UsedPercent,
- "until", decision.Until.UTC())
- return true
- }
- func (s *RateLimitService) applyAnthropicFableSchedulingThreshold(ctx context.Context, account *Account, thresholds map[string]int, now time.Time) {
- decision := evaluateAnthropicFableSchedulingThreshold(account, thresholds, now)
- if !decision.ShouldPause || decision.Until == nil || !decision.Until.After(now) {
- return
- }
- if account.isRateLimitActiveForKey(anthropicFableRateLimitKey) {
- return
- }
- reason := BuildDetailedAccountSchedulingThresholdReason(AccountSchedulingThresholdReasonInput{
- Platform: decision.Platform,
- Window: decision.Window,
- Scope: decision.Scope,
- ThresholdPercent: decision.ThresholdPercent,
- UsedPercent: decision.UsedPercent,
- Until: *decision.Until,
- Now: now,
- })
- setAccountModelRateLimitSnapshot(account, anthropicFableRateLimitKey, *decision.Until, reason, now)
- if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, anthropicFableRateLimitKey, *decision.Until, reason); err != nil {
- slog.Warn("anthropic_fable_scheduling_threshold_set_model_limit_failed",
- "account_id", account.ID,
- "threshold_percent", decision.ThresholdPercent,
- "used_percent", decision.UsedPercent,
- "until", decision.Until.UTC(),
- "error", err)
- return
- }
- slog.Info("anthropic_fable_scheduling_threshold_model_limited",
- "account_id", account.ID,
- "scope", anthropicFableRateLimitKey,
- "threshold_percent", decision.ThresholdPercent,
- "used_percent", decision.UsedPercent,
- "until", decision.Until.UTC())
- }
- func accountHasSameSchedulingThresholdPause(account *Account, until time.Time, reason string) bool {
- if account == nil || account.TempUnschedulableUntil == nil {
- return false
- }
- if account.TempUnschedulableUntil.UTC().Unix() != until.UTC().Unix() {
- return false
- }
- existing, ok := parseTempUnschedReasonPayload(account.TempUnschedulableReason)
- if !ok || existing.Source != AccountSchedulingThresholdReasonSource {
- return false
- }
- next, ok := parseTempUnschedReasonPayload(reason)
- if !ok || next.Source != AccountSchedulingThresholdReasonSource {
- return false
- }
- existing.TriggeredAtUnix = 0
- next.TriggeredAtUnix = 0
- return existing == next
- }
- // ErrorPolicyResult 表示错误策略检查的结果
- type ErrorPolicyResult int
- const (
- ErrorPolicyNone ErrorPolicyResult = iota // 未命中任何策略,继续默认逻辑
- ErrorPolicySkipped // 自定义错误码开启但未命中,跳过处理
- ErrorPolicyMatched // 自定义错误码命中,应停止调度
- ErrorPolicyTempUnscheduled // 临时不可调度规则命中
- )
- // CheckErrorPolicy 检查自定义错误码和临时不可调度规则。
- // 自定义错误码开启时覆盖后续所有逻辑(包括临时不可调度)。
- func (s *RateLimitService) CheckErrorPolicy(ctx context.Context, account *Account, statusCode int, responseBody []byte, requestedModel ...string) ErrorPolicyResult {
- ctx = withTempUnschedulableModel(ctx, requestedModel)
- if account.IsCustomErrorCodesEnabled() {
- if account.ShouldHandleErrorCode(statusCode) {
- return ErrorPolicyMatched
- }
- slog.Info("account_error_code_skipped", "account_id", account.ID, "status_code", statusCode)
- return ErrorPolicySkipped
- }
- if account.IsPoolMode() {
- // 池模式只跳过默认账号状态处理;管理员显式配置的临时不可调度规则仍应生效。
- // 401 保留现有认证错误语义,避免改变重复 401 的升级行为。
- if statusCode != http.StatusUnauthorized && s.tryTempUnschedulable(ctx, account, statusCode, responseBody) {
- return ErrorPolicyTempUnscheduled
- }
- return ErrorPolicySkipped
- }
- // The global overload cooldown is the default for ordinary accounts. Explicit
- // account policies above retain precedence over this fallback.
- if statusCode == 529 {
- return ErrorPolicyMatched
- }
- if s.tryTempUnschedulable(ctx, account, statusCode, responseBody, firstRequestedModel(requestedModel)) {
- return ErrorPolicyTempUnscheduled
- }
- return ErrorPolicyNone
- }
- // HandleUpstreamError 处理上游错误响应,标记账号状态
- // 返回是否应该停止该账号的调度
- func (s *RateLimitService) HandleUpstreamError(ctx context.Context, account *Account, statusCode int, headers http.Header, responseBody []byte, requestedModel ...string) (shouldDisable bool) {
- ctx = withTempUnschedulableModel(ctx, requestedModel)
- // Team 联动熔断必须先于池模式/自定义错误码/临时不可调度的各类早退;
- // 同请求内与 fastpath 调用点的重复触发由方法内去重吸收。
- s.maybeHandleOpenAITeamLinkedError(ctx, account, statusCode, responseBody)
- customErrorCodesEnabled := account.IsCustomErrorCodesEnabled()
- // 池模式默认不标记本地账号状态;但管理员显式配置的临时不可调度规则优先。
- // 401 保留现有认证错误语义,不在这里改变池模式的认证处理。
- if account.IsPoolMode() && !customErrorCodesEnabled {
- if statusCode != http.StatusUnauthorized && s.tryTempUnschedulable(ctx, account, statusCode, responseBody) {
- return true
- }
- slog.Info("pool_mode_error_skipped", "account_id", account.ID, "status_code", statusCode)
- return false
- }
- // apikey 类型账号:检查自定义错误码配置
- // 如果启用且错误码不在列表中,则不处理(不停止调度、不标记限流/过载)
- if !account.ShouldHandleErrorCode(statusCode) {
- slog.Info("account_error_code_skipped", "account_id", account.ID, "status_code", statusCode)
- return false
- }
- if statusCode == 529 {
- if customErrorCodesEnabled {
- s.handleCustomErrorCode(ctx, account, statusCode, extractUpstreamErrorMessage(responseBody))
- return true
- }
- s.handle529(ctx, account)
- return false
- }
- if len(requestedModel) > 0 && s.HandleUpstreamModelNotFound(ctx, account, requestedModel[0], statusCode, responseBody) {
- return true
- }
- // Anthropic official 5h / 7d window exhaustion is a hard account limit.
- // It must take precedence over user-configured 429 temp-unsched rules,
- // otherwise a broad "rate limit" keyword rule can shorten a multi-hour
- // cooldown to a local temporary pause.
- if statusCode == http.StatusTooManyRequests && account.Platform == PlatformAnthropic {
- // Fable may be rejected because the organization has no usage credits for
- // this model. Anthropic reports that as 429, but it is a model entitlement
- // failure rather than a shared account window exhaustion.
- fableCreditsRequired := s.persistAnthropicFableCreditsRequired(ctx, account, headers, responseBody, firstRequestedModel(requestedModel))
- // 7d_oi 是 Fable 模型专属的 7d 窗口:只标记模型级限流,账号对其他模型仍可调度。
- fableLimited := s.persistAnthropicFableWindowLimit(ctx, account, headers)
- if s.persistAnthropicExhaustedWindowLimit(ctx, account, headers) {
- return false
- }
- if fableCreditsRequired || fableLimited {
- return false
- }
- }
- // 先尝试临时不可调度规则(401除外)
- // 如果匹配成功,直接返回,不执行后续禁用逻辑
- if statusCode != 401 {
- if s.tryTempUnschedulable(ctx, account, statusCode, responseBody, firstRequestedModel(requestedModel)) {
- return true
- }
- }
- upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(responseBody))
- upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg)
- if upstreamMsg != "" {
- upstreamMsg = truncateForLog([]byte(upstreamMsg), 512)
- }
- switch statusCode {
- case 400:
- // "organization has been disabled" → 永久禁用
- if strings.Contains(strings.ToLower(upstreamMsg), "organization has been disabled") {
- msg := "Organization disabled (400): " + upstreamMsg
- s.handleAuthError(ctx, account, msg)
- shouldDisable = true
- } else if account.Platform == PlatformAnthropic && strings.Contains(strings.ToLower(upstreamMsg), "credit balance") {
- // Anthropic API key 余额不足(语义等同 402),停止调度
- msg := "Credit balance exhausted (400): " + upstreamMsg
- s.handleAuthError(ctx, account, msg)
- shouldDisable = true
- } else if strings.Contains(strings.ToLower(upstreamMsg), "identity verification is required") {
- // KYC 身份验证要求 → 永久禁用,账号需完成身份验证后才能恢复
- msg := "Identity verification required (400): " + upstreamMsg
- s.handleAuthError(ctx, account, msg)
- shouldDisable = true
- }
- // 其他 400 错误(如参数问题)不处理,不禁用账号
- case 401:
- // 外审第9轮:Spark 影子无独立凭据,401 是母账号 token 问题——失效缓存 / refresh_token 判断 /
- // 永久禁用 / 临时不可调度都必须落到凭据 owner(母账号),否则影子(无 refresh_token)必中
- // "refresh_token missing"永久禁用分支、母账号 token cache 也不会被清,把母账号可恢复的 token
- // 问题变成影子永久死亡。母账号被标记 temp-unschedulable 后由 parentHealthyForShadow 级联排除影子。
- // 非影子时 resolveCredentialAccount 返回自身;母账号缺失/损坏(orphan 影子,罕见)时回退到原 account。
- authAccount := account
- if resolved, rerr := resolveCredentialAccount(ctx, s.accountRepo, account); rerr == nil && resolved != nil {
- authAccount = resolved
- }
- // OpenAI: token_invalidated / token_revoked 表示 token 被永久作废(非过期),直接标记 error
- openai401Code := extractUpstreamErrorCode(responseBody)
- if authAccount.Platform == PlatformOpenAI && (openai401Code == "token_invalidated" || openai401Code == "token_revoked") {
- msg := "Token revoked (401): account authentication permanently revoked"
- if upstreamMsg != "" {
- msg = "Token revoked (401): " + upstreamMsg
- }
- s.handleAuthError(ctx, authAccount, msg)
- shouldDisable = true
- break
- }
- // OpenAI: {"detail":"Unauthorized"} 表示 token 完全无效(非标准 OpenAI 错误格式),直接标记 error
- if authAccount.Platform == PlatformOpenAI && gjson.GetBytes(responseBody, "detail").String() == "Unauthorized" {
- msg := "Unauthorized (401): account authentication failed permanently"
- if upstreamMsg != "" {
- msg = "Unauthorized (401): " + upstreamMsg
- }
- s.handleAuthError(ctx, authAccount, msg)
- shouldDisable = true
- break
- }
- // OAuth 账号在 401 错误时临时不可调度(给 token 刷新窗口);非 OAuth 账号保持原有 SetError 行为。
- if authAccount.Type == AccountTypeOAuth {
- // 1. 失效缓存
- if s.tokenCacheInvalidator != nil {
- if err := s.tokenCacheInvalidator.InvalidateToken(ctx, authAccount); err != nil {
- slog.Warn("oauth_401_invalidate_cache_failed", "account_id", authAccount.ID, "error", err)
- }
- }
- // 缺少 refresh_token 的 OAuth 账号无法在冷却期内自愈(后台刷新服务也会跳过),
- // 直接走 SetError 永久禁用,避免冷却结束后再被选中产生一发无意义的 502。
- if strings.TrimSpace(authAccount.GetCredential("refresh_token")) == "" {
- msg := "Authentication failed (401): refresh_token missing, cannot recover"
- if upstreamMsg != "" {
- msg = "OAuth 401 (no refresh_token): " + upstreamMsg
- }
- s.handleAuthError(ctx, authAccount, msg)
- shouldDisable = true
- break
- }
- // 2. 临时不可调度,替代 SetError(保持 status=active 让刷新服务能拾取)
- // 注意:此处不再写回 account.Credentials/expires_at。
- // 原实现使用请求开始时的 account 快照整列覆盖 credentials JSONB(见
- // persistAccountCredentials → accountRepository.UpdateCredentials → SetCredentials),
- // 在另一个 worker 刚刷新完 refresh_token 的窄窗口内会把新 refresh_token 回滚为旧值,
- // 导致下一周期用旧 refresh_token 调上游拿到 invalid_grant 后,
- // tryRecoverFromRefreshRace 重读 DB 发现 currentRT == usedRT 也救不回来,账号被错误 disable。
- // 这里仅依赖 InvalidateToken + SetTempUnschedulable 让账号在冷却期内不被调度,
- // 冷却结束后由 token_provider 的 NeedsRefresh / token_refresh_service 走带分布式锁的正路刷新。
- msg := "Authentication failed (401): invalid or expired credentials"
- if upstreamMsg != "" {
- msg = "OAuth 401: " + upstreamMsg
- }
- if authAccount.Platform == PlatformAntigravity {
- extraUpdates := antigravityForceTokenRefreshExtra("401_invalid")
- if err := s.accountRepo.UpdateExtra(ctx, authAccount.ID, extraUpdates); err != nil {
- slog.Warn("antigravity_401_force_refresh_mark_failed", "account_id", authAccount.ID, "error", err)
- } else {
- if authAccount.Extra == nil {
- authAccount.Extra = make(map[string]any, len(extraUpdates))
- }
- for k, v := range extraUpdates {
- authAccount.Extra[k] = v
- }
- slog.Info("antigravity_401_force_refresh_marked", "account_id", authAccount.ID)
- }
- }
- cooldownMinutes := s.cfg.RateLimit.OAuth401CooldownMinutes
- if cooldownMinutes <= 0 {
- cooldownMinutes = 10
- }
- until := time.Now().Add(time.Duration(cooldownMinutes) * time.Minute)
- s.notifyAccountSchedulingBlocked(authAccount, until, "oauth_401")
- if err := s.accountRepo.SetTempUnschedulable(ctx, authAccount.ID, until, msg); err != nil {
- slog.Warn("oauth_401_set_temp_unschedulable_failed", "account_id", authAccount.ID, "error", err)
- }
- shouldDisable = true
- } else {
- // 非 OAuth:保持 SetError 行为
- msg := "Authentication failed (401): invalid or expired credentials"
- if upstreamMsg != "" {
- msg = "Authentication failed (401): " + upstreamMsg
- }
- s.handleAuthError(ctx, authAccount, msg)
- shouldDisable = true
- }
- case 402:
- // 国产供应商:余额不足是可恢复状态(充值/检测恢复后由周期任务自动解除),
- // 不能走 handleAuthError 永久置 status=error。改为可恢复的临时停调。
- if account.IsCNProvider() {
- s.handleCNProviderInsufficientBalance(ctx, account, upstreamMsg)
- shouldDisable = true
- break
- }
- // OpenAI: deactivated_workspace 表示工作区已停用,直接标记 error
- if account.Platform == PlatformOpenAI && gjson.GetBytes(responseBody, "detail.code").String() == "deactivated_workspace" {
- msg := "Workspace deactivated (402): workspace has been deactivated"
- s.handleAuthError(ctx, account, msg)
- shouldDisable = true
- break
- }
- // 支付要求:余额不足或计费问题,停止调度
- msg := "Payment required (402): insufficient balance or billing issue"
- if upstreamMsg != "" {
- msg = "Payment required (402): " + upstreamMsg
- }
- s.handleAuthError(ctx, account, msg)
- shouldDisable = true
- case 403:
- logger.LegacyPrintf(
- "service.ratelimit",
- "[HandleUpstreamErrorRaw] account_id=%d platform=%s type=%s status=403 request_id=%s cf_ray=%s upstream_msg=%s raw_body=%s",
- account.ID,
- account.Platform,
- account.Type,
- strings.TrimSpace(headers.Get("x-request-id")),
- strings.TrimSpace(headers.Get("cf-ray")),
- upstreamMsg,
- truncateForLog(responseBody, 1024),
- )
- shouldDisable = s.handle403(ctx, account, upstreamMsg, responseBody)
- case 429:
- s.handle429(ctx, account, headers, responseBody)
- shouldDisable = false
- case 529:
- // Handled after pool/custom-code policy gates above.
- shouldDisable = false
- default:
- // 自定义错误码启用时:在列表中的错误码都应该停止调度
- if customErrorCodesEnabled {
- msg := "Custom error code triggered"
- if upstreamMsg != "" {
- msg = upstreamMsg
- }
- s.handleCustomErrorCode(ctx, account, statusCode, msg)
- shouldDisable = true
- } else if statusCode >= 500 {
- // 未启用自定义错误码时:仅记录5xx错误
- slog.Warn("account_upstream_error", "account_id", account.ID, "status_code", statusCode)
- shouldDisable = false
- }
- }
- return shouldDisable
- }
- // PreCheckUsage proactively checks local quota before dispatching a request.
- // Returns false when the account should be skipped.
- func (s *RateLimitService) PreCheckUsage(ctx context.Context, account *Account, requestedModel string) (bool, error) {
- if account == nil || account.Platform != PlatformGemini {
- return true, nil
- }
- if s.usageRepo == nil || s.geminiQuotaService == nil {
- return true, nil
- }
- quota, ok := s.geminiQuotaService.QuotaForAccount(ctx, account)
- if !ok {
- return true, nil
- }
- now := time.Now()
- modelClass := geminiModelClassFromName(requestedModel)
- // 1) Daily quota precheck (RPD; resets at PST midnight)
- {
- var limit int64
- if quota.SharedRPD > 0 {
- limit = quota.SharedRPD
- } else {
- switch modelClass {
- case geminiModelFlash:
- limit = quota.FlashRPD
- default:
- limit = quota.ProRPD
- }
- }
- if limit > 0 {
- start := geminiDailyWindowStart(now)
- totals, ok := s.getGeminiUsageTotals(account.ID, start, now)
- if !ok {
- stats, err := s.usageRepo.GetModelStatsWithFilters(ctx, start, now, 0, 0, account.ID, 0, nil, nil, nil)
- if err != nil {
- return true, err
- }
- totals = geminiAggregateUsage(stats)
- s.setGeminiUsageTotals(account.ID, start, now, totals)
- }
- var used int64
- if quota.SharedRPD > 0 {
- used = totals.ProRequests + totals.FlashRequests
- } else {
- switch modelClass {
- case geminiModelFlash:
- used = totals.FlashRequests
- default:
- used = totals.ProRequests
- }
- }
- if used >= limit {
- resetAt := geminiDailyResetTime(now)
- // NOTE:
- // - This is a local precheck to reduce upstream 429s.
- // - Do NOT mark the account as rate-limited here; rate_limit_reset_at should reflect real upstream 429s.
- slog.Info("gemini_precheck_daily_quota_reached", "account_id", account.ID, "used", used, "limit", limit, "reset_at", resetAt)
- return false, nil
- }
- }
- }
- // 2) Minute quota precheck (RPM; fixed window current minute)
- {
- var limit int64
- if quota.SharedRPM > 0 {
- limit = quota.SharedRPM
- } else {
- switch modelClass {
- case geminiModelFlash:
- limit = quota.FlashRPM
- default:
- limit = quota.ProRPM
- }
- }
- if limit > 0 {
- start := now.Truncate(time.Minute)
- stats, err := s.usageRepo.GetModelStatsWithFilters(ctx, start, now, 0, 0, account.ID, 0, nil, nil, nil)
- if err != nil {
- return true, err
- }
- totals := geminiAggregateUsage(stats)
- var used int64
- if quota.SharedRPM > 0 {
- used = totals.ProRequests + totals.FlashRequests
- } else {
- switch modelClass {
- case geminiModelFlash:
- used = totals.FlashRequests
- default:
- used = totals.ProRequests
- }
- }
- if used >= limit {
- resetAt := start.Add(time.Minute)
- // Do not persist "rate limited" status from local precheck. See note above.
- slog.Info("gemini_precheck_minute_quota_reached", "account_id", account.ID, "used", used, "limit", limit, "reset_at", resetAt)
- return false, nil
- }
- }
- }
- return true, nil
- }
- // PreCheckUsageBatch performs quota precheck for multiple accounts in one request.
- // Returned map value=false means the account should be skipped.
- func (s *RateLimitService) PreCheckUsageBatch(ctx context.Context, accounts []*Account, requestedModel string) (map[int64]bool, error) {
- result := make(map[int64]bool, len(accounts))
- for _, account := range accounts {
- if account == nil {
- continue
- }
- result[account.ID] = true
- }
- if len(accounts) == 0 || requestedModel == "" {
- return result, nil
- }
- if s.usageRepo == nil || s.geminiQuotaService == nil {
- return result, nil
- }
- modelClass := geminiModelClassFromName(requestedModel)
- now := time.Now()
- dailyStart := geminiDailyWindowStart(now)
- minuteStart := now.Truncate(time.Minute)
- type quotaAccount struct {
- account *Account
- quota GeminiQuota
- }
- quotaAccounts := make([]quotaAccount, 0, len(accounts))
- for _, account := range accounts {
- if account == nil || account.Platform != PlatformGemini {
- continue
- }
- quota, ok := s.geminiQuotaService.QuotaForAccount(ctx, account)
- if !ok {
- continue
- }
- quotaAccounts = append(quotaAccounts, quotaAccount{
- account: account,
- quota: quota,
- })
- }
- if len(quotaAccounts) == 0 {
- return result, nil
- }
- // 1) Daily precheck (cached + batch DB fallback)
- dailyTotalsByID := make(map[int64]GeminiUsageTotals, len(quotaAccounts))
- dailyMissIDs := make([]int64, 0, len(quotaAccounts))
- for _, item := range quotaAccounts {
- limit := geminiDailyLimit(item.quota, modelClass)
- if limit <= 0 {
- continue
- }
- accountID := item.account.ID
- if totals, ok := s.getGeminiUsageTotals(accountID, dailyStart, now); ok {
- dailyTotalsByID[accountID] = totals
- continue
- }
- dailyMissIDs = append(dailyMissIDs, accountID)
- }
- if len(dailyMissIDs) > 0 {
- totalsBatch, err := s.getGeminiUsageTotalsBatch(ctx, dailyMissIDs, dailyStart, now)
- if err != nil {
- return result, err
- }
- for _, accountID := range dailyMissIDs {
- totals := totalsBatch[accountID]
- dailyTotalsByID[accountID] = totals
- s.setGeminiUsageTotals(accountID, dailyStart, now, totals)
- }
- }
- for _, item := range quotaAccounts {
- limit := geminiDailyLimit(item.quota, modelClass)
- if limit <= 0 {
- continue
- }
- accountID := item.account.ID
- used := geminiUsedRequests(item.quota, modelClass, dailyTotalsByID[accountID], true)
- if used >= limit {
- resetAt := geminiDailyResetTime(now)
- slog.Info("gemini_precheck_daily_quota_reached_batch", "account_id", accountID, "used", used, "limit", limit, "reset_at", resetAt)
- result[accountID] = false
- }
- }
- // 2) Minute precheck (batch DB)
- minuteIDs := make([]int64, 0, len(quotaAccounts))
- for _, item := range quotaAccounts {
- accountID := item.account.ID
- if !result[accountID] {
- continue
- }
- if geminiMinuteLimit(item.quota, modelClass) <= 0 {
- continue
- }
- minuteIDs = append(minuteIDs, accountID)
- }
- if len(minuteIDs) == 0 {
- return result, nil
- }
- minuteTotalsByID, err := s.getGeminiUsageTotalsBatch(ctx, minuteIDs, minuteStart, now)
- if err != nil {
- return result, err
- }
- for _, item := range quotaAccounts {
- accountID := item.account.ID
- if !result[accountID] {
- continue
- }
- limit := geminiMinuteLimit(item.quota, modelClass)
- if limit <= 0 {
- continue
- }
- used := geminiUsedRequests(item.quota, modelClass, minuteTotalsByID[accountID], false)
- if used >= limit {
- resetAt := minuteStart.Add(time.Minute)
- slog.Info("gemini_precheck_minute_quota_reached_batch", "account_id", accountID, "used", used, "limit", limit, "reset_at", resetAt)
- result[accountID] = false
- }
- }
- return result, nil
- }
- func (s *RateLimitService) getGeminiUsageTotalsBatch(ctx context.Context, accountIDs []int64, start, end time.Time) (map[int64]GeminiUsageTotals, error) {
- result := make(map[int64]GeminiUsageTotals, len(accountIDs))
- if len(accountIDs) == 0 {
- return result, nil
- }
- ids := make([]int64, 0, len(accountIDs))
- seen := make(map[int64]struct{}, len(accountIDs))
- for _, accountID := range accountIDs {
- if accountID <= 0 {
- continue
- }
- if _, ok := seen[accountID]; ok {
- continue
- }
- seen[accountID] = struct{}{}
- ids = append(ids, accountID)
- }
- if len(ids) == 0 {
- return result, nil
- }
- if batchReader, ok := s.usageRepo.(geminiUsageTotalsBatchProvider); ok {
- stats, err := batchReader.GetGeminiUsageTotalsBatch(ctx, ids, start, end)
- if err != nil {
- return nil, err
- }
- for _, accountID := range ids {
- result[accountID] = stats[accountID]
- }
- return result, nil
- }
- for _, accountID := range ids {
- stats, err := s.usageRepo.GetModelStatsWithFilters(ctx, start, end, 0, 0, accountID, 0, nil, nil, nil)
- if err != nil {
- return nil, err
- }
- result[accountID] = geminiAggregateUsage(stats)
- }
- return result, nil
- }
- func geminiDailyLimit(quota GeminiQuota, modelClass geminiModelClass) int64 {
- if quota.SharedRPD > 0 {
- return quota.SharedRPD
- }
- switch modelClass {
- case geminiModelFlash:
- return quota.FlashRPD
- default:
- return quota.ProRPD
- }
- }
- func geminiMinuteLimit(quota GeminiQuota, modelClass geminiModelClass) int64 {
- if quota.SharedRPM > 0 {
- return quota.SharedRPM
- }
- switch modelClass {
- case geminiModelFlash:
- return quota.FlashRPM
- default:
- return quota.ProRPM
- }
- }
- func geminiUsedRequests(quota GeminiQuota, modelClass geminiModelClass, totals GeminiUsageTotals, daily bool) int64 {
- if daily {
- if quota.SharedRPD > 0 {
- return totals.ProRequests + totals.FlashRequests
- }
- } else {
- if quota.SharedRPM > 0 {
- return totals.ProRequests + totals.FlashRequests
- }
- }
- switch modelClass {
- case geminiModelFlash:
- return totals.FlashRequests
- default:
- return totals.ProRequests
- }
- }
- func (s *RateLimitService) getGeminiUsageTotals(accountID int64, windowStart, now time.Time) (GeminiUsageTotals, bool) {
- s.usageCacheMu.RLock()
- defer s.usageCacheMu.RUnlock()
- if s.usageCache == nil {
- return GeminiUsageTotals{}, false
- }
- entry, ok := s.usageCache[accountID]
- if !ok || entry == nil {
- return GeminiUsageTotals{}, false
- }
- if !entry.windowStart.Equal(windowStart) {
- return GeminiUsageTotals{}, false
- }
- if now.Sub(entry.cachedAt) >= geminiPrecheckCacheTTL {
- return GeminiUsageTotals{}, false
- }
- return entry.totals, true
- }
- func (s *RateLimitService) setGeminiUsageTotals(accountID int64, windowStart, now time.Time, totals GeminiUsageTotals) {
- s.usageCacheMu.Lock()
- defer s.usageCacheMu.Unlock()
- if s.usageCache == nil {
- s.usageCache = make(map[int64]*geminiUsageCacheEntry)
- }
- s.usageCache[accountID] = &geminiUsageCacheEntry{
- windowStart: windowStart,
- cachedAt: now,
- totals: totals,
- }
- }
- // GeminiCooldown returns the fallback cooldown duration for Gemini 429s based on tier.
- func (s *RateLimitService) GeminiCooldown(ctx context.Context, account *Account) time.Duration {
- if account == nil {
- return 5 * time.Minute
- }
- if s.geminiQuotaService == nil {
- return 5 * time.Minute
- }
- return s.geminiQuotaService.CooldownForAccount(ctx, account)
- }
- // handleAuthError 处理认证类错误(401/403),停止账号调度
- func (s *RateLimitService) handleAuthError(ctx context.Context, account *Account, errorMsg string) {
- s.notifyAccountSchedulingBlocked(account, time.Time{}, "auth_error")
- if err := s.accountRepo.SetError(ctx, account.ID, errorMsg); err != nil {
- slog.Warn("account_set_error_failed", "account_id", account.ID, "error", err)
- return
- }
- slog.Warn("account_disabled_auth_error", "account_id", account.ID, "error", errorMsg)
- }
- func buildForbiddenErrorMessage(prefix string, upstreamMsg string, responseBody []byte, fallback string) string {
- prefix = strings.TrimSpace(prefix)
- if prefix != "" && !strings.HasSuffix(prefix, " ") {
- prefix += " "
- }
- if msg := strings.TrimSpace(upstreamMsg); msg != "" {
- return prefix + msg
- }
- rawBody := bytes.TrimSpace(responseBody)
- if len(rawBody) > 0 {
- if json.Valid(rawBody) {
- var compact bytes.Buffer
- if err := json.Compact(&compact, rawBody); err == nil {
- return prefix + truncateForLog(compact.Bytes(), 512)
- }
- }
- return prefix + truncateForLog(rawBody, 512)
- }
- return prefix + fallback
- }
- // handle403 处理 403 Forbidden 错误
- // Antigravity 平台区分 validation/violation/generic 三种类型,均 SetError 永久禁用;
- // 其他平台保持原有 SetError 行为。
- func (s *RateLimitService) handle403(ctx context.Context, account *Account, upstreamMsg string, responseBody []byte) (shouldDisable bool) {
- if account.Platform == PlatformAntigravity {
- return s.handleAntigravity403(ctx, account, upstreamMsg, responseBody)
- }
- // Kimi reports its transient per-account concurrency/business limit as a 403.
- // Keep the normal 403 failover signal (true), but never feed this exact message
- // into the escalating 403 counter that can permanently mark the account error.
- if isCNProviderConcurrencyLimit403(account, upstreamMsg) {
- s.handleCNProviderConcurrencyLimit403(ctx, account)
- return true
- }
- // 国产供应商与 openai 同口径:HTML 403(CDN/代理拦截页)不构成账号失效证据,
- // 且 403 在 failover 状态集里会被逐账号重放——直接 SetError 会让一个坏请求/
- // 一层坏代理连环永久禁用整组账号。走 HTML 豁免 + N 次累计 + 临时冷却。
- if account.Platform == PlatformOpenAI || IsCNProvider(account.Platform) {
- return s.handleOpenAI403(ctx, account, upstreamMsg, responseBody)
- }
- // 非 Antigravity 平台:保持原有行为
- msg := buildForbiddenErrorMessage(
- "Access forbidden (403):",
- upstreamMsg,
- responseBody,
- "account may be suspended or lack permissions",
- )
- s.handleAuthError(ctx, account, msg)
- return true
- }
- func (s *RateLimitService) handleOpenAI403(ctx context.Context, account *Account, upstreamMsg string, responseBody []byte) (shouldDisable bool) {
- // 上游代理 / CDN 在请求到达 OpenAI API 之前就拦下时,回的是 HTML 403 页面而不是
- // {"error":{...}} 结构化错误。这类响应描述的是「这条链路 / 这个端点被挡了」,
- // 不构成账号凭据或权限失效的证据——例如无效的 /v1/responses 子路径(#5334)。
- //
- // 据此写账号状态会把请求级错误放大成账号级处罚:首次即 temp-unschedulable,
- // 连续 openAI403DisableThreshold 次直接永久禁用;而 403 又在 failover 状态集里,
- // 同一个坏请求会被逐个账号重放,足以把整组账号打下线。
- //
- // 与既有口径一致:count_tokens 路径的 isOpenAIOAuthInputTokensUnsupported 已把
- // 「HTML 403 page without a structured error」按端点级响应处理;
- // shouldApplyOpenAIAlphaSearchAccountErrorSideEffects 的不变式也是端点级错误
- // 只换号、不写账号错误状态。这里只跳过账号处罚,不改变 failover 行为——
- // 换个走不同代理的账号仍有可能成功。
- if isHTMLResponse(responseBody) {
- slog.Warn(
- "openai_403_html_body_skips_account_penalty",
- "account_id", account.ID,
- "upstream_message", upstreamMsg,
- )
- return false
- }
- msg := buildForbiddenErrorMessage(
- "Access forbidden (403):",
- upstreamMsg,
- responseBody,
- "account may be suspended or lack permissions",
- )
- if s.openAI403CounterCache == nil {
- s.handleAuthError(ctx, account, msg)
- return true
- }
- count, err := s.openAI403CounterCache.IncrementOpenAI403Count(ctx, account.ID, openAI403CounterWindowMinutes)
- if err != nil {
- slog.Warn("openai_403_increment_failed", "account_id", account.ID, "error", err)
- s.handleAuthError(ctx, account, msg)
- return true
- }
- if count >= openAI403DisableThreshold {
- msg = fmt.Sprintf("%s | consecutive_403=%d/%d", msg, count, openAI403DisableThreshold)
- s.handleAuthError(ctx, account, msg)
- return true
- }
- until := time.Now().Add(time.Duration(openAI403CooldownMinutesDefault) * time.Minute)
- reason := fmt.Sprintf("OpenAI 403 temporary cooldown (%d/%d): %s", count, openAI403DisableThreshold, msg)
- s.notifyAccountSchedulingBlocked(account, until, "openai_403_temp")
- if err := s.accountRepo.SetTempUnschedulable(ctx, account.ID, until, reason); err != nil {
- slog.Warn("openai_403_set_temp_unschedulable_failed", "account_id", account.ID, "error", err)
- s.handleAuthError(ctx, account, msg)
- return true
- }
- slog.Warn(
- "openai_403_temp_unschedulable",
- "account_id", account.ID,
- "until", until,
- "count", count,
- "threshold", openAI403DisableThreshold,
- )
- return true
- }
- // handleAntigravity403 处理 Antigravity 平台的 403 错误
- // validation(需要验证)→ 永久 SetError(需人工去 Google 验证后恢复)
- // violation(违规封号)→ 永久 SetError(需人工处理)
- // generic(通用禁止)→ 永久 SetError
- func (s *RateLimitService) handleAntigravity403(ctx context.Context, account *Account, upstreamMsg string, responseBody []byte) (shouldDisable bool) {
- fbType := classifyForbiddenType(string(responseBody))
- switch fbType {
- case forbiddenTypeValidation:
- // VALIDATION_REQUIRED: 永久禁用,需人工去 Google 验证后手动恢复
- msg := buildForbiddenErrorMessage(
- "Validation required (403):",
- upstreamMsg,
- responseBody,
- "account needs Google verification",
- )
- if validationURL := extractValidationURL(string(responseBody)); validationURL != "" {
- msg += " | validation_url: " + validationURL
- }
- s.handleAuthError(ctx, account, msg)
- return true
- case forbiddenTypeViolation:
- // 违规封号: 永久禁用,需人工处理
- msg := buildForbiddenErrorMessage(
- "Account violation (403):",
- upstreamMsg,
- responseBody,
- "terms of service violation",
- )
- s.handleAuthError(ctx, account, msg)
- return true
- default:
- // 通用 403: 保持原有行为
- msg := buildForbiddenErrorMessage(
- "Access forbidden (403):",
- upstreamMsg,
- responseBody,
- "account may be suspended or lack permissions",
- )
- s.handleAuthError(ctx, account, msg)
- return true
- }
- }
- // handleCustomErrorCode 处理自定义错误码,停止账号调度
- func (s *RateLimitService) handleCustomErrorCode(ctx context.Context, account *Account, statusCode int, errorMsg string) {
- msg := "Custom error code " + strconv.Itoa(statusCode) + ": " + errorMsg
- s.notifyAccountSchedulingBlocked(account, time.Time{}, "custom_error_code")
- if err := s.accountRepo.SetError(ctx, account.ID, msg); err != nil {
- slog.Warn("account_set_error_failed", "account_id", account.ID, "status_code", statusCode, "error", err)
- return
- }
- slog.Warn("account_disabled_custom_error", "account_id", account.ID, "status_code", statusCode, "error", errorMsg)
- }
- // handle429 处理429限流错误
- // 解析响应头获取重置时间,标记账号为限流状态
- func (s *RateLimitService) handle429(ctx context.Context, account *Account, headers http.Header, responseBody []byte) {
- // OpenAI OAuth stays on the same account for the gateway's bounded retry
- // window. Persisting a rate-limit reset on the first 429 would make the next
- // retry ineligible and silently turn same-account recovery into a switch.
- if account != nil && isOpenAIOAuthAccount(account) && s.runtimeBlocker != nil {
- if checker, ok := s.runtimeBlocker.(interface {
- ShouldRetryOpenAIOAuth429(*Account, http.Header, []byte) bool
- }); ok && checker.ShouldRetryOpenAIOAuth429(account, headers, responseBody) {
- return
- }
- }
- // Spark 影子:限流/熔断状态 100% 由 QueryUsage(/wham/usage body 的 codex_bengalfox)驱动。
- // /responses 的 429 携带的 x-codex-*/usage_limit_reached 是 global codex 道(plan/spec §8),
- // 套到影子会把 spark 误耦合到 global 窗口——即便 spark 仍有配额也会被冷却到 global reset,
- // 单影子场景直接变成无可用账号(外审第8轮 P1)。整段跳过;影子的 codex_* 仅由 account_usage 的
- // QueryUsage→persistOpenAICodexProbeSnapshot 维护,枯竭由调度守卫处理。
- if account.IsShadow() {
- if account.ParentAccountID != nil {
- notifyOpenAIAutoReset(*account.ParentAccountID)
- }
- return
- }
- // 国产供应商(kimi/zhipu/deepseek)的 429 走专用可恢复路径:余额不足 → 临时停调,
- // Coding Plan 窗口耗尽 → 冷却到快照重置点。未命中则继续默认 429 逻辑。
- if account.IsCNProvider() {
- if s.applyCNProviderReactive429(ctx, account, headers, responseBody) {
- return
- }
- }
- // 1. OpenAI 平台:优先尝试解析 x-codex-* 响应头(用于 rate_limit_exceeded)
- if account.Platform == PlatformOpenAI {
- persistOpenAI429PlanType(ctx, s.accountRepo, account, responseBody)
- s.persistOpenAICodexSnapshot(ctx, account, headers)
- notifyOpenAIAutoReset(account.ID)
- if resetAt := s.calculateOpenAI429ResetTime(headers); resetAt != nil {
- s.notifyAccountSchedulingBlocked(account, *resetAt, "429")
- if err := s.accountRepo.SetRateLimited(ctx, account.ID, *resetAt); err != nil {
- slog.Warn("rate_limit_set_failed", "account_id", account.ID, "error", err)
- return
- }
- slog.Info("openai_account_rate_limited", "account_id", account.ID, "reset_at", *resetAt)
- return
- }
- }
- // 2. Anthropic 平台:尝试解析 per-window 头(5h / 7d),选择实际触发的窗口
- if result := calculateAnthropic429ResetTime(headers); result != nil {
- s.notifyAccountSchedulingBlocked(account, result.resetAt, "429")
- if err := s.accountRepo.SetRateLimited(ctx, account.ID, result.resetAt); err != nil {
- slog.Warn("rate_limit_set_failed", "account_id", account.ID, "error", err)
- return
- }
- // 更新 session window:优先使用 5h-reset 头精确计算,否则从 resetAt 反推
- windowEnd := result.resetAt
- if result.fiveHourReset != nil {
- windowEnd = *result.fiveHourReset
- }
- windowStart := windowEnd.Add(-5 * time.Hour)
- if err := s.accountRepo.UpdateSessionWindow(ctx, account.ID, &windowStart, &windowEnd, "rejected"); err != nil {
- slog.Warn("rate_limit_update_session_window_failed", "account_id", account.ID, "error", err)
- }
- slog.Info("anthropic_account_rate_limited", "account_id", account.ID, "reset_at", result.resetAt, "reset_in", time.Until(result.resetAt).Truncate(time.Second))
- return
- }
- // 3. 尝试从响应头解析重置时间(Anthropic 聚合头,向后兼容)
- resetTimestamp := headers.Get("anthropic-ratelimit-unified-reset")
- // 4. 如果响应头没有,尝试从响应体解析(OpenAI usage_limit_reached, Gemini)
- if resetTimestamp == "" {
- switch account.Platform {
- case PlatformOpenAI:
- // 尝试解析 OpenAI 的 usage_limit_reached 错误
- if resetAt := parseOpenAIRateLimitResetTime(responseBody); resetAt != nil {
- resetTime := time.Unix(*resetAt, 0)
- s.notifyAccountSchedulingBlocked(account, resetTime, "429")
- if err := s.accountRepo.SetRateLimited(ctx, account.ID, resetTime); err != nil {
- slog.Warn("rate_limit_set_failed", "account_id", account.ID, "error", err)
- return
- }
- slog.Info("account_rate_limited", "account_id", account.ID, "platform", account.Platform, "reset_at", resetTime, "reset_in", time.Until(resetTime).Truncate(time.Second))
- return
- }
- case PlatformGemini, PlatformAntigravity:
- // 尝试解析 Gemini 格式(用于其他平台)
- if resetAt := ParseGeminiRateLimitResetTime(responseBody); resetAt != nil {
- resetTime := time.Unix(*resetAt, 0)
- s.notifyAccountSchedulingBlocked(account, resetTime, "429")
- if err := s.accountRepo.SetRateLimited(ctx, account.ID, resetTime); err != nil {
- slog.Warn("rate_limit_set_failed", "account_id", account.ID, "error", err)
- return
- }
- slog.Info("account_rate_limited", "account_id", account.ID, "platform", account.Platform, "reset_at", resetTime, "reset_in", time.Until(resetTime).Truncate(time.Second))
- return
- }
- }
- // Anthropic 平台:没有限流重置时间的 429 可能是非真实限流(如 Extra usage required),
- // 不适合按 5h/7d 窗口长时间封禁;但完全不标记会导致账号永不冷却,
- // 调度器让每个请求反复撞同一批持续 429 的账号(failover 预算被白白烧掉,
- // 客户端稳定收到 429)。因此同样走可配置的秒级兜底回避,管理端可调大或关闭。
- if resetAt := parseRetryAfterResetTime(headers, time.Now()); resetAt != nil && resetAt.After(time.Now()) {
- s.notifyAccountSchedulingBlocked(account, *resetAt, "429_retry_after")
- if err := s.accountRepo.SetRateLimited(ctx, account.ID, *resetAt); err != nil {
- slog.Warn("rate_limit_set_failed", "account_id", account.ID, "error", err)
- }
- return
- }
- if account.Platform == PlatformAnthropic {
- slog.Warn("rate_limit_429_no_reset_time",
- "account_id", account.ID,
- "platform", account.Platform,
- "reason", "no rate limit reset time in headers, likely not a real rate limit")
- s.apply429FallbackRateLimit(ctx, account, "anthropic_no_reset_time")
- return
- }
- // 其他平台:没有重置时间,使用可配置的秒级默认回避,避免误伤长时间不可调度。
- s.apply429FallbackRateLimit(ctx, account, "no_reset_time")
- return
- }
- // 解析Unix时间戳
- ts, err := strconv.ParseInt(resetTimestamp, 10, 64)
- if err != nil {
- slog.Warn("rate_limit_reset_parse_failed", "reset_timestamp", resetTimestamp, "error", err)
- s.apply429FallbackRateLimit(ctx, account, "reset_parse_failed")
- return
- }
- resetAt := time.Unix(ts, 0)
- // 标记限流状态
- s.notifyAccountSchedulingBlocked(account, resetAt, "429")
- if err := s.accountRepo.SetRateLimited(ctx, account.ID, resetAt); err != nil {
- slog.Warn("rate_limit_set_failed", "account_id", account.ID, "error", err)
- return
- }
- // 根据重置时间反推5h窗口
- windowEnd := resetAt
- windowStart := resetAt.Add(-5 * time.Hour)
- if err := s.accountRepo.UpdateSessionWindow(ctx, account.ID, &windowStart, &windowEnd, "rejected"); err != nil {
- slog.Warn("rate_limit_update_session_window_failed", "account_id", account.ID, "error", err)
- }
- slog.Info("account_rate_limited", "account_id", account.ID, "reset_at", resetAt)
- }
- func (s *RateLimitService) apply429FallbackRateLimit(ctx context.Context, account *Account, reason string) {
- cooldown, enabled := s.get429FallbackCooldown(ctx, account)
- if !enabled {
- slog.Info("rate_limit_429_fallback_ignored", "account_id", account.ID, "platform", account.Platform, "reason", reason)
- return
- }
- resetAt := time.Now().Add(cooldown)
- slog.Warn("rate_limit_429_fallback_used", "account_id", account.ID, "platform", account.Platform, "reason", reason, "using_default", cooldown.String())
- s.notifyAccountSchedulingBlocked(account, resetAt, "429_fallback")
- if err := s.accountRepo.SetRateLimited(ctx, account.ID, resetAt); err != nil {
- slog.Warn("rate_limit_set_failed", "account_id", account.ID, "error", err)
- }
- }
- func (s *RateLimitService) get429FallbackCooldown(ctx context.Context, account *Account) (time.Duration, bool) {
- if s.settingService != nil {
- settings, err := s.settingService.GetRateLimit429CooldownSettings(ctx)
- if err == nil && settings != nil {
- if !settings.Enabled {
- return 0, false
- }
- seconds := clampRateLimit429CooldownSeconds(settings.CooldownSeconds)
- return time.Duration(seconds) * time.Second, true
- }
- slog.Warn("rate_limit_429_settings_read_failed", "account_id", account.ID, "error", err)
- }
- seconds := defaultRateLimit429CooldownSeconds
- seconds = clampRateLimit429CooldownSeconds(seconds)
- return time.Duration(seconds) * time.Second, true
- }
- func clampRateLimit429CooldownSeconds(seconds int) int {
- if seconds < 1 {
- return 1
- }
- if seconds > maxRateLimit429CooldownSeconds {
- return maxRateLimit429CooldownSeconds
- }
- return seconds
- }
- // calculateOpenAI429ResetTime 从 OpenAI 429 响应头计算正确的重置时间
- // 返回 nil 表示无法从响应头中确定重置时间
- func calculateOpenAI429ResetTime(headers http.Header) *time.Time {
- snapshot := ParseCodexRateLimitHeaders(headers)
- if snapshot == nil {
- return nil
- }
- normalized := snapshot.Normalize()
- if normalized == nil {
- return nil
- }
- now := time.Now()
- // 判断哪个限制被触发(used_percent >= 100)
- is7dExhausted := normalized.Used7dPercent != nil && *normalized.Used7dPercent >= 100
- is5hExhausted := normalized.Used5hPercent != nil && *normalized.Used5hPercent >= 100
- // 优先使用被触发限制的重置时间
- if is7dExhausted && normalized.Reset7dSeconds != nil {
- resetAt := now.Add(time.Duration(*normalized.Reset7dSeconds) * time.Second)
- slog.Info("openai_429_7d_limit_exhausted", "reset_after_seconds", *normalized.Reset7dSeconds, "reset_at", resetAt)
- return &resetAt
- }
- if is5hExhausted && normalized.Reset5hSeconds != nil {
- resetAt := now.Add(time.Duration(*normalized.Reset5hSeconds) * time.Second)
- slog.Info("openai_429_5h_limit_exhausted", "reset_after_seconds", *normalized.Reset5hSeconds, "reset_at", resetAt)
- return &resetAt
- }
- // 未达到100%时,reset-after 只代表窗口信息,不能证明账号配额耗尽。
- // 这类瞬时429必须回到可配置的兜底路径,避免未耗尽账号被长时间排除。
- return nil
- }
- func (s *RateLimitService) calculateOpenAI429ResetTime(headers http.Header) *time.Time {
- return calculateOpenAI429ResetTime(headers)
- }
- // anthropic429Result holds the parsed Anthropic 429 rate-limit information.
- type anthropic429Result struct {
- resetAt time.Time // The correct reset time to use for SetRateLimited
- fiveHourReset *time.Time // 5h window reset timestamp (for session window calculation), nil if not available
- }
- type anthropicWindowLimit struct {
- window string
- resetAt time.Time
- reason string
- }
- func selectAnthropicExhaustedWindow(headers http.Header, now time.Time) *anthropicWindowLimit {
- reset5h, ok5hReset := parseAnthropicWindowReset(headers, "5h", now)
- reset7d, ok7dReset := parseAnthropicWindowReset(headers, "7d", now)
- exceeded5h := isAnthropic5hRejected(headers) || isAnthropicWindowExceeded(headers, "5h")
- exceeded7d := isAnthropicWindowExceeded(headers, "7d")
- if exceeded7d && ok7dReset {
- return &anthropicWindowLimit{
- window: "7d",
- resetAt: reset7d,
- reason: "anthropic_7d_window_exhausted",
- }
- }
- if exceeded5h && ok5hReset {
- return &anthropicWindowLimit{
- window: "5h",
- resetAt: reset5h,
- reason: "anthropic_5h_window_exhausted",
- }
- }
- return nil
- }
- func isAnthropic5hRejected(headers http.Header) bool {
- return isAnthropicWindowRejected(headers, "5h")
- }
- func isAnthropicWindowRejected(headers http.Header, window string) bool {
- return strings.EqualFold(strings.TrimSpace(headers.Get("anthropic-ratelimit-unified-"+window+"-status")), "rejected")
- }
- func parseAnthropicWindowReset(headers http.Header, window string, now time.Time) (time.Time, bool) {
- maxAge := 8 * 24 * time.Hour
- if window == "5h" {
- maxAge = 6 * time.Hour
- }
- return parseAnthropicResetTimestamp(headers.Get("anthropic-ratelimit-unified-"+window+"-reset"), now, maxAge)
- }
- // parseAnthropicResetTimestamp 解析 Anthropic reset 头的 Unix 时间戳(自动识别毫秒),
- // 并校验落在 (now, now+maxAge] 的合理区间内。
- func parseAnthropicResetTimestamp(raw string, now time.Time, maxAge time.Duration) (time.Time, bool) {
- raw = strings.TrimSpace(raw)
- if raw == "" {
- return time.Time{}, false
- }
- ts, err := strconv.ParseInt(raw, 10, 64)
- if err != nil {
- return time.Time{}, false
- }
- if ts > 1e11 {
- ts = ts / 1000
- }
- resetAt := time.Unix(ts, 0)
- if !resetAt.After(now) || resetAt.After(now.Add(maxAge)) {
- return time.Time{}, false
- }
- return resetAt, true
- }
- func shouldPersistAnthropicWindowLimit(account *Account, limit *anthropicWindowLimit, now time.Time) bool {
- if account == nil || limit == nil || !limit.resetAt.After(now) {
- return false
- }
- if account.RateLimitResetAt == nil {
- return true
- }
- if !account.RateLimitResetAt.After(now) {
- return true
- }
- return limit.resetAt.After(*account.RateLimitResetAt)
- }
- func (s *RateLimitService) persistAnthropicExhaustedWindowLimit(ctx context.Context, account *Account, headers http.Header) bool {
- if s == nil || s.accountRepo == nil || account == nil {
- return false
- }
- now := time.Now()
- limit := selectAnthropicExhaustedWindow(headers, now)
- if limit == nil {
- return false
- }
- if !shouldPersistAnthropicWindowLimit(account, limit, now) {
- slog.Info("anthropic_window_rate_limit_kept",
- "account_id", account.ID,
- "window", limit.window,
- "reset_at", limit.resetAt,
- "existing_reset_at", account.RateLimitResetAt)
- return true
- }
- s.notifyAccountSchedulingBlocked(account, limit.resetAt, limit.reason)
- if err := s.accountRepo.SetRateLimited(ctx, account.ID, limit.resetAt); err != nil {
- slog.Warn("anthropic_window_rate_limit_set_failed",
- "account_id", account.ID,
- "window", limit.window,
- "reset_at", limit.resetAt,
- "error", err)
- return true
- }
- slog.Info("anthropic_window_rate_limited",
- "account_id", account.ID,
- "window", limit.window,
- "reset_at", limit.resetAt,
- "reset_in", time.Until(limit.resetAt).Truncate(time.Second))
- return true
- }
- const (
- anthropicFableWindowReason = "anthropic_7d_oi_window_exhausted"
- anthropicFableCreditsRequiredReason = "anthropic_fable_credits_required"
- )
- // persistAnthropicFableCreditsRequired handles Anthropic's credits_required
- // response for Fable. Although the upstream status is 429, this response only
- // says that the organization cannot use Fable; marking the whole account rate
- // limited would unnecessarily stop Sonnet, Opus, and Haiku scheduling.
- func (s *RateLimitService) persistAnthropicFableCreditsRequired(ctx context.Context, account *Account, headers http.Header, responseBody []byte, requestedModel string) bool {
- if s == nil || s.accountRepo == nil || account == nil {
- return false
- }
- if !strings.EqualFold(strings.TrimSpace(gjson.GetBytes(responseBody, "error.details.error_code").String()), "credits_required") {
- return false
- }
- model := strings.TrimSpace(gjson.GetBytes(responseBody, "error.details.model").String())
- if model == "" {
- model = strings.TrimSpace(requestedModel)
- }
- if !isAnthropicFableModel(model) {
- return false
- }
- now := time.Now()
- resetAt, ok := parseAnthropicResetTimestamp(headers.Get("anthropic-ratelimit-unified-reset"), now, 366*24*time.Hour)
- if !ok {
- cooldown, enabled := s.get429FallbackCooldown(ctx, account)
- if !enabled {
- slog.Info("anthropic_fable_credits_required_cooldown_ignored", "account_id", account.ID)
- return true
- }
- resetAt = now.Add(cooldown)
- }
- if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, anthropicFableRateLimitKey, resetAt, anthropicFableCreditsRequiredReason); err != nil {
- slog.Warn("anthropic_fable_credits_required_rate_limit_set_failed",
- "account_id", account.ID,
- "scope", anthropicFableRateLimitKey,
- "reset_at", resetAt,
- "error", err)
- // The response is still known to be Fable-specific. Do not widen a
- // persistence failure into an account-level rate limit.
- return true
- }
- slog.Info("anthropic_fable_credits_required_model_rate_limited",
- "account_id", account.ID,
- "scope", anthropicFableRateLimitKey,
- "reset_at", resetAt,
- "reset_in", time.Until(resetAt).Truncate(time.Second))
- return true
- }
- // selectAnthropicFableWindowLimit parses the Anthropic 7d_oi per-model window
- // headers (the Fable-only 7d window, e.g. anthropic-ratelimit-unified-7d_oi-*).
- // Unlike 5h/7d, exhaustion of this window only limits the Fable model family —
- // the account must stay schedulable for other models.
- //
- // The 7d_oi surpassed-threshold header carries a float ("1.0") rather than
- // "true", so exhaustion is detected via status=rejected or utilization >= 1.0.
- // When the 7d_oi reset header is missing, the aggregated
- // anthropic-ratelimit-unified-reset is used (it mirrors the binding claim's
- // reset when 7d_oi is the representative claim).
- func selectAnthropicFableWindowLimit(headers http.Header, now time.Time) *anthropicWindowLimit {
- if !isAnthropicWindowRejected(headers, "7d_oi") && !isAnthropicWindowExceeded(headers, "7d_oi") {
- return nil
- }
- resetAt, ok := parseAnthropicWindowReset(headers, "7d_oi", now)
- if !ok {
- resetAt, ok = parseAnthropicAggregateReset(headers, now)
- }
- if !ok {
- return nil
- }
- return &anthropicWindowLimit{
- window: "7d_oi",
- resetAt: resetAt,
- reason: anthropicFableWindowReason,
- }
- }
- // parseAnthropicAggregateReset parses the aggregated
- // anthropic-ratelimit-unified-reset header with the same sanity checks as the
- // per-window variant (7d scale).
- func parseAnthropicAggregateReset(headers http.Header, now time.Time) (time.Time, bool) {
- return parseAnthropicResetTimestamp(headers.Get("anthropic-ratelimit-unified-reset"), now, 8*24*time.Hour)
- }
- // persistAnthropicFableWindowLimit marks the Fable model family as rate limited
- // when the 7d_oi window is exhausted. Returns true when the 7d_oi window was the
- // (or a) trigger of this 429, so the caller must not fall through to logic that
- // would mark the whole account as rate limited.
- func (s *RateLimitService) persistAnthropicFableWindowLimit(ctx context.Context, account *Account, headers http.Header) bool {
- if s == nil || s.accountRepo == nil || account == nil {
- return false
- }
- now := time.Now()
- limit := selectAnthropicFableWindowLimit(headers, now)
- if limit == nil {
- return false
- }
- // 429 响应头本身携带最新的窗口用量(7d_oi utilization=1.0)。限流期内
- // Fable 请求不再调度到该账号,若不在此处采样,7d F 进度条会冻结在
- // 限流前的旧值直到窗口重置。
- s.samplePassiveUsageFromHeaders(ctx, account, headers)
- if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, anthropicFableRateLimitKey, limit.resetAt, limit.reason); err != nil {
- slog.Warn("anthropic_fable_window_rate_limit_set_failed",
- "account_id", account.ID,
- "scope", anthropicFableRateLimitKey,
- "reset_at", limit.resetAt,
- "error", err)
- return true
- }
- slog.Info("anthropic_fable_window_model_rate_limited",
- "account_id", account.ID,
- "scope", anthropicFableRateLimitKey,
- "reset_at", limit.resetAt,
- "reset_in", time.Until(limit.resetAt).Truncate(time.Second))
- return true
- }
- // calculateAnthropic429ResetTime parses Anthropic's per-window rate-limit headers
- // to determine which window (5h or 7d) actually triggered the 429.
- //
- // Headers used:
- // - anthropic-ratelimit-unified-5h-utilization / anthropic-ratelimit-unified-5h-surpassed-threshold
- // - anthropic-ratelimit-unified-5h-reset
- // - anthropic-ratelimit-unified-7d-utilization / anthropic-ratelimit-unified-7d-surpassed-threshold
- // - anthropic-ratelimit-unified-7d-reset
- //
- // Returns nil when the per-window headers are absent (caller should fall back to
- // the aggregated anthropic-ratelimit-unified-reset header).
- func calculateAnthropic429ResetTime(headers http.Header) *anthropic429Result {
- reset5hStr := headers.Get("anthropic-ratelimit-unified-5h-reset")
- reset7dStr := headers.Get("anthropic-ratelimit-unified-7d-reset")
- if reset5hStr == "" && reset7dStr == "" {
- return nil
- }
- var reset5h, reset7d *time.Time
- if ts, err := strconv.ParseInt(reset5hStr, 10, 64); err == nil {
- t := time.Unix(ts, 0)
- reset5h = &t
- }
- if ts, err := strconv.ParseInt(reset7dStr, 10, 64); err == nil {
- t := time.Unix(ts, 0)
- reset7d = &t
- }
- is5hExceeded := isAnthropicWindowExceeded(headers, "5h")
- is7dExceeded := isAnthropicWindowExceeded(headers, "7d")
- slog.Info("anthropic_429_window_analysis",
- "is_5h_exceeded", is5hExceeded,
- "is_7d_exceeded", is7dExceeded,
- "reset_5h", reset5hStr,
- "reset_7d", reset7dStr,
- )
- // Select the correct reset time based on which window(s) are exceeded.
- var chosen *time.Time
- switch {
- case is5hExceeded && is7dExceeded:
- // Both exceeded → prefer 7d (longer cooldown), fall back to 5h
- chosen = reset7d
- if chosen == nil {
- chosen = reset5h
- }
- case is5hExceeded:
- chosen = reset5h
- case is7dExceeded:
- chosen = reset7d
- default:
- // Neither flag clearly exceeded — pick the sooner reset as best guess
- chosen = pickSooner(reset5h, reset7d)
- }
- if chosen == nil {
- return nil
- }
- return &anthropic429Result{resetAt: *chosen, fiveHourReset: reset5h}
- }
- // isAnthropicWindowExceeded checks whether a given Anthropic rate-limit window
- // (e.g. "5h" or "7d") has been exceeded, using utilization and surpassed-threshold headers.
- func isAnthropicWindowExceeded(headers http.Header, window string) bool {
- prefix := "anthropic-ratelimit-unified-" + window + "-"
- // Check surpassed-threshold first (most explicit signal)
- if st := headers.Get(prefix + "surpassed-threshold"); strings.EqualFold(st, "true") {
- return true
- }
- // Fall back to utilization >= 1.0
- if utilStr := headers.Get(prefix + "utilization"); utilStr != "" {
- if util, err := strconv.ParseFloat(utilStr, 64); err == nil && util >= 1.0-1e-9 {
- // Use a small epsilon to handle floating point: treat 0.9999999... as >= 1.0
- return true
- }
- }
- return false
- }
- // pickSooner returns whichever of the two time pointers is earlier.
- // If only one is non-nil, it is returned. If both are nil, returns nil.
- func pickSooner(a, b *time.Time) *time.Time {
- switch {
- case a != nil && b != nil:
- if a.Before(*b) {
- return a
- }
- return b
- case a != nil:
- return a
- default:
- return b
- }
- }
- func (s *RateLimitService) persistOpenAICodexSnapshot(ctx context.Context, account *Account, headers http.Header) {
- if s == nil || s.accountRepo == nil || account == nil || headers == nil {
- return
- }
- // spark 影子的 codex_* 仅由 QueryUsage(/wham/usage bengalfox 道)更新,不能被 /responses 的
- // x-codex-* 全局头快照污染(外审第7轮 P1,与 updateCodexUsageSnapshot 同口径)。
- if account.IsShadow() {
- return
- }
- snapshot := ParseCodexRateLimitHeaders(headers)
- if snapshot == nil {
- return
- }
- updates := buildCodexUsageExtraUpdates(snapshot, time.Now())
- if len(updates) == 0 {
- return
- }
- if err := s.accountRepo.UpdateExtra(ctx, account.ID, updates); err != nil {
- slog.Warn("openai_codex_snapshot_persist_failed", "account_id", account.ID, "error", err)
- return
- }
- notifyOpenAIAutoReset(account.ID)
- }
- // parseOpenAIRateLimitResetTime 解析 OpenAI 兼容格式的 429 响应,返回重置时间的 Unix 时间戳
- // OpenAI 的 usage_limit_reached 错误格式:
- //
- // {
- // "error": {
- // "message": "The usage limit has been reached",
- // "type": "usage_limit_reached",
- // "resets_at": 1769404154,
- // "resets_in_seconds": 133107
- // }
- // }
- func parseOpenAIRateLimitResetTime(body []byte) *int64 {
- var parsed map[string]any
- if err := json.Unmarshal(body, &parsed); err != nil {
- return nil
- }
- errObj, ok := parsed["error"].(map[string]any)
- if !ok {
- return nil
- }
- // 检查是否为已知的账号用量限制类型。
- errType, _ := errObj["type"].(string)
- if errType != "usage_limit_reached" && errType != "rate_limit_exceeded" && errType != "GoUsageLimitError" {
- return nil
- }
- // 优先使用 resets_at(Unix 时间戳)
- if resetsAt, ok := errObj["resets_at"].(float64); ok {
- ts := int64(resetsAt)
- return &ts
- }
- if resetsAt, ok := errObj["resets_at"].(string); ok {
- if ts, err := strconv.ParseInt(resetsAt, 10, 64); err == nil {
- return &ts
- }
- }
- // 如果没有 resets_at,尝试使用 resets_in_seconds
- if resetsInSeconds, ok := errObj["resets_in_seconds"].(float64); ok {
- ts := time.Now().Unix() + int64(resetsInSeconds)
- return &ts
- }
- if resetsInSeconds, ok := errObj["resets_in_seconds"].(string); ok {
- if sec, err := strconv.ParseInt(resetsInSeconds, 10, 64); err == nil {
- ts := time.Now().Unix() + sec
- return &ts
- }
- }
- // OpenCode Go subscriptions expose the reset only in a human-readable message,
- // for example: "Weekly usage limit reached. Resets in 2 days."
- if errType == "GoUsageLimitError" {
- message, _ := errObj["message"].(string)
- if resetAfter := parseOpenCodeGoUsageLimitResetDuration(message); resetAfter > 0 {
- ts := time.Now().Add(resetAfter).Unix()
- return &ts
- }
- }
- return nil
- }
- func parseOpenCodeGoUsageLimitResetDuration(message string) time.Duration {
- resetPrefix := openCodeGoUsageLimitResetPattern.FindStringIndex(message)
- if resetPrefix == nil {
- return 0
- }
- remainder := message[resetPrefix[1]:]
- var total time.Duration
- for {
- remainder = strings.TrimSpace(remainder)
- matches := openCodeGoUsageLimitDurationPartPattern.FindStringSubmatchIndex(remainder)
- if matches == nil {
- break
- }
- value, err := strconv.ParseFloat(remainder[matches[2]:matches[3]], 64)
- if err != nil || value <= 0 {
- return 0
- }
- unit := openCodeGoUsageLimitDurationUnit(remainder[matches[4]:matches[5]])
- if unit <= 0 {
- return 0
- }
- const maxDuration = time.Duration(1<<63 - 1)
- if value >= float64(maxDuration)/float64(unit) {
- return 0
- }
- part := time.Duration(value * float64(unit))
- if part <= 0 || total > maxDuration-part {
- return 0
- }
- total += part
- remainder = remainder[matches[1]:]
- }
- return total
- }
- func openCodeGoUsageLimitDurationUnit(raw string) time.Duration {
- switch strings.ToLower(raw) {
- case "s", "sec", "secs", "second", "seconds":
- return time.Second
- case "m", "min", "mins", "minute", "minutes":
- return time.Minute
- case "h", "hr", "hrs", "hour", "hours":
- return time.Hour
- case "d", "day", "days":
- return 24 * time.Hour
- case "w", "week", "weeks":
- return 7 * 24 * time.Hour
- default:
- return 0
- }
- }
- func parseOpenAIRateLimitPlanType(body []byte) string {
- var parsed map[string]any
- if err := json.Unmarshal(body, &parsed); err != nil {
- return ""
- }
- errObj, ok := parsed["error"].(map[string]any)
- if !ok {
- return ""
- }
- errType, _ := errObj["type"].(string)
- if errType != "usage_limit_reached" && errType != "rate_limit_exceeded" {
- return ""
- }
- planType, _ := errObj["plan_type"].(string)
- return strings.ToLower(strings.TrimSpace(planType))
- }
- func persistOpenAI429PlanType(ctx context.Context, repo AccountRepository, account *Account, body []byte) {
- if repo == nil || account == nil || account.Platform != PlatformOpenAI {
- return
- }
- // spark 影子账号恒不持凭据:即便收到带 plan_type 的 429,也不能把 plan_type 写进影子 credentials
- // ——该路径走 repo.BulkUpdate 直写、不经 persistAccountCredentials 守卫(外审第7轮 P1)。
- // plan_type 由母账号在自己的请求上维护,影子跳过。
- if account.IsCredentialShadow() {
- return
- }
- planType := parseOpenAIRateLimitPlanType(body)
- if planType == "" {
- return
- }
- current := strings.TrimSpace(account.GetCredential("plan_type"))
- if strings.EqualFold(current, planType) {
- return
- }
- if _, err := repo.BulkUpdate(ctx, []int64{account.ID}, AccountBulkUpdate{
- Credentials: map[string]any{"plan_type": planType},
- }); err != nil {
- slog.Warn("openai_429_plan_type_sync_failed", "account_id", account.ID, "plan_type", planType, "error", err)
- return
- }
- if account.Credentials == nil {
- account.Credentials = make(map[string]any, 1)
- }
- account.Credentials["plan_type"] = planType
- slog.Info("openai_429_plan_type_synced", "account_id", account.ID, "previous_plan_type", current, "plan_type", planType)
- }
- // handle529 处理529过载错误
- // 根据配置决定是否暂停账号调度及冷却时长
- func (s *RateLimitService) handle529(ctx context.Context, account *Account) {
- var settings *OverloadCooldownSettings
- if s.settingService != nil {
- var err error
- settings, err = s.settingService.GetOverloadCooldownSettings(ctx)
- if err != nil {
- slog.Warn("overload_settings_read_failed", "account_id", account.ID, "error", err)
- settings = nil
- }
- }
- // 回退到配置文件
- if settings == nil {
- cooldown := s.cfg.RateLimit.OverloadCooldownMinutes
- if cooldown <= 0 {
- cooldown = 10
- }
- settings = &OverloadCooldownSettings{Enabled: true, CooldownMinutes: cooldown}
- }
- if !settings.Enabled {
- slog.Info("account_529_ignored", "account_id", account.ID, "reason", "overload_cooldown_disabled")
- return
- }
- cooldownMinutes := settings.CooldownMinutes
- if cooldownMinutes <= 0 {
- cooldownMinutes = 10
- }
- until := time.Now().Add(time.Duration(cooldownMinutes) * time.Minute)
- s.notifyAccountSchedulingBlocked(account, until, "529")
- if err := s.accountRepo.SetOverloaded(ctx, account.ID, until); err != nil {
- slog.Warn("overload_set_failed", "account_id", account.ID, "error", err)
- return
- }
- slog.Info("account_overloaded", "account_id", account.ID, "until", until)
- }
- // UpdateSessionWindow 从成功响应更新5h窗口状态
- func (s *RateLimitService) UpdateSessionWindow(ctx context.Context, account *Account, headers http.Header) {
- status := headers.Get("anthropic-ratelimit-unified-5h-status")
- if status == "" {
- return
- }
- // 检查是否需要初始化时间窗口
- // 对于 Setup Token 账号,首次成功请求时需要预测时间窗口
- var windowStart, windowEnd *time.Time
- needInitWindow := account.SessionWindowEnd == nil || time.Now().After(*account.SessionWindowEnd)
- // 优先使用响应头中的真实重置时间(比预测更准确)
- if resetStr := headers.Get("anthropic-ratelimit-unified-5h-reset"); resetStr != "" {
- if ts, err := strconv.ParseInt(resetStr, 10, 64); err == nil {
- // 检测可能的毫秒时间戳(秒级约为 1e9,毫秒约为 1e12)
- if ts > 1e11 {
- slog.Warn("account_session_window_header_millis_detected", "account_id", account.ID, "raw_reset", resetStr)
- ts = ts / 1000
- }
- end := time.Unix(ts, 0)
- // 校验时间戳是否在合理范围内(不早于 5h 前,不晚于 7 天后)
- minAllowed := time.Now().Add(-5 * time.Hour)
- maxAllowed := time.Now().Add(7 * 24 * time.Hour)
- if end.Before(minAllowed) || end.After(maxAllowed) {
- slog.Warn("account_session_window_header_out_of_range", "account_id", account.ID, "raw_reset", resetStr, "parsed_end", end)
- } else if needInitWindow || account.SessionWindowEnd == nil || !end.Equal(*account.SessionWindowEnd) {
- // 窗口需要初始化,或者真实重置时间与已存储的不同,则更新
- start := end.Add(-5 * time.Hour)
- windowStart = &start
- windowEnd = &end
- slog.Info("account_session_window_from_header", "account_id", account.ID, "window_start", start, "window_end", end, "status", status)
- }
- } else {
- slog.Warn("account_session_window_header_parse_failed", "account_id", account.ID, "raw_reset", resetStr, "error", err)
- }
- }
- // 回退:如果没有真实重置时间且需要初始化窗口,使用预测
- if windowEnd == nil && needInitWindow && (status == "allowed" || status == "allowed_warning") {
- now := time.Now()
- start := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location())
- end := start.Add(5 * time.Hour)
- windowStart = &start
- windowEnd = &end
- slog.Info("account_session_window_initialized", "account_id", account.ID, "window_start", start, "window_end", end, "status", status)
- }
- // 窗口重置时清除旧的 utilization 和被动采样数据,避免残留上个窗口的数据
- if windowEnd != nil && needInitWindow {
- _ = s.accountRepo.UpdateExtra(ctx, account.ID, map[string]any{
- "session_window_utilization": nil,
- "passive_usage_7d_utilization": nil,
- "passive_usage_7d_reset": nil,
- "passive_usage_7d_oi_utilization": nil,
- "passive_usage_7d_oi_reset": nil,
- "passive_usage_sampled_at": nil,
- })
- }
- if err := s.accountRepo.UpdateSessionWindow(ctx, account.ID, windowStart, windowEnd, status); err != nil {
- slog.Warn("session_window_update_failed", "account_id", account.ID, "error", err)
- }
- // 被动采样:从响应头收集 5h + 7d + 7d_oi utilization,合并为一次 DB 写入
- s.samplePassiveUsageFromHeaders(ctx, account, headers)
- // 如果状态为allowed且之前有限流,说明窗口已重置,清除限流状态
- if status == "allowed" && account.IsRateLimited() {
- if err := s.ClearRateLimit(ctx, account.ID); err != nil {
- slog.Warn("rate_limit_clear_failed", "account_id", account.ID, "error", err)
- }
- }
- }
- // samplePassiveUsageFromHeaders 从 Anthropic 响应头收集 5h/7d/7d_oi 的
- // utilization 与 reset 被动采样数据,合并为一次 Extra 写入。无数据时不写。
- func (s *RateLimitService) samplePassiveUsageFromHeaders(ctx context.Context, account *Account, headers http.Header) {
- extraUpdates := make(map[string]any, 6)
- // 5h utilization(0-1 小数),供 estimateSetupTokenUsage 使用
- if utilStr := headers.Get("anthropic-ratelimit-unified-5h-utilization"); utilStr != "" {
- if util, err := strconv.ParseFloat(utilStr, 64); err == nil {
- extraUpdates["session_window_utilization"] = util
- }
- }
- // 7d utilization(0-1 小数)
- if utilStr := headers.Get("anthropic-ratelimit-unified-7d-utilization"); utilStr != "" {
- if util, err := strconv.ParseFloat(utilStr, 64); err == nil {
- extraUpdates["passive_usage_7d_utilization"] = util
- }
- }
- // 7d reset timestamp
- if resetStr := headers.Get("anthropic-ratelimit-unified-7d-reset"); resetStr != "" {
- if ts, err := strconv.ParseInt(resetStr, 10, 64); err == nil {
- if ts > 1e11 {
- ts = ts / 1000
- }
- extraUpdates["passive_usage_7d_reset"] = ts
- }
- }
- // 7d_oi (Fable 专属 7d 窗口) utilization(0-1 小数)
- if utilStr := headers.Get("anthropic-ratelimit-unified-7d_oi-utilization"); utilStr != "" {
- if util, err := strconv.ParseFloat(utilStr, 64); err == nil {
- extraUpdates["passive_usage_7d_oi_utilization"] = util
- }
- }
- // 7d_oi reset timestamp
- if resetStr := headers.Get("anthropic-ratelimit-unified-7d_oi-reset"); resetStr != "" {
- if ts, err := strconv.ParseInt(resetStr, 10, 64); err == nil {
- if ts > 1e11 {
- ts = ts / 1000
- }
- extraUpdates["passive_usage_7d_oi_reset"] = ts
- }
- }
- if len(extraUpdates) > 0 {
- extraUpdates["passive_usage_sampled_at"] = time.Now().UTC().Format(time.RFC3339)
- if err := s.accountRepo.UpdateExtra(ctx, account.ID, extraUpdates); err != nil {
- slog.Warn("passive_usage_update_failed", "account_id", account.ID, "error", err)
- }
- }
- }
- // ClearRateLimit 清除账号的限流状态
- func (s *RateLimitService) ClearRateLimit(ctx context.Context, accountID int64) error {
- if err := s.accountRepo.ClearRateLimit(ctx, accountID); err != nil {
- return err
- }
- if err := s.accountRepo.ClearAntigravityQuotaScopes(ctx, accountID); err != nil {
- return err
- }
- if err := s.accountRepo.ClearModelRateLimits(ctx, accountID); err != nil {
- return err
- }
- // 清除限流时一并清理临时不可调度状态,避免周限/窗口重置后仍被本地临时状态阻断。
- if err := s.accountRepo.ClearTempUnschedulable(ctx, accountID); err != nil {
- return err
- }
- if s.tempUnschedCache != nil {
- if err := s.tempUnschedCache.DeleteTempUnsched(ctx, accountID); err != nil {
- slog.Warn("temp_unsched_cache_delete_failed", "account_id", accountID, "error", err)
- }
- }
- s.ResetOpenAI403Counter(ctx, accountID)
- s.notifyAccountSchedulingBlockCleared(accountID)
- return nil
- }
- func (s *RateLimitService) ResetOpenAI403Counter(ctx context.Context, accountID int64) {
- if s == nil || s.openAI403CounterCache == nil || accountID <= 0 {
- return
- }
- if err := s.openAI403CounterCache.ResetOpenAI403Count(ctx, accountID); err != nil {
- slog.Warn("openai_403_reset_failed", "account_id", accountID, "error", err)
- }
- }
- // RecoverAccountState 按需恢复账号的可恢复运行时状态。
- func (s *RateLimitService) RecoverAccountState(ctx context.Context, accountID int64, options AccountRecoveryOptions) (*SuccessfulTestRecoveryResult, error) {
- account, err := s.accountRepo.GetByID(ctx, accountID)
- if err != nil {
- return nil, err
- }
- result := &SuccessfulTestRecoveryResult{}
- if account.Status == StatusError {
- if err := s.accountRepo.ClearError(ctx, accountID); err != nil {
- return nil, err
- }
- result.ClearedError = true
- if options.InvalidateToken && s.tokenCacheInvalidator != nil && account.IsOAuth() {
- if invalidateErr := s.tokenCacheInvalidator.InvalidateToken(ctx, account); invalidateErr != nil {
- slog.Warn("recover_account_state_invalidate_token_failed", "account_id", accountID, "error", invalidateErr)
- }
- }
- }
- if hasRecoverableRuntimeState(account) {
- if err := s.ClearRateLimit(ctx, accountID); err != nil {
- return nil, err
- }
- result.ClearedRateLimit = true
- }
- if result.ClearedError || result.ClearedRateLimit {
- s.ResetOpenAI403Counter(ctx, accountID)
- if result.ClearedError && !result.ClearedRateLimit {
- s.notifyAccountSchedulingBlockCleared(accountID)
- }
- }
- return result, nil
- }
- // A successful probe describes only that request. It cannot clear unrelated
- // model limits or newer failures written by another request. Administrators
- // can explicitly recover state through RecoverAccountState after reviewing it.
- func (s *RateLimitService) RecoverAccountAfterSuccessfulTest(ctx context.Context, accountID int64) (*SuccessfulTestRecoveryResult, error) {
- if _, err := s.accountRepo.GetByID(ctx, accountID); err != nil {
- return nil, err
- }
- return &SuccessfulTestRecoveryResult{}, nil
- }
- func (s *RateLimitService) ClearTempUnschedulable(ctx context.Context, accountID int64) error {
- if err := s.accountRepo.ClearTempUnschedulable(ctx, accountID); err != nil {
- return err
- }
- if s.tempUnschedCache != nil {
- if err := s.tempUnschedCache.DeleteTempUnsched(ctx, accountID); err != nil {
- slog.Warn("temp_unsched_cache_delete_failed", "account_id", accountID, "error", err)
- }
- }
- // 同时清除模型级别限流
- if err := s.accountRepo.ClearModelRateLimits(ctx, accountID); err != nil {
- slog.Warn("clear_model_rate_limits_on_temp_unsched_reset_failed", "account_id", accountID, "error", err)
- }
- s.notifyAccountSchedulingBlockCleared(accountID)
- return nil
- }
- func hasRecoverableRuntimeState(account *Account) bool {
- if account == nil {
- return false
- }
- if account.RateLimitedAt != nil || account.RateLimitResetAt != nil || account.OverloadUntil != nil || account.TempUnschedulableUntil != nil {
- return true
- }
- if len(account.Extra) == 0 {
- return false
- }
- return hasNonEmptyMapValue(account.Extra, "model_rate_limits") ||
- hasNonEmptyMapValue(account.Extra, "antigravity_quota_scopes")
- }
- func hasNonEmptyMapValue(extra map[string]any, key string) bool {
- raw, ok := extra[key]
- if !ok || raw == nil {
- return false
- }
- switch typed := raw.(type) {
- case map[string]any:
- return len(typed) > 0
- case map[string]string:
- return len(typed) > 0
- case []any:
- return len(typed) > 0
- default:
- return true
- }
- }
- func (s *RateLimitService) GetTempUnschedStatus(ctx context.Context, accountID int64) (*TempUnschedState, error) {
- now := time.Now().Unix()
- if s.tempUnschedCache != nil {
- state, err := s.tempUnschedCache.GetTempUnsched(ctx, accountID)
- if err != nil {
- return nil, err
- }
- if state != nil && state.UntilUnix > now {
- return state, nil
- }
- }
- account, err := s.accountRepo.GetByID(ctx, accountID)
- if err != nil {
- return nil, err
- }
- if account.TempUnschedulableUntil == nil {
- return nil, nil
- }
- if account.TempUnschedulableUntil.Unix() <= now {
- return nil, nil
- }
- state := &TempUnschedState{
- UntilUnix: account.TempUnschedulableUntil.Unix(),
- }
- if account.TempUnschedulableReason != "" {
- var parsed TempUnschedState
- if err := json.Unmarshal([]byte(account.TempUnschedulableReason), &parsed); err == nil {
- if parsed.UntilUnix == 0 {
- parsed.UntilUnix = state.UntilUnix
- }
- state = &parsed
- } else {
- state.ErrorMessage = account.TempUnschedulableReason
- }
- }
- if s.tempUnschedCache != nil {
- if err := s.tempUnschedCache.SetTempUnsched(ctx, accountID, state); err != nil {
- slog.Warn("temp_unsched_cache_set_failed", "account_id", accountID, "error", err)
- }
- }
- return state, nil
- }
- func (s *RateLimitService) HandleTempUnschedulable(ctx context.Context, account *Account, statusCode int, responseBody []byte, requestedModel ...string) bool {
- if account == nil {
- return false
- }
- if account.IsPoolMode() && !account.IsCustomErrorCodesEnabled() {
- return false
- }
- if !account.ShouldHandleErrorCode(statusCode) {
- return false
- }
- ctx = withTempUnschedulableModel(ctx, requestedModel)
- return s.tryTempUnschedulable(ctx, account, statusCode, responseBody, firstRequestedModel(requestedModel))
- }
- func (s *RateLimitService) HandleOpenAIImageRateLimit(ctx context.Context, account *Account, statusCode int, headers http.Header, responseBody []byte) bool {
- if s == nil || account == nil || s.accountRepo == nil {
- return false
- }
- if account.Platform != PlatformOpenAI {
- return false
- }
- if !account.ShouldHandleErrorCode(statusCode) {
- slog.Info("openai_image_rate_limit_skipped_by_error_code_policy", "account_id", account.ID, "status_code", statusCode)
- return false
- }
- if !isOpenAIImageRateLimitError(statusCode, responseBody) {
- return false
- }
- resetAt := openAIImageRateLimitResetAt(headers, responseBody)
- if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, openAIImageGenerationRateLimitKey, resetAt, openAIImageRateLimitReason); err != nil {
- slog.Warn("openai_image_rate_limit_set_model_rate_limit_failed", "account_id", account.ID, "scope", openAIImageGenerationRateLimitKey, "error", err)
- return true
- }
- slog.Info("openai_image_rate_limited", "account_id", account.ID, "scope", openAIImageGenerationRateLimitKey, "reset_at", resetAt, "reset_in", time.Until(resetAt).Truncate(time.Second))
- return true
- }
- // HandleOpenAICodexSparkRateLimit 将 Spark 独立配额窗口记录为模型级限流。
- // Spark 的 x-codex-* 使用率和 reset 时间只代表 Spark 模型维度,不能写入账号级
- // RateLimitResetAt,否则同一 OAuth 账号上的其他模型也会被错误停调。
- func (s *RateLimitService) HandleOpenAICodexSparkRateLimit(ctx context.Context, account *Account, requestedModel string, statusCode int, headers http.Header, responseBody []byte) bool {
- if s == nil || account == nil || s.accountRepo == nil || statusCode != http.StatusTooManyRequests || !isOpenAIOAuthAccount(account) {
- return false
- }
- if !isCodexSparkModel(requestedModel) || !account.ShouldHandleErrorCode(statusCode) {
- return false
- }
- modelKey := normalizeCodexModel(modelRateLimitKeyForUpstreamModelNotFound(ctx, account, requestedModel))
- if modelKey == "" {
- return false
- }
- now := time.Now()
- disposition, resetAt := classifyOpenAIOAuth429(headers, responseBody)
- // Spark 只有明确耗尽 5h/7d 窗口时才能使用上游长 reset;普通瞬时 429
- // 即使携带全局 reset 头,也只能使用短时回避,避免错误冷却数天。
- if disposition != openAIOAuth429Quota5h && disposition != openAIOAuth429Quota7d {
- resetAt = nil
- }
- if resetAt == nil || !resetAt.After(now) {
- cooldown, ok := s.get429FallbackCooldown(ctx, account)
- if !ok || cooldown <= 0 {
- cooldown = time.Duration(defaultRateLimit429CooldownSeconds) * time.Second
- }
- reset := now.Add(cooldown)
- resetAt = &reset
- }
- if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, modelKey, *resetAt, openAICodexSparkRateLimitReason); err != nil {
- slog.Warn("openai_codex_spark_model_rate_limit_set_failed", "account_id", account.ID, "model", modelKey, "error", err)
- }
- slog.Info("openai_codex_spark_model_rate_limited", "account_id", account.ID, "model", modelKey, "reset_at", *resetAt)
- return true
- }
- func (s *RateLimitService) HandleOpenAIImageCapabilityLoss(ctx context.Context, account *Account, statusCode int, responseBody []byte) bool {
- if s == nil || account == nil || s.accountRepo == nil {
- return false
- }
- if account.Platform != PlatformOpenAI {
- return false
- }
- if !account.ShouldHandleErrorCode(statusCode) {
- slog.Info("openai_image_capability_loss_skipped_by_error_code_policy", "account_id", account.ID, "status_code", statusCode)
- return false
- }
- if !isOpenAIImageCapabilityLossError(statusCode, responseBody) {
- return false
- }
- resetAt := time.Now().Add(openAIImageCapabilityLossCooldown)
- if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, openAIImageGenerationRateLimitKey, resetAt, openAIImageCapabilityLossReason); err != nil {
- slog.Warn("openai_image_capability_loss_set_model_rate_limit_failed", "account_id", account.ID, "scope", openAIImageGenerationRateLimitKey, "error", err)
- return true
- }
- slog.Info("openai_image_capability_lost", "account_id", account.ID, "scope", openAIImageGenerationRateLimitKey, "reset_at", resetAt, "reset_in", time.Until(resetAt).Truncate(time.Second))
- return true
- }
- // isOpenAIImageCapabilityLossError reports whether upstream rejected the
- // image_generation tool choice that sub2api itself put into the request body.
- // Only meaningful for self-built images requests, where tools always carries a
- // matching image_generation entry — upstream saying otherwise means the account
- // lost the capability.
- func isOpenAIImageCapabilityLossError(statusCode int, body []byte) bool {
- if statusCode != http.StatusBadRequest || len(body) == 0 {
- return false
- }
- lower := strings.ToLower(string(body))
- return strings.Contains(lower, "image_generation") &&
- strings.Contains(lower, "not found in 'tools' parameter")
- }
- func isOpenAIImageRateLimitError(statusCode int, body []byte) bool {
- if statusCode != http.StatusTooManyRequests || len(body) == 0 {
- return false
- }
- lower := strings.ToLower(string(body))
- for _, marker := range []string{
- "for limit gpt-image",
- "input-images per min",
- "gpt-image-2-codex",
- "gpt-image",
- } {
- if strings.Contains(lower, marker) {
- return true
- }
- }
- return false
- }
- func openAIImageRateLimitResetAt(headers http.Header, body []byte) time.Time {
- now := time.Now()
- if resetAt := parseRetryAfterResetTime(headers, now); resetAt != nil && resetAt.After(now) {
- return *resetAt
- }
- if resetAt := calculateOpenAI429ResetTime(headers); resetAt != nil && resetAt.After(now) {
- return *resetAt
- }
- if resetUnix := parseOpenAIRateLimitResetTime(body); resetUnix != nil {
- if resetAt := time.Unix(*resetUnix, 0); resetAt.After(now) {
- return resetAt
- }
- }
- if cooldown := parseOpenAIImageTryAgainCooldown(body); cooldown > 0 {
- return now.Add(cooldown)
- }
- return now.Add(openAIImageRateLimitDefaultCooldown)
- }
- func parseRetryAfterResetTime(headers http.Header, now time.Time) *time.Time {
- if headers == nil {
- return nil
- }
- raw := strings.TrimSpace(headers.Get("Retry-After"))
- if raw == "" {
- return nil
- }
- if seconds, err := strconv.ParseFloat(raw, 64); err == nil {
- resetAt := now.Add(time.Duration(seconds * float64(time.Second)))
- return &resetAt
- }
- if parsed, err := http.ParseTime(raw); err == nil {
- return &parsed
- }
- return nil
- }
- func parseOpenAIImageTryAgainCooldown(body []byte) time.Duration {
- if len(body) == 0 {
- return 0
- }
- match := openAIImageTryAgainPattern.FindSubmatch(body)
- if len(match) != 3 {
- return 0
- }
- value, err := strconv.ParseFloat(string(match[1]), 64)
- if err != nil || value <= 0 {
- return 0
- }
- switch strings.ToLower(string(match[2])) {
- case "ms":
- return time.Duration(value * float64(time.Millisecond))
- case "s", "sec", "secs", "second", "seconds":
- return time.Duration(value * float64(time.Second))
- case "m", "min", "mins", "minute", "minutes":
- return time.Duration(value * float64(time.Minute))
- default:
- return 0
- }
- }
- const upstreamModelNotFoundCooldown = 30 * time.Minute
- const upstreamModelNotFoundReason = "upstream_404_model_not_found"
- const upstreamCodexPlanGatedModelCooldown = 30 * time.Minute
- const upstreamCodexPlanGatedModelReason = "upstream_400_codex_plan_gated_model"
- const tempUnschedBodyMaxBytes = 64 << 10
- const tempUnschedMessageMaxBytes = 2048
- // HandleUpstreamModelNotFound marks the requested model as temporarily
- // unavailable on the account when the upstream deterministically reports it
- // cannot serve that model: a 404 model-not-found, or the Codex 400 rejecting a
- // plan-gated model on a ChatGPT OAuth account. Returning true tells the caller
- // to fail the current attempt over to another account; the scheduler skips the
- // (account, model) pair via IsSchedulableForModelWithContext until the
- // cooldown expires, instead of re-selecting an account that can never serve
- // the model.
- func (s *RateLimitService) HandleUpstreamModelNotFound(ctx context.Context, account *Account, requestedModel string, statusCode int, responseBody []byte) bool {
- if s == nil || account == nil || s.accountRepo == nil {
- return false
- }
- if !account.ShouldHandleErrorCode(statusCode) {
- return false
- }
- var cooldown time.Duration
- var reason string
- switch {
- case isUpstreamModelNotFoundError(statusCode, responseBody):
- cooldown, reason = upstreamModelNotFoundCooldown, upstreamModelNotFoundReason
- case isOpenAIOAuthAccount(account) && isOpenAICodexPlanGatedModelError(statusCode, responseBody):
- cooldown, reason = upstreamCodexPlanGatedModelCooldown, upstreamCodexPlanGatedModelReason
- default:
- return false
- }
- modelKey := modelRateLimitKeyForUpstreamModelNotFound(ctx, account, requestedModel)
- if modelKey == "" {
- return false
- }
- if shouldSkipCodexPlanGatedImageModelCooldown(ctx, reason, requestedModel, modelKey) {
- return true
- }
- resetAt := time.Now().Add(cooldown)
- if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, modelKey, resetAt, reason); err != nil {
- slog.Warn("upstream_model_not_found_set_model_rate_limit_failed", "account_id", account.ID, "model", modelKey, "reason", reason, "error", err)
- return true
- }
- slog.Info("upstream_model_not_found_model_rate_limited", "account_id", account.ID, "model", modelKey, "reason", reason, "reset_at", resetAt)
- return true
- }
- // shouldSkipCodexPlanGatedImageModelCooldown 判断这次 Codex plan-gated 400 是否
- // 属于"图片模型被文本端点拒绝"。
- //
- // 这类错误是确定性的端点错配,不是账号能力缺失:同一账号通过 /v1/images/* 依然
- // 能出图。在这里写 per-model 冷却,会让一次用错端点的请求把整个号池对正确的生图
- // 端点下线(#4828)。
- //
- // 但请求本身就从 /v1/images/* 入站时不适用——那种情况下被拒说明账号确实不具备
- // 该模型能力,冷却是必要的刹车:没有它,每个生图请求都会完整走一遍号池,对上游
- // 形成无上界的 400 放大。
- //
- // 请求模型与最终冷却键都要判:冷却键走的是 account.GetMappedModel,账号可能把
- // 文本别名映射到 gpt-image-*,只判请求模型会漏掉这种形态。
- func shouldSkipCodexPlanGatedImageModelCooldown(ctx context.Context, reason, requestedModel, modelKey string) bool {
- if reason != upstreamCodexPlanGatedModelReason {
- return false
- }
- if OpenAIImagesEndpointFromContext(ctx) {
- return false
- }
- return IsGPTImageGenerationModel(requestedModel) || IsGPTImageGenerationModel(modelKey)
- }
- func modelRateLimitKeyForUpstreamModelNotFound(ctx context.Context, account *Account, requestedModel string) string {
- modelKey := strings.TrimSpace(requestedModel)
- if account == nil || modelKey == "" {
- return modelKey
- }
- if account.Platform == PlatformAntigravity {
- if resolved := strings.TrimSpace(resolveFinalAntigravityModelKey(ctx, account, modelKey)); resolved != "" {
- return resolved
- }
- return modelKey
- }
- if mapped := strings.TrimSpace(account.GetMappedModel(modelKey)); mapped != "" {
- return mapped
- }
- return modelKey
- }
- func firstRequestedModel(requestedModel []string) string {
- if len(requestedModel) == 0 {
- return ""
- }
- return strings.TrimSpace(requestedModel[0])
- }
- type tempUnschedulableModelContextKey struct{}
- func withTempUnschedulableModel(ctx context.Context, requestedModel []string) context.Context {
- model := firstRequestedModel(requestedModel)
- if model == "" {
- return ctx
- }
- if ctx == nil {
- ctx = context.Background()
- }
- return context.WithValue(ctx, tempUnschedulableModelContextKey{}, model)
- }
- func tempUnschedulableModel(ctx context.Context, requestedModel []string) string {
- if model := firstRequestedModel(requestedModel); model != "" {
- return model
- }
- if ctx == nil {
- return ""
- }
- model, _ := ctx.Value(tempUnschedulableModelContextKey{}).(string)
- return strings.TrimSpace(model)
- }
- type tempUnschedulableRuleMatch struct {
- rule TempUnschedulableRule
- ruleIndex int
- matchedKeyword string
- }
- func matchTempUnschedulableRules(account *Account, statusCode int, responseBody []byte) []tempUnschedulableRuleMatch {
- if account == nil || !account.IsTempUnschedulableEnabled() || statusCode <= 0 || len(responseBody) == 0 {
- return nil
- }
- rules := account.GetTempUnschedulableRules()
- if len(rules) == 0 {
- return nil
- }
- body := responseBody
- if len(body) > tempUnschedBodyMaxBytes {
- body = body[:tempUnschedBodyMaxBytes]
- }
- bodyLower := strings.ToLower(string(body))
- matches := make([]tempUnschedulableRuleMatch, 0, 1)
- for idx, rule := range rules {
- if rule.ErrorCode != statusCode || len(rule.Keywords) == 0 {
- continue
- }
- matchedKeyword := matchTempUnschedKeyword(bodyLower, rule.Keywords)
- if matchedKeyword == "" {
- continue
- }
- matches = append(matches, tempUnschedulableRuleMatch{rule: rule, ruleIndex: idx, matchedKeyword: matchedKeyword})
- }
- return matches
- }
- func (s *RateLimitService) tryTempUnschedulable(ctx context.Context, account *Account, statusCode int, responseBody []byte, requestedModel ...string) bool {
- if account == nil {
- return false
- }
- if !account.IsTempUnschedulableEnabled() {
- return false
- }
- // 401 首次命中可临时不可调度(给 token 刷新窗口);
- // 若历史上已因 401 进入过临时不可调度,则本次应升级为 error(返回 false 交由默认错误逻辑处理)。
- // Antigravity 跳过:其 401 由 applyErrorPolicy 的 temp_unschedulable_rules 自行控制,无需升级逻辑。
- if statusCode == http.StatusUnauthorized && account.Platform != PlatformAntigravity {
- reason := account.TempUnschedulableReason
- // 缓存可能没有 reason,从 DB 回退读取
- if reason == "" {
- if dbAcc, err := s.accountRepo.GetByID(ctx, account.ID); err == nil && dbAcc != nil {
- reason = dbAcc.TempUnschedulableReason
- }
- }
- if wasTempUnschedByStatusCode(reason, statusCode) {
- slog.Info("401_escalated_to_error", "account_id", account.ID,
- "reason", "previous temp-unschedulable was also 401")
- return false
- }
- }
- for _, match := range matchTempUnschedulableRules(account, statusCode, responseBody) {
- if s.triggerTempUnschedulable(ctx, account, match.rule, match.ruleIndex, statusCode, match.matchedKeyword, responseBody, tempUnschedulableModel(ctx, requestedModel)) {
- return true
- }
- }
- return false
- }
- func wasTempUnschedByStatusCode(reason string, statusCode int) bool {
- if statusCode <= 0 {
- return false
- }
- reason = strings.TrimSpace(reason)
- if reason == "" {
- return false
- }
- var state TempUnschedState
- if err := json.Unmarshal([]byte(reason), &state); err != nil {
- return false
- }
- return state.StatusCode == statusCode
- }
- func matchTempUnschedKeyword(bodyLower string, keywords []string) string {
- if bodyLower == "" {
- return ""
- }
- for _, keyword := range keywords {
- k := strings.TrimSpace(keyword)
- if k == "" {
- continue
- }
- if strings.Contains(bodyLower, strings.ToLower(k)) {
- return k
- }
- }
- return ""
- }
- func (s *RateLimitService) triggerTempUnschedulable(ctx context.Context, account *Account, rule TempUnschedulableRule, ruleIndex int, statusCode int, matchedKeyword string, responseBody []byte, requestedModel ...string) bool {
- if account == nil {
- return false
- }
- if rule.DurationMinutes <= 0 {
- return false
- }
- now := time.Now()
- until := now.Add(time.Duration(rule.DurationMinutes) * time.Minute)
- state := &TempUnschedState{
- UntilUnix: until.Unix(),
- TriggeredAtUnix: now.Unix(),
- StatusCode: statusCode,
- MatchedKeyword: matchedKeyword,
- RuleIndex: ruleIndex,
- ErrorMessage: truncateTempUnschedMessage(responseBody, tempUnschedMessageMaxBytes),
- }
- reason := ""
- if raw, err := json.Marshal(state); err == nil {
- reason = string(raw)
- }
- if reason == "" {
- reason = strings.TrimSpace(state.ErrorMessage)
- }
- // Persist known-model failures under the model key so the scheduler excludes
- // only this (account, model) pair. Authentication and model-unknown failures
- // retain the legacy account-wide temporary-unschedulable behavior below.
- modelKey := firstRequestedModel(requestedModel)
- if modelKey != "" && statusCode != http.StatusUnauthorized {
- if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, modelKey, until, reason); err != nil {
- slog.Warn("temp_unsched_model_rate_limit_set_failed", "account_id", account.ID, "model", modelKey, "error", err)
- // The rule matched, so fail over the current request even if persistence
- // failed; never widen a model-scoped failure into an account-wide block.
- return true
- }
- slog.Info("account_model_temp_unschedulable", "account_id", account.ID, "model", modelKey, "until", until, "rule_index", ruleIndex, "status_code", statusCode)
- return true
- }
- s.notifyAccountSchedulingBlocked(account, until, "temp_unschedulable")
- if err := s.accountRepo.SetTempUnschedulable(ctx, account.ID, until, reason); err != nil {
- slog.Warn("temp_unsched_set_failed", "account_id", account.ID, "error", err)
- return false
- }
- if s.tempUnschedCache != nil {
- if err := s.tempUnschedCache.SetTempUnsched(ctx, account.ID, state); err != nil {
- slog.Warn("temp_unsched_cache_set_failed", "account_id", account.ID, "error", err)
- }
- }
- slog.Info("account_temp_unschedulable", "account_id", account.ID, "until", until, "rule_index", ruleIndex, "status_code", statusCode)
- return true
- }
- func truncateTempUnschedMessage(body []byte, maxBytes int) string {
- if maxBytes <= 0 || len(body) == 0 {
- return ""
- }
- if len(body) > maxBytes {
- body = body[:maxBytes]
- }
- return strings.TrimSpace(string(body))
- }
- // HandleStreamTimeout 处理流数据超时
- // 根据系统设置决定是否标记账户为临时不可调度或错误状态
- // 返回是否应该停止该账号的调度
- func (s *RateLimitService) HandleStreamTimeout(ctx context.Context, account *Account, model string) bool {
- if account == nil {
- return false
- }
- // 获取系统设置
- if s.settingService == nil {
- slog.Warn("stream_timeout_setting_service_missing", "account_id", account.ID)
- return false
- }
- settings, err := s.settingService.GetStreamTimeoutSettings(ctx)
- if err != nil {
- slog.Warn("stream_timeout_get_settings_failed", "account_id", account.ID, "error", err)
- return false
- }
- if !settings.Enabled {
- return false
- }
- if settings.Action == StreamTimeoutActionNone {
- return false
- }
- // 增加超时计数
- var count int64 = 1
- if s.timeoutCounterCache != nil {
- count, err = s.timeoutCounterCache.IncrementTimeoutCount(ctx, account.ID, settings.ThresholdWindowMinutes)
- if err != nil {
- slog.Warn("stream_timeout_increment_count_failed", "account_id", account.ID, "error", err)
- // 继续处理,使用 count=1
- count = 1
- }
- }
- slog.Info("stream_timeout_count", "account_id", account.ID, "count", count, "threshold", settings.ThresholdCount, "window_minutes", settings.ThresholdWindowMinutes, "model", model)
- // 检查是否达到阈值
- if count < int64(settings.ThresholdCount) {
- return false
- }
- // 达到阈值,执行相应操作
- switch settings.Action {
- case StreamTimeoutActionTempUnsched:
- return s.triggerStreamTimeoutTempUnsched(ctx, account, settings, model)
- case StreamTimeoutActionError:
- return s.triggerStreamTimeoutError(ctx, account, model)
- default:
- return false
- }
- }
- // triggerStreamTimeoutTempUnsched 触发流超时临时不可调度
- func (s *RateLimitService) triggerStreamTimeoutTempUnsched(ctx context.Context, account *Account, settings *StreamTimeoutSettings, model string) bool {
- now := time.Now()
- until := now.Add(time.Duration(settings.TempUnschedMinutes) * time.Minute)
- state := &TempUnschedState{
- UntilUnix: until.Unix(),
- TriggeredAtUnix: now.Unix(),
- StatusCode: 0, // 超时没有状态码
- MatchedKeyword: "stream_timeout",
- RuleIndex: -1, // 表示系统级规则
- ErrorMessage: "Stream data interval timeout for model: " + model,
- }
- reason := ""
- if raw, err := json.Marshal(state); err == nil {
- reason = string(raw)
- }
- if reason == "" {
- reason = state.ErrorMessage
- }
- s.notifyAccountSchedulingBlocked(account, until, "stream_timeout_temp_unschedulable")
- if err := s.accountRepo.SetTempUnschedulable(ctx, account.ID, until, reason); err != nil {
- slog.Warn("stream_timeout_set_temp_unsched_failed", "account_id", account.ID, "error", err)
- return false
- }
- if s.tempUnschedCache != nil {
- if err := s.tempUnschedCache.SetTempUnsched(ctx, account.ID, state); err != nil {
- slog.Warn("stream_timeout_set_temp_unsched_cache_failed", "account_id", account.ID, "error", err)
- }
- }
- // 重置超时计数
- if s.timeoutCounterCache != nil {
- if err := s.timeoutCounterCache.ResetTimeoutCount(ctx, account.ID); err != nil {
- slog.Warn("stream_timeout_reset_count_failed", "account_id", account.ID, "error", err)
- }
- }
- slog.Info("stream_timeout_temp_unschedulable", "account_id", account.ID, "until", until, "model", model)
- return true
- }
- // triggerStreamTimeoutError 触发流超时错误状态
- func (s *RateLimitService) triggerStreamTimeoutError(ctx context.Context, account *Account, model string) bool {
- errorMsg := "Stream data interval timeout (repeated failures) for model: " + model
- s.notifyAccountSchedulingBlocked(account, time.Time{}, "stream_timeout_error")
- if err := s.accountRepo.SetError(ctx, account.ID, errorMsg); err != nil {
- slog.Warn("stream_timeout_set_error_failed", "account_id", account.ID, "error", err)
- return false
- }
- // 重置超时计数
- if s.timeoutCounterCache != nil {
- if err := s.timeoutCounterCache.ResetTimeoutCount(ctx, account.ID); err != nil {
- slog.Warn("stream_timeout_reset_count_failed", "account_id", account.ID, "error", err)
- }
- }
- slog.Warn("stream_timeout_account_error", "account_id", account.ID, "model", model)
- return true
- }
|