openai_team_linked_error.go 3.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899
  1. package service
  2. import (
  3. "context"
  4. "fmt"
  5. "log/slog"
  6. "net/http"
  7. "strings"
  8. "time"
  9. "github.com/tidwall/gjson"
  10. )
  11. const (
  12. openAITeamLinkedErrorDedupTTL = 60 * time.Second
  13. openAITeamLinkedErrorFanoutTimeout = 30 * time.Second
  14. openAITeamLinkedErrorBlockReason = "team_linked_error"
  15. )
  16. // maybeHandleOpenAITeamLinkedError 在 OpenAI OAuth 账户收到 402 deactivated_workspace
  17. // (ChatGPT Team 工作区被停用)时,把同一 Team(credentials.chatgpt_account_id 相同)
  18. // 的其余 active 账户一并置为 error 并立即熔断。触发账户自身不在 fan-out 范围内,
  19. // 仍由常规 402 处理标记。
  20. func (s *RateLimitService) maybeHandleOpenAITeamLinkedError(ctx context.Context, account *Account, statusCode int, responseBody []byte) {
  21. if s == nil || s.accountRepo == nil || statusCode != http.StatusPaymentRequired || !isOpenAIOAuthAccount(account) {
  22. return
  23. }
  24. if gjson.GetBytes(responseBody, "detail.code").String() != "deactivated_workspace" {
  25. return
  26. }
  27. teamID := strings.TrimSpace(account.GetChatGPTAccountID())
  28. if teamID == "" {
  29. return
  30. }
  31. if !s.markOpenAITeamLinkedFired(teamID) {
  32. return
  33. }
  34. // 上游报错场景请求 ctx 往往已被取消,落库需要独立生命周期。
  35. opCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), openAITeamLinkedErrorFanoutTimeout)
  36. defer cancel()
  37. accounts, err := s.accountRepo.ListByPlatform(opCtx, PlatformOpenAI)
  38. if err != nil {
  39. slog.Warn("openai_team_linked_error_list_failed", "trigger_account_id", account.ID, "error", err)
  40. return
  41. }
  42. var targets []*Account
  43. for i := range accounts {
  44. acc := &accounts[i]
  45. if acc.ID == account.ID || acc.IsShadow() || strings.TrimSpace(acc.GetChatGPTAccountID()) != teamID {
  46. continue
  47. }
  48. targets = append(targets, acc)
  49. }
  50. if len(targets) == 0 {
  51. return
  52. }
  53. // 先全部进程内熔断(微秒级生效),再逐个落库,避免后面的账户等待前面的 DB 写入。
  54. for _, acc := range targets {
  55. s.notifyAccountSchedulingBlocked(acc, time.Time{}, openAITeamLinkedErrorBlockReason)
  56. }
  57. errorMsg := fmt.Sprintf("Workspace deactivated (402): team-linked error triggered by account #%d", account.ID)
  58. marked := 0
  59. for _, acc := range targets {
  60. // 单账户写入失败不中断其余账户;进程内熔断已先行,且该账户仍为 active,
  61. // 下一个 402 在去重 TTL 过期后会重新触发 fan-out。
  62. if err := s.accountRepo.SetError(opCtx, acc.ID, errorMsg); err != nil {
  63. slog.Warn("openai_team_linked_error_set_error_failed", "account_id", acc.ID, "error", err)
  64. continue
  65. }
  66. marked++
  67. }
  68. slog.Warn("openai_team_linked_error_fanout",
  69. "trigger_account_id", account.ID,
  70. "chatgpt_account_id", teamID,
  71. "affected", marked,
  72. "targets", len(targets),
  73. )
  74. }
  75. // markOpenAITeamLinkedFired 以 teamID 为键做进程内去重:TTL 内同一 Team 只允许一次 fan-out。
  76. func (s *RateLimitService) markOpenAITeamLinkedFired(teamID string) bool {
  77. now := time.Now()
  78. s.openaiTeamLinkedMu.Lock()
  79. defer s.openaiTeamLinkedMu.Unlock()
  80. if expiry, ok := s.openaiTeamLinkedRecent[teamID]; ok && expiry.After(now) {
  81. return false
  82. }
  83. if s.openaiTeamLinkedRecent == nil {
  84. s.openaiTeamLinkedRecent = make(map[string]time.Time)
  85. }
  86. for k, v := range s.openaiTeamLinkedRecent {
  87. if !v.After(now) {
  88. delete(s.openaiTeamLinkedRecent, k)
  89. }
  90. }
  91. s.openaiTeamLinkedRecent[teamID] = now.Add(openAITeamLinkedErrorDedupTTL)
  92. return true
  93. }