openai_quota_auto_reset_test.go 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336
  1. package service
  2. import (
  3. "context"
  4. "encoding/json"
  5. "sync"
  6. "sync/atomic"
  7. "testing"
  8. "time"
  9. "github.com/stretchr/testify/require"
  10. )
  11. func TestNormalizeOpenAIAutoResetCreditExtra(t *testing.T) {
  12. t.Run("历史账号默认关闭", func(t *testing.T) {
  13. account := &Account{Platform: PlatformOpenAI, Type: AccountTypeOAuth}
  14. config := ResolveOpenAIAutoResetCreditConfig(account)
  15. require.False(t, config.Enabled)
  16. require.Equal(t, 1.0, config.Threshold5h)
  17. require.Equal(t, 1.0, config.Threshold7d)
  18. })
  19. t.Run("开启时补齐两个百分百阈值并剥离运行态", func(t *testing.T) {
  20. extra, err := normalizeOpenAIAutoResetCreditExtra(PlatformOpenAI, AccountTypeOAuth, false, map[string]any{
  21. OpenAIAutoResetCreditEnabledExtraKey: true,
  22. OpenAIAutoResetCreditStateExtraKey: map[string]any{"status": "success"},
  23. })
  24. require.NoError(t, err)
  25. require.Equal(t, 1.0, extra[OpenAIAutoResetCredit5hThresholdExtraKey])
  26. require.Equal(t, 1.0, extra[OpenAIAutoResetCredit7dThresholdExtraKey])
  27. require.NotContains(t, extra, OpenAIAutoResetCreditStateExtraKey)
  28. })
  29. t.Run("阈值和账号类型严格校验", func(t *testing.T) {
  30. _, err := normalizeOpenAIAutoResetCreditExtra(PlatformOpenAI, AccountTypeOAuth, false, map[string]any{
  31. OpenAIAutoResetCreditEnabledExtraKey: true,
  32. OpenAIAutoResetCredit5hThresholdExtraKey: 0.0009,
  33. })
  34. require.Error(t, err)
  35. _, err = normalizeOpenAIAutoResetCreditExtra(PlatformOpenAI, AccountTypeOAuth, true, map[string]any{
  36. OpenAIAutoResetCreditEnabledExtraKey: true,
  37. })
  38. require.Error(t, err)
  39. })
  40. }
  41. func TestShouldAutoPauseOpenAIAccountByQuota_AutoResetCreditStates(t *testing.T) {
  42. now := time.Now().UTC()
  43. baseExtra := map[string]any{
  44. OpenAIAutoResetCreditEnabledExtraKey: true,
  45. OpenAIAutoResetCredit5hThresholdExtraKey: 1.0,
  46. OpenAIAutoResetCredit7dThresholdExtraKey: 1.0,
  47. "auto_pause_5h_threshold": 0.8,
  48. "auto_pause_7d_disabled": true,
  49. "codex_5h_used_percent": 90.0,
  50. "codex_usage_updated_at": now.Format(time.RFC3339),
  51. "codex_5h_reset_at": now.Add(time.Hour).Format(time.RFC3339),
  52. }
  53. t.Run("卡状态未知时暂停并触发异步查询", func(t *testing.T) {
  54. account := &Account{ID: 1, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Extra: cloneOpenAIAutoResetExtra(baseExtra)}
  55. paused, decision := shouldAutoPauseOpenAIAccountByQuota(context.Background(), account)
  56. require.True(t, paused)
  57. require.Equal(t, "quota_auto_reset_credit_check_5h", decision.reason)
  58. })
  59. t.Run("明确有卡时允许继续到用卡阈值", func(t *testing.T) {
  60. extra := cloneOpenAIAutoResetExtra(baseExtra)
  61. extra[OpenAIAutoResetCreditStateExtraKey] = OpenAIAutoResetCreditState{
  62. Status: OpenAIAutoResetStatusAvailable, AvailableCount: 1, CheckedAt: now.Format(time.RFC3339),
  63. }
  64. account := &Account{ID: 2, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Extra: extra}
  65. paused, _ := shouldAutoPauseOpenAIAccountByQuota(context.Background(), account)
  66. require.False(t, paused)
  67. })
  68. t.Run("达到用卡阈值后即使有卡也退出调度", func(t *testing.T) {
  69. extra := cloneOpenAIAutoResetExtra(baseExtra)
  70. extra["codex_5h_used_percent"] = 100.0
  71. extra[OpenAIAutoResetCreditStateExtraKey] = OpenAIAutoResetCreditState{
  72. Status: OpenAIAutoResetStatusAvailable, AvailableCount: 1, CheckedAt: now.Format(time.RFC3339),
  73. }
  74. account := &Account{ID: 3, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Extra: extra}
  75. paused, decision := shouldAutoPauseOpenAIAccountByQuota(context.Background(), account)
  76. require.True(t, paused)
  77. require.Equal(t, "quota_auto_reset_pending_5h", decision.reason)
  78. })
  79. t.Run("自然窗口重置后清除动态阻塞", func(t *testing.T) {
  80. extra := cloneOpenAIAutoResetExtra(baseExtra)
  81. extra["codex_5h_used_percent"] = 100.0
  82. extra["codex_5h_reset_at"] = now.Add(-time.Second).Format(time.RFC3339)
  83. extra[OpenAIAutoResetCreditStateExtraKey] = OpenAIAutoResetCreditState{
  84. Status: OpenAIAutoResetStatusFailed, TriggerWindow: "5h", ErrorCode: "RESET_FAILED",
  85. }
  86. account := &Account{ID: 4, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Extra: extra}
  87. paused, _ := shouldAutoPauseOpenAIAccountByQuota(context.Background(), account)
  88. require.False(t, paused)
  89. })
  90. }
  91. func TestSelectOpenAIAutoResetCandidate_FailsClosed(t *testing.T) {
  92. candidates := []openAIAutoResetCreditCandidate{
  93. {ID: "later", ExpiresAt: "2026-09-02T00:00:00Z"},
  94. {ID: "earlier", ExpiresAt: "2026-09-01T00:00:00Z"},
  95. }
  96. selected, err := selectOpenAIAutoResetCandidate(candidates, 2, nil, "cycle-a")
  97. require.NoError(t, err)
  98. require.Equal(t, "earlier", selected.ID)
  99. _, err = selectOpenAIAutoResetCandidate([]openAIAutoResetCreditCandidate{
  100. {ExpiresAt: "2026-09-01T00:00:00Z"},
  101. }, 1, nil, "cycle-a")
  102. require.Error(t, err)
  103. _, err = selectOpenAIAutoResetCandidate(candidates, 2, &OpenAIAutoResetCreditState{
  104. AttemptCycleHash: "cycle-a", AttemptCreditHash: shortOpenAIAutoResetHash("missing"),
  105. }, "cycle-a")
  106. require.Error(t, err, "模糊结果后原卡消失时不得切换下一张卡")
  107. }
  108. func TestOpenAIQuotaAutoResetService_AssessesIndependentWindows(t *testing.T) {
  109. service := &OpenAIQuotaAutoResetService{}
  110. account := &Account{Extra: map[string]any{
  111. "auto_pause_5h_disabled": true,
  112. "auto_pause_7d_disabled": true,
  113. }}
  114. config := OpenAIAutoResetCreditConfig{Enabled: true, Threshold5h: 0.8, Threshold7d: 0.9}
  115. tests := []struct {
  116. name string
  117. fiveHour float64
  118. sevenDay float64
  119. wantWindow string
  120. }{
  121. {name: "5h", fiveHour: 0.8, sevenDay: 0.2, wantWindow: "5h"},
  122. {name: "7d", fiveHour: 0.2, sevenDay: 0.9, wantWindow: "7d"},
  123. {name: "同时触发", fiveHour: 0.95, sevenDay: 0.95, wantWindow: "5h+7d"},
  124. }
  125. for _, test := range tests {
  126. t.Run(test.name, func(t *testing.T) {
  127. assessment := service.buildAssessment(account, config, test.fiveHour, test.sevenDay)
  128. require.True(t, assessment.resetReached)
  129. require.Equal(t, test.wantWindow, assessment.triggerWindow)
  130. })
  131. }
  132. }
  133. type autoResetTestAccountRepo struct {
  134. AccountRepository
  135. mu sync.Mutex
  136. account *Account
  137. }
  138. func (r *autoResetTestAccountRepo) GetByID(_ context.Context, id int64) (*Account, error) {
  139. r.mu.Lock()
  140. defer r.mu.Unlock()
  141. copy := *r.account
  142. copy.Extra = cloneOpenAIAutoResetExtra(r.account.Extra)
  143. return &copy, nil
  144. }
  145. func (r *autoResetTestAccountRepo) UpdateExtra(_ context.Context, id int64, updates map[string]any) error {
  146. r.mu.Lock()
  147. defer r.mu.Unlock()
  148. if r.account.Extra == nil {
  149. r.account.Extra = make(map[string]any)
  150. }
  151. for key, value := range updates {
  152. r.account.Extra[key] = value
  153. }
  154. return nil
  155. }
  156. type autoResetTestQuota struct {
  157. usage *OpenAIQuotaUsage
  158. resetCalls atomic.Int32
  159. resetEntered chan struct{}
  160. releaseReset chan struct{}
  161. enterOnce sync.Once
  162. mu sync.Mutex
  163. resetArgs [][2]string
  164. failFirst bool
  165. }
  166. func (q *autoResetTestQuota) QueryUsage(context.Context, int64) (*OpenAIQuotaUsage, error) {
  167. copy := *q.usage
  168. return &copy, nil
  169. }
  170. func (q *autoResetTestQuota) CacheResetCreditsSnapshot(context.Context, int64, *OpenAIRateLimitResetCredits) error {
  171. return nil
  172. }
  173. func (q *autoResetTestQuota) CachePostResetSnapshot(context.Context, int64, *OpenAIQuotaUsage) error {
  174. return nil
  175. }
  176. func (q *autoResetTestQuota) ResetCreditTargeted(_ context.Context, _ int64, creditID, redeemRequestID string) (*OpenAIQuotaResetResult, error) {
  177. if creditID == "" || redeemRequestID == "" {
  178. panic("targeted reset identifiers must be present")
  179. }
  180. call := q.resetCalls.Add(1)
  181. q.mu.Lock()
  182. q.resetArgs = append(q.resetArgs, [2]string{creditID, redeemRequestID})
  183. q.mu.Unlock()
  184. if q.failFirst && call == 1 {
  185. return nil, context.DeadlineExceeded
  186. }
  187. if q.resetEntered != nil {
  188. q.enterOnce.Do(func() { close(q.resetEntered) })
  189. }
  190. if q.releaseReset != nil {
  191. <-q.releaseReset
  192. }
  193. return &OpenAIQuotaResetResult{Code: "ok", WindowsReset: 2}, nil
  194. }
  195. type autoResetTestRecoverer struct{}
  196. func (autoResetTestRecoverer) RecoverAccountState(context.Context, int64, AccountRecoveryOptions) (*SuccessfulTestRecoveryResult, error) {
  197. return &SuccessfulTestRecoveryResult{ClearedRateLimit: true}, nil
  198. }
  199. func TestOpenAIQuotaAutoResetService_ConcurrentInstancesConsumeOnce(t *testing.T) {
  200. now := time.Now().UTC()
  201. account := &Account{
  202. ID: 99, Platform: PlatformOpenAI, Type: AccountTypeOAuth,
  203. Status: StatusActive, Schedulable: true,
  204. Extra: map[string]any{
  205. OpenAIAutoResetCreditEnabledExtraKey: true,
  206. OpenAIAutoResetCredit5hThresholdExtraKey: 1.0,
  207. OpenAIAutoResetCredit7dThresholdExtraKey: 1.0,
  208. "codex_5h_used_percent": 100.0,
  209. "codex_7d_used_percent": 10.0,
  210. "codex_usage_updated_at": now.Format(time.RFC3339),
  211. "codex_5h_reset_at": now.Add(time.Hour).Format(time.RFC3339),
  212. "codex_7d_reset_at": now.Add(24 * time.Hour).Format(time.RFC3339),
  213. },
  214. }
  215. repo := &autoResetTestAccountRepo{account: account}
  216. usage := &OpenAIQuotaUsage{
  217. FetchedAt: now.Unix(),
  218. RateLimit: &OpenAIRateLimit{
  219. PrimaryWindow: &OpenAIRateLimitWindow{UsedPercent: 100, LimitWindowSeconds: 5 * 60 * 60, ResetAfterSeconds: 3600, ResetAt: now.Add(time.Hour).Unix()},
  220. SecondaryWindow: &OpenAIRateLimitWindow{UsedPercent: 10, LimitWindowSeconds: 7 * 24 * 60 * 60, ResetAfterSeconds: 86400, ResetAt: now.Add(24 * time.Hour).Unix()},
  221. },
  222. RateLimitResetCredits: &OpenAIRateLimitResetCredits{
  223. AvailableCount: 1,
  224. Credits: []OpenAIRateLimitResetCreditDetail{{ExpiresAt: now.Add(48 * time.Hour).Format(time.RFC3339)}},
  225. },
  226. autoResetCandidates: []openAIAutoResetCreditCandidate{{ID: "credit-sensitive-id", ExpiresAt: now.Add(48 * time.Hour).Format(time.RFC3339)}},
  227. }
  228. quota := &autoResetTestQuota{usage: usage, resetEntered: make(chan struct{}), releaseReset: make(chan struct{})}
  229. idempotencyRepo := newInMemoryIdempotencyRepo()
  230. config := DefaultIdempotencyConfig()
  231. config.ObserveOnly = false
  232. config.ProcessingTimeout = time.Second
  233. serviceA := NewOpenAIQuotaAutoResetService(repo, quota, autoResetTestRecoverer{}, NewIdempotencyCoordinator(idempotencyRepo, config), nil, nil, nil)
  234. serviceB := NewOpenAIQuotaAutoResetService(repo, quota, autoResetTestRecoverer{}, NewIdempotencyCoordinator(idempotencyRepo, config), nil, nil, nil)
  235. var wg sync.WaitGroup
  236. wg.Add(2)
  237. go func() {
  238. defer wg.Done()
  239. _ = serviceA.evaluateAccount(context.Background(), account.ID)
  240. }()
  241. <-quota.resetEntered
  242. go func() {
  243. defer wg.Done()
  244. _ = serviceB.evaluateAccount(context.Background(), account.ID)
  245. }()
  246. time.Sleep(50 * time.Millisecond)
  247. close(quota.releaseReset)
  248. wg.Wait()
  249. require.Equal(t, int32(1), quota.resetCalls.Load())
  250. repo.mu.Lock()
  251. state := openAIAutoResetStateFromExtra(repo.account.Extra)
  252. repo.mu.Unlock()
  253. require.NotNil(t, state)
  254. require.Equal(t, OpenAIAutoResetStatusSuccess, state.Status)
  255. encodedState, err := json.Marshal(state)
  256. require.NoError(t, err)
  257. require.NotContains(t, string(encodedState), "credit-sensitive-id")
  258. }
  259. func TestOpenAIQuotaAutoResetService_TimeoutRetryReusesRequestBody(t *testing.T) {
  260. now := time.Now().UTC()
  261. account := &Account{
  262. ID: 100, Platform: PlatformOpenAI, Type: AccountTypeOAuth,
  263. Status: StatusActive, Schedulable: true,
  264. Extra: map[string]any{
  265. OpenAIAutoResetCreditEnabledExtraKey: true,
  266. OpenAIAutoResetCredit5hThresholdExtraKey: 1.0,
  267. OpenAIAutoResetCredit7dThresholdExtraKey: 1.0,
  268. "codex_5h_used_percent": 100.0,
  269. "codex_usage_updated_at": now.Format(time.RFC3339),
  270. "codex_5h_reset_at": now.Add(time.Hour).Format(time.RFC3339),
  271. },
  272. }
  273. repo := &autoResetTestAccountRepo{account: account}
  274. expiresAt := now.Add(48 * time.Hour).Format(time.RFC3339)
  275. quota := &autoResetTestQuota{
  276. failFirst: true,
  277. usage: &OpenAIQuotaUsage{
  278. FetchedAt: now.Unix(),
  279. RateLimit: &OpenAIRateLimit{
  280. PrimaryWindow: &OpenAIRateLimitWindow{UsedPercent: 100, LimitWindowSeconds: 5 * 60 * 60, ResetAfterSeconds: 3600, ResetAt: now.Add(time.Hour).Unix()},
  281. },
  282. RateLimitResetCredits: &OpenAIRateLimitResetCredits{
  283. AvailableCount: 1,
  284. Credits: []OpenAIRateLimitResetCreditDetail{{ExpiresAt: expiresAt}},
  285. },
  286. autoResetCandidates: []openAIAutoResetCreditCandidate{{ID: "retry-credit", ExpiresAt: expiresAt}},
  287. },
  288. }
  289. idempotencyConfig := DefaultIdempotencyConfig()
  290. idempotencyConfig.ObserveOnly = false
  291. idempotencyConfig.FailedRetryBackoff = 0
  292. service := NewOpenAIQuotaAutoResetService(
  293. repo,
  294. quota,
  295. autoResetTestRecoverer{},
  296. NewIdempotencyCoordinator(newInMemoryIdempotencyRepo(), idempotencyConfig),
  297. nil, nil, nil,
  298. )
  299. require.Error(t, service.evaluateAccount(context.Background(), account.ID))
  300. require.NoError(t, service.evaluateAccount(context.Background(), account.ID))
  301. quota.mu.Lock()
  302. args := append([][2]string(nil), quota.resetArgs...)
  303. quota.mu.Unlock()
  304. require.Len(t, args, 2)
  305. require.Equal(t, args[0], args[1], "超时重试必须复用相同 credit_id 与 redeem_request_id")
  306. }