openai_quota_auto_reset.go 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810
  1. package service
  2. import (
  3. "context"
  4. "crypto/sha256"
  5. "encoding/hex"
  6. "encoding/json"
  7. "errors"
  8. "fmt"
  9. "log/slog"
  10. "net/http"
  11. "sort"
  12. "strings"
  13. "sync"
  14. "time"
  15. infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
  16. "github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
  17. "github.com/google/uuid"
  18. )
  19. const (
  20. openAIAutoResetScanInterval = time.Minute
  21. openAIAutoResetSnapshotTTL = openAIProbeCacheTTL
  22. openAIAutoResetBatchSize = 100
  23. openAIAutoResetWorkerCount = 4
  24. openAIAutoResetQueueCapacity = 1024
  25. openAIAutoResetAttemptTTL = 8 * 24 * time.Hour
  26. openAIAutoResetLeaderLockKey = "jobs:openai-auto-reset-credit"
  27. )
  28. const (
  29. OpenAIAutoResetStatusChecking = "checking"
  30. OpenAIAutoResetStatusAvailable = "available"
  31. OpenAIAutoResetStatusResetting = "resetting"
  32. OpenAIAutoResetStatusSuccess = "success"
  33. OpenAIAutoResetStatusNoCredit = "no_credit"
  34. OpenAIAutoResetStatusFailed = "failed"
  35. )
  36. // OpenAIAutoResetCreditState 是可返回管理端的脱敏运行态。Attempt* 仅保存不可逆
  37. // 指纹,用于重启后拒绝切换到另一张卡;不会保存卡 ID 或兑换 ID。
  38. type OpenAIAutoResetCreditState struct {
  39. Status string `json:"status"`
  40. TriggerWindow string `json:"trigger_window,omitempty"`
  41. AvailableCount int `json:"available_count"`
  42. CheckedAt string `json:"checked_at,omitempty"`
  43. LastResultAt string `json:"last_result_at,omitempty"`
  44. ErrorCode string `json:"error_code,omitempty"`
  45. AttemptCycleHash string `json:"attempt_cycle_hash,omitempty"`
  46. AttemptCreditHash string `json:"attempt_credit_hash,omitempty"`
  47. }
  48. type openAIAutoResetQuota interface {
  49. QueryUsage(ctx context.Context, accountID int64) (*OpenAIQuotaUsage, error)
  50. CacheResetCreditsSnapshot(ctx context.Context, accountID int64, credits *OpenAIRateLimitResetCredits) error
  51. CachePostResetSnapshot(ctx context.Context, accountID int64, usage *OpenAIQuotaUsage) error
  52. ResetCreditTargeted(ctx context.Context, accountID int64, creditID, redeemRequestID string) (*OpenAIQuotaResetResult, error)
  53. }
  54. type openAIAutoResetContextKey struct{}
  55. func withOpenAIAutoResetContext(ctx context.Context) context.Context {
  56. if ctx == nil {
  57. ctx = context.Background()
  58. }
  59. return context.WithValue(ctx, openAIAutoResetContextKey{}, true)
  60. }
  61. func isOpenAIAutoResetContext(ctx context.Context) bool {
  62. if ctx == nil {
  63. return false
  64. }
  65. value, _ := ctx.Value(openAIAutoResetContextKey{}).(bool)
  66. return value
  67. }
  68. type openAIAutoResetRecovery interface {
  69. RecoverAccountState(ctx context.Context, accountID int64, options AccountRecoveryOptions) (*SuccessfulTestRecoveryResult, error)
  70. }
  71. // OpenAIQuotaAutoResetService 通过小型去重队列承接实时信号,并用分钟扫描补偿
  72. // 重启、漏事件和多实例读取;真正消费仍由 PostgreSQL 幂等记录串行化。
  73. type OpenAIQuotaAutoResetService struct {
  74. accountRepo AccountRepository
  75. quota openAIAutoResetQuota
  76. recoverer openAIAutoResetRecovery
  77. idempotency *IdempotencyCoordinator
  78. audit *AuditLogService
  79. settings *SettingService
  80. leaderLock LeaderLockCache
  81. ctx context.Context
  82. cancel context.CancelFunc
  83. queue chan int64
  84. pending sync.Map
  85. owner string
  86. start sync.Once
  87. stop sync.Once
  88. wg sync.WaitGroup
  89. }
  90. func NewOpenAIQuotaAutoResetService(
  91. accountRepo AccountRepository,
  92. quota openAIAutoResetQuota,
  93. recoverer openAIAutoResetRecovery,
  94. idempotency *IdempotencyCoordinator,
  95. audit *AuditLogService,
  96. settings *SettingService,
  97. leaderLock LeaderLockCache,
  98. ) *OpenAIQuotaAutoResetService {
  99. ctx, cancel := context.WithCancel(context.Background())
  100. return &OpenAIQuotaAutoResetService{
  101. accountRepo: accountRepo,
  102. quota: quota,
  103. recoverer: recoverer,
  104. idempotency: idempotency,
  105. audit: audit,
  106. settings: settings,
  107. leaderLock: leaderLock,
  108. ctx: ctx,
  109. cancel: cancel,
  110. queue: make(chan int64, openAIAutoResetQueueCapacity),
  111. owner: uuid.NewString(),
  112. }
  113. }
  114. func (s *OpenAIQuotaAutoResetService) Start() {
  115. if s == nil || s.accountRepo == nil || s.quota == nil || s.idempotency == nil {
  116. return
  117. }
  118. s.start.Do(func() {
  119. setOpenAIAutoResetNotifier(s)
  120. for range openAIAutoResetWorkerCount {
  121. s.wg.Add(1)
  122. go s.runWorker()
  123. }
  124. s.wg.Add(1)
  125. go s.runScanner()
  126. })
  127. }
  128. func (s *OpenAIQuotaAutoResetService) Stop() {
  129. if s == nil {
  130. return
  131. }
  132. s.stop.Do(func() {
  133. clearOpenAIAutoResetNotifier(s)
  134. s.cancel()
  135. s.wg.Wait()
  136. })
  137. }
  138. // Notify 是请求热路径的非阻塞入口。同一账号尚在队列时只保留一个任务;队列
  139. // 满时丢弃本次信号,分钟扫描仍会补偿,因此不会反向拖慢网关请求。
  140. func (s *OpenAIQuotaAutoResetService) Notify(accountID int64) {
  141. if s == nil || accountID <= 0 {
  142. return
  143. }
  144. if _, loaded := s.pending.LoadOrStore(accountID, struct{}{}); loaded {
  145. return
  146. }
  147. select {
  148. case <-s.ctx.Done():
  149. s.pending.Delete(accountID)
  150. case s.queue <- accountID:
  151. default:
  152. s.pending.Delete(accountID)
  153. slog.Warn("openai_auto_reset_queue_full", "account_id", accountID)
  154. }
  155. }
  156. func (s *OpenAIQuotaAutoResetService) runWorker() {
  157. defer s.wg.Done()
  158. for {
  159. select {
  160. case <-s.ctx.Done():
  161. return
  162. case accountID := <-s.queue:
  163. ctx, cancel := context.WithTimeout(s.ctx, 50*time.Second)
  164. if err := s.evaluateAccount(ctx, accountID); err != nil && !errors.Is(err, context.Canceled) {
  165. slog.Warn("openai_auto_reset_evaluate_failed", "account_id", accountID, "error_code", infraerrors.Reason(err))
  166. }
  167. cancel()
  168. s.pending.Delete(accountID)
  169. }
  170. }
  171. }
  172. func (s *OpenAIQuotaAutoResetService) runScanner() {
  173. defer s.wg.Done()
  174. timer := time.NewTimer(2 * time.Second)
  175. defer timer.Stop()
  176. select {
  177. case <-s.ctx.Done():
  178. return
  179. case <-timer.C:
  180. s.scanEnabledAccounts(s.ctx)
  181. }
  182. ticker := time.NewTicker(openAIAutoResetScanInterval)
  183. defer ticker.Stop()
  184. for {
  185. select {
  186. case <-s.ctx.Done():
  187. return
  188. case <-ticker.C:
  189. s.scanEnabledAccounts(s.ctx)
  190. }
  191. }
  192. }
  193. func (s *OpenAIQuotaAutoResetService) scanEnabledAccounts(ctx context.Context) {
  194. release, scan := s.tryAcquireScanLock(ctx)
  195. if !scan {
  196. return
  197. }
  198. if release != nil {
  199. defer release()
  200. }
  201. for page := 1; ; page++ {
  202. accounts, pageInfo, err := s.accountRepo.ListWithFilters(ctx, pagination.PaginationParams{
  203. Page: page, PageSize: openAIAutoResetBatchSize,
  204. }, PlatformOpenAI, AccountTypeOAuth, StatusActive, "", 0, "")
  205. if err != nil {
  206. slog.Warn("openai_auto_reset_scan_failed", "page", page, "error", err)
  207. return
  208. }
  209. for i := range accounts {
  210. account := &accounts[i]
  211. if account.Schedulable && ResolveOpenAIAutoResetCreditConfig(account).Enabled {
  212. s.Notify(account.ID)
  213. }
  214. }
  215. if len(accounts) < openAIAutoResetBatchSize || pageInfo == nil || page >= pageInfo.Pages {
  216. return
  217. }
  218. }
  219. }
  220. // Redis 锁异常时允许重复扫描,避免协调设施故障导致所有实例同时停止补偿;
  221. // 消费唯一性由数据库幂等记录负责,扫描锁只用于削减重复查询。
  222. func (s *OpenAIQuotaAutoResetService) tryAcquireScanLock(ctx context.Context) (func(), bool) {
  223. if s.leaderLock == nil {
  224. return func() {}, true
  225. }
  226. ok, err := s.leaderLock.TryAcquireLeaderLock(ctx, openAIAutoResetLeaderLockKey, s.owner, 55*time.Second)
  227. if err != nil {
  228. slog.Warn("openai_auto_reset_leader_lock_unavailable", "error", err)
  229. return func() {}, true
  230. }
  231. if !ok {
  232. return nil, false
  233. }
  234. return func() {
  235. releaseCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
  236. defer cancel()
  237. _ = s.leaderLock.ReleaseLeaderLock(releaseCtx, openAIAutoResetLeaderLockKey, s.owner)
  238. }, true
  239. }
  240. type openAIAutoResetAssessment struct {
  241. triggerWindow string
  242. resetReached bool
  243. pauseReached bool
  244. utilization5h float64
  245. utilization7d float64
  246. threshold5h float64
  247. threshold7d float64
  248. }
  249. func (s *OpenAIQuotaAutoResetService) evaluateAccount(ctx context.Context, accountID int64) error {
  250. ctx = withOpenAIAutoResetContext(ctx)
  251. account, err := s.accountRepo.GetByID(ctx, accountID)
  252. if err != nil || account == nil {
  253. return err
  254. }
  255. if account.IsShadow() {
  256. if account.ParentAccountID != nil {
  257. s.Notify(*account.ParentAccountID)
  258. }
  259. return nil
  260. }
  261. config := ResolveOpenAIAutoResetCreditConfig(account)
  262. if !config.Enabled || !account.IsActive() || !account.Schedulable {
  263. return nil
  264. }
  265. now := time.Now()
  266. assessment := s.assessExtra(account, config, now)
  267. state := openAIAutoResetStateFromExtra(account.Extra)
  268. needsQuery := openAIAutoResetSnapshotStale(account.Extra, now) || assessment.resetReached
  269. if assessment.pauseReached && !assessment.resetReached {
  270. needsQuery = needsQuery || state == nil || state.Status == OpenAIAutoResetStatusChecking || state.Status == OpenAIAutoResetStatusFailed || openAIAutoResetStateStale(state, now)
  271. }
  272. if !needsQuery {
  273. if !assessment.pauseReached && state != nil && state.TriggerWindow != "" {
  274. state.TriggerWindow = ""
  275. state.ErrorCode = ""
  276. state.CheckedAt = now.UTC().Format(time.RFC3339)
  277. if state.AvailableCount > 0 {
  278. state.Status = OpenAIAutoResetStatusAvailable
  279. } else {
  280. state.Status = OpenAIAutoResetStatusNoCredit
  281. }
  282. return s.persistState(ctx, accountID, state)
  283. }
  284. return nil
  285. }
  286. checking := &OpenAIAutoResetCreditState{
  287. Status: OpenAIAutoResetStatusChecking,
  288. TriggerWindow: assessment.triggerWindow,
  289. AvailableCount: stateAvailableCount(state),
  290. CheckedAt: now.UTC().Format(time.RFC3339),
  291. }
  292. copyOpenAIAutoResetAttempt(checking, state)
  293. if err := s.persistState(ctx, accountID, checking); err != nil {
  294. return err
  295. }
  296. usage, err := s.quota.QueryUsage(ctx, accountID)
  297. if err != nil || usage == nil {
  298. return s.failState(ctx, accountID, checking, "RESET_CREDIT_QUERY_FAILED", err)
  299. }
  300. if err := s.persistFreshUsage(ctx, accountID, usage, now); err != nil {
  301. return s.failState(ctx, accountID, checking, "USAGE_SNAPSHOT_WRITE_FAILED", err)
  302. }
  303. if usage.RateLimitResetCredits == nil {
  304. return s.failState(ctx, accountID, checking, "RESET_CREDIT_DETAILS_UNAVAILABLE", nil)
  305. }
  306. // 查询期间管理员可能关闭开关;消费前重新读取账号,确保尚未发出的任务可取消。
  307. account, err = s.accountRepo.GetByID(ctx, accountID)
  308. if err != nil || account == nil {
  309. return err
  310. }
  311. config = ResolveOpenAIAutoResetCreditConfig(account)
  312. if !config.Enabled {
  313. return nil
  314. }
  315. assessment = s.assessUsage(usage, account, config, now)
  316. available := usage.RateLimitResetCredits.AvailableCount
  317. if !assessment.resetReached {
  318. status := OpenAIAutoResetStatusNoCredit
  319. if available > 0 {
  320. status = OpenAIAutoResetStatusAvailable
  321. }
  322. return s.persistState(ctx, accountID, &OpenAIAutoResetCreditState{
  323. Status: status,
  324. TriggerWindow: assessment.triggerWindow,
  325. AvailableCount: available,
  326. CheckedAt: now.UTC().Format(time.RFC3339),
  327. })
  328. }
  329. if available <= 0 {
  330. return s.persistState(ctx, accountID, &OpenAIAutoResetCreditState{
  331. Status: OpenAIAutoResetStatusNoCredit,
  332. TriggerWindow: assessment.triggerWindow,
  333. AvailableCount: 0,
  334. CheckedAt: now.UTC().Format(time.RFC3339),
  335. LastResultAt: now.UTC().Format(time.RFC3339),
  336. ErrorCode: "NO_RESET_CREDIT",
  337. })
  338. }
  339. cycleSeed := openAIAutoResetCycleSeed(usage)
  340. cycleHash := shortOpenAIAutoResetHash(cycleSeed)
  341. candidate, selectErr := selectOpenAIAutoResetCandidate(usage.autoResetCandidates, available, state, cycleHash)
  342. if selectErr != nil {
  343. failed := checking
  344. failed.AvailableCount = available
  345. failed.TriggerWindow = assessment.triggerWindow
  346. failed.AttemptCycleHash = cycleHash
  347. return s.failState(ctx, accountID, failed, infraerrors.Reason(selectErr), selectErr)
  348. }
  349. creditHash := shortOpenAIAutoResetHash(candidate.ID)
  350. stableKey := fmt.Sprintf("oarc:%d:%s:%s", accountID, creditHash, cycleHash)
  351. redeemRequestID := uuid.NewSHA1(uuid.NameSpaceURL, []byte(stableKey)).String()
  352. resetting := &OpenAIAutoResetCreditState{
  353. Status: OpenAIAutoResetStatusResetting,
  354. TriggerWindow: assessment.triggerWindow,
  355. AvailableCount: available,
  356. CheckedAt: now.UTC().Format(time.RFC3339),
  357. AttemptCycleHash: cycleHash,
  358. AttemptCreditHash: creditHash,
  359. }
  360. if err := s.persistState(ctx, accountID, resetting); err != nil {
  361. return err
  362. }
  363. account, err = s.accountRepo.GetByID(ctx, accountID)
  364. if err != nil || account == nil || !ResolveOpenAIAutoResetCreditConfig(account).Enabled {
  365. return err
  366. }
  367. result, err := s.idempotency.Execute(ctx, IdempotencyExecuteOptions{
  368. Scope: "openai_auto_reset_credit",
  369. ActorScope: fmt.Sprintf("account:%d", accountID),
  370. Method: http.MethodPost,
  371. Route: "/system/openai/reset-credit/auto",
  372. IdempotencyKey: stableKey,
  373. Payload: map[string]any{
  374. "account_id": accountID,
  375. "credit_hash": creditHash,
  376. "cycle_hash": cycleHash,
  377. },
  378. TTL: openAIAutoResetAttemptTTL,
  379. RequireKey: true,
  380. }, func(execCtx context.Context) (any, error) {
  381. resetResult, resetErr := s.quota.ResetCreditTargeted(execCtx, accountID, candidate.ID, redeemRequestID)
  382. if resetErr != nil {
  383. return nil, resetErr
  384. }
  385. if resetResult == nil {
  386. return nil, infraerrors.InternalServer("OPENAI_AUTO_RESET_EMPTY_RESULT", "automatic reset returned an empty result")
  387. }
  388. // 幂等表只保存脱敏结果,避免上游返回的卡 ID 被持久化到响应体列。
  389. return openAIAutoResetConsumeResult{Code: resetResult.Code, WindowsReset: resetResult.WindowsReset}, nil
  390. })
  391. if err != nil {
  392. // 另一个实例已持有同一周期的兑换时保持 resetting,等待下一轮读取同一
  393. // 幂等结果;不能把并发冲突误报成上游消费失败,更不能改选下一张卡。
  394. reason := infraerrors.Reason(err)
  395. if reason == infraerrors.Reason(ErrIdempotencyInProgress) || reason == infraerrors.Reason(ErrIdempotencyRetryBackoff) {
  396. return nil
  397. }
  398. s.recordAudit(accountID, assessment, available, "failed", 0, infraerrors.Reason(err))
  399. return s.failState(ctx, accountID, resetting, infraerrors.Reason(err), err)
  400. }
  401. consumeResult := decodeOpenAIAutoResetConsumeResult(result.Data)
  402. if strings.EqualFold(strings.TrimSpace(consumeResult.Code), "no_credit") {
  403. noCreditAt := time.Now().UTC().Format(time.RFC3339)
  404. noCredit := &OpenAIAutoResetCreditState{
  405. Status: OpenAIAutoResetStatusNoCredit,
  406. TriggerWindow: assessment.triggerWindow,
  407. AvailableCount: 0,
  408. CheckedAt: noCreditAt,
  409. LastResultAt: noCreditAt,
  410. ErrorCode: "NO_RESET_CREDIT",
  411. AttemptCycleHash: cycleHash,
  412. AttemptCreditHash: creditHash,
  413. }
  414. s.recordAudit(accountID, assessment, available, "no_credit", 0, noCredit.ErrorCode)
  415. return s.persistState(ctx, accountID, noCredit)
  416. }
  417. postCtx, cancelPost := context.WithTimeout(context.WithoutCancel(ctx), 8*time.Second)
  418. post := RunOpenAIQuotaResetPostProcess(postCtx, accountID, s.quota, s.recoverer, s.accountRepo.GetByID)
  419. cancelPost()
  420. if !post.AccountStateRecovered || post.WarningCode != "" {
  421. code := post.WarningCode
  422. if code == "" {
  423. code = OpenAIQuotaResetWarningAccountRecoveryFailed
  424. }
  425. s.recordAudit(accountID, assessment, available, "recovery_failed", consumeResult.WindowsReset, code)
  426. return s.failState(ctx, accountID, resetting, code, nil)
  427. }
  428. successAt := time.Now().UTC().Format(time.RFC3339)
  429. success := &OpenAIAutoResetCreditState{
  430. Status: OpenAIAutoResetStatusSuccess,
  431. TriggerWindow: assessment.triggerWindow,
  432. AvailableCount: max(0, available-1),
  433. CheckedAt: successAt,
  434. LastResultAt: successAt,
  435. AttemptCycleHash: cycleHash,
  436. AttemptCreditHash: creditHash,
  437. }
  438. if post.Quota != nil && post.Quota.RateLimitResetCredits != nil {
  439. success.AvailableCount = post.Quota.RateLimitResetCredits.AvailableCount
  440. }
  441. if err := s.persistState(ctx, accountID, success); err != nil {
  442. return err
  443. }
  444. s.recordAudit(accountID, assessment, available, "success", consumeResult.WindowsReset, "")
  445. slog.Info("openai_auto_reset_credit_success",
  446. "account_id", accountID,
  447. "trigger_window", assessment.triggerWindow,
  448. "threshold_5h", assessment.threshold5h,
  449. "threshold_7d", assessment.threshold7d,
  450. "utilization_5h", assessment.utilization5h,
  451. "utilization_7d", assessment.utilization7d,
  452. "windows_reset", consumeResult.WindowsReset,
  453. )
  454. return nil
  455. }
  456. type openAIAutoResetConsumeResult struct {
  457. Code string `json:"code"`
  458. WindowsReset int `json:"windows_reset"`
  459. }
  460. func decodeOpenAIAutoResetConsumeResult(value any) openAIAutoResetConsumeResult {
  461. if typed, ok := value.(openAIAutoResetConsumeResult); ok {
  462. return typed
  463. }
  464. raw, _ := json.Marshal(value)
  465. var decoded openAIAutoResetConsumeResult
  466. _ = json.Unmarshal(raw, &decoded)
  467. return decoded
  468. }
  469. func (s *OpenAIQuotaAutoResetService) assessExtra(account *Account, config OpenAIAutoResetCreditConfig, now time.Time) openAIAutoResetAssessment {
  470. utilization5h, _ := resolveOpenAIQuotaUtilization(account.Extra, "5h", now)
  471. utilization7d, _ := resolveOpenAIQuotaUtilization(account.Extra, "7d", now)
  472. return s.buildAssessment(account, config, utilization5h, utilization7d)
  473. }
  474. func (s *OpenAIQuotaAutoResetService) assessUsage(usage *OpenAIQuotaUsage, account *Account, config OpenAIAutoResetCreditConfig, now time.Time) openAIAutoResetAssessment {
  475. updates := buildOpenAIAutoResetUsageUpdates(usage, now)
  476. utilization5h := readOpenAIQuotaUsedPercent(updates, "5h") / 100
  477. utilization7d := readOpenAIQuotaUsedPercent(updates, "7d") / 100
  478. return s.buildAssessment(account, config, utilization5h, utilization7d)
  479. }
  480. func (s *OpenAIQuotaAutoResetService) buildAssessment(account *Account, config OpenAIAutoResetCreditConfig, utilization5h, utilization7d float64) openAIAutoResetAssessment {
  481. assessment := openAIAutoResetAssessment{
  482. utilization5h: utilization5h,
  483. utilization7d: utilization7d,
  484. threshold5h: config.Threshold5h,
  485. threshold7d: config.Threshold7d,
  486. }
  487. reset5h := utilization5h >= config.Threshold5h
  488. reset7d := utilization7d >= config.Threshold7d
  489. assessment.resetReached = reset5h || reset7d
  490. assessment.triggerWindow = joinOpenAIAutoResetWindows(reset5h, reset7d)
  491. pause5h, pause7d := resolveOpenAIQuotaAutoPauseThresholds(context.Background(), account)
  492. if s.settings != nil {
  493. pause5h, pause7d = resolveOpenAIQuotaAutoPauseThresholds(
  494. withOpenAIQuotaAutoPauseSettings(context.Background(), s.settings.GetOpenAIQuotaAutoPauseSettings(context.Background())),
  495. account,
  496. )
  497. }
  498. pauseReached5h := !resolveAccountExtraBool(account.Extra, "auto_pause_5h_disabled") && pause5h > 0 && utilization5h >= pause5h
  499. pauseReached7d := !resolveAccountExtraBool(account.Extra, "auto_pause_7d_disabled") && pause7d > 0 && utilization7d >= pause7d
  500. assessment.pauseReached = pauseReached5h || pauseReached7d || assessment.resetReached
  501. if assessment.triggerWindow == "" {
  502. assessment.triggerWindow = joinOpenAIAutoResetWindows(pauseReached5h, pauseReached7d)
  503. }
  504. return assessment
  505. }
  506. func joinOpenAIAutoResetWindows(fiveHour, sevenDay bool) string {
  507. switch {
  508. case fiveHour && sevenDay:
  509. return "5h+7d"
  510. case fiveHour:
  511. return "5h"
  512. case sevenDay:
  513. return "7d"
  514. default:
  515. return ""
  516. }
  517. }
  518. func buildOpenAIAutoResetUsageUpdates(usage *OpenAIQuotaUsage, now time.Time) map[string]any {
  519. if usage == nil || usage.RateLimit == nil {
  520. return nil
  521. }
  522. rateLimit := usage.RateLimit
  523. snapshot := &OpenAICodexUsageSnapshot{UpdatedAt: now.UTC().Format(time.RFC3339)}
  524. applyWindow := func(window *OpenAIRateLimitWindow, primary bool) {
  525. if window == nil {
  526. return
  527. }
  528. used := window.UsedPercent
  529. resetAfter := int(window.ResetAfterSeconds)
  530. windowMinutes := int(window.LimitWindowSeconds / 60)
  531. if primary {
  532. snapshot.PrimaryUsedPercent = &used
  533. snapshot.PrimaryResetAfterSeconds = &resetAfter
  534. snapshot.PrimaryWindowMinutes = &windowMinutes
  535. } else {
  536. snapshot.SecondaryUsedPercent = &used
  537. snapshot.SecondaryResetAfterSeconds = &resetAfter
  538. snapshot.SecondaryWindowMinutes = &windowMinutes
  539. }
  540. }
  541. applyWindow(rateLimit.PrimaryWindow, true)
  542. applyWindow(rateLimit.SecondaryWindow, false)
  543. return buildCodexUsageExtraUpdates(snapshot, now)
  544. }
  545. func (s *OpenAIQuotaAutoResetService) persistFreshUsage(ctx context.Context, accountID int64, usage *OpenAIQuotaUsage, now time.Time) error {
  546. updates := buildOpenAIAutoResetUsageUpdates(usage, now)
  547. if len(updates) > 0 {
  548. if err := s.accountRepo.UpdateExtra(ctx, accountID, updates); err != nil {
  549. return err
  550. }
  551. }
  552. return s.quota.CacheResetCreditsSnapshot(ctx, accountID, usage.RateLimitResetCredits)
  553. }
  554. func selectOpenAIAutoResetCandidate(candidates []openAIAutoResetCreditCandidate, available int, previous *OpenAIAutoResetCreditState, cycleHash string) (openAIAutoResetCreditCandidate, error) {
  555. if available <= 0 {
  556. return openAIAutoResetCreditCandidate{}, infraerrors.Conflict("OPENAI_AUTO_RESET_NO_CREDIT", "no reset credit is available")
  557. }
  558. if len(candidates) < available {
  559. return openAIAutoResetCreditCandidate{}, infraerrors.Conflict("OPENAI_AUTO_RESET_CREDIT_DETAILS_INCOMPLETE", "reset credit details are incomplete")
  560. }
  561. for _, candidate := range candidates {
  562. if _, err := time.Parse(time.RFC3339, candidate.ExpiresAt); err != nil {
  563. return openAIAutoResetCreditCandidate{}, infraerrors.Conflict("OPENAI_AUTO_RESET_CREDIT_EXPIRY_INVALID", "reset credit expiration is invalid")
  564. }
  565. }
  566. sorted := append([]openAIAutoResetCreditCandidate(nil), candidates...)
  567. sort.SliceStable(sorted, func(i, j int) bool {
  568. left, leftErr := time.Parse(time.RFC3339, sorted[i].ExpiresAt)
  569. right, rightErr := time.Parse(time.RFC3339, sorted[j].ExpiresAt)
  570. if leftErr != nil {
  571. return false
  572. }
  573. if rightErr != nil {
  574. return true
  575. }
  576. return left.Before(right)
  577. })
  578. if previous != nil && previous.AttemptCycleHash == cycleHash && previous.AttemptCreditHash != "" {
  579. for _, candidate := range sorted {
  580. if shortOpenAIAutoResetHash(candidate.ID) == previous.AttemptCreditHash {
  581. if strings.TrimSpace(candidate.ID) == "" {
  582. break
  583. }
  584. return candidate, nil
  585. }
  586. }
  587. return openAIAutoResetCreditCandidate{}, infraerrors.Conflict("OPENAI_AUTO_RESET_ORIGINAL_CREDIT_UNAVAILABLE", "the original reset credit cannot be confirmed; refusing to switch credits")
  588. }
  589. if len(sorted) == 0 || strings.TrimSpace(sorted[0].ID) == "" {
  590. return openAIAutoResetCreditCandidate{}, infraerrors.Conflict("OPENAI_AUTO_RESET_CREDIT_ID_MISSING", "the earliest reset credit has no official id")
  591. }
  592. return sorted[0], nil
  593. }
  594. func openAIAutoResetCycleSeed(usage *OpenAIQuotaUsage) string {
  595. if usage == nil || usage.RateLimit == nil {
  596. return "5h:0|7d:0"
  597. }
  598. var fiveHour, sevenDay int64
  599. for _, window := range []*OpenAIRateLimitWindow{usage.RateLimit.PrimaryWindow, usage.RateLimit.SecondaryWindow} {
  600. if window == nil {
  601. continue
  602. }
  603. resetAt := window.ResetAt
  604. if resetAt <= 0 {
  605. resetAt = usage.FetchedAt + window.ResetAfterSeconds
  606. }
  607. if window.LimitWindowSeconds <= 6*60*60 {
  608. fiveHour = resetAt
  609. } else {
  610. sevenDay = resetAt
  611. }
  612. }
  613. return fmt.Sprintf("5h:%d|7d:%d", fiveHour, sevenDay)
  614. }
  615. func shortOpenAIAutoResetHash(value string) string {
  616. sum := sha256.Sum256([]byte(value))
  617. return hex.EncodeToString(sum[:12])
  618. }
  619. func openAIAutoResetSnapshotStale(extra map[string]any, now time.Time) bool {
  620. if len(extra) == 0 {
  621. return true
  622. }
  623. raw, ok := extra["codex_usage_updated_at"]
  624. if !ok {
  625. return true
  626. }
  627. updatedAt, err := parseTime(fmt.Sprint(raw))
  628. return err != nil || now.Sub(updatedAt) >= openAIAutoResetSnapshotTTL
  629. }
  630. func openAIAutoResetStateFromExtra(extra map[string]any) *OpenAIAutoResetCreditState {
  631. if len(extra) == 0 {
  632. return nil
  633. }
  634. raw, ok := extra[OpenAIAutoResetCreditStateExtraKey]
  635. if !ok || raw == nil {
  636. return nil
  637. }
  638. encoded, err := json.Marshal(raw)
  639. if err != nil {
  640. return nil
  641. }
  642. var state OpenAIAutoResetCreditState
  643. if err := json.Unmarshal(encoded, &state); err != nil || state.Status == "" {
  644. return nil
  645. }
  646. return &state
  647. }
  648. func openAIAutoResetStateStale(state *OpenAIAutoResetCreditState, now time.Time) bool {
  649. if state == nil || state.CheckedAt == "" {
  650. return true
  651. }
  652. checkedAt, err := time.Parse(time.RFC3339, state.CheckedAt)
  653. return err != nil || now.Sub(checkedAt) >= openAIAutoResetSnapshotTTL
  654. }
  655. func stateAvailableCount(state *OpenAIAutoResetCreditState) int {
  656. if state == nil {
  657. return 0
  658. }
  659. return state.AvailableCount
  660. }
  661. func copyOpenAIAutoResetAttempt(target, source *OpenAIAutoResetCreditState) {
  662. if target == nil || source == nil {
  663. return
  664. }
  665. target.AttemptCycleHash = source.AttemptCycleHash
  666. target.AttemptCreditHash = source.AttemptCreditHash
  667. }
  668. func (s *OpenAIQuotaAutoResetService) persistState(ctx context.Context, accountID int64, state *OpenAIAutoResetCreditState) error {
  669. if state == nil {
  670. return nil
  671. }
  672. return s.accountRepo.UpdateExtra(ctx, accountID, map[string]any{OpenAIAutoResetCreditStateExtraKey: state})
  673. }
  674. func (s *OpenAIQuotaAutoResetService) failState(ctx context.Context, accountID int64, state *OpenAIAutoResetCreditState, code string, cause error) error {
  675. if state == nil {
  676. state = &OpenAIAutoResetCreditState{}
  677. }
  678. if strings.TrimSpace(code) == "" {
  679. code = "OPENAI_AUTO_RESET_FAILED"
  680. }
  681. state.Status = OpenAIAutoResetStatusFailed
  682. state.ErrorCode = code
  683. state.LastResultAt = time.Now().UTC().Format(time.RFC3339)
  684. if err := s.persistState(ctx, accountID, state); err != nil {
  685. return err
  686. }
  687. slog.Warn("openai_auto_reset_credit_failed",
  688. "account_id", accountID,
  689. "trigger_window", state.TriggerWindow,
  690. "available_count", state.AvailableCount,
  691. "error_code", code,
  692. )
  693. if cause != nil {
  694. return cause
  695. }
  696. return infraerrors.Conflict(code, "automatic reset credit operation failed")
  697. }
  698. func (s *OpenAIQuotaAutoResetService) recordAudit(accountID int64, assessment openAIAutoResetAssessment, available int, resultCode string, windowsReset int, errorCode string) {
  699. if s.audit == nil {
  700. return
  701. }
  702. statusCode := http.StatusOK
  703. if resultCode != "success" {
  704. statusCode = http.StatusConflict
  705. }
  706. s.audit.Record(&AuditLog{
  707. ActorEmail: "system",
  708. ActorRole: "system",
  709. AuthMethod: "system",
  710. Action: "system.openai.reset_credit.auto",
  711. Method: "SYSTEM",
  712. Path: fmt.Sprintf("/system/openai/accounts/%d/auto-reset-credit", accountID),
  713. StatusCode: statusCode,
  714. Extra: map[string]any{
  715. "account_id": accountID,
  716. "trigger_window": assessment.triggerWindow,
  717. "threshold_5h": assessment.threshold5h,
  718. "threshold_7d": assessment.threshold7d,
  719. "utilization_5h": assessment.utilization5h,
  720. "utilization_7d": assessment.utilization7d,
  721. "available_count": available,
  722. "result_code": resultCode,
  723. "windows_reset": windowsReset,
  724. "error_code": errorCode,
  725. },
  726. })
  727. }
  728. var openAIAutoResetNotifierRegistry struct {
  729. sync.RWMutex
  730. service *OpenAIQuotaAutoResetService
  731. }
  732. func setOpenAIAutoResetNotifier(service *OpenAIQuotaAutoResetService) {
  733. openAIAutoResetNotifierRegistry.Lock()
  734. openAIAutoResetNotifierRegistry.service = service
  735. openAIAutoResetNotifierRegistry.Unlock()
  736. }
  737. func clearOpenAIAutoResetNotifier(service *OpenAIQuotaAutoResetService) {
  738. openAIAutoResetNotifierRegistry.Lock()
  739. if openAIAutoResetNotifierRegistry.service == service {
  740. openAIAutoResetNotifierRegistry.service = nil
  741. }
  742. openAIAutoResetNotifierRegistry.Unlock()
  743. }
  744. func notifyOpenAIAutoReset(accountID int64) {
  745. openAIAutoResetNotifierRegistry.RLock()
  746. service := openAIAutoResetNotifierRegistry.service
  747. openAIAutoResetNotifierRegistry.RUnlock()
  748. if service != nil {
  749. service.Notify(accountID)
  750. }
  751. }
  752. // NotifyOpenAIAutoResetCredit 供额度查询入口发送轻量信号;不执行同步上游请求。
  753. func NotifyOpenAIAutoResetCredit(accountID int64) {
  754. notifyOpenAIAutoReset(accountID)
  755. }