grok_import_probe.go 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234
  1. package admin
  2. import (
  3. "context"
  4. "log/slog"
  5. "sync"
  6. "time"
  7. "github.com/Wei-Shaw/sub2api/internal/config"
  8. infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
  9. "github.com/Wei-Shaw/sub2api/internal/service"
  10. )
  11. const (
  12. grokImportProbeConcurrency = 3
  13. grokImportProbeTimeout = 25 * time.Second
  14. grokImportProbeQueueLimit = 64
  15. )
  16. type grokImportProber interface {
  17. QueryQuota(ctx context.Context, accountID int64) (*service.GrokQuotaProbeResult, error)
  18. }
  19. type grokImportProbeTask struct {
  20. prober grokImportProber
  21. accountID int64
  22. }
  23. type grokImportProbeScheduler struct {
  24. mu sync.Mutex
  25. queue []grokImportProbeTask
  26. pending map[int64]struct{}
  27. inFlight map[int64]struct{}
  28. concurrency int
  29. workers int
  30. maxWorkers int
  31. timeout time.Duration
  32. }
  33. var defaultGrokImportProbeScheduler = newGrokImportProbeScheduler(
  34. grokImportProbeConcurrency,
  35. grokImportProbeTimeout,
  36. )
  37. func newGrokImportProbeScheduler(concurrency int, timeout time.Duration) *grokImportProbeScheduler {
  38. if concurrency <= 0 {
  39. concurrency = 1
  40. }
  41. if timeout <= 0 {
  42. timeout = grokImportProbeTimeout
  43. }
  44. return &grokImportProbeScheduler{
  45. concurrency: concurrency,
  46. timeout: timeout,
  47. pending: make(map[int64]struct{}),
  48. inFlight: make(map[int64]struct{}),
  49. }
  50. }
  51. func (s *grokImportProbeScheduler) schedule(prober grokImportProber, account *service.Account) {
  52. if s == nil || prober == nil || account == nil || account.ID <= 0 {
  53. return
  54. }
  55. if account.Platform != service.PlatformGrok || account.Type != service.AccountTypeOAuth {
  56. return
  57. }
  58. s.mu.Lock()
  59. if _, exists := s.pending[account.ID]; exists {
  60. s.mu.Unlock()
  61. return
  62. }
  63. if _, exists := s.inFlight[account.ID]; exists {
  64. s.mu.Unlock()
  65. return
  66. }
  67. if len(s.queue) >= grokImportProbeQueueLimit {
  68. s.mu.Unlock()
  69. slog.Debug("grok_import_active_probe_dropped", "account_id", account.ID, "reason", "queue_full")
  70. return
  71. }
  72. s.queue = append(s.queue, grokImportProbeTask{prober: prober, accountID: account.ID})
  73. s.pending[account.ID] = struct{}{}
  74. if s.workers < s.concurrency {
  75. s.workers++
  76. if s.workers > s.maxWorkers {
  77. s.maxWorkers = s.workers
  78. }
  79. go s.worker()
  80. }
  81. s.mu.Unlock()
  82. }
  83. func (s *grokImportProbeScheduler) worker() {
  84. for {
  85. task, ok := s.nextTask()
  86. if !ok {
  87. return
  88. }
  89. s.run(task.prober, task.accountID)
  90. s.finish(task.accountID)
  91. }
  92. }
  93. func (s *grokImportProbeScheduler) nextTask() (grokImportProbeTask, bool) {
  94. s.mu.Lock()
  95. defer s.mu.Unlock()
  96. if len(s.queue) == 0 {
  97. s.workers--
  98. return grokImportProbeTask{}, false
  99. }
  100. task := s.queue[0]
  101. s.queue[0] = grokImportProbeTask{}
  102. s.queue = s.queue[1:]
  103. if len(s.queue) == 0 {
  104. s.queue = nil
  105. }
  106. delete(s.pending, task.accountID)
  107. s.inFlight[task.accountID] = struct{}{}
  108. return task, true
  109. }
  110. func (s *grokImportProbeScheduler) finish(accountID int64) {
  111. s.mu.Lock()
  112. delete(s.inFlight, accountID)
  113. s.mu.Unlock()
  114. }
  115. func (s *grokImportProbeScheduler) run(prober grokImportProber, accountID int64) {
  116. defer func() {
  117. if recovered := recover(); recovered != nil {
  118. slog.Error(
  119. "grok_import_active_probe_panic",
  120. "account_id", accountID,
  121. "recovery_type", panicType(recovered),
  122. )
  123. }
  124. }()
  125. ctx, cancel := context.WithTimeout(context.Background(), s.timeout)
  126. defer cancel()
  127. result, err := prober.QueryQuota(ctx, accountID)
  128. if err != nil {
  129. slog.Warn(
  130. "grok_import_active_probe_failed",
  131. "account_id", accountID,
  132. "status", infraerrors.Code(err),
  133. "reason", infraerrors.Reason(err),
  134. )
  135. return
  136. }
  137. if result == nil {
  138. slog.Warn(
  139. "grok_import_active_probe_failed",
  140. "account_id", accountID,
  141. "reason", "empty_result",
  142. )
  143. return
  144. }
  145. slog.Info(
  146. "grok_import_active_probe_completed",
  147. "account_id", accountID,
  148. "model", result.Model,
  149. "status", result.StatusCode,
  150. "headers_observed", result.HeadersObserved,
  151. )
  152. }
  153. func panicType(value any) string {
  154. switch value.(type) {
  155. case string:
  156. return "string"
  157. case error:
  158. return "error"
  159. default:
  160. return "unknown"
  161. }
  162. }
  163. func (h *AccountHandler) scheduleGrokImportProbe(account *service.Account) {
  164. if h == nil {
  165. return
  166. }
  167. defaultGrokImportProbeScheduler.schedule(h.grokImportProber, account)
  168. }
  169. func (h *GrokOAuthHandler) scheduleGrokImportProbe(account *service.Account) {
  170. if h == nil {
  171. return
  172. }
  173. defaultGrokImportProbeScheduler.schedule(h.importProber, account)
  174. }
  175. // ProvideAccountHandler injects the Grok active prober for production while
  176. // keeping NewAccountHandler convenient for focused unit tests.
  177. func ProvideAccountHandler(
  178. cfg *config.Config,
  179. adminService service.AdminService,
  180. oauthService *service.OAuthService,
  181. openaiOAuthService *service.OpenAIOAuthService,
  182. geminiOAuthService *service.GeminiOAuthService,
  183. antigravityOAuthService *service.AntigravityOAuthService,
  184. grokOAuthService service.GrokOAuthTokenService,
  185. rateLimitService *service.RateLimitService,
  186. accountUsageService *service.AccountUsageService,
  187. accountTestService *service.AccountTestService,
  188. concurrencyService *service.ConcurrencyService,
  189. crsSyncService *service.CRSSyncService,
  190. sessionLimitCache service.SessionLimitCache,
  191. rpmCache service.RPMCache,
  192. tokenCacheInvalidator service.TokenCacheInvalidator,
  193. grokQuotaService *service.GrokQuotaService,
  194. ) *AccountHandler {
  195. handler := NewAccountHandler(
  196. adminService,
  197. oauthService,
  198. openaiOAuthService,
  199. geminiOAuthService,
  200. antigravityOAuthService,
  201. grokOAuthService,
  202. rateLimitService,
  203. accountUsageService,
  204. accountTestService,
  205. concurrencyService,
  206. crsSyncService,
  207. sessionLimitCache,
  208. rpmCache,
  209. tokenCacheInvalidator,
  210. )
  211. handler.grokImportProber = grokQuotaService
  212. handler.cfg = cfg
  213. return handler
  214. }